MCPcopy Create free account
hub / github.com/apache/arrow / CheckStreamingCompressor

Function CheckStreamingCompressor

cpp/src/arrow/util/compression_test.cc:110–173  ·  view source on GitHub ↗

Source from the content-addressed store, hash-verified

108// Check the streaming compressor against one-shot decompression
109
110void CheckStreamingCompressor(Codec* codec, const std::vector<uint8_t>& data) {
111 std::shared_ptr<Compressor> compressor;
112 ASSERT_OK_AND_ASSIGN(compressor, codec->MakeCompressor());
113
114 std::vector<uint8_t> compressed;
115 int64_t compressed_size = 0;
116 const uint8_t* input = data.data();
117 int64_t remaining = data.size();
118
119 compressed.resize(10);
120 bool do_flush = false;
121
122 while (remaining > 0) {
123 // Feed a small amount each time
124 int64_t input_len = std::min(remaining, static_cast<int64_t>(1111));
125 int64_t output_len = compressed.size() - compressed_size;
126 uint8_t* output = compressed.data() + compressed_size;
127 ASSERT_OK_AND_ASSIGN(auto result,
128 compressor->Compress(input_len, input, output_len, output));
129 ASSERT_LE(result.bytes_read, input_len);
130 ASSERT_LE(result.bytes_written, output_len);
131 compressed_size += result.bytes_written;
132 input += result.bytes_read;
133 remaining -= result.bytes_read;
134 if (result.bytes_read == 0) {
135 compressed.resize(compressed.capacity() * 2);
136 }
137 // Once every two iterations, do a flush
138 if (do_flush) {
139 Compressor::FlushResult result;
140 do {
141 output_len = compressed.size() - compressed_size;
142 output = compressed.data() + compressed_size;
143 ASSERT_OK_AND_ASSIGN(result, compressor->Flush(output_len, output));
144 ASSERT_LE(result.bytes_written, output_len);
145 compressed_size += result.bytes_written;
146 if (result.should_retry) {
147 compressed.resize(compressed.capacity() * 2);
148 }
149 } while (result.should_retry);
150 }
151 do_flush = !do_flush;
152 }
153
154 // End the compressed stream
155 Compressor::EndResult result;
156 do {
157 int64_t output_len = compressed.size() - compressed_size;
158 uint8_t* output = compressed.data() + compressed_size;
159 ASSERT_OK_AND_ASSIGN(result, compressor->End(output_len, output));
160 ASSERT_LE(result.bytes_written, output_len);
161 compressed_size += result.bytes_written;
162 if (result.should_retry) {
163 compressed.resize(compressed.capacity() * 2);
164 }
165 } while (result.should_retry);
166
167 // Check decompressing the compressed data

Callers 1

TEST_PFunction · 0.85

Calls 9

resizeMethod · 0.80
ASSERT_OK_AND_ASSIGNFunction · 0.70
MakeCompressorMethod · 0.45
dataMethod · 0.45
sizeMethod · 0.45
capacityMethod · 0.45
FlushMethod · 0.45
EndMethod · 0.45
DecompressMethod · 0.45

Tested by

no test coverage detected