MCPcopy Create free account
hub / github.com/apache/arrow / GetReadRecordBatchReadRanges

Function GetReadRecordBatchReadRanges

cpp/src/arrow/ipc/read_write_test.cc:3026–3099  ·  view source on GitHub ↗

Source from the content-addressed store, hash-verified

3024}
3025
3026void GetReadRecordBatchReadRanges(
3027 uint32_t num_rows, const std::vector<int>& included_fields,
3028 const std::vector<int64_t>& expected_body_read_lengths) {
3029 auto buffer = MakeBooleanInt32Int64File(num_rows, /*num_batches=*/1);
3030
3031 io::BufferReader buffer_reader(buffer);
3032 std::unique_ptr<io::TrackedRandomAccessFile> tracked =
3033 io::TrackedRandomAccessFile::Make(&buffer_reader);
3034
3035 auto read_options = IpcReadOptions::Defaults();
3036 // if empty, return all fields
3037 read_options.included_fields = included_fields;
3038 ASSERT_OK_AND_ASSIGN(auto reader,
3039 RecordBatchFileReader::Open(tracked.get(), read_options));
3040 ASSERT_OK_AND_ASSIGN(auto out_batch, reader->ReadRecordBatch(0));
3041
3042 ASSERT_EQ(out_batch->num_rows(), num_rows);
3043 ASSERT_EQ(out_batch->num_columns(),
3044 included_fields.empty() ? 3 : included_fields.size());
3045
3046 auto read_ranges = tracked->get_read_ranges();
3047
3048 const int32_t magic_size = static_cast<int>(ipc::internal::kArrowMagicBytes.size());
3049 // read magic and footer length IO
3050 auto file_end_size = magic_size + sizeof(int32_t);
3051 auto footer_length_offset = buffer->size() - file_end_size;
3052 auto footer_length = bit_util::FromLittleEndian(
3053 util::SafeLoadAs<int32_t>(buffer->data() + footer_length_offset));
3054
3055 // there are at least 2 read IOs before reading body:
3056 // 1) read magic and footer length IO
3057 // 2) footer IO
3058 EXPECT_GE(read_ranges.size(), 2);
3059
3060 // read magic and footer length IO
3061 EXPECT_EQ(read_ranges[0].length, file_end_size);
3062 // read footer IO
3063 EXPECT_EQ(read_ranges[1].length, footer_length);
3064
3065 if (included_fields.empty()) {
3066 // When no fields are explicitly included, the reader optimizes by
3067 // reading metadata and the entire body in a single IO.
3068 // Thus, there are exactly 3 read IOs in total:
3069 // 1) magic and footer length
3070 // 2) footer
3071 // 3) record batch metadata + body
3072 EXPECT_EQ(read_ranges.size(), 3);
3073
3074 int64_t total_body = 0;
3075 for (auto len : expected_body_read_lengths) total_body += len;
3076
3077 // In the optimized path (included_fields is empty), the 3rd read operation
3078 // fetches both the message metadata (flatbuffer) and the entire message body
3079 // in one contiguous block. Therefore, its length must at least exceed the
3080 // total body length by the size of the metadata.
3081 EXPECT_GT(read_ranges[2].length, total_body);
3082 EXPECT_LE(read_ranges[2].length, total_body + footer_length);
3083 } else {

Callers 1

TESTFunction · 0.85

Calls 9

MakeFunction · 0.50
DefaultsFunction · 0.50
FromLittleEndianFunction · 0.50
num_rowsMethod · 0.45
num_columnsMethod · 0.45
emptyMethod · 0.45
sizeMethod · 0.45
dataMethod · 0.45

Tested by

no test coverage detected