MCPcopy Create free account
hub / github.com/apache/arrow / PutInternal

Method PutInternal

cpp/src/parquet/encoder.cc:1486–1529  ·  view source on GitHub ↗

Source from the content-addressed store, hash-verified

1484 protected:
1485 template <typename VisitorType>
1486 void PutInternal(const T* src, int num_values, const VisitorType visitor) {
1487 if (num_values == 0) {
1488 return;
1489 }
1490
1491 std::string_view last_value_view = last_value_;
1492 constexpr int kBatchSize = 256;
1493 std::array<int32_t, kBatchSize> prefix_lengths;
1494 std::array<ByteArray, kBatchSize> suffixes;
1495
1496 for (int i = 0; i < num_values; i += kBatchSize) {
1497 const int batch_size = std::min(kBatchSize, num_values - i);
1498
1499 for (int j = 0; j < batch_size; ++j) {
1500 const int idx = i + j;
1501 const auto view = visitor[idx];
1502 const auto len = static_cast<const uint32_t>(view.length());
1503
1504 uint32_t common_prefix_length = 0;
1505 const uint32_t maximum_common_prefix_length =
1506 std::min(len, static_cast<uint32_t>(last_value_view.length()));
1507 while (common_prefix_length < maximum_common_prefix_length) {
1508 if (last_value_view[common_prefix_length] != view[common_prefix_length]) {
1509 break;
1510 }
1511 common_prefix_length++;
1512 }
1513
1514 last_value_view = view;
1515 prefix_lengths[j] = common_prefix_length;
1516 const uint32_t suffix_length = len - common_prefix_length;
1517 const uint8_t* suffix_ptr = src[idx].ptr + common_prefix_length;
1518
1519 // Convert to ByteArray, so it can be passed to the suffix_encoder_.
1520 const ByteArray suffix(suffix_length, suffix_ptr);
1521 suffixes[j] = suffix;
1522
1523 unencoded_byte_array_data_bytes_ += len;
1524 }
1525 suffix_encoder_.Put(suffixes.data(), batch_size);
1526 prefix_length_encoder_.Put(prefix_lengths.data(), batch_size);
1527 }
1528 last_value_ = last_value_view;
1529 }
1530
1531 template <typename ArrayType>
1532 void PutBinaryArray(const ArrayType& array) {

Callers

nothing calls this directly

Calls 3

lengthMethod · 0.45
PutMethod · 0.45
dataMethod · 0.45

Tested by

no test coverage detected