Method TestDeltaDict

cpp/src/arrow/ipc/read_write_test.cc:2345–2411

  }

  void TestDeltaDict() {
    auto type = dictionary(int8(), utf8());
    auto batch1 = MakeBatch(ArrayFromJSON(type, R"(["foo", "foo", "bar", null])"));
    // Potential delta
    auto batch2 = MakeBatch(ArrayFromJSON(type, R"(["foo", "bar", "quux", "foo"])"));
    // Potential delta
    auto batch3 =
        MakeBatch(ArrayFromJSON(type, R"(["foo", "bar", "quux", "zzz", "foo"])"));
    auto batch4 = MakeBatch(ArrayFromJSON(type, R"(["bar", null, "quux", "foo"])"));
    RecordBatchVector batches{batch1, batch2, batch3, batch4};
    RecordBatchVector only_deltas{batch1, batch2, batch3};

    // Emit replacements
    if (WriterHelper::kIsFileFormat) {
      CheckWritingFails(batches, 1);
    } else {
      CheckRoundtrip(batches);
      EXPECT_EQ(read_stats_.num_messages, 9);  // including schema message
      EXPECT_EQ(read_stats_.num_record_batches, 4);
      EXPECT_EQ(read_stats_.num_dictionary_batches, 4);
      EXPECT_EQ(read_stats_.num_replaced_dictionaries, 3);
      EXPECT_EQ(read_stats_.num_dictionary_deltas, 0);
    }

    // Emit deltas
    write_options_.emit_dictionary_deltas = true;
    if (WriterHelper::kIsFileFormat) {
      // batch4 is incompatible with the previous batches and would emit
      // a replacement
      CheckWritingFails(batches, 3);
    } else {
      CheckRoundtrip(batches);
      EXPECT_EQ(read_stats_.num_messages, 9);  // including schema message
      EXPECT_EQ(read_stats_.num_record_batches, 4);
      EXPECT_EQ(read_stats_.num_dictionary_batches, 4);
      EXPECT_EQ(read_stats_.num_replaced_dictionaries, 1);
      EXPECT_EQ(read_stats_.num_dictionary_deltas, 2);
    }

    CheckRoundtrip(only_deltas,
                   /*expect_expanded_dictionary=*/WriterHelper::kIsFileFormat);
    EXPECT_EQ(read_stats_.num_messages, 7);  // including schema message
    EXPECT_EQ(read_stats_.num_record_batches, 3);
    EXPECT_EQ(read_stats_.num_dictionary_batches, 3);
    EXPECT_EQ(read_stats_.num_replaced_dictionaries, 0);
    EXPECT_EQ(read_stats_.num_dictionary_deltas, 2);

    // IPC file format: WriteTable should unify dicts
    RecordBatchVector actual;
    write_options_.unify_dictionaries = true;
    ASSERT_OK(RoundTripTable(batches, &actual));
    if (WriterHelper::kIsFileFormat) {
      EXPECT_EQ(read_stats_.num_messages, 6);  // including schema message
      EXPECT_EQ(read_stats_.num_record_batches, 4);
      EXPECT_EQ(read_stats_.num_dictionary_batches, 1);
      EXPECT_EQ(read_stats_.num_replaced_dictionaries, 0);
      EXPECT_EQ(read_stats_.num_dictionary_deltas, 0);
      CheckBatchesLogical(batches, actual);