| 553 | |
| 554 | template <typename SpecializedOptions> |
| 555 | Status ParseSpecialized(const std::vector<std::string_view>& views, bool is_final, |
| 556 | uint32_t* out_size) { |
| 557 | internal::PreferredBulkFilterType<SpecializedOptions> bulk_filter(options_); |
| 558 | |
| 559 | batch_ = DataBatch{batch_.num_cols_}; |
| 560 | values_size_ = 0; |
| 561 | |
| 562 | size_t total_view_length = 0; |
| 563 | for (const auto& view : views) { |
| 564 | total_view_length += view.length(); |
| 565 | } |
| 566 | if (total_view_length > std::numeric_limits<uint32_t>::max()) { |
| 567 | return Status::Invalid("CSV block too large"); |
| 568 | } |
| 569 | |
| 570 | PresizedDataWriter parsed_writer(pool_, static_cast<uint32_t>(total_view_length)); |
| 571 | uint32_t total_parsed_length = 0; |
| 572 | |
| 573 | for (const auto& view : views) { |
| 574 | const char* data = view.data(); |
| 575 | const char* data_end = view.data() + view.length(); |
| 576 | bool finished_parsing = false; |
| 577 | |
| 578 | if (batch_.num_cols_ == -1) { |
| 579 | // Can't presize values when the number of columns is not known, first parse |
| 580 | // a single line |
| 581 | const int32_t rows_in_chunk = 1; |
| 582 | ARROW_ASSIGN_OR_RAISE(auto values_writer, ResizableValueDescWriter::Make(pool_)); |
| 583 | values_writer.Start(parsed_writer); |
| 584 | |
| 585 | RETURN_NOT_OK(ParseChunk<SpecializedOptions>( |
| 586 | &values_writer, &parsed_writer, data, data_end, is_final, rows_in_chunk, |
| 587 | &data, &finished_parsing, bulk_filter)); |
| 588 | if (batch_.num_cols_ == -1) { |
| 589 | return ParseError("Empty CSV file or block: cannot infer number of columns"); |
| 590 | } |
| 591 | } |
| 592 | |
| 593 | while (!finished_parsing && data < data_end && batch_.num_rows_ < max_num_rows_) { |
| 594 | // We know the number of columns, so can presize a values array for |
| 595 | // a given number of rows |
| 596 | DCHECK_GE(batch_.num_cols_, 0); |
| 597 | |
| 598 | int32_t rows_in_chunk; |
| 599 | constexpr int32_t kTargetChunkSize = 32768; // in number of values |
| 600 | if (batch_.num_cols_ > 0) { |
| 601 | rows_in_chunk = std::min(std::max(kTargetChunkSize / batch_.num_cols_, 512), |
| 602 | max_num_rows_ - batch_.num_rows_); |
| 603 | } else { |
| 604 | rows_in_chunk = std::min(kTargetChunkSize, max_num_rows_ - batch_.num_rows_); |
| 605 | } |
| 606 | |
| 607 | ARROW_ASSIGN_OR_RAISE( |
| 608 | auto values_writer, |
| 609 | PresizedValueDescWriter::Make(pool_, rows_in_chunk, batch_.num_cols_)); |
| 610 | values_writer.Start(parsed_writer); |
| 611 | |
| 612 | RETURN_NOT_OK(ParseChunk<SpecializedOptions>( |