In the future we should support async scanning of fragments. The Dataset class doesn't support this yet but we pretend it does here to ease future adoption of the feature.
| 57 | // Dataset class doesn't support this yet but we pretend it does here to |
| 58 | // ease future adoption of the feature. |
| 59 | Future<AsyncGenerator<std::shared_ptr<Fragment>>> GetFragments( |
| 60 | Dataset* dataset, compute::Expression predicate) { |
| 61 | // In the future the dataset should be responsible for figuring out |
| 62 | // the I/O context. This will allow different I/O contexts to be used |
| 63 | // when scanning different datasets. For example, if we are scanning a |
| 64 | // union of a remote dataset and a local dataset. |
| 65 | const auto& io_context = io::default_io_context(); |
| 66 | auto io_executor = io_context.executor(); |
| 67 | return DeferNotOk( |
| 68 | io_executor->Submit( |
| 69 | [dataset, predicate]() -> Result<std::shared_ptr<FragmentIterator>> { |
| 70 | ARROW_ASSIGN_OR_RAISE(FragmentIterator fragments_iter, |
| 71 | dataset->GetFragments(predicate)); |
| 72 | return std::make_shared<FragmentIterator>(std::move(fragments_iter)); |
| 73 | })) |
| 74 | .Then([](const std::shared_ptr<FragmentIterator>& fragments_it) |
| 75 | -> Result<AsyncGenerator<std::shared_ptr<Fragment>>> { |
| 76 | ARROW_ASSIGN_OR_RAISE(std::vector<std::shared_ptr<Fragment>> fragments, |
| 77 | fragments_it->ToVector()); |
| 78 | return MakeVectorGenerator(std::move(fragments)); |
| 79 | }); |
| 80 | } |
| 81 | |
| 82 | /// \brief A node that scans a dataset |
| 83 | /// |