Read (multiple) Parquet files as a single pyarrow.Table. Parameters ---------- columns : List[str] Names of columns to read from the dataset. The partition fields are not automatically included. use_threads : bool, default True
(self, columns=None, use_threads=True, use_pandas_metadata=False)
| 1518 | return self._dataset.schema |
| 1519 | |
| 1520 | def read(self, columns=None, use_threads=True, use_pandas_metadata=False): |
| 1521 | """ |
| 1522 | Read (multiple) Parquet files as a single pyarrow.Table. |
| 1523 | |
| 1524 | Parameters |
| 1525 | ---------- |
| 1526 | columns : List[str] |
| 1527 | Names of columns to read from the dataset. The partition fields |
| 1528 | are not automatically included. |
| 1529 | use_threads : bool, default True |
| 1530 | Perform multi-threaded column reads. |
| 1531 | use_pandas_metadata : bool, default False |
| 1532 | If True and file has custom pandas schema metadata, ensure that |
| 1533 | index columns are also loaded. |
| 1534 | |
| 1535 | Returns |
| 1536 | ------- |
| 1537 | pyarrow.Table |
| 1538 | Content of the file as a table (of columns). |
| 1539 | |
| 1540 | Examples |
| 1541 | -------- |
| 1542 | Generate an example dataset: |
| 1543 | |
| 1544 | >>> import pyarrow as pa |
| 1545 | >>> table = pa.table({'year': [2020, 2022, 2021, 2022, 2019, 2021], |
| 1546 | ... 'n_legs': [2, 2, 4, 4, 5, 100], |
| 1547 | ... 'animal': ["Flamingo", "Parrot", "Dog", "Horse", |
| 1548 | ... "Brittle stars", "Centipede"]}) |
| 1549 | >>> import pyarrow.parquet as pq |
| 1550 | >>> pq.write_to_dataset(table, root_path='dataset_v2_read', |
| 1551 | ... partition_cols=['year']) |
| 1552 | >>> dataset = pq.ParquetDataset('dataset_v2_read/') |
| 1553 | |
| 1554 | Read the dataset: |
| 1555 | |
| 1556 | >>> dataset.read(columns=["n_legs"]) |
| 1557 | pyarrow.Table |
| 1558 | n_legs: int64 |
| 1559 | ---- |
| 1560 | n_legs: [[5],[2],[4,100],[2,4]] |
| 1561 | """ |
| 1562 | # if use_pandas_metadata, we need to include index columns in the |
| 1563 | # column selection, to be able to restore those in the pandas DataFrame |
| 1564 | metadata = self.schema.metadata or {} |
| 1565 | |
| 1566 | if use_pandas_metadata: |
| 1567 | # if the dataset schema metadata itself doesn't have pandas |
| 1568 | # then try to get this from common file (for backwards compat) |
| 1569 | if b"pandas" not in metadata: |
| 1570 | common_metadata = self._get_common_pandas_metadata() |
| 1571 | if common_metadata: |
| 1572 | metadata = common_metadata |
| 1573 | |
| 1574 | if columns is not None and use_pandas_metadata: |
| 1575 | if metadata and b'pandas' in metadata: |
| 1576 | # RangeIndex can be represented as dict instead of column name |
| 1577 | index_columns = [ |