| 3676 | |
| 3677 | # Test if a dataset with non-utf8 chars in the column names is properly handled |
| 3678 | def test_column_names_encoding(tempdir, dataset_reader): |
| 3679 | path = str(tempdir / 'test.csv') |
| 3680 | |
| 3681 | with open(path, 'wb') as sink: |
| 3682 | sink.write(b"\xe9,b\nun,\xe9l\xe9phant") |
| 3683 | |
| 3684 | # Interpret as utf8: |
| 3685 | expected_schema = pa.schema([("é", pa.string()), ("b", pa.string())]) |
| 3686 | expected_table = pa.table({'é': ["un"], |
| 3687 | 'b': ["éléphant"]}, schema=expected_schema) |
| 3688 | |
| 3689 | # Reading as string without specifying encoding should produce an error |
| 3690 | dataset = ds.dataset(path, format='csv', schema=expected_schema) |
| 3691 | with pytest.raises(pyarrow.lib.ArrowInvalid, match="invalid UTF8"): |
| 3692 | dataset_reader.to_table(dataset) |
| 3693 | |
| 3694 | # Setting the encoding in the read_options should transcode the data |
| 3695 | read_options = pa.csv.ReadOptions(encoding='latin-1') |
| 3696 | file_format = ds.CsvFileFormat(read_options=read_options) |
| 3697 | dataset_transcoded = ds.dataset(path, format=file_format) |
| 3698 | assert dataset_transcoded.schema.equals(expected_schema) |
| 3699 | assert dataset_transcoded.to_table().equals(expected_table) |
| 3700 | |
| 3701 | |
| 3702 | def test_feather_format(tempdir, dataset_reader): |