Build a PyArrow buffer from NaN or sentinel values. Parameters ---------- data_pa_buffer : pa.Buffer PyArrow buffer for the column data. data_type : Dtype Dtype description as a tuple ``(kind, bit-width, format string, endianness)``. describe_null :
(
data_pa_buffer: BufferObject,
data_type: Dtype,
describe_null: ColumnNullType,
length: int,
offset: int = 0,
allow_copy: bool = True,
)
| 502 | |
| 503 | |
| 504 | def validity_buffer_nan_sentinel( |
| 505 | data_pa_buffer: BufferObject, |
| 506 | data_type: Dtype, |
| 507 | describe_null: ColumnNullType, |
| 508 | length: int, |
| 509 | offset: int = 0, |
| 510 | allow_copy: bool = True, |
| 511 | ) -> pa.Buffer: |
| 512 | """ |
| 513 | Build a PyArrow buffer from NaN or sentinel values. |
| 514 | |
| 515 | Parameters |
| 516 | ---------- |
| 517 | data_pa_buffer : pa.Buffer |
| 518 | PyArrow buffer for the column data. |
| 519 | data_type : Dtype |
| 520 | Dtype description as a tuple ``(kind, bit-width, format string, |
| 521 | endianness)``. |
| 522 | describe_null : ColumnNullType |
| 523 | Null representation the column dtype uses, |
| 524 | as a tuple ``(kind, value)`` |
| 525 | length : int |
| 526 | The number of values in the array. |
| 527 | offset : int, default: 0 |
| 528 | Number of elements to offset from the start of the buffer. |
| 529 | allow_copy : bool, default: True |
| 530 | Whether to allow copying the memory to perform the conversion |
| 531 | (if false then zero-copy approach is requested). |
| 532 | |
| 533 | Returns |
| 534 | ------- |
| 535 | pa.Buffer |
| 536 | """ |
| 537 | kind, bit_width, _, _ = data_type |
| 538 | data_dtype = map_date_type(data_type) |
| 539 | null_kind, sentinel_val = describe_null |
| 540 | |
| 541 | # Check for float NaN values |
| 542 | if null_kind == ColumnNullType.USE_NAN: |
| 543 | if not allow_copy: |
| 544 | raise RuntimeError( |
| 545 | "To create a bitmask a copy of the data is " |
| 546 | "required which is forbidden by allow_copy=False" |
| 547 | ) |
| 548 | |
| 549 | if kind == DtypeKind.FLOAT and bit_width == 16: |
| 550 | # 'pyarrow.compute.is_nan' kernel not yet implemented |
| 551 | # for float16 |
| 552 | raise NotImplementedError( |
| 553 | f"{data_type} with {null_kind} is not yet supported.") |
| 554 | else: |
| 555 | pyarrow_data = pa.Array.from_buffers( |
| 556 | data_dtype, |
| 557 | length, |
| 558 | [None, data_pa_buffer], |
| 559 | offset=offset, |
| 560 | ) |
| 561 | mask = pc.is_nan(pyarrow_data) |
no test coverage detected