* Initialize the tokenizer. We may want to copy all important config * variables into the tokenizer. This would improve the cache locality during * tokenizing. */
| 423 | * tokenizing. |
| 424 | */ |
| 425 | NPY_NO_EXPORT int |
| 426 | npy_tokenizer_init(tokenizer_state *ts, parser_config *config) |
| 427 | { |
| 428 | /* State and buf_state could be moved into tokenize if we go by row */ |
| 429 | ts->buf_state = BUFFER_MAY_CONTAIN_NEWLINE; |
| 430 | ts->state = TOKENIZE_INIT; |
| 431 | if (config->delimiter_is_whitespace) { |
| 432 | ts->unquoted_state = TOKENIZE_UNQUOTED_WHITESPACE; |
| 433 | } |
| 434 | else { |
| 435 | ts->unquoted_state = TOKENIZE_UNQUOTED; |
| 436 | } |
| 437 | ts->num_fields = 0; |
| 438 | |
| 439 | ts->buf_state = 0; |
| 440 | ts->pos = nullptr; |
| 441 | ts->end = nullptr; |
| 442 | |
| 443 | ts->field_buffer = (Py_UCS4 *)PyMem_Malloc(32 * sizeof(Py_UCS4)); |
| 444 | if (ts->field_buffer == nullptr) { |
| 445 | PyErr_NoMemory(); |
| 446 | return -1; |
| 447 | } |
| 448 | ts->field_buffer_length = 32; |
| 449 | |
| 450 | ts->fields = (field_info *)PyMem_Malloc(4 * sizeof(*ts->fields)); |
| 451 | if (ts->fields == nullptr) { |
| 452 | PyMem_Free(ts->field_buffer); |
| 453 | ts->field_buffer = nullptr; |
| 454 | PyErr_NoMemory(); |
| 455 | return -1; |
| 456 | } |
| 457 | ts->fields_size = 4; |
| 458 | return 0; |
| 459 | } |