Function npy_tokenize

numpy/core/src/multiarray/textreading/tokenize.cpp:289–404

* This tokenizer always copies the full "row" (all tokens). This makes * two things easier: * 1. It means that every word is guaranteed to be followed by a NUL character * (although it can include one as well). * 2. If usecols are used we can sniff the first row easier by parsing it * fully. Further, usecols can be negative so we may not know which row we * need up-front. * * Th

 * unicode flavors UCS1, UCS2, and UCS4 as a worthwhile optimization.
 */
NPY_NO_EXPORT int
npy_tokenize(stream *s, tokenizer_state *ts, parser_config *const config)
{
    assert(ts->fields_size >= 2);
    assert(ts->field_buffer_length >= 2*(Py_ssize_t)sizeof(Py_UCS4));

    int finished_reading_file = 0;

    /* Reset to start of buffer */
    ts->field_buffer_pos = 0;
    ts->num_fields = 0;

    while (true) {
        /*
         * This loop adds new fields to the result (to make up a full row)
         * until the row ends (typically a line end or the file end)
         */
        if (ts->state == TOKENIZE_INIT) {
            /* Start a new field */
            if (add_field(ts) < 0) {
                return -1;
            }
            ts->state = TOKENIZE_CHECK_QUOTED;
        }

        if (NPY_UNLIKELY(ts->pos >= ts->end)) {
            if (ts->buf_state == BUFFER_IS_LINEND &&
                    ts->state != TOKENIZE_QUOTED) {
                /*
                 * Finished line, do not read anymore (also do not eat \n).
                 * If we are in a quoted field and the "line" does not end with
                 * a newline, the quoted field will not have it either.
                 * I.e. `np.loadtxt(['"a', 'b"'], dtype="S2", quotechar='"')`
                 * reads "ab". This matches `next(csv.reader(['"a', 'b"']))`.
                 */
                break;
            }
            /* fetch new data */
            ts->buf_state = stream_nextbuf(s,
                    &ts->pos, &ts->end, &ts->unicode_kind);
            if (ts->buf_state < 0) {
                return -1;
            }
            if (ts->buf_state == BUFFER_IS_FILEEND) {
                finished_reading_file = 1;
                ts->pos = ts->end;  /* stream should ensure this. */
                break;
            }
            else if (ts->pos == ts->end) {
                /* This must be an empty line (and it must be indicated!). */
                assert(ts->buf_state == BUFFER_IS_LINEND);
                break;
            }
        }
        int status;
        if (ts->unicode_kind == PyUnicode_1BYTE_KIND) {
            status = tokenizer_core<Py_UCS1>(ts, config);
        }