Function read_rows

numpy/core/src/multiarray/textreading/rows.c:155–500

* Read a file into the provided array, or create (and possibly grow) an * array to read into. * * @param s The stream object/struct providing reading capabilities used by * the tokenizer. * @param max_rows The number of rows to read, or -1. If negative * all rows are read. * @param num_field_types The number of field types stored in `field_types`. * @param field_types Inform

 *          is always a new reference (even when `data_array` was passed in).
 */
NPY_NO_EXPORT PyArrayObject *
read_rows(stream *s,
        npy_intp max_rows, Py_ssize_t num_field_types, field_type *field_types,
        parser_config *pconfig, Py_ssize_t num_usecols, Py_ssize_t *usecols,
        Py_ssize_t skiplines, PyObject *converters,
        PyArrayObject *data_array, PyArray_Descr *out_descr,
        bool homogeneous)
{
    char *data_ptr = NULL;
    Py_ssize_t current_num_fields;
    npy_intp row_size = out_descr->elsize;
    PyObject **conv_funcs = NULL;

    bool needs_init = PyDataType_FLAGCHK(out_descr, NPY_NEEDS_INIT);

    int ndim = homogeneous ? 2 : 1;
    npy_intp result_shape[2] = {0, 1};

    bool data_array_allocated = data_array == NULL;
    /* Make sure we own `data_array` for the purpose of error handling */
    Py_XINCREF(data_array);
    size_t rows_per_block = 1;  /* will be increased depending on row size */
    npy_intp data_allocated_rows = 0;

    /* We give a warning if max_rows is used and an empty line is encountered */
    bool give_empty_row_warning = max_rows >= 0;

    int ts_result = 0;
    tokenizer_state ts;
    if (npy_tokenizer_init(&ts, pconfig) < 0) {
        goto error;
    }

    /* Set the actual number of fields if it is already known, otherwise -1 */
    Py_ssize_t actual_num_fields = -1;
    if (usecols != NULL) {
        assert(homogeneous || num_field_types == num_usecols);
        actual_num_fields = num_usecols;
    }
    else if (!homogeneous) {
        assert(usecols == NULL || num_field_types == num_usecols);
        actual_num_fields = num_field_types;
    }

    for (Py_ssize_t i = 0; i < skiplines; i++) {
        ts.state = TOKENIZE_GOTO_LINE_END;
        ts_result = npy_tokenize(s, &ts, pconfig);
        if (ts_result < 0) {
            goto error;
        }
        else if (ts_result != 0) {
            /* Fewer lines than skiplines is acceptable */
            break;
        }
    }

    Py_ssize_t row_count = 0;  /* number of rows actually processed */
    while ((max_rows < 0 || row_count < max_rows) && ts_result == 0) {