pandas/src/parser/tokenizer.c
/*
  nrows : number of rows to tokenize (or until reach EOF)
  all : tokenize all the data vs. certain number of rows
 */
nrows : number of rows to tokenize (or until reach EOF)
all : tokenize all the data vs. certain number of rows
nrows : number of rows to tokenize ( or until reach EOF ) all : tokenize all the data vs . certain number of rows
int _tokenize_helper(parser_t *self, size_t nrows, int all) {
    parser_op tokenize_bytes;
    int status = 0;
    int start_lines = self->lines;
    if (self->delim_whitespace) {
        tokenize_bytes = tokenize_whitespace;
    } else if (self->lineterminator == '\0') {
        tokenize_bytes = tokenize_delimited;
    } else {
        tokenize_bytes = tokenize_delim_customterm;
    }
    if (self->state == FINISHED) {
        return 0;
    }
    TRACE(("Asked to tokenize %d rows\n", (int) nrows));
    while (1) {
        if (!all && self->lines - start_lines >= nrows)
            break;
        if (self->datapos == self->datalen) {
            status = parser_buffer_bytes(self, self->chunksize);
            if (status == REACHED_EOF) {
                status = parser_handle_eof(self);
                self->state = FINISHED;
                break;
            } else if (status != 0) {
                return status;
            }
        }
        TRACE(("Trying to process %d bytes\n", self->datalen - self->datapos));
        status = tokenize_bytes(self, nrows);
        if (status < 0) {
            TRACE(("Status %d returned from tokenize_bytes, breaking\n",
                   status));
            status = -1;
            break;
        }
    }
    TRACE(("leaving tokenize_helper\n"));
    return status;
}
int _tokenize_helper ( parser_t * self , size_t nrows , int all ) { parser_op tokenize_bytes ; int status = 0 ; int start_lines = self -> lines ; if ( self -> delim_whitespace ) { tokenize_bytes = tokenize_whitespace ; } else if ( self -> lineterminator == ' \0 ' ) { tokenize_bytes = tokenize_delimited ; } else { tokenize_bytes = tokenize_delim_customterm ; } if ( self -> state == FINISHED ) { return 0 ; } TRACE ( ( " \n " , ( int ) nrows ) ) ; while ( 1 ) { if ( ! all && self -> lines - start_lines >= nrows ) break ; if ( self -> datapos == self -> datalen ) { status = parser_buffer_bytes ( self , self -> chunksize ) ; if ( status == REACHED_EOF ) { status = parser_handle_eof ( self ) ; self -> state = FINISHED ; break ; } else if ( status != 0 ) { return status ; } } TRACE ( ( " \n " , self -> datalen - self -> datapos ) ) ; status = tokenize_bytes ( self , nrows ) ; if ( status < 0 ) { TRACE ( ( " \n " , status ) ) ; status = -1 ; break ; } } TRACE ( ( " \n " ) ) ; return status ; }
nrows : number of rows to tokenize (or until reach EOF)
all : tokenize all the data vs. certain number of rows
nrows : number of rows to tokenize ( or until reach EOF ) all : tokenize all the data vs . certain number of rows
// close out last line /* TRACE(("sourcetype: %c, status: %d\n", self->sourcetype, status)); */ /* debug_print_parser(self); */ // XXX