astropy/io/ascii/src/tokenizer.c
/*
  First, backtrack to eliminate trailing whitespace if strip_whitespace_fields
  is true. If the field is empty, push '\x01' as a marker.
  Append a null byte to the end of the column string as a field delimiting marker.
  Increment the variable col if we are tokenizing data.
*/
First, backtrack to eliminate trailing whitespace if strip_whitespace_fields
is true. If the field is empty, push '\x01' as a marker.
Append a null byte to the end of the column string as a field delimiting marker.
Increment the variable col if we are tokenizing data.
First backtrack to eliminate trailing whitespace if strip_whitespace_fields is true . If the field is empty push ' \ x01 ' as a marker . Append a null byte to the end of the column string as a field delimiting marker . Increment the variable col if we are tokenizing data .
static inline void end_field(tokenizer_t *self, int *col, int header)
{
    if (self->strip_whitespace_fields &&
            self->col_ptrs[*col] != self->output_cols[*col])
    {
        --self->col_ptrs[*col];
        while (*self->col_ptrs[*col] == ' ' || *self->col_ptrs[*col] == '\t')
        {
            *self->col_ptrs[*col]-- = '\x00';
        }
        ++self->col_ptrs[*col];
    }
    if (self->col_ptrs[*col] == self->output_cols[*col] ||
            self->col_ptrs[*col][-1] == '\x00')
    {
        push(self, '\x01', *col);
    }
    push(self, '\x00', *col);
    if (!header) {
        ++*col;
    }
}
static inline void end_field ( tokenizer_t * self , int * col , int header ) { if ( self -> strip_whitespace_fields && self -> col_ptrs [ * col ] != self -> output_cols [ * col ] ) { -- self -> col_ptrs [ * col ] ; while ( * self -> col_ptrs [ * col ] == ' ' || * self -> col_ptrs [ * col ] == ' \t ' ) { * self -> col_ptrs [ * col ] -- = ' \x00 ' ; } ++ self -> col_ptrs [ * col ] ; } if ( self -> col_ptrs [ * col ] == self -> output_cols [ * col ] || self -> col_ptrs [ * col ] [ -1 ] == ' \x00 ' ) { push ( self , ' \x01 ' , * col ) ; } push ( self , ' \x00 ' , * col ) ; if ( ! header ) { ++ * col ; } }
First, backtrack to eliminate trailing whitespace if strip_whitespace_fields
is true.
First backtrack to eliminate trailing whitespace if strip_whitespace_fields is true .