src/lexer.c
/* emit a token built from characters accumulated in token_buffer. */
emit a token built from characters accumulated in token_buffer.
emit a token built from characters accumulated in token_buffer .
static void emit(struct lexer* l, struct token* next_tok, int token_kind, int token_type)
{
  expect(l, l->token_length > 0);
  l->token_buffer[l->token_length] = 0;
  next_tok->symbol  = str_copy(l->token_buffer);
  next_tok->symlen  = l->token_length;
  next_tok->tokknd  = token_kind;
  next_tok->toktyp  = token_type;
  next_tok->lineno  = l->cur_lineno;
  next_tok->byteno  = l->cur_byteno;
  next_tok->foffset = (l->buffer - l->buffer_original);
  if (next_tok->tokknd == TOK_IDENTIFIER) 
  {
    uint32_t i;
    for (i = idx_first_tok_keyword; i < l->nkeywords; ++i)
    {
      if (    (l->lexemes[i].tokknd == TOK_KEYWORD)
           && (l->lexemes[i].symlen == l->token_length)
           && (strncmp(l->token_buffer, l->lexemes[i].symbol, l->token_length) == 0))
      {
        next_tok->tokknd = l->lexemes[i].tokknd;
        next_tok->toktyp = l->lexemes[i].toktyp;
        break;
      }
    }
  }
  l->token_length = 0;
}
static void emit ( struct lexer * l , struct token * next_tok , int token_kind , int token_type ) { expect ( l , l -> token_length > 0 ) ; l -> token_buffer [ l -> token_length ] = 0 ; next_tok -> symbol = str_copy ( l -> token_buffer ) ; next_tok -> symlen = l -> token_length ; next_tok -> tokknd = token_kind ; next_tok -> toktyp = token_type ; next_tok -> lineno = l -> cur_lineno ; next_tok -> byteno = l -> cur_byteno ; next_tok -> foffset = ( l -> buffer - l -> buffer_original ) ; if ( next_tok -> tokknd == TOK_IDENTIFIER ) { uint32_t i ; for ( i = idx_first_tok_keyword ; i < l -> nkeywords ; ++ i ) { if ( ( l -> lexemes [ i ] . tokknd == TOK_KEYWORD ) && ( l -> lexemes [ i ] . symlen == l -> token_length ) && ( strncmp ( l -> token_buffer , l -> lexemes [ i ] . symbol , l -> token_length ) == 0 ) ) { next_tok -> tokknd = l -> lexemes [ i ] . tokknd ; next_tok -> toktyp = l -> lexemes [ i ] . toktyp ; break ; } } } l -> token_length = 0 ; }
emit a token built from characters accumulated in token_buffer.
emit a token built from characters accumulated in token_buffer .
/* Did we match a keyword? */