lexer.cpp
/* Skip whitespace and comments and return the first
 * non-white character.
 * As this makes use of the above getch() ungetch() functions,
 * we don't need to care at all about line numbering anymore.
 *
 * In theory, this function should only be used at the beginning
 * of lexing, or when we *know* the next character is part of the token.
 * Otherwise, if the parser throws an error, the linenumber may not be
 * the line of the error, but the line of the next token AFTER the error.
 *
 * This is currently only problematic when using c-like string-continuation,
 * since comments and whitespaces are allowed between 2 such strings.
 * Example:
printf(   "line one\n"
// A comment
          "A continuation of the previous string"
// This line is skipped
      , foo);

 * In this case, if the parse decides it didn't actually want a string,
 * and uses lex->line to print an error, it will show the ', foo);' line's
 * linenumber.
 *
 * On the other hand, the parser is supposed to remember the line of the next
 * token's beginning. In this case we would want skipwhite() to be called
 * AFTER reading a token, so that the parser, before reading the NEXT token,
 * doesn't store teh *comment's* linenumber, but the actual token's linenumber.
 *
 * THIS SOLUTION
 *    here is to store the line of the first character after skipping
 *    the initial whitespace in lex->sline, this happens in lex_do.
 */
Skip whitespace and comments and return the first
non-white character.
As this makes use of the above getch() ungetch() functions,
we don't need to care at all about line numbering anymore.

In theory, this function should only be used at the beginning
of lexing, or when we *know* the next character is part of the token.
Otherwise, if the parser throws an error, the linenumber may not be
the line of the error, but the line of the next token AFTER the error.

This is currently only problematic when using c-like string-continuation,
since comments and whitespaces are allowed between 2 such strings.
Example:
printf(   "line one\n"
A comment
"A continuation of the previous string"
This line is skipped
, foo).

In this case, if the parse decides it didn't actually want a string,
and uses lex->line to print an error, it will show the ', foo);' line's
linenumber.

On the other hand, the parser is supposed to remember the line of the next
token's beginning. In this case we would want skipwhite() to be called
AFTER reading a token, so that the parser, before reading the NEXT token,
doesn't store teh *comment's* linenumber, but the actual token's linenumber.

THIS SOLUTION
here is to store the line of the first character after skipping
the initial whitespace in lex->sline, this happens in lex_do.
Skip whitespace and comments and return the first non - white character . As this makes use of the above getch () ungetch () functions we don ' t need to care at all about line numbering anymore . In theory this function should only be used at the beginning of lexing or when we * know * the next character is part of the token . Otherwise if the parser throws an error the linenumber may not be the line of the error but the line of the next token AFTER the error . This is currently only problematic when using c - like string - continuation since comments and whitespaces are allowed between 2 such strings . Example : printf ( " line one \ n " A comment " A continuation of the previous string " This line is skipped foo ) . In this case if the parse decides it didn ' t actually want a string and uses lex - > line to print an error it will show the ' foo ) ; ' line ' s linenumber . On the other hand the parser is supposed to remember the line of the next token ' s beginning . In this case we would want skipwhite () to be called AFTER reading a token so that the parser before reading the NEXT token doesn ' t store teh * comment ' s * linenumber but the actual token ' s linenumber . THIS SOLUTION here is to store the line of the first character after skipping the initial whitespace in lex - > sline this happens in lex_do .
static int lex_skipwhite(lex_file *lex, bool hadwhite)
{
    int ch = 0;
    bool haswhite = hadwhite;
    do
    {
        ch = lex_getch(lex);
        while (ch != EOF && util_isspace(ch)) {
            if (ch == '\n') {
                if (lex_try_pragma(lex))
                    continue;
            }
            if (lex->flags.preprocessing) {
                if (ch == '\n') {
                    if (haswhite) { 
                        lex_ungetch(lex, ch);
                        lex_endtoken(lex);
                        return TOKEN_WHITE;
                    }
                    return TOKEN_EOL;
                }
                haswhite = true;
                lex_tokench(lex, ch);
            }
            ch = lex_getch(lex);
        }
        if (ch == '/') {
            ch = lex_getch(lex);
            if (ch == '/')
            {
                ch = lex_getch(lex);
                if (lex->flags.preprocessing) {
                    haswhite = true;
                    lex_tokench(lex, ' ');
                    lex_tokench(lex, ' ');
                }
                while (ch != EOF && ch != '\n') {
                    if (lex->flags.preprocessing)
                        lex_tokench(lex, ' '); 
                    ch = lex_getch(lex);
                }
                if (lex->flags.preprocessing) {
                    lex_ungetch(lex, '\n');
                    lex_endtoken(lex);
                    return TOKEN_WHITE;
                }
                continue;
            }
            if (ch == '*')
            {
                if (lex->flags.preprocessing) {
                    haswhite = true;
                    lex_tokench(lex, ' ');
                    lex_tokench(lex, ' ');
                }
                while (ch != EOF)
                {
                    ch = lex_getch(lex);
                    if (ch == '*') {
                        ch = lex_getch(lex);
                        if (ch == '/') {
                            if (lex->flags.preprocessing) {
                                lex_tokench(lex, ' ');
                                lex_tokench(lex, ' ');
                            }
                            break;
                        }
                        lex_ungetch(lex, ch);
                    }
                    if (lex->flags.preprocessing) {
                        if (ch == '\n')
                            lex_tokench(lex, '\n');
                        else
                            lex_tokench(lex, ' ');
                    }
                }
                ch = ' '; 
                continue;
            }
            lex_ungetch(lex, ch);
            ch = '/';
            break;
        }
    } while (ch != EOF && util_isspace(ch));
    if (haswhite) {
        lex_endtoken(lex);
        lex_ungetch(lex, ch);
        return TOKEN_WHITE;
    }
    return ch;
}
static int lex_skipwhite ( lex_file * lex , bool hadwhite ) { int ch = 0 ; bool haswhite = hadwhite ; do { ch = lex_getch ( lex ) ; while ( ch != EOF && util_isspace ( ch ) ) { if ( ch == ' \n ' ) { if ( lex_try_pragma ( lex ) ) continue ; } if ( lex -> flags . preprocessing ) { if ( ch == ' \n ' ) { if ( haswhite ) { lex_ungetch ( lex , ch ) ; lex_endtoken ( lex ) ; return TOKEN_WHITE ; } return TOKEN_EOL ; } haswhite = true ; lex_tokench ( lex , ch ) ; } ch = lex_getch ( lex ) ; } if ( ch == ' ' ) { ch = lex_getch ( lex ) ; if ( ch == ' ' ) { ch = lex_getch ( lex ) ; if ( lex -> flags . preprocessing ) { haswhite = true ; lex_tokench ( lex , ' ' ) ; lex_tokench ( lex , ' ' ) ; } while ( ch != EOF && ch != ' \n ' ) { if ( lex -> flags . preprocessing ) lex_tokench ( lex , ' ' ) ; ch = lex_getch ( lex ) ; } if ( lex -> flags . preprocessing ) { lex_ungetch ( lex , ' \n ' ) ; lex_endtoken ( lex ) ; return TOKEN_WHITE ; } continue ; } if ( ch == ' ' ) { if ( lex -> flags . preprocessing ) { haswhite = true ; lex_tokench ( lex , ' ' ) ; lex_tokench ( lex , ' ' ) ; } while ( ch != EOF ) { ch = lex_getch ( lex ) ; if ( ch == ' ' ) { ch = lex_getch ( lex ) ; if ( ch == ' ' ) { if ( lex -> flags . preprocessing ) { lex_tokench ( lex , ' ' ) ; lex_tokench ( lex , ' ' ) ; } break ; } lex_ungetch ( lex , ch ) ; } if ( lex -> flags . preprocessing ) { if ( ch == ' \n ' ) lex_tokench ( lex , ' \n ' ) ; else lex_tokench ( lex , ' ' ) ; } } ch = ' ' ; continue ; } lex_ungetch ( lex , ch ) ; ch = ' ' ; break ; } } while ( ch != EOF && util_isspace ( ch ) ) ; if ( haswhite ) { lex_endtoken ( lex ) ; lex_ungetch ( lex , ch ) ; return TOKEN_WHITE ; } return ch ; }
Skip whitespace and comments and return the first
 non-white character.
Skip whitespace and comments and return the first non - white character .
/* end-of-line */ /* see if there was whitespace first */ /* (vec_size(lex->tok.value)) { */ /* otherwise return EOL */ /* one line comment */ /* ch); */ /* multiline comment */ /* cause TRUE in the isspace check */ /* Otherwise roll back to the slash and break out of the loop */