@SuppressWarnings("fallthrough")
    @Override
    public Token<CppTokenId> nextToken() {
        while (true) {
            // special handling for escaped lines
            if (lastTokenEndedByEscapedLine > 0) {
                int c = read(false);
                lastTokenEndedByEscapedLine--;
                assert c == '\\' : "there must be \\";
                c = read(false);
                assert c == '\n' || c == '\r' : "there must be \r or \n";
                if (c == '\r') {
                    lastTokenEndedByEscapedLine--;
                    if (input.consumeNewline()) {
                        lastTokenEndedByEscapedLine--;
                    }
                    return token(CppTokenId.ESCAPED_LINE);
                } else {
                    lastTokenEndedByEscapedLine--;
                    return token(CppTokenId.ESCAPED_LINE, "\\\n", PartType.COMPLETE); // NOI18N
                }
            } else {
                int c = read(true);
                // if read of the first char caused skipping escaped line
                // do we need to backup and create escaped lines first?
                switch (c) {
                    case '"': {
                        Token<CppTokenId> out = finishDblQuote();
                        assert out != null : "not handled dobule quote";
                        return out;
                    }
                    case '\'': {// char literal
                        Token<CppTokenId> out = finishSingleQuote();
                        assert out != null : "not handled single quote";
                        return out;
                    }
                    case '#': {
                        Token<CppTokenId> out = finishSharp();
                        assert out != null : "not handled #";
                        return out;
                    }
                    case '/':
                        switch (read(true)) {
                            case '/': // in single-line or doxygen comment
                            {
                                Token<CppTokenId> out = finishLineComment(true);
                                assert out != null : "not handled //";
                                return out;
                            }
                            case '=': // found /=
                                return token(CppTokenId.SLASHEQ);
                            case '*': // in multi-line or doxygen comment
                            {
                                Token<CppTokenId> out = finishBlockComment(true);
                                assert out != null : "not handled /*";
                                return out;
                            }
                        } // end of switch()
                        backup(1);
                        return token(CppTokenId.SLASH);
                    case '=':
                        if (read(true) == '=') {
                            return token(CppTokenId.EQEQ);
                        }
                        backup(1);
                        return token(CppTokenId.EQ);
                    case '>':
                        switch (read(true)) {
                            case '>': // >>
                                if (read(true) == '=') {
                                    return token(CppTokenId.GTGTEQ);
                                }
                                backup(1);
                                return token(CppTokenId.GTGT);
                            case '=': // >=
                                return token(CppTokenId.GTEQ);
                        }
                        backup(1);
                        return token(CppTokenId.GT);
                    case '<': {
                        Token<CppTokenId> out = finishLT();
                        assert out != null : "not handled '<'";
                        return out;
                    }
                    case '+':
                        switch (read(true)) {
                            case '+':
                                return token(CppTokenId.PLUSPLUS);
                            case '=':
                                return token(CppTokenId.PLUSEQ);
                        }
                        backup(1);
                        return token(CppTokenId.PLUS);
                    case '-':
                        switch (read(true)) {
                            case '-':
                                return token(CppTokenId.MINUSMINUS);
                            case '>':
                                if (read(true) == '*') {
                                    return token(CppTokenId.ARROWMBR);
                                }
                                backup(1);
                                return token(CppTokenId.ARROW);
                            case '=':
                                return token(CppTokenId.MINUSEQ);
                        }
                        backup(1);
                        return token(CppTokenId.MINUS);
                    case '*':
                        switch (read(true)) {
                            case '/': // invalid comment end - */ or int*/* */
                                if (read(true) == '*') {
                                    backup(2);
                                    return token(CppTokenId.STAR);
                                }
                                backup(1);
                                return token(CppTokenId.INVALID_COMMENT_END);
                            case '=':
                                return token(CppTokenId.STAREQ);
                        }
                        backup(1);
                        return token(CppTokenId.STAR);
                    case '|':
                        switch (read(true)) {
                            case '|':
                                return token(CppTokenId.BARBAR);
                            case '=':
                                return token(CppTokenId.BAREQ);
                        }
                        backup(1);
                        return token(CppTokenId.BAR);
                    case '&':
                        switch (read(true)) {
                            case '&':
                                return token(CppTokenId.AMPAMP);
                            case '=':
                                return token(CppTokenId.AMPEQ);
                        }
                        backup(1);
                        return token(CppTokenId.AMP);
                    case '%': {
                        Token<CppTokenId> out = finishPercent();
                        assert out != null : "not handled %";
                        return out;
                    }
                    case '^':
                        if (read(true) == '=') {
                            return token(CppTokenId.CARETEQ);
                        }
                        backup(1);
                        return token(CppTokenId.CARET);
                    case '!':
                        if (read(true) == '=') {
                            return token(CppTokenId.NOTEQ);
                        }
                        backup(1);
                        return token(CppTokenId.NOT);
                    case '.':
                        if ((c = read(true)) == '.') {
                            if (read(true) == '.') { // ellipsis ...
                                return token(CppTokenId.ELLIPSIS);
                            } else {
                                input.backup(2);
                            }
                        } else if ('0' <= c && c <= '9') { // float literal
                            return finishNumberLiteral(read(true), true);
                        } else if (c == '*') {
                            return token(CppTokenId.DOTMBR);
                        } else {
                            backup(1);
                        }
                        return token(CppTokenId.DOT);
                    case ':':
                        if (read(true) == ':') {
                            return token(CppTokenId.SCOPE);
                        }
                        backup(1);
                        return token(CppTokenId.COLON);
                    case '~':
                        return token(CppTokenId.TILDE);
                    case ',':
                        return token(CppTokenId.COMMA);
                    case ';':
                        return token(CppTokenId.SEMICOLON);
                    case '?':
                        return token(CppTokenId.QUESTION);
                    case '(':
                        return token(CppTokenId.LPAREN);
                    case ')':
                        return token(CppTokenId.RPAREN);
                    case '[':
                        return token(CppTokenId.LBRACKET);
                    case ']':
                        return token(CppTokenId.RBRACKET);
                    case '{':
                        return token(CppTokenId.LBRACE);
                    case '}':
                        return token(CppTokenId.RBRACE);
                    case '`':
                        return token(CppTokenId.GRAVE_ACCENT);
                    case '@':
                        return token(CppTokenId.AT);
                    case '0': // in a number literal
                        c = read(true);
                        if (c == 'x' || c == 'X' || // in hexadecimal (possibly floating-point) literal
                                c == 'b' || c == 'B' ) { // in bianry literal
                            boolean inFraction = false;
                            while (true) {
                                switch (read(true)) {
                                    case '0':
                                    case '1':
                                    case '2':
                                    case '3':
                                    case '4':
                                    case '5':
                                    case '6':
                                    case '7':
                                    case '8':
                                    case '9':
                                    case 'a':
                                    case 'b':
                                    case 'c':
                                    case 'd':
                                    case 'e':
                                    case 'f':
                                    case 'A':
                                    case 'B':
                                    case 'C':
                                    case 'D':
                                    case 'E':
                                    case 'F':
                                        break;
                                    case '.': // hex float literal
                                        if (!inFraction) {
                                            inFraction = true;
                                        } else { // two dots in the float literal
                                            return token(CppTokenId.FLOAT_LITERAL_INVALID);
                                        }
                                        break;
                                    case 'l':
                                    case 'L': // 0x1234l or 0x1234L
                                        return finishLongLiteral(read(true));
                                    case 'p':
                                    case 'P': // binary exponent
                                        return finishFloatExponent();
                                    case 'u':
                                    case 'U':
                                        return finishUnsignedLiteral(read(true));
                                    default:
                                        backup(1);
                                        // if float then before mandatory binary exponent => invalid
                                        return token(inFraction ? CppTokenId.FLOAT_LITERAL_INVALID
                                                : CppTokenId.INT_LITERAL);
                                }
                            } // end of while(true)
                        }
                        return finishNumberLiteral(c, false);
                    case '1':
                    case '2':
                    case '3':
                    case '4':
                    case '5':
                    case '6':
                    case '7':
                    case '8':
                    case '9':
                        return finishNumberLiteral(read(true), false);
                    case '\\':
                        return token(CppTokenId.BACK_SLASH);
                    case '\r':
                        consumeNewline();
                        return token(CppTokenId.NEW_LINE);
                    case '\n':
                        return token(CppTokenId.NEW_LINE, "\n", PartType.COMPLETE); // NOI18N
                    // All Character.isWhitespace(c) below 0x80 follow
                    // ['\t' - '\f'] and [0x1c - ' ']
                    case '\t':
                    case 0x0b:
                    case '\f':
                    case 0x1c:
                    case 0x1d:
                    case 0x1e:
                    case 0x1f:
                        return finishWhitespace();
                    case ' ':
                        c = read(true);
                        if (c == EOF || !Character.isWhitespace(c) || c == '\n' || c == '\r') { // Return single space as flyweight token
                            backup(1);
                            return token(CppTokenId.WHITESPACE, " ", PartType.COMPLETE); // NOI18N
                        }
                        return finishWhitespace();
                    case EOF:
                        if (isTokenSplittedByEscapedLine()) {
                            backup(1);
                            assert lastTokenEndedByEscapedLine > 0 : "lastTokenEndedByEscapedLine is " + lastTokenEndedByEscapedLine;
                            break;
                        }
                        return null;
                    case '$':
                        // dollar is extension in gcc and msvc $ is a valid start of identifiers
//                        return token(CppTokenId.DOLLAR);
                    default:
                        c = translateSurrogates(c);
                        if (CndLexerUtilities.isCppIdentifierStart(c)) {
                            if (c == 'L' || c == 'U' || c == 'u' || c == 'R') {
                                int next = read(true);
                                boolean raw_string = (c == 'R');
                                if (next == 'R' && (c == 'u' || c == 'U' || c == 'L')) {
                                    // uR, UR or LR
                                    raw_string = true;
                                    next = read(true);
                                } else if (next == '8' && c == 'u') {
                                    // u8
                                    next = read(true);
                                    if (next == 'R') {
                                        // u8R
                                        raw_string = true;
                                        next = read(true);
                                    }
                                }
                                if (next == '"') {
                                    // string with L/U/u/R prefixes
                                    Token<CppTokenId> out = raw_string ? finishRawString() : finishDblQuote();
                                    assert out != null : "not handled dobule quote";
                                    return out;
                                } else if (next == '\'' && !raw_string) {
                                    // char with L or U/u prefix
                                    Token<CppTokenId> out = finishSingleQuote();
                                    assert out != null : "not handled single quote";
                                    return out;
                                } else {
                                    backup(1);
                                }
                            }
                            if (c == 'E') {
                                if(isExecSQL(c)) {
                                    Token<CppTokenId> out = finishExecSQL();
                                    assert out != null : "not handled exec sql";
                                    return out;
                                }
                            }
                            return keywordOrIdentifier(c);
                        }
                        if (Character.isWhitespace(c)) {
                            return finishWhitespace();
                        }
                        // Invalid char
                        return token(CppTokenId.ERROR);
                }
            } // end of switch (c)
        } // end of while(true)
    }
@SuppressWarnings("fallthrough")
    @Override
    public Token<CppTokenId> nextToken() {
        while (true) {
           
            if (lastTokenEndedByEscapedLine > 0) {
                int c = read(false);
                lastTokenEndedByEscapedLine--;
                assert c == '\\' : "there must be \\";
                c = read(false);
                assert c == '\n' || c == '\r' : "there must be \r or \n";
                if (c == '\r') {
                    lastTokenEndedByEscapedLine--;
                    if (input.consumeNewline()) {
                        lastTokenEndedByEscapedLine--;
                    }
                    return token(CppTokenId.ESCAPED_LINE);
                } else {
                    lastTokenEndedByEscapedLine--;
                    return token(CppTokenId.ESCAPED_LINE, "\\\n", PartType.COMPLETE);
                }
            } else {
                int c = read(true);
               
               
                switch (c) {
                    case '"': {
                        Token<CppTokenId> out = finishDblQuote();
                        assert out != null : "not handled dobule quote";
                        return out;
                    }
                    case '\'': 
                        Token<CppTokenId> out = finishSingleQuote();
                        assert out != null : "not handled single quote";
                        return out;
                    }
                    case '#': {
                        Token<CppTokenId> out = finishSharp();
                        assert out != null : "not handled #";
                        return out;
                    }
                    case '/':
                        switch (read(true)) {
                            case '/':
                            {
                                Token<CppTokenId> out = finishLineComment(true);
                                assert out != null : "not handled //";
                                return out;
                            }
                            case '=':
                                return token(CppTokenId.SLASHEQ);
                            case '*':
                            {
                                Token<CppTokenId> out = finishBlockComment(true);
                                assert out != null : "not handled /*";
                                return out;
                            }
                        }
                        backup(1);
                        return token(CppTokenId.SLASH);
                    case '=':
                        if (read(true) == '=') {
                            return token(CppTokenId.EQEQ);
                        }
                        backup(1);
                        return token(CppTokenId.EQ);
                    case '>':
                        switch (read(true)) {
                            case '>':
                                if (read(true) == '=') {
                                    return token(CppTokenId.GTGTEQ);
                                }
                                backup(1);
                                return token(CppTokenId.GTGT);
                            case '=':
                                return token(CppTokenId.GTEQ);
                        }
                        backup(1);
                        return token(CppTokenId.GT);
                    case '<': {
                        Token<CppTokenId> out = finishLT();
                        assert out != null : "not handled '<'";
                        return out;
                    }
                    case '+':
                        switch (read(true)) {
                            case '+':
                                return token(CppTokenId.PLUSPLUS);
                            case '=':
                                return token(CppTokenId.PLUSEQ);
                        }
                        backup(1);
                        return token(CppTokenId.PLUS);
                    case '-':
                        switch (read(true)) {
                            case '-':
                                return token(CppTokenId.MINUSMINUS);
                            case '>':
                                if (read(true) == '*') {
                                    return token(CppTokenId.ARROWMBR);
                                }
                                backup(1);
                                return token(CppTokenId.ARROW);
                            case '=':
                                return token(CppTokenId.MINUSEQ);
                        }
                        backup(1);
                        return token(CppTokenId.MINUS);
                    case '*':
                        switch (read(true)) {
                            case '/':
                                if (read(true) == '*') {
                                    backup(2);
                                    return token(CppTokenId.STAR);
                                }
                                backup(1);
                                return token(CppTokenId.INVALID_COMMENT_END);
                            case '=':
                                return token(CppTokenId.STAREQ);
                        }
                        backup(1);
                        return token(CppTokenId.STAR);
                    case '|':
                        switch (read(true)) {
                            case '|':
                                return token(CppTokenId.BARBAR);
                            case '=':
                                return token(CppTokenId.BAREQ);
                        }
                        backup(1);
                        return token(CppTokenId.BAR);
                    case '&':
                        switch (read(true)) {
                            case '&':
                                return token(CppTokenId.AMPAMP);
                            case '=':
                                return token(CppTokenId.AMPEQ);
                        }
                        backup(1);
                        return token(CppTokenId.AMP);
                    case '%': {
                        Token<CppTokenId> out = finishPercent();
                        assert out != null : "not handled %";
                        return out;
                    }
                    case '^':
                        if (read(true) == '=') {
                            return token(CppTokenId.CARETEQ);
                        }
                        backup(1);
                        return token(CppTokenId.CARET);
                    case '!':
                        if (read(true) == '=') {
                            return token(CppTokenId.NOTEQ);
                        }
                        backup(1);
                        return token(CppTokenId.NOT);
                    case '.':
                        if ((c = read(true)) == '.') {
                            if (read(true) == '.') {
                                return token(CppTokenId.ELLIPSIS);
                            } else {
                                input.backup(2);
                            }
                        } else if ('0' <= c && c <= '9') {
                            return finishNumberLiteral(read(true), true);
                        } else if (c == '*') {
                            return token(CppTokenId.DOTMBR);
                        } else {
                            backup(1);
                        }
                        return token(CppTokenId.DOT);
                    case ':':
                        if (read(true) == ':') {
                            return token(CppTokenId.SCOPE);
                        }
                        backup(1);
                        return token(CppTokenId.COLON);
                    case '~':
                        return token(CppTokenId.TILDE);
                    case ',':
                        return token(CppTokenId.COMMA);
                    case ';':
                        return token(CppTokenId.SEMICOLON);
                    case '?':
                        return token(CppTokenId.QUESTION);
                    case '(':
                        return token(CppTokenId.LPAREN);
                    case ')':
                        return token(CppTokenId.RPAREN);
                    case '[':
                        return token(CppTokenId.LBRACKET);
                    case ']':
                        return token(CppTokenId.RBRACKET);
                    case '{':
                        return token(CppTokenId.LBRACE);
                    case '}':
                        return token(CppTokenId.RBRACE);
                    case '`':
                        return token(CppTokenId.GRAVE_ACCENT);
                    case '@':
                        return token(CppTokenId.AT);
                    case '0':
                        c = read(true);
                        if (c == 'x' || c == 'X' ||
                                c == 'b' || c == 'B' ) {
                            boolean inFraction = false;
                            while (true) {
                                switch (read(true)) {
                                    case '0':
                                    case '1':
                                    case '2':
                                    case '3':
                                    case '4':
                                    case '5':
                                    case '6':
                                    case '7':
                                    case '8':
                                    case '9':
                                    case 'a':
                                    case 'b':
                                    case 'c':
                                    case 'd':
                                    case 'e':
                                    case 'f':
                                    case 'A':
                                    case 'B':
                                    case 'C':
                                    case 'D':
                                    case 'E':
                                    case 'F':
                                        break;
                                    case '.':
                                        if (!inFraction) {
                                            inFraction = true;
                                        } else {
                                            return token(CppTokenId.FLOAT_LITERAL_INVALID);
                                        }
                                        break;
                                    case 'l':
                                    case 'L':
                                        return finishLongLiteral(read(true));
                                    case 'p':
                                    case 'P':
                                        return finishFloatExponent();
                                    case 'u':
                                    case 'U':
                                        return finishUnsignedLiteral(read(true));
                                    default:
                                        backup(1);
                                       
                                        return token(inFraction ? CppTokenId.FLOAT_LITERAL_INVALID
                                                : CppTokenId.INT_LITERAL);
                                }
                            }
                        }
                        return finishNumberLiteral(c, false);
                    case '1':
                    case '2':
                    case '3':
                    case '4':
                    case '5':
                    case '6':
                    case '7':
                    case '8':
                    case '9':
                        return finishNumberLiteral(read(true), false);
                    case '\\':
                        return token(CppTokenId.BACK_SLASH);
                    case '\r':
                        consumeNewline();
                        return token(CppTokenId.NEW_LINE);
                    case '\n':
                        return token(CppTokenId.NEW_LINE, "\n", PartType.COMPLETE);
                   
                   
                    case '\t':
                    case 0x0b:
                    case '\f':
                    case 0x1c:
                    case 0x1d:
                    case 0x1e:
                    case 0x1f:
                        return finishWhitespace();
                    case ' ':
                        c = read(true);
                        if (c == EOF || !Character.isWhitespace(c) || c == '\n' || c == '\r') {
                            backup(1);
                            return token(CppTokenId.WHITESPACE, " ", PartType.COMPLETE);
                        }
                        return finishWhitespace();
                    case EOF:
                        if (isTokenSplittedByEscapedLine()) {
                            backup(1);
                            assert lastTokenEndedByEscapedLine > 0 : "lastTokenEndedByEscapedLine is " + lastTokenEndedByEscapedLine;
                            break;
                        }
                        return null;
                    case '$':
                       
                    default:
                        c = translateSurrogates(c);
                        if (CndLexerUtilities.isCppIdentifierStart(c)) {
                            if (c == 'L' || c == 'U' || c == 'u' || c == 'R') {
                                int next = read(true);
                                boolean raw_string = (c == 'R');
                                if (next == 'R' && (c == 'u' || c == 'U' || c == 'L')) {
                                   
                                    raw_string = true;
                                    next = read(true);
                                } else if (next == '8' && c == 'u') {
                                   
                                    next = read(true);
                                    if (next == 'R') {
                                       
                                        raw_string = true;
                                        next = read(true);
                                    }
                                }
                                if (next == '"') {
                                   
                                    Token<CppTokenId> out = raw_string ? finishRawString() : finishDblQuote();
                                    assert out != null : "not handled dobule quote";
                                    return out;
                                } else if (next == '\'' && !raw_string) {
                                   
                                    Token<CppTokenId> out = finishSingleQuote();
                                    assert out != null : "not handled single quote";
                                    return out;
                                } else {
                                    backup(1);
                                }
                            }
                            if (c == 'E') {
                                if(isExecSQL(c)) {
                                    Token<CppTokenId> out = finishExecSQL();
                                    assert out != null : "not handled exec sql";
                                    return out;
                                }
                            }
                            return keywordOrIdentifier(c);
                        }
                        if (Character.isWhitespace(c)) {
                            return finishWhitespace();
                        }
                       
                        return token(CppTokenId.ERROR);
                }
            }
        }
    }
@suppresswarnings("fallthrough") @override public token<cpptokenid> nexttoken() { while (true) { if (lasttokenendedbyescapedline > 0) { int c = read(false); lasttokenendedbyescapedline--; assert c == '\\' : "there must be \\"; c = read(false); assert c == '\n' || c == '\r' : "there must be \r or \n"; if (c == '\r') { lasttokenendedbyescapedline--; if (input.consumenewline()) { lasttokenendedbyescapedline--; } return token(cpptokenid.escaped_line); } else { lasttokenendedbyescapedline--; return token(cpptokenid.escaped_line, "\\\n", parttype.complete); } } else { int c = read(true); switch (c) { case '"': { token<cpptokenid> out = finishdblquote(); assert out != null : "not handled dobule quote"; return out; } case '\'': token<cpptokenid> out = finishsinglequote(); assert out != null : "not handled single quote"; return out; } case '#': { token<cpptokenid> out = finishsharp(); assert out != null : "not handled #"; return out; } case '/': switch (read(true)) { case '/': { token<cpptokenid> out = finishlinecomment(true); assert out != null : "not handled //"; return out; } case '=': return token(cpptokenid.slasheq); case '*': { token<cpptokenid> out = finishblockcomment(true); assert out != null : "not handled /*"; return out; } } backup(1); return token(cpptokenid.slash); case '=': if (read(true) == '=') { return token(cpptokenid.eqeq); } backup(1); return token(cpptokenid.eq); case '>': switch (read(true)) { case '>': if (read(true) == '=') { return token(cpptokenid.gtgteq); } backup(1); return token(cpptokenid.gtgt); case '=': return token(cpptokenid.gteq); } backup(1); return token(cpptokenid.gt); case '<': { token<cpptokenid> out = finishlt(); assert out != null : "not handled '<'"; return out; } case '+': switch (read(true)) { case '+': return token(cpptokenid.plusplus); case '=': return token(cpptokenid.pluseq); } backup(1); return token(cpptokenid.plus); case '-': switch (read(true)) { case '-': return token(cpptokenid.minusminus); case '>': if (read(true) == '*') { return token(cpptokenid.arrowmbr); } backup(1); return token(cpptokenid.arrow); case '=': return token(cpptokenid.minuseq); } backup(1); return token(cpptokenid.minus); case '*': switch (read(true)) { case '/': if (read(true) == '*') { backup(2); return token(cpptokenid.star); } backup(1); return token(cpptokenid.invalid_comment_end); case '=': return token(cpptokenid.stareq); } backup(1); return token(cpptokenid.star); case '|': switch (read(true)) { case '|': return token(cpptokenid.barbar); case '=': return token(cpptokenid.bareq); } backup(1); return token(cpptokenid.bar); case '&': switch (read(true)) { case '&': return token(cpptokenid.ampamp); case '=': return token(cpptokenid.ampeq); } backup(1); return token(cpptokenid.amp); case '%': { token<cpptokenid> out = finishpercent(); assert out != null : "not handled %"; return out; } case '^': if (read(true) == '=') { return token(cpptokenid.careteq); } backup(1); return token(cpptokenid.caret); case '!': if (read(true) == '=') { return token(cpptokenid.noteq); } backup(1); return token(cpptokenid.not); case '.': if ((c = read(true)) == '.') { if (read(true) == '.') { return token(cpptokenid.ellipsis); } else { input.backup(2); } } else if ('0' <= c && c <= '9') { return finishnumberliteral(read(true), true); } else if (c == '*') { return token(cpptokenid.dotmbr); } else { backup(1); } return token(cpptokenid.dot); case ':': if (read(true) == ':') { return token(cpptokenid.scope); } backup(1); return token(cpptokenid.colon); case '~': return token(cpptokenid.tilde); case ',': return token(cpptokenid.comma); case ';': return token(cpptokenid.semicolon); case '?': return token(cpptokenid.question); case '(': return token(cpptokenid.lparen); case ')': return token(cpptokenid.rparen); case '[': return token(cpptokenid.lbracket); case ']': return token(cpptokenid.rbracket); case '{': return token(cpptokenid.lbrace); case '}': return token(cpptokenid.rbrace); case '`': return token(cpptokenid.grave_accent); case '@': return token(cpptokenid.at); case '0': c = read(true); if (c == 'x' || c == 'x' || c == 'b' || c == 'b' ) { boolean infraction = false; while (true) { switch (read(true)) { case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': case 'a': case 'b': case 'c': case 'd': case 'e': case 'f': case 'a': case 'b': case 'c': case 'd': case 'e': case 'f': break; case '.': if (!infraction) { infraction = true; } else { return token(cpptokenid.float_literal_invalid); } break; case 'l': case 'l': return finishlongliteral(read(true)); case 'p': case 'p': return finishfloatexponent(); case 'u': case 'u': return finishunsignedliteral(read(true)); default: backup(1); return token(infraction ? cpptokenid.float_literal_invalid : cpptokenid.int_literal); } } } return finishnumberliteral(c, false); case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': return finishnumberliteral(read(true), false); case '\\': return token(cpptokenid.back_slash); case '\r': consumenewline(); return token(cpptokenid.new_line); case '\n': return token(cpptokenid.new_line, "\n", parttype.complete); case '\t': case 0x0b: case '\f': case 0x1c: case 0x1d: case 0x1e: case 0x1f: return finishwhitespace(); case ' ': c = read(true); if (c == eof || !character.iswhitespace(c) || c == '\n' || c == '\r') { backup(1); return token(cpptokenid.whitespace, " ", parttype.complete); } return finishwhitespace(); case eof: if (istokensplittedbyescapedline()) { backup(1); assert lasttokenendedbyescapedline > 0 : "lasttokenendedbyescapedline is " + lasttokenendedbyescapedline; break; } return null; case '$': default: c = translatesurrogates(c); if (cndlexerutilities.iscppidentifierstart(c)) { if (c == 'l' || c == 'u' || c == 'u' || c == 'r') { int next = read(true); boolean raw_string = (c == 'r'); if (next == 'r' && (c == 'u' || c == 'u' || c == 'l')) { raw_string = true; next = read(true); } else if (next == '8' && c == 'u') { next = read(true); if (next == 'r') { raw_string = true; next = read(true); } } if (next == '"') { token<cpptokenid> out = raw_string ? finishrawstring() : finishdblquote(); assert out != null : "not handled dobule quote"; return out; } else if (next == '\'' && !raw_string) { token<cpptokenid> out = finishsinglequote(); assert out != null : "not handled single quote"; return out; } else { backup(1); } } if (c == 'e') { if(isexecsql(c)) { token<cpptokenid> out = finishexecsql(); assert out != null : "not handled exec sql"; return out; } } return keywordoridentifier(c); } if (character.iswhitespace(c)) { return finishwhitespace(); } return token(cpptokenid.error); } } } }