src/sys/dotnet/fanx/serial/Tokenizer.cs
/// <summary>
/// Read the next token, set the 'val' field but return
/// type without worrying setting the 'type' field.
/// </summary>
Read the next token, set the 'val' field but return
type without worrying setting the 'type' field.
Read the next token set the ' val ' field but return type without worrying setting the ' type ' field .
private int doNext()
    {
      while (true)
      {
        while (curt == SPACE) consume();
        if (cur < 0) return Token.EOF;
        if (curt == ALPHA) return id();
        if (curt == DIGIT) return number(false);
        switch (cur)
        {
          case '+':   consume(); return number(false);
          case '-':   consume(); return number(true);
          case '"':   return str();
          case '\'':  return ch();
          case '`':   return uri();
          case '(':   consume(); return Token.LPAREN;
          case ')':   consume(); return Token.RPAREN;
          case ',':   consume(); return Token.COMMA;
          case ';':   consume(); return Token.SEMICOLON;
          case '=':   consume(); return Token.EQ;
          case '{':   consume(); return Token.LBRACE;
          case '}':   consume(); return Token.RBRACE;
          case '#':   consume(); return Token.POUND;
          case '?':   consume(); return Token.QUESTION;
          case '@':   consume(); return Token.AT;
          case '.':
            if (peekt == DIGIT) return number(false);
            consume();
            return Token.DOT;
          case '[':
            consume();
            if (cur == ']') { consume(); return Token.LRBRACKET; }
            return Token.LBRACKET;
          case ']':
            consume();
            return Token.RBRACKET;
          case ':':
            consume();
            if (cur == ':') { consume(); return Token.DOUBLE_COLON; }
            return Token.COLON;
          case '*':
            if (peek == '*') { skipCommentSL(); continue; }
            break;
          case '/':
            if (peek == '/') { skipCommentSL(); continue; }
            if (peek == '*') { skipCommentML(); continue; }
            break;
        }
        throw err("Unexpected symbol: " + (char)cur + " (0x" + System.Convert.ToInt32(cur).ToString("X").ToLower() + ")");
      }
    }
private int doNext ( ) { while ( true ) { while ( curt == SPACE ) consume ( ) ; if ( cur < 0 ) return Token . EOF ; if ( curt == ALPHA ) return id ( ) ; if ( curt == DIGIT ) return number ( false ) ; switch ( cur ) { case ' ' : consume ( ) ; return number ( false ) ; case ' ' : consume ( ) ; return number ( true ) ; case ' ' : return str ( ) ; case ' \' ' : return ch ( ) ; case ' ' : return uri ( ) ; case ' ' : consume ( ) ; return Token . LPAREN ; case ' ' : consume ( ) ; return Token . RPAREN ; case ' ' : consume ( ) ; return Token . COMMA ; case ' ' : consume ( ) ; return Token . SEMICOLON ; case ' ' : consume ( ) ; return Token . EQ ; case ' ' : consume ( ) ; return Token . LBRACE ; case ' ' : consume ( ) ; return Token . RBRACE ; case ' ' : consume ( ) ; return Token . POUND ; case ' ' : consume ( ) ; return Token . QUESTION ; case ' ' : consume ( ) ; return Token . AT ; case ' ' : if ( peekt == DIGIT ) return number ( false ) ; consume ( ) ; return Token . DOT ; case ' ' : consume ( ) ; if ( cur == ' ' ) { consume ( ) ; return Token . LRBRACKET ; } return Token . LBRACKET ; case ' ' : consume ( ) ; return Token . RBRACKET ; case ' ' : consume ( ) ; if ( cur == ' ' ) { consume ( ) ; return Token . DOUBLE_COLON ; } return Token . COLON ; case ' ' : if ( peek == ' ' ) { skipCommentSL ( ) ; continue ; } break ; case ' ' : if ( peek == ' ' ) { skipCommentSL ( ) ; continue ; } if ( peek == ' ' ) { skipCommentML ( ) ; continue ; } break ; } throw err ( " " + ( char ) cur + " " + System . Convert . ToInt32 ( cur ) . ToString ( " " ) . ToLower ( ) + " " ) ; } }
Read the next token, set the 'val' field but return
type without worrying setting the 'type' field.
Read the next token set the ' val ' field but return type without worrying setting the ' type ' field .
// skip whitespace // alpha means identifier // number // symbol // invalid character