public String readToken(String delimiters) throws IOException {
  int space=0;
  char ch, ch2, quoteChar='\0';
  boolean done=false, first=true, quoted=false, isSpace;
  nextCharacter();
  StringBuffer token=new StringBuffer();
  while (!done) {
    ch=read();
    try {
      isSpace=Character.isWhitespace(ch);
      if (quoted && ch == quoteChar) {
        ch2=read();
        if (ch == ch2) {
          token.append(ch);
        }
 else {
          lastDelimiter=' ';
          unreadCharacter(ch2);
          done=true;
          quoted=false;
        }
      }
 else       if (first && (ch == '\'' || ch == '"')) {
        quoted=true;
        quoteChar=ch;
        first=false;
        space=0;
      }
 else       if (ch == startComment || ch == lineComment) {
        skipComments(ch);
        lastDelimiter=' ';
        done=true;
      }
 else {
        if (quoted) {
          if (isSpace) {
            space++;
            ch=' ';
          }
 else {
            space=0;
          }
          if (space < 2) {
            token.append(ch);
          }
        }
 else         if (isSpace) {
          lastDelimiter=' ';
          done=true;
        }
 else         if (delimiters.indexOf(ch) != -1) {
          done=true;
          lastDelimiter=ch;
        }
 else {
          token.append(ch);
          first=false;
        }
      }
    }
 catch (    EOFException e) {
      done=true;
    }
  }
  if (Character.isWhitespace((char)lastDelimiter)) {
    ch=nextCharacter();
    while (Character.isWhitespace(ch)) {
      read();
      ch=nextCharacter();
    }
    if (delimiters.indexOf(ch) != -1) {
      lastDelimiter=readCharacter();
    }
  }
  return token.toString();
}

public String readToken ( String delimiters ) throws IOException { int space = 0 ; char ch , ch2 , quoteChar = '\0' ; boolean done = false , first = true , quoted = false , isSpace ; nextCharacter ( ) ; StringBuffer token = new StringBuffer ( ) ; while ( ! done ) { ch = read ( ) ; try { isSpace = Character . isWhitespace ( ch ) ; if ( quoted && ch == quoteChar ) { ch2 = read ( ) ; if ( ch == ch2 ) { token . append ( ch ) ; } else { lastDelimiter = ' ' ; unreadCharacter ( ch2 ) ; done = true ; quoted = false ; } } else if ( first && ( ch == '\'' || ch == '"' ) ) { quoted = true ; quoteChar = ch ; first = false ; space = 0 ; } else if ( ch == startComment || ch == lineComment ) { skipComments ( ch ) ; lastDelimiter = ' ' ; done = true ; } else { if ( quoted ) { if ( isSpace ) { space ++ ; ch = ' ' ; } else { space = 0 ; } if ( space < 2 ) { token . append ( ch ) ; } } else if ( isSpace ) { lastDelimiter = ' ' ; done = true ; } else if ( delimiters . indexOf ( ch ) != - 1 ) { done = true ; lastDelimiter = ch ; } else { token . append ( ch ) ; first = false ; } } } catch ( EOFException e ) { done = true ; } } if ( Character . isWhitespace ( ( char ) lastDelimiter ) ) { ch = nextCharacter ( ) ; while ( Character . isWhitespace ( ch ) ) { read ( ) ; ch = nextCharacter ( ) ; } if ( delimiters . indexOf ( ch ) != - 1 ) { lastDelimiter = readCharacter ( ) ; } } return token . toString ( ) ; }
reads a token stopping when any whitespace , a comment or when any character in delimiters is found . if the token begins with a quote char then all characters will be included in token until a matching quote is found ( including whitespace or comments ) .
