private Token pollNext(){
  Token result=currentToken;
  if (matcher.find()) {
    if (index < matcher.start()) {
      currentToken=new Token(TokenKind.TEXT,source.substring(index,matcher.start()),lineNum);
      index=matcher.start();
      matcher.region(index,matcher.regionEnd());
    }
 else {
      currentToken=createTokenFromMatcher();
      index=matcher.end();
    }
  }
 else {
    if (index < source.length()) {
      currentToken=new Token(TokenKind.TEXT,source.substring(index),lineNum);
      index=source.length();
    }
 else {
      currentToken=null;
    }
  }
  return result;
}

private Token pollNext ( ) { Token result = currentToken ; if ( matcher . find ( ) ) { if ( index < matcher . start ( ) ) { currentToken = new Token ( TokenKind . TEXT , source . substring ( index , matcher . start ( ) ) , lineNum ) ; index = matcher . start ( ) ; matcher . region ( index , matcher . regionEnd ( ) ) ; } else { currentToken = createTokenFromMatcher ( ) ; index = matcher . end ( ) ; } } else { if ( index < source . length ( ) ) { currentToken = new Token ( TokenKind . TEXT , source . substring ( index ) , lineNum ) ; index = source . length ( ) ; } else { currentToken = null ; } } return result ; }
returns next token . the matcher will move forward .
