private void parse(){
  if (lexer == null) {
    tokens=null;
    return;
  }
  List<Token> toks=new ArrayList<Token>(getLength() / 10);
  long ts=System.nanoTime();
  int len=getLength();
  try {
    Segment seg=new Segment();
    getText(0,getLength(),seg);
    CharArrayReader reader=new CharArrayReader(seg.array,seg.offset,seg.count);
    lexer.yyreset(reader);
    Token token;
    while ((token=lexer.yylex()) != null) {
      toks.add(token);
    }
  }
 catch (  BadLocationException ex) {
    log.log(Level.SEVERE,null,ex);
  }
catch (  IOException ex) {
    log.log(Level.SEVERE,null,ex);
  }
 finally {
    if (log.isLoggable(Level.FINEST)) {
      log.finest(String.format("Parsed %d in %d ms, giving %d tokens\n",len,(System.nanoTime() - ts) / 1000000,toks.size()));
    }
    tokens=toks;
  }
}

private void parse ( ) { if ( lexer == null ) { tokens = null ; return ; } List < Token > toks = new ArrayList < Token > ( getLength ( ) / 10 ) ; long ts = System . nanoTime ( ) ; int len = getLength ( ) ; try { Segment seg = new Segment ( ) ; getText ( 0 , getLength ( ) , seg ) ; CharArrayReader reader = new CharArrayReader ( seg . array , seg . offset , seg . count ) ; lexer . yyreset ( reader ) ; Token token ; while ( ( token = lexer . yylex ( ) ) != null ) { toks . add ( token ) ; } } catch ( BadLocationException ex ) { log . log ( Level . SEVERE , null , ex ) ; } catch ( IOException ex ) { log . log ( Level . SEVERE , null , ex ) ; } finally { if ( log . isLoggable ( Level . FINEST ) ) { log . finest ( String . format ( "Parsed %d in %d ms, giving %d tokens\n" , len , ( System . nanoTime ( ) - ts ) / 1000000 , toks . size ( ) ) ) ; } tokens = toks ; } }
parse the entire document and return list of tokens that do not already exist in the tokens list . there may be overlaps , and replacements , which we will cleanup later .
