question: private void parse ( ) { if ( lexer == null ) { tokens = null ; return ; } List < Token > toks = new ArrayList < Token > ( getLength ( ) / 10 ) ; long ts = System . nanoTime ( ) ; int len = getLength ( ) ; try { Segment seg = new Segment ( ) ; getText ( 0 , getLength ( ) , seg ) ; CharArrayReader reader = new CharArrayReader ( seg . array , seg . offset , seg . count ) ; lexer . yyreset ( reader ) ; Token token ; while ( ( token = lexer . yylex ( ) ) != null ) { toks . add ( token ) ; } } catch ( BadLocationException ex ) { log . log ( Level . SEVERE , null , ex ) ; } catch ( IOException ex ) { log . log ( Level . SEVERE , null , ex ) ; } finally { if ( log . isLoggable ( Level . FINEST ) ) { log . finest ( String . format ( "Parsed %d in %d ms, giving %d tokens\n" , len , ( System . nanoTime ( ) - ts ) / 1000000 , toks . size ( ) ) ) ; } tokens = toks ; } }
target:  Parse the entire document and return list of tokens that do not already exist in the tokens list.  There may be overlaps, and replacements,  which we will cleanup later.