@Override public boolean incrementToken() throws IOException {
  if (savedState != null) {
    restoreState(savedState);
    savedState=null;
    saveTermBuffer();
    return true;
  }
 else   if (!input.incrementToken()) {
    return false;
  }
  if (lastWasCommon || (isCommon() && buffer.length() > 0)) {
    savedState=captureState();
    gramToken();
    return true;
  }
  saveTermBuffer();
  return true;
}

@ Override public boolean incrementToken ( ) throws IOException { if ( savedState != null ) { restoreState ( savedState ) ; savedState = null ; saveTermBuffer ( ) ; return true ; } else if ( ! input . incrementToken ( ) ) { return false ; } if ( lastWasCommon || ( isCommon ( ) && buffer . length ( ) > 0 ) ) { savedState = captureState ( ) ; gramToken ( ) ; return true ; } saveTermBuffer ( ) ; return true ; }
inserts bigrams for common words into a token stream . for each input token , output the token . if the token and / or the following token are in the list of common words also output a bigram with position increment 0 and type = " gram " todo : consider adding an option to not emit unigram stopwords as in cdl xtf bigramstopfilter , commongramsqueryfilter would need to be changed to work with this . todo : consider optimizing for the case of three commongrams i . e " man of the year " normally produces 3 bigrams : " man - of " , " of - the " , " the - year " but with proper management of positions we could eliminate the middle bigram " of - the " and save a disk seek and a whole set of position lookups .
