public int emitCommentsToTheLeft(int tokenIndexInStream,int injectNL_WS){
  List<Token> hiddenTokensToLeft=originalTokens.getHiddenTokensToLeft(tokenIndexInStream);
  if (hiddenTokensToLeft != null) {
    boolean hasComment=Trainer.hasCommentToken(hiddenTokensToLeft);
    if (hasComment) {
      int last=-1;
      for (int i=hiddenTokensToLeft.size() - 1; i >= 0; i--) {
        Token hidden=hiddenTokensToLeft.get(i);
        String hiddenText=hidden.getText();
        if (!hiddenText.matches("\\s+")) {
          last=i;
          break;
        }
      }
      Token commentToken=hiddenTokensToLeft.get(last);
      List<Token> truncated=hiddenTokensToLeft.subList(0,last + 1);
      for (      Token hidden : truncated) {
        String hiddenText=hidden.getText();
        output.append(hiddenText);
        if (hiddenText.matches("\\n+")) {
          line+=Tool.count(hiddenText,'\n');
          charPosInLine=0;
        }
 else {
          charPosInLine+=hiddenText.length();
        }
      }
      if (commentToken.getType() == corpus.language.singleLineCommentType && (injectNL_WS & 0xFF) != CAT_INJECT_NL) {
        return nlcat(1);
      }
    }
  }
  return injectNL_WS;
}

public int emitCommentsToTheLeft ( int tokenIndexInStream , int injectNL_WS ) { List < Token > hiddenTokensToLeft = originalTokens . getHiddenTokensToLeft ( tokenIndexInStream ) ; if ( hiddenTokensToLeft != null ) { boolean hasComment = Trainer . hasCommentToken ( hiddenTokensToLeft ) ; if ( hasComment ) { int last = - 1 ; for ( int i = hiddenTokensToLeft . size ( ) - 1 ; i >= 0 ; i -- ) { Token hidden = hiddenTokensToLeft . get ( i ) ; String hiddenText = hidden . getText ( ) ; if ( ! hiddenText . matches ( "\\s+" ) ) { last = i ; break ; } } Token commentToken = hiddenTokensToLeft . get ( last ) ; List < Token > truncated = hiddenTokensToLeft . subList ( 0 , last + 1 ) ; for ( Token hidden : truncated ) { String hiddenText = hidden . getText ( ) ; output . append ( hiddenText ) ; if ( hiddenText . matches ( "\\n+" ) ) { line += Tool . count ( hiddenText , '\n' ) ; charPosInLine = 0 ; } else { charPosInLine += hiddenText . length ( ) ; } } if ( commentToken . getType ( ) == corpus . language . singleLineCommentType && ( injectNL_WS & 0xFF ) != CAT_INJECT_NL ) { return nlcat ( 1 ) ; } } } return injectNL_WS ; }
look into the originaltokens stream to get the comments to the left of current token . emit all whitespace and comments except for whitespace at the end as we ' ll inject that per newline prediction . we able to see original input stream for comment purposes only . with all whitespace removed , we can ' t emit this stuff properly . this is the only place that examines the original token stream during formatting .
