question: public int emitCommentsToTheLeft ( int tokenIndexInStream , int injectNL_WS ) { List < Token > hiddenTokensToLeft = originalTokens . getHiddenTokensToLeft ( tokenIndexInStream ) ; if ( hiddenTokensToLeft != null ) { boolean hasComment = Trainer . hasCommentToken ( hiddenTokensToLeft ) ; if ( hasComment ) { int last = - 1 ; for ( int i = hiddenTokensToLeft . size ( ) - 1 ; i >= 0 ; i -- ) { Token hidden = hiddenTokensToLeft . get ( i ) ; String hiddenText = hidden . getText ( ) ; if ( ! hiddenText . matches ( "\\s+" ) ) { last = i ; break ; } } Token commentToken = hiddenTokensToLeft . get ( last ) ; List < Token > truncated = hiddenTokensToLeft . subList ( 0 , last + 1 ) ; for ( Token hidden : truncated ) { String hiddenText = hidden . getText ( ) ; output . append ( hiddenText ) ; if ( hiddenText . matches ( "\\n+" ) ) { line += Tool . count ( hiddenText , '\n' ) ; charPosInLine = 0 ; } else { charPosInLine += hiddenText . length ( ) ; } } if ( commentToken . getType ( ) == corpus . language . singleLineCommentType && ( injectNL_WS & 0xFF ) != CAT_INJECT_NL ) { return nlcat ( 1 ) ; } } } return injectNL_WS ; }
target:  Look into the originalTokens stream to get the comments to the left of current token. Emit all whitespace and comments except for whitespace at the end as we'll inject that per newline prediction. We able to see original input stream for comment purposes only. With all whitespace removed, we can't emit this stuff properly. This is the only place that examines the original token stream during formatting.