public static final String cleanupTokens(final String input){
  final StringBuilder output_data=new StringBuilder();
  String current_token, current_font="";
  int pointer=0;
  boolean next_font_is_identical=false;
  final StringTokenizer data_As_tokens=new StringTokenizer(input,"<>",true);
  String next_item=data_As_tokens.nextToken();
  while (data_As_tokens.hasMoreTokens()) {
    if (next_item.equals("<") && data_As_tokens.hasMoreTokens()) {
      current_token=next_item + data_As_tokens.nextToken() + data_As_tokens.nextToken();
      pointer+=current_token.length();
      next_item="";
      if ((current_token.startsWith(fb))) {
        current_font=current_token;
      }
      if ((current_token.equals(fe))) {
        final int nextToken=input.indexOf('<',pointer - 1);
        final int nextEndToken=input.indexOf("</",pointer - 1);
        if (nextToken == nextEndToken) {
          output_data.append(current_token);
        }
 else {
          final int next_font_pointer_s=input.indexOf(fb,pointer - 1);
          final int next_font_pointer_e=input.indexOf('>',next_font_pointer_s);
          next_font_is_identical=false;
          if ((next_font_pointer_s != -1) && (next_font_pointer_e != -1)) {
            final String next_font=input.substring(next_font_pointer_s,next_font_pointer_e + 1);
            if (next_font.equals(current_font)) {
              next_font_is_identical=true;
            }
          }
          if (!next_font_is_identical) {
            output_data.append(current_token);
          }
        }
      }
 else       if ((current_token.startsWith(fb)) & next_font_is_identical) {
        next_font_is_identical=false;
      }
 else {
        output_data.append(current_token);
      }
    }
 else {
      output_data.append(next_item);
      pointer+=next_item.length();
      next_item="";
    }
    if ((data_As_tokens.hasMoreTokens())) {
      next_item=data_As_tokens.nextToken();
      if (!data_As_tokens.hasMoreTokens()) {
        output_data.append(next_item);
        pointer+=next_item.length();
      }
    }
  }
  return output_data.toString();
}

public static final String cleanupTokens ( final String input ) { final StringBuilder output_data = new StringBuilder ( ) ; String current_token , current_font = "" ; int pointer = 0 ; boolean next_font_is_identical = false ; final StringTokenizer data_As_tokens = new StringTokenizer ( input , "<>" , true ) ; String next_item = data_As_tokens . nextToken ( ) ; while ( data_As_tokens . hasMoreTokens ( ) ) { if ( next_item . equals ( "<" ) && data_As_tokens . hasMoreTokens ( ) ) { current_token = next_item + data_As_tokens . nextToken ( ) + data_As_tokens . nextToken ( ) ; pointer += current_token . length ( ) ; next_item = "" ; if ( ( current_token . startsWith ( fb ) ) ) { current_font = current_token ; } if ( ( current_token . equals ( fe ) ) ) { final int nextToken = input . indexOf ( '<' , pointer - 1 ) ; final int nextEndToken = input . indexOf ( "</" , pointer - 1 ) ; if ( nextToken == nextEndToken ) { output_data . append ( current_token ) ; } else { final int next_font_pointer_s = input . indexOf ( fb , pointer - 1 ) ; final int next_font_pointer_e = input . indexOf ( '>' , next_font_pointer_s ) ; next_font_is_identical = false ; if ( ( next_font_pointer_s != - 1 ) && ( next_font_pointer_e != - 1 ) ) { final String next_font = input . substring ( next_font_pointer_s , next_font_pointer_e + 1 ) ; if ( next_font . equals ( current_font ) ) { next_font_is_identical = true ; } } if ( ! next_font_is_identical ) { output_data . append ( current_token ) ; } } } else if ( ( current_token . startsWith ( fb ) ) & next_font_is_identical ) { next_font_is_identical = false ; } else { output_data . append ( current_token ) ; } } else { output_data . append ( next_item ) ; pointer += next_item . length ( ) ; next_item = "" ; } if ( ( data_As_tokens . hasMoreTokens ( ) ) ) { next_item = data_As_tokens . nextToken ( ) ; if ( ! data_As_tokens . hasMoreTokens ( ) ) { output_data . append ( next_item ) ; pointer += next_item . length ( ) ; } } } return output_data . toString ( ) ; }
take data which has just been joined and tidy up fonts < br > removes duplicate font commands where a font is turned off with / font tag and then turned back on again < br > this arises because we put font tokens into the data and then may merge it together .
