public CSVTokenizer(char separator){
  addJoinedType(CSVTokenType.TEXT);
  addIgnoredType(CSVTokenType.QUOTE);
  putRules(new Rule("\n|\r\n|\r",CSVTokenType.ROW_SEPARATOR),new Rule(Pattern.quote(String.valueOf(separator)),CSVTokenType.COLUMN_SEPARATOR),new Rule("\"",CSVTokenType.QUOTE,"quoted"),new Rule("[ \t]+",CSVTokenType.EMPTY_SPACE),new Rule(".",CSVTokenType.TEXT));
  putRules("quoted",new Rule("(\")\"",CSVTokenType.TEXT),new Rule("\"",CSVTokenType.QUOTE,"#pop"),new Rule(".",CSVTokenType.TEXT));
}

public CSVTokenizer ( char separator ) { addJoinedType ( CSVTokenType . TEXT ) ; addIgnoredType ( CSVTokenType . QUOTE ) ; putRules ( new Rule ( "\n|\r\n|\r" , CSVTokenType . ROW_SEPARATOR ) , new Rule ( Pattern . quote ( String . valueOf ( separator ) ) , CSVTokenType . COLUMN_SEPARATOR ) , new Rule ( "\"" , CSVTokenType . QUOTE , "quoted" ) , new Rule ( "[ \t]+" , CSVTokenType . EMPTY_SPACE ) , new Rule ( "." , CSVTokenType . TEXT ) ) ; putRules ( "quoted" , new Rule ( "(\")\"" , CSVTokenType . TEXT ) , new Rule ( "\"" , CSVTokenType . QUOTE , "#pop" ) , new Rule ( "." , CSVTokenType . TEXT ) ) ; }
initializes a new tokenizer instance with a grammar to analyze csv or tsv content . the character that separates columns must be provided .
