private void initTokenizer(StreamTokenizer tokenizer){
  tokenizer.resetSyntax();
  tokenizer.whitespaceChars(0,(' ' - 1));
  tokenizer.wordChars(' ','\u00FF');
  tokenizer.whitespaceChars(',',',');
  tokenizer.whitespaceChars(':',':');
  tokenizer.commentChar('|');
  tokenizer.whitespaceChars('\t','\t');
  tokenizer.quoteChar('"');
  tokenizer.quoteChar('\'');
  tokenizer.eolIsSignificant(true);
}

private void initTokenizer ( StreamTokenizer tokenizer ) { tokenizer . resetSyntax ( ) ; tokenizer . whitespaceChars ( 0 , ( ' ' - 1 ) ) ; tokenizer . wordChars ( ' ' , 'ÿ' ) ; tokenizer . whitespaceChars ( ',' , ',' ) ; tokenizer . whitespaceChars ( ':' , ':' ) ; tokenizer . commentChar ( '|' ) ; tokenizer . whitespaceChars ( '\t' , '\t' ) ; tokenizer . quoteChar ( '"' ) ; tokenizer . quoteChar ( '\'' ) ; tokenizer . eolIsSignificant ( true ) ; }
initializes the stream tokenizer
