public Tokenizer(Reader reader,int maxStringLength){
  this.reader=reader;
  this.maxStringLength=Math.max(1,maxStringLength);
}

public Tokenizer ( Reader reader , int maxStringLength ) { this . reader = reader ; this . maxStringLength = Math . max ( 1 , maxStringLength ) ; }
creates a hutn tokenizer with the given maximum string length . longer strings will be reported as a sequence of partial_string and a final string token .
