static public List<Sequence<IString>> tokenizeFile(String filename,boolean doNIST){
  List<Sequence<IString>> sequences=new ArrayList<>();
  LineNumberReader reader=IOTools.getReaderFromFile(filename);
  try {
    for (String line; (line=reader.readLine()) != null; ) {
      if (doNIST)       line=NISTTokenizer.tokenize(line);
      sequences.add(IStrings.tokenize(line));
    }
    reader.close();
  }
 catch (  IOException e) {
    e.printStackTrace();
  }
  return sequences;
}

static public List < Sequence < IString > > tokenizeFile ( String filename , boolean doNIST ) { List < Sequence < IString > > sequences = new ArrayList < > ( ) ; LineNumberReader reader = IOTools . getReaderFromFile ( filename ) ; try { for ( String line ; ( line = reader . readLine ( ) ) != null ; ) { if ( doNIST ) line = NISTTokenizer . tokenize ( line ) ; sequences . add ( IStrings . tokenize ( line ) ) ; } reader . close ( ) ; } catch ( IOException e ) { e . printStackTrace ( ) ; } return sequences ; }
convert a newline - delimited file to a list of sequences , optionally applying nist tokenization .
