private void loadCorpus(String source,String target,String align) throws IOException {
  logger.info("Counting the number of corpus positions");
  TimeKeeper timer=TimingUtils.start();
  int numSourcePositions=0;
  int numTargetPositions=0;
  numSentences=0;
  ParallelCorpus corpus=new ParallelCorpus(1);
  try (LineNumberReader fReader=IOTools.getReaderFromFile(source)){
    LineNumberReader eReader=IOTools.getReaderFromFile(target);
    LineNumberReader aReader=IOTools.getReaderFromFile(align);
    for (String fLine; (fLine=fReader.readLine()) != null; ) {
      String eLine=eReader.readLine();
      String aLine=aReader.readLine();
      AlignedSentence example=corpus.getSentence(fLine,eLine,aLine);
      if (example != null) {
        numSourcePositions+=example.sourceLength();
        numTargetPositions+=example.targetLength();
        ++numSentences;
      }
    }
  }
   final int initialVocabularySize=corpus.getVocabulary().size();
  timer.mark("Counting corpus positions");
  logger.info("Source positions: {}  Target positions: {}  Sentences: {}",numSourcePositions,numTargetPositions,numSentences);
  final int srcLength=numSourcePositions + numSentences;
  if (srcLength < 0)   throw new RuntimeException("Maximum source bitext size exceeded");
  srcBitext=new int[srcLength];
  f2e=new int[srcLength];
  final int tgtLength=numTargetPositions + numSentences;
  if (tgtLength < 0)   throw new RuntimeException("Maximum target bitext size exceeded");
  tgtBitext=new int[tgtLength];
  e2f=new int[tgtLength];
  try (LineNumberReader fReader=IOTools.getReaderFromFile(source)){
    LineNumberReader eReader=IOTools.getReaderFromFile(target);
    LineNumberReader aReader=IOTools.getReaderFromFile(align);
    int srcOffset=0;
    int tgtOffset=0;
    for (String fLine; (fLine=fReader.readLine()) != null; ) {
      String eLine=eReader.readLine();
      String aLine=aReader.readLine();
      AlignedSentence sentence=corpus.getSentence(fLine,eLine,aLine);
      if (sentence == null) {
        logger.info("Discarding parallel example {}",fReader.getLineNumber());
      }
 else {
        System.arraycopy(sentence.source,0,srcBitext,srcOffset,sentence.sourceLength());
        System.arraycopy(sentence.f2e,0,f2e,srcOffset,sentence.f2e.length);
        System.arraycopy(sentence.target,0,tgtBitext,tgtOffset,sentence.targetLength());
        System.arraycopy(sentence.e2f,0,e2f,tgtOffset,sentence.e2f.length);
        srcOffset+=sentence.sourceLength();
        tgtOffset+=sentence.targetLength();
        srcBitext[srcOffset]=toSentenceOffset(tgtOffset);
        tgtBitext[tgtOffset]=toSentenceOffset(srcOffset);
        ++srcOffset;
        ++tgtOffset;
      }
    }
  }
   this.vocabulary=corpus.getVocabulary();
  assert initialVocabularySize == vocabulary.size();
  timer.mark("Loading corpus");
  logger.info("Done loading corpus: {}",timer);
}

private void loadCorpus ( String source , String target , String align ) throws IOException { logger . info ( "Counting the number of corpus positions" ) ; TimeKeeper timer = TimingUtils . start ( ) ; int numSourcePositions = 0 ; int numTargetPositions = 0 ; numSentences = 0 ; ParallelCorpus corpus = new ParallelCorpus ( 1 ) ; try ( LineNumberReader fReader = IOTools . getReaderFromFile ( source ) ) { LineNumberReader eReader = IOTools . getReaderFromFile ( target ) ; LineNumberReader aReader = IOTools . getReaderFromFile ( align ) ; for ( String fLine ; ( fLine = fReader . readLine ( ) ) != null ; ) { String eLine = eReader . readLine ( ) ; String aLine = aReader . readLine ( ) ; AlignedSentence example = corpus . getSentence ( fLine , eLine , aLine ) ; if ( example != null ) { numSourcePositions += example . sourceLength ( ) ; numTargetPositions += example . targetLength ( ) ; ++ numSentences ; } } } final int initialVocabularySize = corpus . getVocabulary ( ) . size ( ) ; timer . mark ( "Counting corpus positions" ) ; logger . info ( "Source positions: {}  Target positions: {}  Sentences: {}" , numSourcePositions , numTargetPositions , numSentences ) ; final int srcLength = numSourcePositions + numSentences ; if ( srcLength < 0 ) throw new RuntimeException ( "Maximum source bitext size exceeded" ) ; srcBitext = new int [ srcLength ] ; f2e = new int [ srcLength ] ; final int tgtLength = numTargetPositions + numSentences ; if ( tgtLength < 0 ) throw new RuntimeException ( "Maximum target bitext size exceeded" ) ; tgtBitext = new int [ tgtLength ] ; e2f = new int [ tgtLength ] ; try ( LineNumberReader fReader = IOTools . getReaderFromFile ( source ) ) { LineNumberReader eReader = IOTools . getReaderFromFile ( target ) ; LineNumberReader aReader = IOTools . getReaderFromFile ( align ) ; int srcOffset = 0 ; int tgtOffset = 0 ; for ( String fLine ; ( fLine = fReader . readLine ( ) ) != null ; ) { String eLine = eReader . readLine ( ) ; String aLine = aReader . readLine ( ) ; AlignedSentence sentence = corpus . getSentence ( fLine , eLine , aLine ) ; if ( sentence == null ) { logger . info ( "Discarding parallel example {}" , fReader . getLineNumber ( ) ) ; } else { System . arraycopy ( sentence . source , 0 , srcBitext , srcOffset , sentence . sourceLength ( ) ) ; System . arraycopy ( sentence . f2e , 0 , f2e , srcOffset , sentence . f2e . length ) ; System . arraycopy ( sentence . target , 0 , tgtBitext , tgtOffset , sentence . targetLength ( ) ) ; System . arraycopy ( sentence . e2f , 0 , e2f , tgtOffset , sentence . e2f . length ) ; srcOffset += sentence . sourceLength ( ) ; tgtOffset += sentence . targetLength ( ) ; srcBitext [ srcOffset ] = toSentenceOffset ( tgtOffset ) ; tgtBitext [ tgtOffset ] = toSentenceOffset ( srcOffset ) ; ++ srcOffset ; ++ tgtOffset ; } } } this . vocabulary = corpus . getVocabulary ( ) ; assert initialVocabularySize == vocabulary . size ( ) ; timer . mark ( "Loading corpus" ) ; logger . info ( "Done loading corpus: {}" , timer ) ; }
streaming loader , which does not double peak memory like the loader that creates a suffix array from a parallel corpus .
