question: private void loadCorpus ( String source , String target , String align ) throws IOException { logger . info ( "Counting the number of corpus positions" ) ; TimeKeeper timer = TimingUtils . start ( ) ; int numSourcePositions = 0 ; int numTargetPositions = 0 ; numSentences = 0 ; ParallelCorpus corpus = new ParallelCorpus ( 1 ) ; try ( LineNumberReader fReader = IOTools . getReaderFromFile ( source ) ) { LineNumberReader eReader = IOTools . getReaderFromFile ( target ) ; LineNumberReader aReader = IOTools . getReaderFromFile ( align ) ; for ( String fLine ; ( fLine = fReader . readLine ( ) ) != null ; ) { String eLine = eReader . readLine ( ) ; String aLine = aReader . readLine ( ) ; AlignedSentence example = corpus . getSentence ( fLine , eLine , aLine ) ; if ( example != null ) { numSourcePositions += example . sourceLength ( ) ; numTargetPositions += example . targetLength ( ) ; ++ numSentences ; } } } final int initialVocabularySize = corpus . getVocabulary ( ) . size ( ) ; timer . mark ( "Counting corpus positions" ) ; logger . info ( "Source positions: {}  Target positions: {}  Sentences: {}" , numSourcePositions , numTargetPositions , numSentences ) ; final int srcLength = numSourcePositions + numSentences ; if ( srcLength < 0 ) throw new RuntimeException ( "Maximum source bitext size exceeded" ) ; srcBitext = new int [ srcLength ] ; f2e = new int [ srcLength ] ; final int tgtLength = numTargetPositions + numSentences ; if ( tgtLength < 0 ) throw new RuntimeException ( "Maximum target bitext size exceeded" ) ; tgtBitext = new int [ tgtLength ] ; e2f = new int [ tgtLength ] ; try ( LineNumberReader fReader = IOTools . getReaderFromFile ( source ) ) { LineNumberReader eReader = IOTools . getReaderFromFile ( target ) ; LineNumberReader aReader = IOTools . getReaderFromFile ( align ) ; int srcOffset = 0 ; int tgtOffset = 0 ; for ( String fLine ; ( fLine = fReader . readLine ( ) ) != null ; ) { String eLine = eReader . readLine ( ) ; String aLine = aReader . readLine ( ) ; AlignedSentence sentence = corpus . getSentence ( fLine , eLine , aLine ) ; if ( sentence == null ) { logger . info ( "Discarding parallel example {}" , fReader . getLineNumber ( ) ) ; } else { System . arraycopy ( sentence . source , 0 , srcBitext , srcOffset , sentence . sourceLength ( ) ) ; System . arraycopy ( sentence . f2e , 0 , f2e , srcOffset , sentence . f2e . length ) ; System . arraycopy ( sentence . target , 0 , tgtBitext , tgtOffset , sentence . targetLength ( ) ) ; System . arraycopy ( sentence . e2f , 0 , e2f , tgtOffset , sentence . e2f . length ) ; srcOffset += sentence . sourceLength ( ) ; tgtOffset += sentence . targetLength ( ) ; srcBitext [ srcOffset ] = toSentenceOffset ( tgtOffset ) ; tgtBitext [ tgtOffset ] = toSentenceOffset ( srcOffset ) ; ++ srcOffset ; ++ tgtOffset ; } } } this . vocabulary = corpus . getVocabulary ( ) ; assert initialVocabularySize == vocabulary . size ( ) ; timer . mark ( "Loading corpus" ) ; logger . info ( "Done loading corpus: {}" , timer ) ; }
target:  Streaming loader, which does not double peak memory like the loader that creates a suffix array from a parallel corpus.