private void loadCorpus(ParallelCorpus corpus){
  logger.info("Flattening parallel corpus");
  TimeKeeper timer=TimingUtils.start();
  numSentences=corpus.size();
  int numSourcePositions=corpus.numSourcePositions();
  int numTargetPositions=corpus.numTargetPositions();
  int srcLength=numSourcePositions + numSentences;
  srcBitext=new int[srcLength];
  f2e=new int[srcLength];
  int tgtLength=numTargetPositions + numSentences;
  tgtBitext=new int[tgtLength];
  e2f=new int[tgtLength];
  int srcOffset=0;
  int tgtOffset=0;
  for (  AlignedSentence sentence : corpus) {
    System.arraycopy(sentence.source,0,srcBitext,srcOffset,sentence.sourceLength());
    System.arraycopy(sentence.f2e,0,f2e,srcOffset,sentence.f2e.length);
    System.arraycopy(sentence.target,0,tgtBitext,tgtOffset,sentence.targetLength());
    System.arraycopy(sentence.e2f,0,e2f,tgtOffset,sentence.e2f.length);
    srcOffset+=sentence.sourceLength();
    tgtOffset+=sentence.targetLength();
    srcBitext[srcOffset]=toSentenceOffset(tgtOffset);
    tgtBitext[tgtOffset]=toSentenceOffset(srcOffset);
    ++srcOffset;
    ++tgtOffset;
  }
  vocabulary=corpus.getVocabulary();
  timer.mark("Corpus loading");
  logger.info("Done loading corpus: {}",timer);
}

private void loadCorpus ( ParallelCorpus corpus ) { logger . info ( "Flattening parallel corpus" ) ; TimeKeeper timer = TimingUtils . start ( ) ; numSentences = corpus . size ( ) ; int numSourcePositions = corpus . numSourcePositions ( ) ; int numTargetPositions = corpus . numTargetPositions ( ) ; int srcLength = numSourcePositions + numSentences ; srcBitext = new int [ srcLength ] ; f2e = new int [ srcLength ] ; int tgtLength = numTargetPositions + numSentences ; tgtBitext = new int [ tgtLength ] ; e2f = new int [ tgtLength ] ; int srcOffset = 0 ; int tgtOffset = 0 ; for ( AlignedSentence sentence : corpus ) { System . arraycopy ( sentence . source , 0 , srcBitext , srcOffset , sentence . sourceLength ( ) ) ; System . arraycopy ( sentence . f2e , 0 , f2e , srcOffset , sentence . f2e . length ) ; System . arraycopy ( sentence . target , 0 , tgtBitext , tgtOffset , sentence . targetLength ( ) ) ; System . arraycopy ( sentence . e2f , 0 , e2f , tgtOffset , sentence . e2f . length ) ; srcOffset += sentence . sourceLength ( ) ; tgtOffset += sentence . targetLength ( ) ; srcBitext [ srcOffset ] = toSentenceOffset ( tgtOffset ) ; tgtBitext [ tgtOffset ] = toSentenceOffset ( srcOffset ) ; ++ srcOffset ; ++ tgtOffset ; } vocabulary = corpus . getVocabulary ( ) ; timer . mark ( "Corpus loading" ) ; logger . info ( "Done loading corpus: {}" , timer ) ; }
load the parallel corpus into a contiguous block of memory . set the corpus reference to null after this call to free memory .
