public Map<Span,SuffixArraySample> lookupFrequentSourceNgrams(int sampleSize,int minOccurrences){
  if (sampleSize >= minOccurrences)   throw new IllegalArgumentException();
  if (srcSuffixArray.length == 0)   return Collections.emptyMap();
  logger.info("Building query cache with threshold {}",minOccurrences);
  Map<Span,SuffixArraySample> queryCache=new HashMap<>(1000);
  int nCnt=1, nnCnt=1, nnnCnt=1;
  int nStart=0, nnStart=0, nnnStart=0;
  Suffix firstSuffix=new Suffix(srcSuffixArray[0],true);
  Span nSpan=new Span(firstSuffix,1), nnSpan=new Span(firstSuffix,2), nnnSpan=new Span(firstSuffix,3);
  for (int i=1; i < srcSuffixArray.length; ++i) {
    Suffix suffix=new Suffix(srcSuffixArray[i],true);
    Span nSpanThis=new Span(suffix,1);
    Span nnSpanThis=new Span(suffix,2);
    Span nnnSpanThis=new Span(suffix,3);
    nCnt=checkSpan(nSpan,nSpanThis,nStart,i,nCnt,minOccurrences,sampleSize,queryCache);
    if (nCnt == 1) {
      nStart=i;
      nSpan=nSpanThis;
    }
    nnCnt=checkSpan(nnSpan,nnSpanThis,nnStart,i,nnCnt,minOccurrences,sampleSize,queryCache);
    if (nnCnt == 1) {
      nnStart=i;
      nnSpan=nnSpanThis;
    }
    nnnCnt=checkSpan(nnnSpan,nnnSpanThis,nnnStart,i,nnnCnt,minOccurrences,sampleSize,queryCache);
    if (nnnCnt == 1) {
      nnnStart=i;
      nnnSpan=nnnSpanThis;
    }
  }
  ;
  logger.info("Query cache size: {}",queryCache.size());
  logger.info("Creating target unigram caches for the count() function...");
  this.tgtCountLBCache=new int[vocabulary.size()];
  Arrays.fill(tgtCountLBCache,-1);
  this.tgtCountUBCache=new int[vocabulary.size()];
  Arrays.fill(tgtCountUBCache,-1);
  int lastId=tgtBitext[tgtSuffixArray[0]];
  for (int i=0; i < tgtSuffixArray.length; ++i) {
    int tgtId=tgtBitext[tgtSuffixArray[i]];
    assert tgtId >= 0;
    if (tgtCountLBCache[tgtId] < 0) {
      tgtCountLBCache[tgtId]=i;
    }
    if (lastId != tgtId) {
      tgtCountUBCache[lastId]=i - 1;
      assert tgtCountUBCache[lastId] >= tgtCountLBCache[lastId] : String.format("%d %d %d",i,lastId,tgtId);
    }
    lastId=tgtId;
  }
  tgtCountUBCache[lastId]=tgtSuffixArray.length;
  assert tgtCountUBCache[lastId] >= tgtCountLBCache[lastId] : String.format("%d %d final",tgtSuffixArray.length,lastId);
  logger.info("Finished building count() cache.");
  return queryCache;
}

public Map < Span , SuffixArraySample > lookupFrequentSourceNgrams ( int sampleSize , int minOccurrences ) { if ( sampleSize >= minOccurrences ) throw new IllegalArgumentException ( ) ; if ( srcSuffixArray . length == 0 ) return Collections . emptyMap ( ) ; logger . info ( "Building query cache with threshold {}" , minOccurrences ) ; Map < Span , SuffixArraySample > queryCache = new HashMap < > ( 1000 ) ; int nCnt = 1 , nnCnt = 1 , nnnCnt = 1 ; int nStart = 0 , nnStart = 0 , nnnStart = 0 ; Suffix firstSuffix = new Suffix ( srcSuffixArray [ 0 ] , true ) ; Span nSpan = new Span ( firstSuffix , 1 ) , nnSpan = new Span ( firstSuffix , 2 ) , nnnSpan = new Span ( firstSuffix , 3 ) ; for ( int i = 1 ; i < srcSuffixArray . length ; ++ i ) { Suffix suffix = new Suffix ( srcSuffixArray [ i ] , true ) ; Span nSpanThis = new Span ( suffix , 1 ) ; Span nnSpanThis = new Span ( suffix , 2 ) ; Span nnnSpanThis = new Span ( suffix , 3 ) ; nCnt = checkSpan ( nSpan , nSpanThis , nStart , i , nCnt , minOccurrences , sampleSize , queryCache ) ; if ( nCnt == 1 ) { nStart = i ; nSpan = nSpanThis ; } nnCnt = checkSpan ( nnSpan , nnSpanThis , nnStart , i , nnCnt , minOccurrences , sampleSize , queryCache ) ; if ( nnCnt == 1 ) { nnStart = i ; nnSpan = nnSpanThis ; } nnnCnt = checkSpan ( nnnSpan , nnnSpanThis , nnnStart , i , nnnCnt , minOccurrences , sampleSize , queryCache ) ; if ( nnnCnt == 1 ) { nnnStart = i ; nnnSpan = nnnSpanThis ; } } ; logger . info ( "Query cache size: {}" , queryCache . size ( ) ) ; logger . info ( "Creating target unigram caches for the count() function..." ) ; this . tgtCountLBCache = new int [ vocabulary . size ( ) ] ; Arrays . fill ( tgtCountLBCache , - 1 ) ; this . tgtCountUBCache = new int [ vocabulary . size ( ) ] ; Arrays . fill ( tgtCountUBCache , - 1 ) ; int lastId = tgtBitext [ tgtSuffixArray [ 0 ] ] ; for ( int i = 0 ; i < tgtSuffixArray . length ; ++ i ) { int tgtId = tgtBitext [ tgtSuffixArray [ i ] ] ; assert tgtId >= 0 ; if ( tgtCountLBCache [ tgtId ] < 0 ) { tgtCountLBCache [ tgtId ] = i ; } if ( lastId != tgtId ) { tgtCountUBCache [ lastId ] = i - 1 ; assert tgtCountUBCache [ lastId ] >= tgtCountLBCache [ lastId ] : String . format ( "%d %d %d" , i , lastId , tgtId ) ; } lastId = tgtId ; } tgtCountUBCache [ lastId ] = tgtSuffixArray . length ; assert tgtCountUBCache [ lastId ] >= tgtCountLBCache [ lastId ] : String . format ( "%d %d final" , tgtSuffixArray . length , lastId ) ; logger . info ( "Finished building count() cache." ) ; return queryCache ; }
find all source spans up to dimension = = 3 . todo ( spenceg ) lopez reports finding a few order = 5 n - grams of high frequency so maybe generalize this lookup .
