question: public Map < Span , SuffixArraySample > lookupFrequentSourceNgrams ( int sampleSize , int minOccurrences ) { if ( sampleSize >= minOccurrences ) throw new IllegalArgumentException ( ) ; if ( srcSuffixArray . length == 0 ) return Collections . emptyMap ( ) ; logger . info ( "Building query cache with threshold {}" , minOccurrences ) ; Map < Span , SuffixArraySample > queryCache = new HashMap < > ( 1000 ) ; int nCnt = 1 , nnCnt = 1 , nnnCnt = 1 ; int nStart = 0 , nnStart = 0 , nnnStart = 0 ; Suffix firstSuffix = new Suffix ( srcSuffixArray [ 0 ] , true ) ; Span nSpan = new Span ( firstSuffix , 1 ) , nnSpan = new Span ( firstSuffix , 2 ) , nnnSpan = new Span ( firstSuffix , 3 ) ; for ( int i = 1 ; i < srcSuffixArray . length ; ++ i ) { Suffix suffix = new Suffix ( srcSuffixArray [ i ] , true ) ; Span nSpanThis = new Span ( suffix , 1 ) ; Span nnSpanThis = new Span ( suffix , 2 ) ; Span nnnSpanThis = new Span ( suffix , 3 ) ; nCnt = checkSpan ( nSpan , nSpanThis , nStart , i , nCnt , minOccurrences , sampleSize , queryCache ) ; if ( nCnt == 1 ) { nStart = i ; nSpan = nSpanThis ; } nnCnt = checkSpan ( nnSpan , nnSpanThis , nnStart , i , nnCnt , minOccurrences , sampleSize , queryCache ) ; if ( nnCnt == 1 ) { nnStart = i ; nnSpan = nnSpanThis ; } nnnCnt = checkSpan ( nnnSpan , nnnSpanThis , nnnStart , i , nnnCnt , minOccurrences , sampleSize , queryCache ) ; if ( nnnCnt == 1 ) { nnnStart = i ; nnnSpan = nnnSpanThis ; } } ; logger . info ( "Query cache size: {}" , queryCache . size ( ) ) ; logger . info ( "Creating target unigram caches for the count() function..." ) ; this . tgtCountLBCache = new int [ vocabulary . size ( ) ] ; Arrays . fill ( tgtCountLBCache , - 1 ) ; this . tgtCountUBCache = new int [ vocabulary . size ( ) ] ; Arrays . fill ( tgtCountUBCache , - 1 ) ; int lastId = tgtBitext [ tgtSuffixArray [ 0 ] ] ; for ( int i = 0 ; i < tgtSuffixArray . length ; ++ i ) { int tgtId = tgtBitext [ tgtSuffixArray [ i ] ] ; assert tgtId >= 0 ; if ( tgtCountLBCache [ tgtId ] < 0 ) { tgtCountLBCache [ tgtId ] = i ; } if ( lastId != tgtId ) { tgtCountUBCache [ lastId ] = i - 1 ; assert tgtCountUBCache [ lastId ] >= tgtCountLBCache [ lastId ] : String . format ( "%d %d %d" , i , lastId , tgtId ) ; } lastId = tgtId ; } tgtCountUBCache [ lastId ] = tgtSuffixArray . length ; assert tgtCountUBCache [ lastId ] >= tgtCountLBCache [ lastId ] : String . format ( "%d %d final" , tgtSuffixArray . length , lastId ) ; logger . info ( "Finished building count() cache." ) ; return queryCache ; }
target:  Find all source spans up to dimension == 3. TODO(spenceg) Lopez reports finding a few order=5 n-grams of high frequency so maybe generalize this lookup.