public static String rewriteTermDP(String term){
  int size=term.length();
  if (size == 0)   return "";
  double[] scoreAtGap=new double[size];
  int[] preBestGap=new int[size];
  Arrays.fill(scoreAtGap,RewriterConstants.MAXIMUM_SEQUENCE_SCORE);
  Arrays.fill(preBestGap,-1);
  scoreAtGap[0]=0;
  preBestGap[0]=0;
  for (int endPosition=1; endPosition < size; endPosition++) {
    for (int spanSize=1; spanSize < RewriterConstants.MAX_WORD_LENGTH && spanSize <= endPosition; spanSize++) {
      double freq=wordBase.getFrequency(term.substring(endPosition - spanSize,endPosition + 1));
      if (freq < 0) {
        if (spanSize == 1)         freq=RewriterConstants.UNKNOWN_CHAR_FREQ;
 else         continue;
      }
      if (freq + scoreAtGap[endPosition - spanSize] + RewriterConstants.TOKEN_LENGTH_PENALTY < scoreAtGap[endPosition]) {
        scoreAtGap[endPosition]=freq + scoreAtGap[endPosition - spanSize] + RewriterConstants.TOKEN_LENGTH_PENALTY;
        preBestGap[endPosition]=endPosition - spanSize;
      }
    }
  }
  String tokenizedQuery=new String();
  for (int pos=size - 1; pos > 0; pos=preBestGap[pos] - 1) {
    tokenizedQuery=" " + term.substring(preBestGap[pos],pos + 1) + tokenizedQuery;
  }
  return tokenizedQuery.substring(1);
}

public static String rewriteTermDP ( String term ) { int size = term . length ( ) ; if ( size == 0 ) return "" ; double [ ] scoreAtGap = new double [ size ] ; int [ ] preBestGap = new int [ size ] ; Arrays . fill ( scoreAtGap , RewriterConstants . MAXIMUM_SEQUENCE_SCORE ) ; Arrays . fill ( preBestGap , - 1 ) ; scoreAtGap [ 0 ] = 0 ; preBestGap [ 0 ] = 0 ; for ( int endPosition = 1 ; endPosition < size ; endPosition ++ ) { for ( int spanSize = 1 ; spanSize < RewriterConstants . MAX_WORD_LENGTH && spanSize <= endPosition ; spanSize ++ ) { double freq = wordBase . getFrequency ( term . substring ( endPosition - spanSize , endPosition + 1 ) ) ; if ( freq < 0 ) { if ( spanSize == 1 ) freq = RewriterConstants . UNKNOWN_CHAR_FREQ ; else continue ; } if ( freq + scoreAtGap [ endPosition - spanSize ] + RewriterConstants . TOKEN_LENGTH_PENALTY < scoreAtGap [ endPosition ] ) { scoreAtGap [ endPosition ] = freq + scoreAtGap [ endPosition - spanSize ] + RewriterConstants . TOKEN_LENGTH_PENALTY ; preBestGap [ endPosition ] = endPosition - spanSize ; } } } String tokenizedQuery = new String ( ) ; for ( int pos = size - 1 ; pos > 0 ; pos = preBestGap [ pos ] - 1 ) { tokenizedQuery = " " + term . substring ( preBestGap [ pos ] , pos + 1 ) + tokenizedQuery ; } return tokenizedQuery . substring ( 1 ) ; }
performs most likely tokenization of a single term uses dynamic programming algorithm approach used for tokenization of chinese strings source - chinese tokenizer in srch2 codebase https : / / github . com / srch2 / srch2 - ngn / blob / master / src / core / analyzer / chinesetokenizer . cpp # l197
