private int toKenLMId(IString token){
{
    int[] map=istringIdToKenLMId.get();
    if (token.id < map.length) {
      return map[token.id];
    }
  }
  if (preventDuplicateWork.tryLock()) {
    try {
      int[] oldTable=istringIdToKenLMId.get();
      if (token.id < oldTable.length) {
        return oldTable[token.id];
      }
      int[] newTable=new int[Vocabulary.systemSize()];
      System.arraycopy(oldTable,0,newTable,0,oldTable.length);
      for (int i=oldTable.length; i < newTable.length; ++i) {
        newTable[i]=model.index(Vocabulary.systemGet(i));
      }
      istringIdToKenLMId.set(newTable);
      return newTable[token.id];
    }
  finally {
      preventDuplicateWork.unlock();
    }
  }
  return model.index(token.toString());
}

private int toKenLMId ( IString token ) { { int [ ] map = istringIdToKenLMId . get ( ) ; if ( token . id < map . length ) { return map [ token . id ] ; } } if ( preventDuplicateWork . tryLock ( ) ) { try { int [ ] oldTable = istringIdToKenLMId . get ( ) ; if ( token . id < oldTable . length ) { return oldTable [ token . id ] ; } int [ ] newTable = new int [ Vocabulary . systemSize ( ) ] ; System . arraycopy ( oldTable , 0 , newTable , 0 , oldTable . length ) ; for ( int i = oldTable . length ; i < newTable . length ; ++ i ) { newTable [ i ] = model . index ( Vocabulary . systemGet ( i ) ) ; } istringIdToKenLMId . set ( newTable ) ; return newTable [ token . id ] ; } finally { preventDuplicateWork . unlock ( ) ; } } return model . index ( token . toString ( ) ) ; }
maps the istring id to a kenlm id . if the istring id is out of range , update the vocab mapping .
