def get_suffixes( txt_file, suffix_file, stdout_file="", threshold=sys.maxint, prefix=u'__' ):

  out = open( stdout_file, 'w' ) if stdout_file else sys.stdout 
  sys.stderr.write('Loading corpus\n') 
  my_corp = corpora.TextCorpus(txt_file)
  sys.stderr.write('Building suffix list\n')
  suffix_list = form_suffix_list(suffix_file)
  sys.stderr.write('Suffix search\n')

  #replace only words that occur in corpus less times than threshold
  #default - no threshold (all words are replaced with suffix)
  dict_copy = dict( [ (token,find_suffix(token, suffix_list, prefix=prefix)) if my_corp.dictionary.dfs[id] < threshold else (token,token) for (id, token) in my_corp.dictionary.items() ] )
  print dict_copy
  sys.stderr.write('Output\n')
 
  cnt = 0
  
  in_file = open(txt_file)
  for line in in_file:
    cnt += 1
    if cnt%10000 == 0:
      sys.stderr.write('.')
    words = line[:-1].decode('utf-8').split()
    for w in words:
      try:
        out.write("%s " % dict_copy[w].encode('utf-8'))
      except KeyError:
        dict_copy[w] = w
        out.write("%s " % dict_copy[w].encode('utf-8'))
    out.write("\n")
  in_file.close()

  if stdout_file: out.close()
Replace all words in <txt_file> with suffixes where possible.
dataset/ETHPy150Open qe-team/marmot/marmot/preprocessing/get_suffixes.py/get_suffixes