def clean_text_by_word(text, deacc=True):
 text_without_acronyms = replace_with_separator(text, '', [AB_ACRONYM_LETTERS])
 original_words = list(tokenize(text_without_acronyms, to_lower=True, deacc=deacc))
 filtered_words = [join_words(word_list, '') for word_list in preprocess_documents(original_words)]
 if HAS_PATTERN:
  tags = tag(join_words(original_words))
 else:
  tags = None
 units = merge_syntactic_units(original_words, filtered_words, tags)
 return dict(((unit.text, unit) for unit in units))

def clean_text_by_word ( text , deacc = True ) : text_without_acronyms = replace_with_separator ( text , '' , [ AB_ACRONYM_LETTERS ] ) original_words = list ( tokenize ( text_without_acronyms , to_lower = True , deacc = deacc ) ) filtered_words = [ join_words ( word_list , '' ) for word_list in preprocess_documents ( original_words ) ] if HAS_PATTERN : tags = tag ( join_words ( original_words ) ) else : tags = None units = merge_syntactic_units ( original_words , filtered_words , tags ) return dict ( ( ( unit . text , unit ) for unit in units ) )
tokenizes a given text into words .