def detokenize(token_rules, words):
 string = ' '.join(words)
 for subtoks in token_rules:
  string = string.replace(subtoks.replace('<SEP>', ' '), subtoks)
 positions = []
 i = 0
 for chunk in string.split():
  subtoks = chunk.split('<SEP>')
  positions.append(tuple(range(i, (i + len(subtoks)))))
  i += len(subtoks)
 return positions

def detokenize ( token_rules , words ) : string = ' ' . join ( words ) for subtoks in token_rules : string = string . replace ( subtoks . replace ( '<SEP>' , ' ' ) , subtoks ) positions = [ ] i = 0 for chunk in string . split ( ) : subtoks = chunk . split ( '<SEP>' ) positions . append ( tuple ( range ( i , ( i + len ( subtoks ) ) ) ) ) i += len ( subtoks ) return positions
to align with treebanks .