def tokenize_snippet_text(snippet_instance, text, indent, allowed_tokens_in_text, allowed_tokens_in_tabstops, token_to_textobject):
 seen_ts = {}
 all_tokens = []
 def _do_parse(parent, text, allowed_tokens):
  'Recursive function that actually creates the objects.'
  tokens = list(tokenize(text, indent, parent.start, allowed_tokens))
  for token in tokens:
   all_tokens.append((parent, token))
   if isinstance(token, TabStopToken):
    ts = TabStop(parent, token)
    seen_ts[token.number] = ts
    _do_parse(ts, token.initial_text, allowed_tokens_in_tabstops)
   else:
    klass = token_to_textobject.get(token.__class__, None)
    if (klass is not None):
     klass(parent, token)
 _do_parse(snippet_instance, text, allowed_tokens_in_text)
 return (all_tokens, seen_ts)

def tokenize_snippet_text ( snippet_instance , text , indent , allowed_tokens_in_text , allowed_tokens_in_tabstops , token_to_textobject ) : seen_ts = { } all_tokens = [ ] def _do_parse ( parent , text , allowed_tokens ) : tokens = list ( tokenize ( text , indent , parent . start , allowed_tokens ) ) for token in tokens : all_tokens . append ( ( parent , token ) ) if isinstance ( token , TabStopToken ) : ts = TabStop ( parent , token ) seen_ts [ token . number ] = ts _do_parse ( ts , token . initial_text , allowed_tokens_in_tabstops ) else : klass = token_to_textobject . get ( token . __class__ , None ) if ( klass is not None ) : klass ( parent , token ) _do_parse ( snippet_instance , text , allowed_tokens_in_text ) return ( all_tokens , seen_ts )
turns text into a stream of tokens and creates the text objects from those tokens that are mentioned in token_to_textobject assuming the current indent .