def entoken(textstream, positions=False, chars=False, start_pos=0, start_char=0, **kwargs):
 pos = start_pos
 char = start_char
 t = Token(positions=positions, chars=chars, **kwargs)
 for text in textstream:
  t.text = text
  if positions:
   t.pos = pos
   pos += 1
  if chars:
   t.startchar = char
   char = (char + len(text))
   t.endchar = char
  (yield t)

def entoken ( textstream , positions = False , chars = False , start_pos = 0 , start_char = 0 , ** kwargs ) : pos = start_pos char = start_char t = Token ( positions = positions , chars = chars , ** kwargs ) for text in textstream : t . text = text if positions : t . pos = pos pos += 1 if chars : t . startchar = char char = ( char + len ( text ) ) t . endchar = char ( yield t )
takes a sequence of unicode strings and yields a series of token objects .