def lex(s, name=None, trim_whitespace=True, line_offset=0, delimeters=None):
 if (delimeters is None):
  delimeters = (Template.default_namespace['start_braces'], Template.default_namespace['end_braces'])
 in_expr = False
 chunks = []
 last = 0
 last_pos = ((line_offset + 1), 1)
 token_re = re.compile(('%s|%s' % (re.escape(delimeters[0]), re.escape(delimeters[1]))))
 for match in token_re.finditer(s):
  expr = match.group(0)
  pos = find_position(s, match.end(), last, last_pos)
  if ((expr == delimeters[0]) and in_expr):
   raise TemplateError(('%s inside expression' % delimeters[0]), position=pos, name=name)
  elif ((expr == delimeters[1]) and (not in_expr)):
   raise TemplateError(('%s outside expression' % delimeters[1]), position=pos, name=name)
  if (expr == delimeters[0]):
   part = s[last:match.start()]
   if part:
    chunks.append(part)
   in_expr = True
  else:
   chunks.append((s[last:match.start()], last_pos))
   in_expr = False
  last = match.end()
  last_pos = pos
 if in_expr:
  raise TemplateError(('No %s to finish last expression' % delimeters[1]), name=name, position=last_pos)
 part = s[last:]
 if part:
  chunks.append(part)
 if trim_whitespace:
  chunks = trim_lex(chunks)
 return chunks

def lex ( s , name = None , trim_whitespace = True , line_offset = 0 , delimeters = None ) : if ( delimeters is None ) : delimeters = ( Template . default_namespace [ 'start_braces' ] , Template . default_namespace [ 'end_braces' ] ) in_expr = False chunks = [ ] last = 0 last_pos = ( ( line_offset + 1 ) , 1 ) token_re = re . compile ( ( '%s|%s' % ( re . escape ( delimeters [ 0 ] ) , re . escape ( delimeters [ 1 ] ) ) ) ) for match in token_re . finditer ( s ) : expr = match . group ( 0 ) pos = find_position ( s , match . end ( ) , last , last_pos ) if ( ( expr == delimeters [ 0 ] ) and in_expr ) : raise TemplateError ( ( '%s inside expression' % delimeters [ 0 ] ) , position = pos , name = name ) elif ( ( expr == delimeters [ 1 ] ) and ( not in_expr ) ) : raise TemplateError ( ( '%s outside expression' % delimeters [ 1 ] ) , position = pos , name = name ) if ( expr == delimeters [ 0 ] ) : part = s [ last : match . start ( ) ] if part : chunks . append ( part ) in_expr = True else : chunks . append ( ( s [ last : match . start ( ) ] , last_pos ) ) in_expr = False last = match . end ( ) last_pos = pos if in_expr : raise TemplateError ( ( 'No %s to finish last expression' % delimeters [ 1 ] ) , name = name , position = last_pos ) part = s [ last : ] if part : chunks . append ( part ) if trim_whitespace : chunks = trim_lex ( chunks ) return chunks
lex code with lexer and return an iterable of tokens .