<s> """<STR_LIT>""" <EOL> from exabgp . configuration . core . format import tokens <EOL> from exabgp . protocol . family import AFI <EOL> from collections import deque <EOL> class Tokeniser ( object ) : <EOL> class Iterator ( object ) : <EOL> fname = '<STR_LIT>' <EOL> def __init__ ( self ) : <EOL> self . next = deque ( ) <EOL> self . tokens = [ ] <EOL> self . generator = iter ( [ ] ) <EOL> def replenish ( self , content ) : <EOL> self . next . clear ( ) <EOL> self . tokens = content <EOL> self . generator = iter ( content ) <EOL> return self <EOL> def clear ( self ) : <EOL> self . replenish ( [ ] ) <EOL> def __call__ ( self ) : <EOL> if self . next : <EOL> return self . next . popleft ( ) <EOL> try : <EOL> return self . generator . next ( ) <EOL> except StopIteration : <EOL> return '<STR_LIT>' <EOL> def peek ( self ) : <EOL> try : <EOL> peaked = self . generator . next ( ) <EOL> self . next . append ( peaked ) <EOL> return peaked <EOL> except StopIteration : <EOL> return '<STR_LIT>' <EOL> @ staticmethod <EOL> def _off ( ) : <EOL> raise StopIteration ( ) <EOL> def __init__ ( self , scope , error , logger ) : <EOL> self . scope = scope <EOL> self . error = error <EOL> self . logger = logger <EOL> self . finished = False <EOL> self . number = <NUM_LIT:0> <EOL> self . line = [ ] <EOL> self . iterate = Tokeniser . Iterator ( ) <EOL> self . end = '<STR_LIT>' <EOL> self . index_column = <NUM_LIT:0> <EOL> self . index_line = <NUM_LIT:0> <EOL> self . fname = '<STR_LIT>' <EOL> self . type = '<STR_LIT>' <EOL> self . afi = AFI . undefined <EOL> self . _tokens = Tokeniser . _off <EOL> self . _next = None <EOL> self . _data = None <EOL> def clear ( self ) : <EOL> self . finished = False <EOL> self . number = <NUM_LIT:0> <EOL> self . line = [ ] <EOL> self . iterate . clear ( ) <EOL> self . end = '<STR_LIT>' <EOL> self . index_column = <NUM_LIT:0> <EOL> self . index_line = <NUM_LIT:0> <EOL> self . fname = '<STR_LIT>' <EOL> self . type = '<STR_LIT>' <EOL> if self . _data : <EOL> self . _set ( self . _data ) <EOL> def params ( self ) : <EOL> if len ( self . line ) <= <NUM_LIT:2> : <EOL> return '<STR_LIT>' <EOL> if self . end in ( '<STR_LIT:{>' , '<STR_LIT:}>' , '<STR_LIT:;>' ) : <EOL> return "<STR_LIT>" % "<STR_LIT>" . join ( self . line [ <NUM_LIT:1> : - <NUM_LIT:1> ] ) <EOL> return "<STR_LIT>" % "<STR_LIT>" . join ( self . line [ <NUM_LIT:1> : ] ) <EOL> def _tokenise ( self , iterator ) : <EOL> for parsed in tokens ( iterator ) : <EOL> words = [ word for y , x , word in parsed ] <EOL> self . line = '<STR_LIT>' . join ( words ) <EOL> yield words <EOL> def _set ( self , function ) : <EOL> try : <EOL> self . _tokens = function <EOL> self . _next = self . _tokens . next ( ) <EOL> except IOError , exc : <EOL> error = str ( exc ) <EOL> if error . count ( '<STR_LIT:]>' ) : <EOL> self . error . set ( error . split ( '<STR_LIT:]>' ) [ <NUM_LIT:1> ] . strip ( ) ) <EOL> else : <EOL> self . error . set ( error ) <EOL> self . _tokens = Tokeniser . _off <EOL> self . _next = [ ] <EOL> return self . error . set ( '<STR_LIT>' ) <EOL> except StopIteration : <EOL> self . _tokens = Tokeniser . _off <EOL> self . _next = [ ] <EOL> return self . error . set ( '<STR_LIT>' ) <EOL> return True <EOL> def set_file ( self , data ) : <EOL> def _source ( fname ) : <EOL> with open ( fname , '<STR_LIT:r>' ) as fileobject : <EOL> def formated ( ) : <EOL> while True : <EOL> line = fileobject . next ( ) . rstrip ( ) <EOL> self . index_line += <NUM_LIT:1> <EOL> while line . endswith ( '<STR_LIT:\\>' ) : <EOL> line = line [ : - <NUM_LIT:1> ] + fileobject . next ( ) . rstrip ( ) <EOL> self . index_line += <NUM_LIT:1> <EOL> yield line <EOL> for _ in self . _tokenise ( formated ( ) ) : <EOL> yield _ <EOL> self . type = '<STR_LIT:file>' <EOL> self . Iterator . fname = data <EOL> return self . _set ( _source ( data ) ) <EOL> def set_text ( self , data ) : <EOL> def _source ( data ) : <EOL> for _ in self . _tokenise ( data . split ( '<STR_LIT:\n>' ) ) : <EOL> yield _ <EOL> self . type = '<STR_LIT:text>' <EOL> return self . _set ( _source ( data ) ) <EOL> def set_api ( self , line ) : <EOL> return self . _set ( self . _tokenise ( iter ( [ line ] ) ) ) <EOL> def __call__ ( self ) : <EOL> self . number += <NUM_LIT:1> <EOL> try : <EOL> self . line , self . _next = self . _next , self . _tokens . next ( ) <EOL> self . end = self . line [ - <NUM_LIT:1> ] <EOL> except StopIteration : <EOL> if not self . finished : <EOL> self . finished = True <EOL> self . line , self . _next = self . _next , [ ] <EOL> self . end = self . line [ - <NUM_LIT:1> ] <EOL> else : <EOL> self . line = [ ] <EOL> self . end = '<STR_LIT>' <EOL> self . iterate . replenish ( self . line [ : - <NUM_LIT:1> ] ) <EOL> return self . line </s>
