<s> import re <EOL> from pygments . lexer import Lexer <EOL> from pygments . token import Token <EOL> HEADING = Token . Generic . Heading <EOL> SETTING = Token . Keyword . Namespace <EOL> IMPORT = Token . Name . Namespace <EOL> TC_KW_NAME = Token . Generic . Subheading <EOL> KEYWORD = Token . Name . Function <EOL> ARGUMENT = Token . String <EOL> VARIABLE = Token . Name . Variable <EOL> COMMENT = Token . Comment <EOL> SEPARATOR = Token . Punctuation <EOL> SYNTAX = Token . Punctuation <EOL> GHERKIN = Token . Generic . Emph <EOL> ERROR = Token . Error <EOL> def normalize ( string , remove = '<STR_LIT>' ) : <EOL> string = string . lower ( ) <EOL> for char in remove + '<STR_LIT:U+0020>' : <EOL> if char in string : <EOL> string = string . replace ( char , '<STR_LIT>' ) <EOL> return string <EOL> class RobotFrameworkLexer ( Lexer ) : <EOL> """<STR_LIT>""" <EOL> name = '<STR_LIT>' <EOL> aliases = [ '<STR_LIT>' , '<STR_LIT>' ] <EOL> filenames = [ '<STR_LIT>' ] <EOL> mimetypes = [ '<STR_LIT>' ] <EOL> def __init__ ( self ) : <EOL> Lexer . __init__ ( self , tabsize = <NUM_LIT:2> , encoding = '<STR_LIT>' ) <EOL> def get_tokens_unprocessed ( self , text ) : <EOL> row_tokenizer = RowTokenizer ( ) <EOL> var_tokenizer = VariableTokenizer ( ) <EOL> index = <NUM_LIT:0> <EOL> for row in text . splitlines ( ) : <EOL> for value , token in row_tokenizer . tokenize ( row ) : <EOL> for value , token in var_tokenizer . tokenize ( value , token ) : <EOL> if value : <EOL> yield index , token , unicode ( value ) <EOL> index += len ( value ) <EOL> class VariableTokenizer ( object ) : <EOL> def tokenize ( self , string , token ) : <EOL> var = VariableSplitter ( string , identifiers = '<STR_LIT>' ) <EOL> if var . start < <NUM_LIT:0> or token in ( COMMENT , ERROR ) : <EOL> yield string , token <EOL> return <EOL> for value , token in self . _tokenize ( var , string , token ) : <EOL> if value : <EOL> yield value , token <EOL> def _tokenize ( self , var , string , orig_token ) : <EOL> before = string [ : var . start ] <EOL> yield before , orig_token <EOL> yield var . identifier + '<STR_LIT:{>' , SYNTAX <EOL> for value , token in self . tokenize ( var . base , VARIABLE ) : <EOL> yield value , token <EOL> yield '<STR_LIT:}>' , SYNTAX <EOL> if var . index : <EOL> yield '<STR_LIT:[>' , SYNTAX <EOL> for value , token in self . tokenize ( var . index , VARIABLE ) : <EOL> yield value , token <EOL> yield '<STR_LIT:]>' , SYNTAX <EOL> for value , token in self . tokenize ( string [ var . end : ] , orig_token ) : <EOL> yield value , token <EOL> class RowTokenizer ( object ) : <EOL> def __init__ ( self ) : <EOL> self . _table = UnknownTable ( ) <EOL> self . _splitter = RowSplitter ( ) <EOL> testcases = TestCaseTable ( ) <EOL> settings = SettingTable ( testcases . set_default_template ) <EOL> variables = VariableTable ( ) <EOL> keywords = KeywordTable ( ) <EOL> self . _tables = { '<STR_LIT>' : settings , '<STR_LIT>' : settings , <EOL> '<STR_LIT>' : settings , <EOL> '<STR_LIT>' : variables , '<STR_LIT>' : variables , <EOL> '<STR_LIT>' : testcases , '<STR_LIT>' : testcases , <EOL> '<STR_LIT>' : keywords , '<STR_LIT>' : keywords , <EOL> '<STR_LIT>' : keywords , '<STR_LIT>' : keywords } <EOL> def tokenize ( self , row ) : <EOL> commented = False <EOL> heading = False <EOL> for index , value in enumerate ( self . _splitter . split ( row ) ) : <EOL> index , separator = divmod ( index - <NUM_LIT:1> , <NUM_LIT:2> ) <EOL> if value . startswith ( '<STR_LIT:#>' ) : <EOL> commented = True <EOL> elif index == <NUM_LIT:0> and value . startswith ( '<STR_LIT:*>' ) : <EOL> self . _table = self . _start_table ( value ) <EOL> heading = True <EOL> for value , token in self . _tokenize ( value , index , commented , <EOL> separator , heading ) : <EOL> yield value , token <EOL> self . _table . end_row ( ) <EOL> def _start_table ( self , header ) : <EOL> name = normalize ( header , remove = '<STR_LIT:*>' ) <EOL> return self . _tables . get ( name , UnknownTable ( ) ) <EOL> def _tokenize ( self , value , index , commented , separator , heading ) : <EOL> if commented : <EOL> yield value , COMMENT <EOL> elif separator : <EOL> yield value , SEPARATOR <EOL> elif heading : <EOL> yield value , HEADING <EOL> else : <EOL> for value , token in self . _table . tokenize ( value , index ) : <EOL> yield value , token <EOL> class RowSplitter ( object ) : <EOL> _space_splitter = re . compile ( '<STR_LIT>' ) <EOL> _pipe_splitter = re . compile ( '<STR_LIT>' ) <EOL> def split ( self , row ) : <EOL> splitter = self . _split_from_spaces if not row . startswith ( '<STR_LIT>' ) else self . _split_from_pipes <EOL> for value in splitter ( row ) : <EOL> yield value <EOL> yield '<STR_LIT:\n>' <EOL> def _split_from_spaces ( self , row ) : <EOL> yield '<STR_LIT>' <EOL> for value in self . _space_splitter . split ( row ) : <EOL> yield value <EOL> def _split_from_pipes ( self , row ) : <EOL> _ , separator , rest = self . _pipe_splitter . split ( row , <NUM_LIT:1> ) <EOL> yield separator <EOL> while self . _pipe_splitter . search ( rest ) : <EOL> cell , separator , rest = self . _pipe_splitter . split ( rest , <NUM_LIT:1> ) <EOL> yield cell <EOL> yield separator <EOL> yield rest <EOL> class Tokenizer ( object ) : <EOL> _tokens = None <EOL> def __init__ ( self ) : <EOL> self . _index = <NUM_LIT:0> <EOL> def tokenize ( self , value ) : <EOL> values_and_tokens = self . _tokenize ( value , self . _index ) <EOL> self . _index += <NUM_LIT:1> <EOL> if isinstance ( values_and_tokens , type ( Token ) ) : <EOL> values_and_tokens = [ ( value , values_and_tokens ) ] <EOL> return values_and_tokens <EOL> def _tokenize ( self , value , index ) : <EOL> index = min ( index , len ( self . _tokens ) - <NUM_LIT:1> ) <EOL> return self . _tokens [ index ] <EOL> def _is_assign ( self , value ) : <EOL> if value . endswith ( '<STR_LIT:=>' ) : <EOL> value = value [ : - <NUM_LIT:1> ] . strip ( ) <EOL> var = VariableSplitter ( value , identifiers = '<STR_LIT>' ) <EOL> return var . start == <NUM_LIT:0> and var . end == len ( value ) <EOL> class Comment ( Tokenizer ) : <EOL> _tokens = ( COMMENT , ) <EOL> class Setting ( Tokenizer ) : <EOL> _tokens = ( SETTING , ARGUMENT ) <EOL> _keyword_settings = ( '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ) <EOL> _import_settings = ( '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ) <EOL> _other_settings = ( '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' ) <EOL> _custom_tokenizer = None <EOL> def __init__ ( self , template_setter = None ) : <EOL> Tokenizer . __init__ ( self ) <EOL> self . _template_setter = template_setter <EOL> def _tokenize ( self , value , index ) : <EOL> if index == <NUM_LIT:1> and self . _template_setter : <EOL> self . _template_setter ( value ) <EOL> if index == <NUM_LIT:0> : <EOL> normalized = normalize ( value ) <EOL> if normalized in self . _keyword_settings : <EOL> self . _custom_tokenizer = KeywordCall ( support_assign = False ) <EOL> elif normalized in self . _import_settings : <EOL> self . _custom_tokenizer = ImportSetting ( ) <EOL> elif normalized not in self . _other_settings : <EOL> return ERROR <EOL> elif self . _custom_tokenizer : <EOL> return self . _custom_tokenizer . tokenize ( value ) <EOL> return Tokenizer . _tokenize ( self , value , index ) <EOL> class ImportSetting ( Tokenizer ) : <EOL> _tokens = ( IMPORT , ARGUMENT ) <EOL> class TestCaseSetting ( Setting ) : <EOL> _keyword_settings = ( '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' ) <EOL> _import_settings = ( ) <EOL> _other_settings = ( '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ) <EOL> def _tokenize ( self , value , index ) : <EOL> if index == <NUM_LIT:0> : <EOL> type = Setting . _tokenize ( self , value [ <NUM_LIT:1> : - <NUM_LIT:1> ] , index ) <EOL> return [ ( '<STR_LIT:[>' , SYNTAX ) , ( value [ <NUM_LIT:1> : - <NUM_LIT:1> ] , type ) , ( '<STR_LIT:]>' , SYNTAX ) ] <EOL> return Setting . _tokenize ( self , value , index ) <EOL> class KeywordSetting ( TestCaseSetting ) : <EOL> _keyword_settings = ( '<STR_LIT>' , ) <EOL> _other_settings = ( '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ) <EOL> class Variable ( Tokenizer ) : <EOL> _tokens = ( SYNTAX , ARGUMENT ) <EOL> def _tokenize ( self , value , index ) : <EOL> if index == <NUM_LIT:0> and not self . _is_assign ( value ) : <EOL> return ERROR <EOL> return Tokenizer . _tokenize ( self , value , index ) <EOL> class KeywordCall ( Tokenizer ) : <EOL> _tokens = ( KEYWORD , ARGUMENT ) <EOL> def __init__ ( self , support_assign = True ) : <EOL> Tokenizer . __init__ ( self ) <EOL> self . _keyword_found = not support_assign <EOL> self . _assigns = <NUM_LIT:0> <EOL> def _tokenize ( self , value , index ) : <EOL> if not self . _keyword_found and self . _is_assign ( value ) : <EOL> self . _assigns += <NUM_LIT:1> <EOL> return SYNTAX <EOL> if self . _keyword_found : <EOL> return Tokenizer . _tokenize ( self , value , index - self . _assigns ) <EOL> self . _keyword_found = True <EOL> return GherkinTokenizer ( ) . tokenize ( value , KEYWORD ) <EOL> class GherkinTokenizer ( object ) : <EOL> _gherkin_prefix = re . compile ( '<STR_LIT>' , re . IGNORECASE ) <EOL> def tokenize ( self , value , token ) : <EOL> match = self . _gherkin_prefix . match ( value ) <EOL> if not match : <EOL> return [ ( value , token ) ] <EOL> end = match . end ( ) <EOL> return [ ( value [ : end ] , GHERKIN ) , ( value [ end : ] , token ) ] <EOL> class TemplatedKeywordCall ( Tokenizer ) : <EOL> _tokens = ( ARGUMENT , ) <EOL> class ForLoop ( Tokenizer ) : <EOL> def __init__ ( self ) : <EOL> Tokenizer . __init__ ( self ) <EOL> self . _in_arguments = False <EOL> def _tokenize ( self , value , index ) : <EOL> token = ARGUMENT if self . _in_arguments else SYNTAX <EOL> if value . upper ( ) in ( '<STR_LIT>' , '<STR_LIT>' ) : <EOL> self . _in_arguments = True <EOL> return token <EOL> class _Table ( object ) : <EOL> _tokenizer_class = None <EOL> def __init__ ( self , prev_tokenizer = None ) : <EOL> self . _tokenizer = self . _tokenizer_class ( ) <EOL> self . _prev_tokenizer = prev_tokenizer <EOL> self . _prev_values_on_row = [ ] <EOL> def tokenize ( self , value , index ) : <EOL> if self . _continues ( value , index ) : <EOL> self . _tokenizer = self . _prev_tokenizer <EOL> yield value , SYNTAX <EOL> else : <EOL> for value_and_token in self . _tokenize ( value , index ) : <EOL> yield value_and_token <EOL> self . _prev_values_on_row . append ( value ) <EOL> def _continues ( self , value , index ) : <EOL> return value == '<STR_LIT>' and all ( self . _is_empty ( t ) <EOL> for t in self . _prev_values_on_row ) <EOL> def _is_empty ( self , value ) : <EOL> return value in ( '<STR_LIT>' , '<STR_LIT:\\>' ) <EOL> def _tokenize ( self , value , index ) : <EOL> return self . _tokenizer . tokenize ( value ) <EOL> def end_row ( self ) : <EOL> self . __init__ ( prev_tokenizer = self . _tokenizer ) <EOL> class UnknownTable ( _Table ) : <EOL> _tokenizer_class = Comment <EOL> def _continues ( self , value , index ) : <EOL> return False <EOL> class VariableTable ( _Table ) : <EOL> _tokenizer_class = Variable <EOL> class SettingTable ( _Table ) : <EOL> _tokenizer_class = Setting <EOL> def __init__ ( self , template_setter , prev_tokenizer = None ) : <EOL> _Table . __init__ ( self , prev_tokenizer ) <EOL> self . _template_setter = template_setter <EOL> def _tokenize ( self , value , index ) : <EOL> if index == <NUM_LIT:0> and normalize ( value ) == '<STR_LIT>' : <EOL> self . _tokenizer = Setting ( self . _template_setter ) <EOL> return _Table . _tokenize ( self , value , index ) <EOL> def end_row ( self ) : <EOL> self . __init__ ( self . _template_setter , prev_tokenizer = self . _tokenizer ) <EOL> class TestCaseTable ( _Table ) : <EOL> _setting_class = TestCaseSetting <EOL> _test_template = None <EOL> _default_template = None <EOL> @ property <EOL> def _tokenizer_class ( self ) : <EOL> if self . _test_template or ( self . _default_template <EOL> and self . _test_template is not False ) : <EOL> return TemplatedKeywordCall <EOL> return KeywordCall <EOL> def _continues ( self , value , index ) : <EOL> return index > <NUM_LIT:0> and _Table . _continues ( self , value , index ) <EOL> def _tokenize ( self , value , index ) : <EOL> if index == <NUM_LIT:0> : <EOL> if value : <EOL> self . _test_template = None <EOL> return GherkinTokenizer ( ) . tokenize ( value , TC_KW_NAME ) <EOL> if index == <NUM_LIT:1> and self . _is_setting ( value ) : <EOL> if self . _is_template ( value ) : <EOL> self . _test_template = False <EOL> self . _tokenizer = self . _setting_class ( self . set_test_template ) <EOL> else : <EOL> self . _tokenizer = self . _setting_class ( ) <EOL> if index == <NUM_LIT:1> and self . _is_for_loop ( value ) : <EOL> self . _tokenizer = ForLoop ( ) <EOL> if index == <NUM_LIT:1> and self . _is_empty ( value ) : <EOL> return [ ( value , SYNTAX ) ] <EOL> return _Table . _tokenize ( self , value , index ) <EOL> def _is_setting ( self , value ) : <EOL> return value . startswith ( '<STR_LIT:[>' ) and value . endswith ( '<STR_LIT:]>' ) <EOL> def _is_template ( self , value ) : <EOL> return normalize ( value ) == '<STR_LIT>' <EOL> def _is_for_loop ( self , value ) : <EOL> return value . startswith ( '<STR_LIT::>' ) and normalize ( value , remove = '<STR_LIT::>' ) == '<STR_LIT>' <EOL> def set_test_template ( self , template ) : <EOL> self . _test_template = self . _is_template_set ( template ) <EOL> def set_default_template ( self , template ) : <EOL> self . _default_template = self . _is_template_set ( template ) <EOL> def _is_template_set ( self , template ) : <EOL> return normalize ( template ) not in ( '<STR_LIT>' , '<STR_LIT:\\>' , '<STR_LIT:none>' , '<STR_LIT>' ) <EOL> class KeywordTable ( TestCaseTable ) : <EOL> _tokenizer_class = KeywordCall <EOL> _setting_class = KeywordSetting <EOL> def _is_template ( self , value ) : <EOL> return False <EOL> class VariableSplitter : <EOL> def __init__ ( self , string , identifiers ) : <EOL> self . identifier = None <EOL> self . base = None <EOL> self . index = None <EOL> self . start = - <NUM_LIT:1> <EOL> self . end = - <NUM_LIT:1> <EOL> self . _identifiers = identifiers <EOL> self . _may_have_internal_variables = False <EOL> try : <EOL> self . _split ( string ) <EOL> except ValueError : <EOL> pass <EOL> else : <EOL> self . _finalize ( ) <EOL> def get_replaced_base ( self , variables ) : <EOL> if self . _may_have_internal_variables : <EOL> return variables . replace_string ( self . base ) <EOL> return self . base <EOL> def _finalize ( self ) : <EOL> self . identifier = self . _variable_chars [ <NUM_LIT:0> ] <EOL> self . base = '<STR_LIT>' . join ( self . _variable_chars [ <NUM_LIT:2> : - <NUM_LIT:1> ] ) <EOL> self . end = self . start + len ( self . _variable_chars ) <EOL> if self . _has_list_variable_index ( ) : <EOL> self . index = '<STR_LIT>' . join ( self . _list_variable_index_chars [ <NUM_LIT:1> : - <NUM_LIT:1> ] ) <EOL> self . end += len ( self . _list_variable_index_chars ) <EOL> def _has_list_variable_index ( self ) : <EOL> return self . _list_variable_index_chars and self . _list_variable_index_chars [ - <NUM_LIT:1> ] == '<STR_LIT:]>' <EOL> def _split ( self , string ) : <EOL> start_index , max_index = self . _find_variable ( string ) <EOL> self . start = start_index <EOL> self . _open_curly = <NUM_LIT:1> <EOL> self . _state = self . _variable_state <EOL> self . _variable_chars = [ string [ start_index ] , '<STR_LIT:{>' ] <EOL> self . _list_variable_index_chars = [ ] <EOL> self . _string = string <EOL> start_index += <NUM_LIT:2> <EOL> for index , char in enumerate ( string [ start_index : ] ) : <EOL> index += start_index <EOL> try : <EOL> self . _state ( char , index ) <EOL> except StopIteration : <EOL> return <EOL> if index == max_index and not self . _scanning_list_variable_index ( ) : <EOL> return <EOL> def _scanning_list_variable_index ( self ) : <EOL> return self . _state in [ self . _waiting_list_variable_index_state , <EOL> self . _list_variable_index_state ] <EOL> def _find_variable ( self , string ) : <EOL> max_end_index = string . rfind ( '<STR_LIT:}>' ) <EOL> if max_end_index == - <NUM_LIT:1> : <EOL> return ValueError ( '<STR_LIT>' ) <EOL> if self . _is_escaped ( string , max_end_index ) : <EOL> return self . _find_variable ( string [ : max_end_index ] ) <EOL> start_index = self . _find_start_index ( string , <NUM_LIT:1> , max_end_index ) <EOL> if start_index == - <NUM_LIT:1> : <EOL> return ValueError ( '<STR_LIT>' ) <EOL> return start_index , max_end_index <EOL> def _find_start_index ( self , string , start , end ) : <EOL> index = string . find ( '<STR_LIT:{>' , start , end ) - <NUM_LIT:1> <EOL> if index < <NUM_LIT:0> : <EOL> return - <NUM_LIT:1> <EOL> if self . _start_index_is_ok ( string , index ) : <EOL> return index <EOL> return self . _find_start_index ( string , index + <NUM_LIT:2> , end ) <EOL> def _start_index_is_ok ( self , string , index ) : <EOL> return string [ index ] in self . _identifiers and not self . _is_escaped ( string , index ) <EOL> def _is_escaped ( self , string , index ) : <EOL> escaped = False <EOL> while index > <NUM_LIT:0> and string [ index - <NUM_LIT:1> ] == '<STR_LIT:\\>' : <EOL> index -= <NUM_LIT:1> <EOL> escaped = not escaped <EOL> return escaped <EOL> def _variable_state ( self , char , index ) : <EOL> self . _variable_chars . append ( char ) <EOL> if char == '<STR_LIT:}>' and not self . _is_escaped ( self . _string , index ) : <EOL> self . _open_curly -= <NUM_LIT:1> <EOL> if self . _open_curly == <NUM_LIT:0> : <EOL> if not self . _is_list_variable ( ) : <EOL> raise StopIteration <EOL> self . _state = self . _waiting_list_variable_index_state <EOL> elif char in self . _identifiers : <EOL> self . _state = self . _internal_variable_start_state <EOL> def _is_list_variable ( self ) : <EOL> return self . _variable_chars [ <NUM_LIT:0> ] == '<STR_LIT:@>' <EOL> def _internal_variable_start_state ( self , char , index ) : <EOL> self . _state = self . _variable_state <EOL> if char == '<STR_LIT:{>' : <EOL> self . _variable_chars . append ( char ) <EOL> self . _open_curly += <NUM_LIT:1> <EOL> self . _may_have_internal_variables = True <EOL> else : <EOL> self . _variable_state ( char , index ) <EOL> def _waiting_list_variable_index_state ( self , char , index ) : <EOL> if char != '<STR_LIT:[>' : <EOL> raise StopIteration <EOL> self . _list_variable_index_chars . append ( char ) <EOL> self . _state = self . _list_variable_index_state <EOL> def _list_variable_index_state ( self , char , index ) : <EOL> self . _list_variable_index_chars . append ( char ) <EOL> if char == '<STR_LIT:]>' : <EOL> raise StopIteration </s>
