<s> from zen import * <EOL> import unittest <EOL> import os <EOL> import os . path as path <EOL> import tempfile <EOL> class GMLTokenizerCase ( unittest . TestCase ) : <EOL> tok = gml_tokenizer . GMLTokenizer ( ) <EOL> def test_basic_correct ( self ) : <EOL> gml_string = '<STR_LIT>' <EOL> got_tokens = self . tok . tokenize ( gml_string ) <EOL> expected_tokens = [ ( '<STR_LIT>' , <NUM_LIT:0> , <NUM_LIT:0> ) , ( '<STR_LIT>' , <NUM_LIT:1> , <NUM_LIT:0> ) , ( '<STR_LIT>' , <NUM_LIT:0> , <NUM_LIT:0> ) , ( '<STR_LIT:2>' , <NUM_LIT:1> , <NUM_LIT:0> ) ] <EOL> self . assertEqual ( got_tokens , expected_tokens ) <EOL> def test_nested_list_correct ( self ) : <EOL> gml_string = '''<STR_LIT>''' <EOL> got_tokens = self . tok . tokenize ( gml_string ) <EOL> expected_tokens = [ <EOL> ( '<STR_LIT>' , <NUM_LIT:0> , <NUM_LIT:1> ) , ( '<STR_LIT:[>' , <NUM_LIT:2> , <NUM_LIT:1> ) , <EOL> ( '<STR_LIT>' , <NUM_LIT:0> , <NUM_LIT:2> ) , ( '<STR_LIT>' , <NUM_LIT:1> , <NUM_LIT:2> ) , <EOL> ( '<STR_LIT>' , <NUM_LIT:0> , <NUM_LIT:3> ) , ( '<STR_LIT:[>' , <NUM_LIT:2> , <NUM_LIT:3> ) , <EOL> ( '<STR_LIT>' , <NUM_LIT:0> , <NUM_LIT:4> ) , ( '<STR_LIT>' , <NUM_LIT:1> , <NUM_LIT:4> ) , <EOL> ( '<STR_LIT>' , <NUM_LIT:0> , <NUM_LIT:5> ) , ( '<STR_LIT:2>' , <NUM_LIT:1> , <NUM_LIT:5> ) , <EOL> ( '<STR_LIT:]>' , <NUM_LIT:3> , <NUM_LIT:6> ) , <EOL> ( '<STR_LIT:]>' , <NUM_LIT:3> , <NUM_LIT:7> ) , <EOL> ( '<STR_LIT>' , <NUM_LIT:0> , <NUM_LIT:8> ) , ( '<STR_LIT>' , <NUM_LIT:1> , <NUM_LIT:8> ) <EOL> ] <EOL> self . assertEqual ( got_tokens , expected_tokens ) <EOL> def test_correct_with_comments ( self ) : <EOL> gml_string = '''<STR_LIT>''' <EOL> got_tokens = self . tok . tokenize ( gml_string ) <EOL> expected_tokens = [ <EOL> ( '<STR_LIT>' , <NUM_LIT:0> , <NUM_LIT:2> ) , ( '<STR_LIT>' , <NUM_LIT:1> , <NUM_LIT:3> ) , <EOL> ( '<STR_LIT>' , <NUM_LIT:0> , <NUM_LIT:4> ) , ( '<STR_LIT:2>' , <NUM_LIT:1> , <NUM_LIT:5> ) <EOL> ] <EOL> self . assertEqual ( got_tokens , expected_tokens ) <EOL> def test_correct_empty_list ( self ) : <EOL> gml_string = '''<STR_LIT>''' <EOL> got_tokens = self . tok . tokenize ( gml_string ) <EOL> expected_tokens = [ <EOL> ( '<STR_LIT>' , <NUM_LIT:0> , <NUM_LIT:1> ) , ( '<STR_LIT:[>' , <NUM_LIT:2> , <NUM_LIT:1> ) , <EOL> ( '<STR_LIT:]>' , <NUM_LIT:3> , <NUM_LIT:2> ) , ( '<STR_LIT>' , <NUM_LIT:0> , <NUM_LIT:3> ) , ( '<STR_LIT>' , <NUM_LIT:1> , <NUM_LIT:3> ) <EOL> ] <EOL> self . assertEqual ( got_tokens , expected_tokens ) <EOL> def test_incorrect_eof_in_string ( self ) : <EOL> gml_string = '<STR_LIT>' <EOL> self . assertRaises ( ZenException , self . tok . tokenize , gml_string ) <EOL> def test_incorrect_string_as_key ( self ) : <EOL> gml_string = '<STR_LIT>' <EOL> self . assertRaises ( ZenException , self . tok . tokenize , gml_string ) <EOL> def test_incorrect_eof_when_expecting_value ( self ) : <EOL> gml_string = '<STR_LIT>' <EOL> self . assertRaises ( ZenException , self . tok . tokenize , gml_string ) <EOL> def test_incorrect_eolist_when_expecting_value ( self ) : <EOL> gml_string = '''<STR_LIT>''' <EOL> self . assertRaises ( ZenException , self . tok . tokenize , gml_string ) <EOL> if __name__ == '<STR_LIT:__main__>' : <EOL> unittest . main ( ) </s>
