<s> doctests = """<STR_LIT>""" <EOL> from test import test_support <EOL> from tokenize import ( untokenize , generate_tokens , NUMBER , NAME , OP , <EOL> STRING , ENDMARKER , tok_name ) <EOL> from StringIO import StringIO <EOL> import os <EOL> def dump_tokens ( s ) : <EOL> """<STR_LIT>""" <EOL> f = StringIO ( s ) <EOL> for type , token , start , end , line in generate_tokens ( f . readline ) : <EOL> if type == ENDMARKER : <EOL> break <EOL> type = tok_name [ type ] <EOL> print ( "<STR_LIT>" % locals ( ) ) <EOL> def decistmt ( s ) : <EOL> """<STR_LIT>""" <EOL> result = [ ] <EOL> g = generate_tokens ( StringIO ( s ) . readline ) <EOL> for toknum , tokval , _ , _ , _ in g : <EOL> if toknum == NUMBER and '<STR_LIT:.>' in tokval : <EOL> result . extend ( [ <EOL> ( NAME , '<STR_LIT>' ) , <EOL> ( OP , '<STR_LIT:(>' ) , <EOL> ( STRING , repr ( tokval ) ) , <EOL> ( OP , '<STR_LIT:)>' ) <EOL> ] ) <EOL> else : <EOL> result . append ( ( toknum , tokval ) ) <EOL> return untokenize ( result ) <EOL> __test__ = { "<STR_LIT>" : doctests , '<STR_LIT>' : decistmt } <EOL> def test_main ( ) : <EOL> from test import test_tokenize <EOL> test_support . run_doctest ( test_tokenize , True ) <EOL> if __name__ == "<STR_LIT:__main__>" : <EOL> test_main ( ) </s>
