<s> import os , glob , random <EOL> from cStringIO import StringIO <EOL> from test . test_support import ( verbose , findfile , is_resource_enabled , <EOL> TestFailed ) <EOL> from tokenize import ( tokenize , generate_tokens , untokenize , <EOL> NUMBER , NAME , OP , STRING ) <EOL> def test_roundtrip ( f ) : <EOL> fobj = open ( f ) <EOL> try : <EOL> fulltok = list ( generate_tokens ( fobj . readline ) ) <EOL> finally : <EOL> fobj . close ( ) <EOL> t1 = [ tok [ : <NUM_LIT:2> ] for tok in fulltok ] <EOL> newtext = untokenize ( t1 ) <EOL> readline = iter ( newtext . splitlines ( <NUM_LIT:1> ) ) . next <EOL> t2 = [ tok [ : <NUM_LIT:2> ] for tok in generate_tokens ( readline ) ] <EOL> if t1 != t2 : <EOL> raise TestFailed ( "<STR_LIT>" % f ) <EOL> def decistmt ( s ) : <EOL> """<STR_LIT>""" <EOL> result = [ ] <EOL> g = generate_tokens ( StringIO ( s ) . readline ) <EOL> for toknum , tokval , _ , _ , _ in g : <EOL> if toknum == NUMBER and '<STR_LIT:.>' in tokval : <EOL> result . extend ( [ <EOL> ( NAME , '<STR_LIT>' ) , <EOL> ( OP , '<STR_LIT:(>' ) , <EOL> ( STRING , repr ( tokval ) ) , <EOL> ( OP , '<STR_LIT:)>' ) <EOL> ] ) <EOL> else : <EOL> result . append ( ( toknum , tokval ) ) <EOL> return untokenize ( result ) <EOL> def test_main ( ) : <EOL> if verbose : <EOL> print '<STR_LIT>' <EOL> f = open ( findfile ( '<STR_LIT>' + os . extsep + '<STR_LIT>' ) ) <EOL> tokenize ( f . readline ) <EOL> f . close ( ) <EOL> f = findfile ( '<STR_LIT>' + os . extsep + '<STR_LIT>' ) <EOL> test_roundtrip ( f ) <EOL> testdir = os . path . dirname ( f ) or os . curdir <EOL> testfiles = glob . glob ( testdir + os . sep + '<STR_LIT>' ) <EOL> if not is_resource_enabled ( '<STR_LIT>' ) : <EOL> testfiles = random . sample ( testfiles , <NUM_LIT:10> ) <EOL> for f in testfiles : <EOL> test_roundtrip ( f ) <EOL> sampleBadText = """<STR_LIT>""" <EOL> try : <EOL> for tok in generate_tokens ( StringIO ( sampleBadText ) . readline ) : <EOL> pass <EOL> except IndentationError : <EOL> pass <EOL> else : <EOL> raise TestFailed ( "<STR_LIT>" ) <EOL> from test import test_tokenize <EOL> from test . test_support import run_doctest <EOL> run_doctest ( test_tokenize ) <EOL> if verbose : <EOL> print '<STR_LIT>' <EOL> if __name__ == "<STR_LIT:__main__>" : <EOL> test_main ( ) </s>
