<s> import os <EOL> import sys <EOL> import codecs <EOL> import glob <EOL> base_path = os . path . split ( __file__ ) [ <NUM_LIT:0> ] <EOL> if os . path . exists ( os . path . join ( base_path , '<STR_LIT>' ) ) : <EOL> test_dir = os . path . join ( base_path , '<STR_LIT>' ) <EOL> else : <EOL> test_dir = os . path . abspath ( <EOL> os . path . join ( base_path , <EOL> os . path . pardir , os . path . pardir , <EOL> os . path . pardir , '<STR_LIT>' ) ) <EOL> assert os . path . exists ( test_dir ) , "<STR_LIT>" <EOL> sys . path . insert ( <NUM_LIT:0> , os . path . abspath ( os . path . join ( base_path , <EOL> os . path . pardir , <EOL> os . path . pardir ) ) ) <EOL> import html5lib <EOL> from html5lib import html5parser , treebuilders <EOL> del base_path <EOL> treeTypes = { "<STR_LIT>" : treebuilders . getTreeBuilder ( "<STR_LIT>" ) , <EOL> "<STR_LIT>" : treebuilders . getTreeBuilder ( "<STR_LIT>" ) } <EOL> try : <EOL> import xml . etree . ElementTree as ElementTree <EOL> treeTypes [ '<STR_LIT>' ] = treebuilders . getTreeBuilder ( "<STR_LIT>" , ElementTree , fullTree = True ) <EOL> except ImportError : <EOL> try : <EOL> import elementtree . ElementTree as ElementTree <EOL> treeTypes [ '<STR_LIT>' ] = treebuilders . getTreeBuilder ( "<STR_LIT>" , ElementTree , fullTree = True ) <EOL> except ImportError : <EOL> pass <EOL> try : <EOL> import xml . etree . cElementTree as cElementTree <EOL> treeTypes [ '<STR_LIT>' ] = treebuilders . getTreeBuilder ( "<STR_LIT>" , cElementTree , fullTree = True ) <EOL> except ImportError : <EOL> try : <EOL> import cElementTree <EOL> treeTypes [ '<STR_LIT>' ] = treebuilders . getTreeBuilder ( "<STR_LIT>" , cElementTree , fullTree = True ) <EOL> except ImportError : <EOL> pass <EOL> try : <EOL> import lxml . etree as lxml <EOL> treeTypes [ '<STR_LIT>' ] = treebuilders . getTreeBuilder ( "<STR_LIT>" , lxml , fullTree = True ) <EOL> except ImportError : <EOL> pass <EOL> try : <EOL> import BeautifulSoup <EOL> treeTypes [ "<STR_LIT>" ] = treebuilders . getTreeBuilder ( "<STR_LIT>" , fullTree = True ) <EOL> except ImportError : <EOL> pass <EOL> def html5lib_test_files ( subdirectory , files = '<STR_LIT>' ) : <EOL> return glob . glob ( os . path . join ( test_dir , subdirectory , files ) ) <EOL> class DefaultDict ( dict ) : <EOL> def __init__ ( self , default , * args , ** kwargs ) : <EOL> self . default = default <EOL> dict . __init__ ( self , * args , ** kwargs ) <EOL> def __getitem__ ( self , key ) : <EOL> return dict . get ( self , key , self . default ) <EOL> class TestData ( object ) : <EOL> def __init__ ( self , filename , newTestHeading = "<STR_LIT:data>" ) : <EOL> self . f = codecs . open ( filename , encoding = "<STR_LIT:utf8>" ) <EOL> self . newTestHeading = newTestHeading <EOL> def __iter__ ( self ) : <EOL> data = DefaultDict ( None ) <EOL> key = None <EOL> for line in self . f : <EOL> heading = self . isSectionHeading ( line ) <EOL> if heading : <EOL> if data and heading == self . newTestHeading : <EOL> data [ key ] = data [ key ] [ : - <NUM_LIT:1> ] <EOL> yield self . normaliseOutput ( data ) <EOL> data = DefaultDict ( None ) <EOL> key = heading <EOL> data [ key ] = "<STR_LIT>" <EOL> elif key is not None : <EOL> data [ key ] += line <EOL> if data : <EOL> yield self . normaliseOutput ( data ) <EOL> def isSectionHeading ( self , line ) : <EOL> """<STR_LIT>""" <EOL> if line . startswith ( "<STR_LIT:#>" ) : <EOL> return line [ <NUM_LIT:1> : ] . strip ( ) <EOL> else : <EOL> return False <EOL> def normaliseOutput ( self , data ) : <EOL> for key , value in data . iteritems ( ) : <EOL> if value . endswith ( "<STR_LIT:\n>" ) : <EOL> data [ key ] = value [ : - <NUM_LIT:1> ] <EOL> return data <EOL> def convert ( stripChars ) : <EOL> def convertData ( data ) : <EOL> """<STR_LIT>""" <EOL> data = data . split ( "<STR_LIT:\n>" ) <EOL> rv = [ ] <EOL> for line in data : <EOL> if line . startswith ( "<STR_LIT:|>" ) : <EOL> rv . append ( line [ stripChars : ] ) <EOL> else : <EOL> rv . append ( line ) <EOL> return "<STR_LIT:\n>" . join ( rv ) <EOL> return convertData <EOL> convertExpected = convert ( <NUM_LIT:2> ) </s>
