<s> """<STR_LIT>""" <EOL> import types <EOL> import csv <EOL> def getMapColumn2Type ( rows , ignore_empty = False , get_max_values = False ) : <EOL> """<STR_LIT>""" <EOL> headers = rows [ <NUM_LIT:0> ] . keys ( ) <EOL> map_column2type = { } <EOL> is_full = { } <EOL> max_values = { } <EOL> for row in rows : <EOL> for h in headers : <EOL> if row [ h ] == "<STR_LIT>" : <EOL> continue <EOL> is_full [ h ] = True <EOL> if isinstance ( row [ h ] , int ) : <EOL> t = types . IntType <EOL> if h not in max_values : <EOL> max_values [ h ] = int ( row [ h ] ) <EOL> else : <EOL> max_values [ h ] = max ( h , int ( row [ h ] ) ) <EOL> elif isinstance ( row [ h ] , float ) : <EOL> t = types . FloatType <EOL> else : <EOL> continue <EOL> map_column2type [ h ] = t <EOL> ignored = [ ] <EOL> for h in headers : <EOL> if h not in map_column2type : <EOL> if h in is_full or not ignore_empty : <EOL> map_column2type [ h ] = types . StringType <EOL> else : <EOL> ignored . append ( h ) <EOL> if get_max_values : <EOL> return map_column2type , ignored , max_values <EOL> else : <EOL> return map_column2type , ignored <EOL> class CommentStripper : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , infile ) : <EOL> self . infile = infile <EOL> def __iter__ ( self ) : <EOL> return self <EOL> def next ( self ) : <EOL> while <NUM_LIT:1> : <EOL> line = self . infile . next ( ) <EOL> if line is None : <EOL> raise StopIteration <EOL> if line . strip ( ) != "<STR_LIT>" and not line . startswith ( "<STR_LIT:#>" ) : <EOL> return line <EOL> class DictReader ( csv . DictReader ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , infile , * args , ** kwargs ) : <EOL> csv . DictReader . __init__ ( self , <EOL> CommentStripper ( infile ) , <EOL> * args , ** kwargs ) <EOL> class UnicodeCsvReader ( object ) : <EOL> def __init__ ( self , f , encoding = "<STR_LIT:utf-8>" , ** kwargs ) : <EOL> self . csv_reader = csv . reader ( f , ** kwargs ) <EOL> self . encoding = encoding <EOL> def __iter__ ( self ) : <EOL> return self <EOL> def next ( self ) : <EOL> row = self . csv_reader . next ( ) <EOL> return [ unicode ( cell , self . encoding ) for cell in row ] <EOL> @ property <EOL> def line_num ( self ) : <EOL> return self . csv_reader . line_num <EOL> class UnicodeDictReader ( csv . DictReader ) : <EOL> def __init__ ( self , f , encoding = "<STR_LIT:utf-8>" , fieldnames = None , ** kwds ) : <EOL> csv . DictReader . __init__ ( self , f , fieldnames = fieldnames , ** kwds ) <EOL> self . reader = UnicodeCsvReader ( f , encoding = encoding , ** kwds ) <EOL> class DictReaderLarge : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , infile , fieldnames , * args , ** kwargs ) : <EOL> self . mFile = infile <EOL> self . mFieldNames = fieldnames <EOL> self . mNFields = len ( fieldnames ) <EOL> def __iter__ ( self ) : <EOL> return self <EOL> def next ( self ) : <EOL> line = self . mFile . next ( ) <EOL> if not line : <EOL> raise StopIteration <EOL> data = line [ : - <NUM_LIT:1> ] . split ( "<STR_LIT:\t>" ) <EOL> assert len ( data ) == self . mNFields <EOL> return dict ( zip ( self . mFieldNames , data ) ) <EOL> def readTable ( infile , <EOL> as_rows = True , <EOL> with_header = True , <EOL> ignore_incomplete = False , <EOL> dialect = "<STR_LIT>" ) : <EOL> """<STR_LIT>""" <EOL> lines = [ x for x in infile if x [ <NUM_LIT:0> ] != "<STR_LIT:#>" ] <EOL> if len ( lines ) == <NUM_LIT:0> : <EOL> return [ ] , [ ] <EOL> if with_header : <EOL> fields = lines [ <NUM_LIT:0> ] [ : - <NUM_LIT:1> ] . split ( "<STR_LIT:\t>" ) <EOL> del lines [ <NUM_LIT:0> ] <EOL> else : <EOL> fields = lines [ <NUM_LIT:0> ] [ : - <NUM_LIT:1> ] . split ( "<STR_LIT:\t>" ) <EOL> fields = map ( str , range ( len ( fields ) ) ) <EOL> nfields = len ( fields ) <EOL> try : <EOL> reader = csv . reader ( lines . __iter__ ( ) , <EOL> dialect = dialect ) <EOL> except TypeError : <EOL> reader = csv . reader ( lines . __iter__ ( ) ) <EOL> table = list ( reader ) <EOL> if ignore_incomplete : <EOL> table = [ x for x in table if len ( x ) == nfields ] <EOL> else : <EOL> for r , row in enumerate ( table ) : <EOL> if len ( row ) != nfields : <EOL> if not ignore_incomplete : <EOL> raise ValueError ( <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" % <EOL> ( r , str ( row ) , str ( fields ) ) ) <EOL> raise ValueError <EOL> if not as_rows : <EOL> table = zip ( * table ) <EOL> return fields , table <EOL> def readTables ( infile , * args , ** kwargs ) : <EOL> """<STR_LIT>""" <EOL> lines = [ x for x in infile if x [ <NUM_LIT:0> ] != "<STR_LIT:#>" ] <EOL> chunks = filter ( lambda x : lines [ x ] [ : <NUM_LIT:2> ] == "<STR_LIT>" , range ( len ( lines ) ) ) <EOL> if not lines [ - <NUM_LIT:1> ] . startswith ( "<STR_LIT>" ) : <EOL> chunks . append ( len ( lines ) ) <EOL> result = [ ] <EOL> start = <NUM_LIT:0> <EOL> for end in chunks : <EOL> fields , table = readTable ( lines [ start : end ] , * args , ** kwargs ) <EOL> result . append ( ( fields , table ) ) <EOL> start = end + <NUM_LIT:1> <EOL> return result <EOL> def __DoGroup ( rows , group_column , group_function , missing_value = "<STR_LIT>" ) : <EOL> values = [ ] <EOL> for x in range ( len ( rows [ <NUM_LIT:0> ] ) ) : <EOL> if x == group_column : <EOL> values . append ( rows [ <NUM_LIT:0> ] [ x ] ) <EOL> else : <EOL> v = filter ( lambda x : x != missing_value , map ( lambda y : y [ x ] , rows ) ) <EOL> if len ( v ) == <NUM_LIT:0> : <EOL> values . append ( missing_value ) <EOL> else : <EOL> values . append ( group_function ( map ( lambda y : y [ x ] , rows ) ) ) <EOL> return values <EOL> def groupTable ( table , <EOL> group_column = <NUM_LIT:0> , <EOL> group_function = min , <EOL> missing_value = "<STR_LIT>" ) : <EOL> '''<STR_LIT>''' <EOL> table . sort ( lambda x , y : cmp ( x [ group_column ] , y [ group_column ] ) ) <EOL> rows = [ ] <EOL> last_value = None <EOL> new_table = [ ] <EOL> for row in table : <EOL> if row [ group_column ] != last_value : <EOL> if last_value is not None : <EOL> new_table . append ( <EOL> __DoGroup ( rows , group_column , group_function , <EOL> missing_value ) ) <EOL> rows = [ ] <EOL> last_value = row [ group_column ] <EOL> rows . append ( row ) <EOL> if last_value is not None : <EOL> new_table . append ( <EOL> __DoGroup ( rows , group_column , group_function , missing_value ) ) <EOL> return new_table <EOL> def convertTable ( table , columns , function = float , <EOL> skip_errors = False ) : <EOL> """<STR_LIT>""" <EOL> new_table = [ ] <EOL> for row in table : <EOL> skip = False <EOL> for c in columns : <EOL> try : <EOL> row [ c ] = float ( row [ c ] ) <EOL> except ValueError , msg : <EOL> if skip_errors : <EOL> skip = True <EOL> break <EOL> else : <EOL> raise ValueError ( msg ) <EOL> if not skip : <EOL> new_table . append ( row ) <EOL> return new_table </s>
