<s> """<STR_LIT>""" <EOL> import re <EOL> from functools import reduce <EOL> from _csv import Error , __version__ , writer , reader , register_dialect , unregister_dialect , get_dialect , list_dialects , field_size_limit , QUOTE_MINIMAL , QUOTE_ALL , QUOTE_NONNUMERIC , QUOTE_NONE , __doc__ <EOL> from _csv import Dialect as _Dialect <EOL> try : <EOL> from cStringIO import StringIO <EOL> except ImportError : <EOL> from StringIO import StringIO <EOL> __all__ = [ "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , <EOL> "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , <EOL> "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , <EOL> "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , <EOL> "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" ] <EOL> class Dialect : <EOL> """<STR_LIT>""" <EOL> _name = "<STR_LIT>" <EOL> _valid = False <EOL> delimiter = None <EOL> quotechar = None <EOL> escapechar = None <EOL> doublequote = None <EOL> skipinitialspace = None <EOL> lineterminator = None <EOL> quoting = None <EOL> def __init__ ( self ) : <EOL> if self . __class__ != Dialect : <EOL> self . _valid = True <EOL> self . _validate ( ) <EOL> def _validate ( self ) : <EOL> try : <EOL> _Dialect ( self ) <EOL> except TypeError , e : <EOL> raise Error ( str ( e ) ) <EOL> class excel ( Dialect ) : <EOL> """<STR_LIT>""" <EOL> delimiter = '<STR_LIT:U+002C>' <EOL> quotechar = '<STR_LIT:">' <EOL> doublequote = True <EOL> skipinitialspace = False <EOL> lineterminator = '<STR_LIT:\r\n>' <EOL> quoting = QUOTE_MINIMAL <EOL> register_dialect ( "<STR_LIT>" , excel ) <EOL> class excel_tab ( excel ) : <EOL> """<STR_LIT>""" <EOL> delimiter = '<STR_LIT:\t>' <EOL> register_dialect ( "<STR_LIT>" , excel_tab ) <EOL> class DictReader : <EOL> def __init__ ( self , f , fieldnames = None , restkey = None , restval = None , <EOL> dialect = "<STR_LIT>" , * args , ** kwds ) : <EOL> self . _fieldnames = fieldnames <EOL> self . restkey = restkey <EOL> self . restval = restval <EOL> self . reader = reader ( f , dialect , * args , ** kwds ) <EOL> self . dialect = dialect <EOL> self . line_num = <NUM_LIT:0> <EOL> def __iter__ ( self ) : <EOL> return self <EOL> @ property <EOL> def fieldnames ( self ) : <EOL> if self . _fieldnames is None : <EOL> try : <EOL> self . _fieldnames = self . reader . next ( ) <EOL> except StopIteration : <EOL> pass <EOL> self . line_num = self . reader . line_num <EOL> return self . _fieldnames <EOL> @ fieldnames . setter <EOL> def fieldnames ( self , value ) : <EOL> self . _fieldnames = value <EOL> def next ( self ) : <EOL> if self . line_num == <NUM_LIT:0> : <EOL> self . fieldnames <EOL> row = self . reader . next ( ) <EOL> self . line_num = self . reader . line_num <EOL> while row == [ ] : <EOL> row = self . reader . next ( ) <EOL> d = dict ( zip ( self . fieldnames , row ) ) <EOL> lf = len ( self . fieldnames ) <EOL> lr = len ( row ) <EOL> if lf < lr : <EOL> d [ self . restkey ] = row [ lf : ] <EOL> elif lf > lr : <EOL> for key in self . fieldnames [ lr : ] : <EOL> d [ key ] = self . restval <EOL> return d <EOL> class DictWriter : <EOL> def __init__ ( self , f , fieldnames , restval = "<STR_LIT>" , extrasaction = "<STR_LIT>" , <EOL> dialect = "<STR_LIT>" , * args , ** kwds ) : <EOL> self . fieldnames = fieldnames <EOL> self . restval = restval <EOL> if extrasaction . lower ( ) not in ( "<STR_LIT>" , "<STR_LIT:ignore>" ) : <EOL> raise ValueError , ( "<STR_LIT>" % <EOL> extrasaction ) <EOL> self . extrasaction = extrasaction <EOL> self . writer = writer ( f , dialect , * args , ** kwds ) <EOL> def writeheader ( self ) : <EOL> header = dict ( zip ( self . fieldnames , self . fieldnames ) ) <EOL> self . writerow ( header ) <EOL> def _dict_to_list ( self , rowdict ) : <EOL> if self . extrasaction == "<STR_LIT>" : <EOL> wrong_fields = [ k for k in rowdict if k not in self . fieldnames ] <EOL> if wrong_fields : <EOL> raise ValueError ( "<STR_LIT>" <EOL> + "<STR_LIT:U+002CU+0020>" . join ( [ repr ( x ) for x in wrong_fields ] ) ) <EOL> return [ rowdict . get ( key , self . restval ) for key in self . fieldnames ] <EOL> def writerow ( self , rowdict ) : <EOL> return self . writer . writerow ( self . _dict_to_list ( rowdict ) ) <EOL> def writerows ( self , rowdicts ) : <EOL> rows = [ ] <EOL> for rowdict in rowdicts : <EOL> rows . append ( self . _dict_to_list ( rowdict ) ) <EOL> return self . writer . writerows ( rows ) <EOL> try : <EOL> complex <EOL> except NameError : <EOL> complex = float <EOL> class Sniffer : <EOL> '''<STR_LIT>''' <EOL> def __init__ ( self ) : <EOL> self . preferred = [ '<STR_LIT:U+002C>' , '<STR_LIT:\t>' , '<STR_LIT:;>' , '<STR_LIT:U+0020>' , '<STR_LIT::>' ] <EOL> def sniff ( self , sample , delimiters = None ) : <EOL> """<STR_LIT>""" <EOL> quotechar , doublequote , delimiter , skipinitialspace = self . _guess_quote_and_delimiter ( sample , delimiters ) <EOL> if not delimiter : <EOL> delimiter , skipinitialspace = self . _guess_delimiter ( sample , <EOL> delimiters ) <EOL> if not delimiter : <EOL> raise Error , "<STR_LIT>" <EOL> class dialect ( Dialect ) : <EOL> _name = "<STR_LIT>" <EOL> lineterminator = '<STR_LIT:\r\n>' <EOL> quoting = QUOTE_MINIMAL <EOL> dialect . doublequote = doublequote <EOL> dialect . delimiter = delimiter <EOL> dialect . quotechar = quotechar or '<STR_LIT:">' <EOL> dialect . skipinitialspace = skipinitialspace <EOL> return dialect <EOL> def _guess_quote_and_delimiter ( self , data , delimiters ) : <EOL> """<STR_LIT>""" <EOL> matches = [ ] <EOL> for restr in ( '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' ) : <EOL> regexp = re . compile ( restr , re . DOTALL | re . MULTILINE ) <EOL> matches = regexp . findall ( data ) <EOL> if matches : <EOL> break <EOL> if not matches : <EOL> return ( '<STR_LIT>' , False , None , <NUM_LIT:0> ) <EOL> quotes = { } <EOL> delims = { } <EOL> spaces = <NUM_LIT:0> <EOL> for m in matches : <EOL> n = regexp . groupindex [ '<STR_LIT>' ] - <NUM_LIT:1> <EOL> key = m [ n ] <EOL> if key : <EOL> quotes [ key ] = quotes . get ( key , <NUM_LIT:0> ) + <NUM_LIT:1> <EOL> try : <EOL> n = regexp . groupindex [ '<STR_LIT>' ] - <NUM_LIT:1> <EOL> key = m [ n ] <EOL> except KeyError : <EOL> continue <EOL> if key and ( delimiters is None or key in delimiters ) : <EOL> delims [ key ] = delims . get ( key , <NUM_LIT:0> ) + <NUM_LIT:1> <EOL> try : <EOL> n = regexp . groupindex [ '<STR_LIT>' ] - <NUM_LIT:1> <EOL> except KeyError : <EOL> continue <EOL> if m [ n ] : <EOL> spaces += <NUM_LIT:1> <EOL> quotechar = reduce ( lambda a , b , quotes = quotes : <EOL> ( quotes [ a ] > quotes [ b ] ) and a or b , quotes . keys ( ) ) <EOL> if delims : <EOL> delim = reduce ( lambda a , b , delims = delims : <EOL> ( delims [ a ] > delims [ b ] ) and a or b , delims . keys ( ) ) <EOL> skipinitialspace = delims [ delim ] == spaces <EOL> if delim == '<STR_LIT:\n>' : <EOL> delim = '<STR_LIT>' <EOL> else : <EOL> delim = '<STR_LIT>' <EOL> skipinitialspace = <NUM_LIT:0> <EOL> dq_regexp = re . compile ( <EOL> r"<STR_LIT>" % { '<STR_LIT>' : re . escape ( delim ) , '<STR_LIT>' : quotechar } , re . MULTILINE ) <EOL> if dq_regexp . search ( data ) : <EOL> doublequote = True <EOL> else : <EOL> doublequote = False <EOL> return ( quotechar , doublequote , delim , skipinitialspace ) <EOL> def _guess_delimiter ( self , data , delimiters ) : <EOL> """<STR_LIT>""" <EOL> data = filter ( None , data . split ( '<STR_LIT:\n>' ) ) <EOL> ascii = [ chr ( c ) for c in range ( <NUM_LIT> ) ] <EOL> chunkLength = min ( <NUM_LIT:10> , len ( data ) ) <EOL> iteration = <NUM_LIT:0> <EOL> charFrequency = { } <EOL> modes = { } <EOL> delims = { } <EOL> start , end = <NUM_LIT:0> , min ( chunkLength , len ( data ) ) <EOL> while start < len ( data ) : <EOL> iteration += <NUM_LIT:1> <EOL> for line in data [ start : end ] : <EOL> for char in ascii : <EOL> metaFrequency = charFrequency . get ( char , { } ) <EOL> freq = line . count ( char ) <EOL> metaFrequency [ freq ] = metaFrequency . get ( freq , <NUM_LIT:0> ) + <NUM_LIT:1> <EOL> charFrequency [ char ] = metaFrequency <EOL> for char in charFrequency . keys ( ) : <EOL> items = charFrequency [ char ] . items ( ) <EOL> if len ( items ) == <NUM_LIT:1> and items [ <NUM_LIT:0> ] [ <NUM_LIT:0> ] == <NUM_LIT:0> : <EOL> continue <EOL> if len ( items ) > <NUM_LIT:1> : <EOL> modes [ char ] = reduce ( lambda a , b : a [ <NUM_LIT:1> ] > b [ <NUM_LIT:1> ] and a or b , <EOL> items ) <EOL> items . remove ( modes [ char ] ) <EOL> modes [ char ] = ( modes [ char ] [ <NUM_LIT:0> ] , modes [ char ] [ <NUM_LIT:1> ] <EOL> - reduce ( lambda a , b : ( <NUM_LIT:0> , a [ <NUM_LIT:1> ] + b [ <NUM_LIT:1> ] ) , <EOL> items ) [ <NUM_LIT:1> ] ) <EOL> else : <EOL> modes [ char ] = items [ <NUM_LIT:0> ] <EOL> modeList = modes . items ( ) <EOL> total = float ( chunkLength * iteration ) <EOL> consistency = <NUM_LIT:1.0> <EOL> threshold = <NUM_LIT> <EOL> while len ( delims ) == <NUM_LIT:0> and consistency >= threshold : <EOL> for k , v in modeList : <EOL> if v [ <NUM_LIT:0> ] > <NUM_LIT:0> and v [ <NUM_LIT:1> ] > <NUM_LIT:0> : <EOL> if ( ( v [ <NUM_LIT:1> ] / total ) >= consistency and <EOL> ( delimiters is None or k in delimiters ) ) : <EOL> delims [ k ] = v <EOL> consistency -= <NUM_LIT> <EOL> if len ( delims ) == <NUM_LIT:1> : <EOL> delim = delims . keys ( ) [ <NUM_LIT:0> ] <EOL> skipinitialspace = ( data [ <NUM_LIT:0> ] . count ( delim ) == <EOL> data [ <NUM_LIT:0> ] . count ( "<STR_LIT>" % delim ) ) <EOL> return ( delim , skipinitialspace ) <EOL> start = end <EOL> end += chunkLength <EOL> if not delims : <EOL> return ( '<STR_LIT>' , <NUM_LIT:0> ) <EOL> if len ( delims ) > <NUM_LIT:1> : <EOL> for d in self . preferred : <EOL> if d in delims . keys ( ) : <EOL> skipinitialspace = ( data [ <NUM_LIT:0> ] . count ( d ) == <EOL> data [ <NUM_LIT:0> ] . count ( "<STR_LIT>" % d ) ) <EOL> return ( d , skipinitialspace ) <EOL> items = [ ( v , k ) for ( k , v ) in delims . items ( ) ] <EOL> items . sort ( ) <EOL> delim = items [ - <NUM_LIT:1> ] [ <NUM_LIT:1> ] <EOL> skipinitialspace = ( data [ <NUM_LIT:0> ] . count ( delim ) == <EOL> data [ <NUM_LIT:0> ] . count ( "<STR_LIT>" % delim ) ) <EOL> return ( delim , skipinitialspace ) <EOL> def has_header ( self , sample ) : <EOL> rdr = reader ( StringIO ( sample ) , self . sniff ( sample ) ) <EOL> header = rdr . next ( ) <EOL> columns = len ( header ) <EOL> columnTypes = { } <EOL> for i in range ( columns ) : columnTypes [ i ] = None <EOL> checked = <NUM_LIT:0> <EOL> for row in rdr : <EOL> if checked > <NUM_LIT:20> : <EOL> break <EOL> checked += <NUM_LIT:1> <EOL> if len ( row ) != columns : <EOL> continue <EOL> for col in columnTypes . keys ( ) : <EOL> for thisType in [ int , long , float , complex ] : <EOL> try : <EOL> thisType ( row [ col ] ) <EOL> break <EOL> except ( ValueError , OverflowError ) : <EOL> pass <EOL> else : <EOL> thisType = len ( row [ col ] ) <EOL> if thisType == long : <EOL> thisType = int <EOL> if thisType != columnTypes [ col ] : <EOL> if columnTypes [ col ] is None : <EOL> columnTypes [ col ] = thisType <EOL> else : <EOL> del columnTypes [ col ] <EOL> hasHeader = <NUM_LIT:0> <EOL> for col , colType in columnTypes . items ( ) : <EOL> if type ( colType ) == type ( <NUM_LIT:0> ) : <EOL> if len ( header [ col ] ) != colType : <EOL> hasHeader += <NUM_LIT:1> <EOL> else : <EOL> hasHeader -= <NUM_LIT:1> <EOL> else : <EOL> try : <EOL> colType ( header [ col ] ) <EOL> except ( ValueError , TypeError ) : <EOL> hasHeader += <NUM_LIT:1> <EOL> else : <EOL> hasHeader -= <NUM_LIT:1> <EOL> return hasHeader > <NUM_LIT:0> </s>
