<s> import os <EOL> import csv <EOL> import codecs <EOL> from os import listdir <EOL> from os . path import isfile , join <EOL> class Recoder ( object ) : <EOL> def __init__ ( self , stream , decoder , encoder , eol = '<STR_LIT:\r\n>' ) : <EOL> self . _stream = stream <EOL> self . _decoder = decoder if isinstance ( decoder , codecs . IncrementalDecoder ) else codecs . getincrementaldecoder ( decoder ) ( ) <EOL> self . _encoder = encoder if isinstance ( encoder , codecs . IncrementalEncoder ) else codecs . getincrementalencoder ( encoder ) ( ) <EOL> self . _buf = '<STR_LIT>' <EOL> self . _eol = eol <EOL> self . _reachedEof = False <EOL> def read ( self , size = None ) : <EOL> r = self . _stream . read ( size ) <EOL> raw = self . _decoder . decode ( r , size is None ) <EOL> return self . _encoder . encode ( raw ) <EOL> def __iter__ ( self ) : <EOL> return self <EOL> def __next__ ( self ) : <EOL> if self . _reachedEof : <EOL> raise StopIteration ( ) <EOL> while True : <EOL> line , eol , rest = self . _buf . partition ( self . _eol ) <EOL> if eol == self . _eol : <EOL> self . _buf = rest <EOL> return self . _encoder . encode ( line + eol ) <EOL> raw = self . _stream . read ( <NUM_LIT> ) <EOL> if raw == '<STR_LIT>' : <EOL> self . _decoder . decode ( b'<STR_LIT>' , True ) <EOL> self . _reachedEof = True <EOL> return self . _encoder . encode ( self . _buf ) <EOL> self . _buf += self . _decoder . decode ( raw ) <EOL> next = __next__ <EOL> def close ( self ) : <EOL> return self . _stream . close ( ) <EOL> def check_encoding ( ) : <EOL> import chardet <EOL> base_path = os . path . abspath ( "<STR_LIT>" ) <EOL> onlyfiles = [ f for f in listdir ( base_path ) if isfile ( join ( base_path , f ) ) ] <EOL> headers = { } <EOL> for filepath in onlyfiles : <EOL> refFile_path = os . path . join ( base_path , filepath ) <EOL> rawdata = open ( refFile_path , "<STR_LIT:r>" ) . read ( ) <EOL> result = chardet . detect ( rawdata ) <EOL> charenc = result [ '<STR_LIT>' ] <EOL> print charenc <EOL> def check_for_differences ( ) : <EOL> base_path = os . path . abspath ( "<STR_LIT>" ) <EOL> onlyfiles = [ f for f in listdir ( base_path ) if isfile ( join ( base_path , f ) ) ] <EOL> headers = { } <EOL> for filepath in onlyfiles : <EOL> refFile_path = os . path . join ( base_path , filepath ) <EOL> filename , _ = os . path . splitext ( filepath ) <EOL> with open ( refFile_path , '<STR_LIT:rb>' ) as f : <EOL> sr = Recoder ( f , '<STR_LIT>' , '<STR_LIT:utf-8>' ) <EOL> headerRow = csv . reader ( sr ) . next ( ) <EOL> headers [ filename ] = headerRow <EOL> columnList = [ ] <EOL> for header in headers : <EOL> columnName = headers [ header ] [ <NUM_LIT:0> ] . split ( "<STR_LIT:\t>" ) <EOL> columnName = filter ( len , columnName ) <EOL> numOfColumn = len ( columnName ) <EOL> columnList . extend ( columnName ) <EOL> uniqueColumns = set ( columnList ) <EOL> print "<STR_LIT>" , len ( uniqueColumns ) <EOL> for i in uniqueColumns : <EOL> if '<STR_LIT>' in i : <EOL> i = '<STR_LIT>' <EOL> print unicode ( i . decode ( '<STR_LIT>' ) ) <EOL> sameColumnList = [ ] <EOL> firstTime = True <EOL> for header in headers : <EOL> if firstTime : <EOL> sameColumnList = headers [ header ] [ <NUM_LIT:0> ] . split ( "<STR_LIT:\t>" ) <EOL> sameColumnList = filter ( len , sameColumnList ) <EOL> firstTime = False <EOL> continue <EOL> columnName = headers [ header ] [ <NUM_LIT:0> ] . split ( "<STR_LIT:\t>" ) <EOL> columnName = filter ( len , columnName ) <EOL> sameColumnList = set ( columnName ) & set ( sameColumnList ) <EOL> sameColumnList = set ( sameColumnList ) <EOL> print "<STR_LIT>" , len ( sameColumnList ) <EOL> for i in sameColumnList : <EOL> if '<STR_LIT>' in i : <EOL> i = '<STR_LIT>' <EOL> print unicode ( i . decode ( '<STR_LIT>' ) ) <EOL> if __name__ == "<STR_LIT:__main__>" : <EOL> check_for_differences ( ) <EOL> print "<STR_LIT>" </s>
