<s> __author__ = '<STR_LIT>' <EOL> import re <EOL> from snakebite . client import Client <EOL> import pyhs2 <EOL> class Loader : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , path , name_node , hive_server , <EOL> user = "<STR_LIT:root>" , hive_db = "<STR_LIT:default>" , password = None , nn_port = <NUM_LIT> , hive_port = <NUM_LIT> ) : <EOL> self . _client = Client ( name_node , nn_port ) <EOL> self . _db = hive_db <EOL> self . _hive = pyhs2 . connect ( host = hive_server , <EOL> port = hive_port , <EOL> authMechanism = "<STR_LIT>" , <EOL> database = hive_db , <EOL> user = user , <EOL> password = password ) <EOL> self . _path = path <EOL> def load ( self ) : <EOL> files = self . _client . ls ( [ self . _path ] ) <EOL> files = [ f for f in files if f [ '<STR_LIT>' ] == '<STR_LIT:f>' ] <EOL> if len ( files ) == <NUM_LIT:0> : <EOL> raise Exception ( "<STR_LIT>" ) <EOL> data = self . head ( files [ <NUM_LIT:0> ] [ '<STR_LIT:path>' ] ) <EOL> res = self . check_separator ( data ) <EOL> if res == None : <EOL> print ( "<STR_LIT>" ) <EOL> return <EOL> sep = res [ <NUM_LIT:0> ] <EOL> num_cols = res [ <NUM_LIT:1> ] <EOL> table_statement , table_name = self . _create_table ( self . _path , sep , num_cols ) <EOL> cursor = self . _hive . cursor ( ) <EOL> cursor . execute ( table_statement ) <EOL> return self . _db , table_name <EOL> def _create_table ( self , path , sep , count ) : <EOL> buf = """<STR_LIT>""" % ( "<STR_LIT:U+002C>" . join ( [ "<STR_LIT>" % x for x in range ( count ) ] ) , sep , path ) <EOL> return buf , "<STR_LIT>" <EOL> def check_separator ( self , data ) : <EOL> """<STR_LIT>""" <EOL> sep_list = [ r'<STR_LIT:\t>' , r'<STR_LIT:;>' , r'<STR_LIT:U+002C>' , r'<STR_LIT:|>' , r'<STR_LIT>' ] <EOL> data_copy = data <EOL> for sep in sep_list : <EOL> splitted = data_copy . split ( "<STR_LIT:\n>" ) <EOL> parts = [ len ( re . split ( sep , line ) ) for line in splitted ] <EOL> if sum ( parts ) == len ( splitted ) : <EOL> continue <EOL> diff = <NUM_LIT:0> <EOL> for i in range ( len ( parts [ <NUM_LIT:1> : - <NUM_LIT:1> ] ) ) : <EOL> diff += abs ( parts [ i ] - parts [ i + <NUM_LIT:1> ] ) <EOL> if diff == <NUM_LIT:0> : <EOL> return sep , parts [ <NUM_LIT:0> ] <EOL> return None <EOL> def head ( self , file_path ) : <EOL> """<STR_LIT>""" <EOL> processor = lambda path , node , tail_only = True , append = False : self . _handle_head ( <EOL> path , node ) <EOL> for item in self . _client . _find_items ( [ file_path ] , processor , <EOL> include_toplevel = True , <EOL> include_children = False , recurse = False ) : <EOL> if item : <EOL> return item <EOL> def _handle_head ( self , path , node , upper = <NUM_LIT> * <NUM_LIT> ) : <EOL> data = '<STR_LIT>' <EOL> for load in self . _client . _read_file ( path , node , tail_only = False , <EOL> check_crc = False ) : <EOL> data += load <EOL> if ( len ( data ) > upper ) : <EOL> return data <EOL> return data </s>
