<s> import sys <EOL> import csv <EOL> import datetime <EOL> import math <EOL> import os <EOL> import logging <EOL> import re <EOL> import time <EOL> import pdb <EOL> moduledir = os . path . abspath ( os . path . dirname ( __file__ ) ) <EOL> sys . path . append ( moduledir ) <EOL> from collections import * <EOL> from dateutil . parser import parse as dateparse <EOL> from infertypes import * <EOL> from parsers . parsers import * <EOL> from util import get_logger , to_utf <EOL> _log = get_logger ( ) <EOL> def file_iterators ( fnames , parser = None , ** kwargs ) : <EOL> if not parser : <EOL> parser = DBTruckParser ( '<STR_LIT>' , '<STR_LIT>' ) <EOL> if isinstance ( fnames , basestring ) : <EOL> fnames = [ fnames ] <EOL> print fnames <EOL> iterfs = parser . get_reader ( fnames ) <EOL> iterfs = [ iterf for iterf in iterfs ] <EOL> for iterf in iterfs : <EOL> iterf . infer_metadata ( ) <EOL> return iterfs <EOL> def import_datafiles ( fnames , new , tablename , errfile , exportmethodsklass , parser = None , ** kwargs ) : <EOL> """<STR_LIT>""" <EOL> _log . info ( "<STR_LIT>" , kwargs ) <EOL> iterfs = file_iterators ( fnames , parser = parser , ** kwargs ) <EOL> new_errfile = False <EOL> if not errfile : <EOL> errfile = file ( '<STR_LIT>' , '<STR_LIT:a>' ) <EOL> new_errfile = True <EOL> try : <EOL> for idx , iterf in enumerate ( iterfs ) : <EOL> try : <EOL> exportmethods = exportmethodsklass ( tablename , errfile , ** kwargs ) <EOL> newtable = new and idx == <NUM_LIT:0> <EOL> exportmethods . setup_table ( iterf . types , iterf . header , iterf . add_id_col , newtable ) <EOL> import_iterator ( iterf , exportmethods ) <EOL> idx += <NUM_LIT:1> <EOL> except Exception as e : <EOL> _log . warn ( traceback . format_exc ( ) ) <EOL> if new_errfile : <EOL> errfile . close ( ) <EOL> except : <EOL> _log . warn ( traceback . format_exc ( ) ) <EOL> def transform_and_validate ( types , row ) : <EOL> row = map ( str2sqlval , zip ( types , row ) ) <EOL> return row <EOL> def import_iterator ( iterf , dbmethods ) : <EOL> """<STR_LIT:U+0020>""" <EOL> rowiter = iterf ( ) <EOL> types = iterf . types <EOL> blocksize = <NUM_LIT> <EOL> buf = [ ] <EOL> if iterf . header_inferred : <EOL> rowiter . next ( ) <EOL> rowidx = dbmethods . get_max_id ( ) or <NUM_LIT:0> <EOL> for row in rowiter : <EOL> rowidx += <NUM_LIT:1> <EOL> row = transform_and_validate ( types , row ) <EOL> if iterf . add_id_col : <EOL> row . append ( str ( rowidx ) ) <EOL> if row is not None and len ( row ) == len ( iterf . types ) : <EOL> buf . append ( row ) <EOL> elif row is not None and len ( row ) != len ( iterf . types ) : <EOL> print >> dbmethods . errfile , '<STR_LIT:U+002C>' . join ( map ( to_utf , row ) ) <EOL> if len ( buf ) > <NUM_LIT:0> and len ( buf ) % blocksize == <NUM_LIT:0> : <EOL> try : <EOL> _log . info ( "<STR_LIT>" , len ( buf ) ) <EOL> success = dbmethods . import_block ( buf , iterf ) <EOL> except Exception as e : <EOL> _log . warn ( traceback . format_exc ( ) ) <EOL> buf = [ ] <EOL> if len ( buf ) > <NUM_LIT:0> : <EOL> try : <EOL> success = dbmethods . import_block ( buf , iterf ) <EOL> _log . info ( "<STR_LIT>" , success , rowidx ) <EOL> except Exception as e : <EOL> _log . warn ( traceback . format_exc ( ) ) </s>
