<s> import logging <EOL> from math import floor <EOL> import time <EOL> from django . conf import settings <EOL> from django . utils . translation import ugettext <EOL> import xlrd <EOL> from livesettings import config_value <EOL> from panda import solr , utils <EOL> from panda . tasks . import_file import ImportFileTask <EOL> from panda . utils . typecoercion import DataTyper <EOL> SOLR_ADD_BUFFER_SIZE = <NUM_LIT> <EOL> class ImportXLSTask ( ImportFileTask ) : <EOL> """<STR_LIT>""" <EOL> name = '<STR_LIT>' <EOL> def run ( self , dataset_slug , upload_id , external_id_field_index = None , * args , ** kwargs ) : <EOL> """<STR_LIT>""" <EOL> from panda . models import Dataset , DataUpload <EOL> log = logging . getLogger ( self . name ) <EOL> log . info ( '<STR_LIT>' % dataset_slug ) <EOL> try : <EOL> dataset = Dataset . objects . get ( slug = dataset_slug ) <EOL> except Dataset . DoesNotExist : <EOL> log . warning ( '<STR_LIT>' % dataset_slug ) <EOL> return <EOL> upload = DataUpload . objects . get ( id = upload_id ) <EOL> task_status = dataset . current_task <EOL> task_status . begin ( ugettext ( '<STR_LIT>' ) ) <EOL> book = xlrd . open_workbook ( upload . get_path ( ) , on_demand = True ) <EOL> sheet = book . sheet_by_index ( <NUM_LIT:0> ) <EOL> row_count = sheet . nrows <EOL> add_buffer = [ ] <EOL> data_typer = DataTyper ( dataset . column_schema ) <EOL> throttle = config_value ( '<STR_LIT>' , '<STR_LIT>' ) <EOL> for i in range ( <NUM_LIT:1> , row_count ) : <EOL> values = sheet . row_values ( i ) <EOL> types = sheet . row_types ( i ) <EOL> normal_values = [ ] <EOL> for v , t in zip ( values , types ) : <EOL> if t == xlrd . biffh . XL_CELL_DATE : <EOL> v = utils . xls . normalize_date ( v , book . datemode ) <EOL> elif t == xlrd . biffh . XL_CELL_NUMBER : <EOL> if v % <NUM_LIT:1> == <NUM_LIT:0> : <EOL> v = int ( v ) <EOL> normal_values . append ( unicode ( v ) ) <EOL> external_id = None <EOL> if external_id_field_index is not None : <EOL> external_id = values [ external_id_field_index ] <EOL> data = utils . solr . make_data_row ( dataset , normal_values , data_upload = upload , external_id = external_id ) <EOL> data = data_typer ( data , normal_values ) <EOL> add_buffer . append ( data ) <EOL> if i % SOLR_ADD_BUFFER_SIZE == <NUM_LIT:0> : <EOL> solr . add ( settings . SOLR_DATA_CORE , add_buffer ) <EOL> add_buffer = [ ] <EOL> task_status . update ( ugettext ( '<STR_LIT>' ) % floor ( float ( i ) / float ( row_count ) * <NUM_LIT:100> ) ) <EOL> if self . is_aborted ( ) : <EOL> task_status . abort ( ugettext ( '<STR_LIT>' ) % floor ( float ( i ) / float ( row_count ) * <NUM_LIT:100> ) ) <EOL> log . warning ( '<STR_LIT>' % dataset_slug ) <EOL> return <EOL> time . sleep ( throttle ) <EOL> if add_buffer : <EOL> solr . add ( settings . SOLR_DATA_CORE , add_buffer ) <EOL> add_buffer = [ ] <EOL> solr . commit ( settings . SOLR_DATA_CORE ) <EOL> task_status . update ( ugettext ( '<STR_LIT>' ) ) <EOL> try : <EOL> dataset = Dataset . objects . get ( slug = dataset_slug ) <EOL> except Dataset . DoesNotExist : <EOL> log . warning ( '<STR_LIT>' % dataset_slug ) <EOL> return <EOL> if not dataset . row_count : <EOL> dataset . row_count = i <EOL> else : <EOL> dataset . row_count += i <EOL> dataset . column_schema = data_typer . schema <EOL> dataset . save ( ) <EOL> upload = DataUpload . objects . get ( id = upload_id ) <EOL> upload . imported = True <EOL> upload . save ( ) <EOL> log . info ( '<STR_LIT>' % dataset_slug ) <EOL> return data_typer </s>
