<s> """<STR_LIT>""" <EOL> import os <EOL> import sys <EOL> import zipfile <EOL> import xlrd <EOL> from retriever . lib . templates import Script <EOL> from retriever . lib . models import Table <EOL> from retriever . lib . excel import Excel <EOL> VERSION = '<STR_LIT>' <EOL> TAX_GROUPS = <NUM_LIT> <EOL> class main ( Script ) : <EOL> def __init__ ( self , ** kwargs ) : <EOL> Script . __init__ ( self , ** kwargs ) <EOL> self . name = "<STR_LIT>" <EOL> self . shortname = "<STR_LIT>" <EOL> self . urls = { "<STR_LIT>" : "<STR_LIT>" , <EOL> "<STR_LIT>" : "<STR_LIT>" , <EOL> "<STR_LIT>" : "<STR_LIT>" , <EOL> "<STR_LIT>" : "<STR_LIT>" } <EOL> self . tags = [ "<STR_LIT>" , "<STR_LIT>" , <EOL> "<STR_LIT>" ] <EOL> self . ref = "<STR_LIT>" <EOL> self . addendum = """<STR_LIT>""" <EOL> def download ( self , engine = None , debug = False ) : <EOL> Script . download ( self , engine , debug ) <EOL> self . engine . auto_create_table ( Table ( "<STR_LIT>" ) , url = self . urls [ "<STR_LIT>" ] ) <EOL> self . engine . insert_data_from_url ( self . urls [ "<STR_LIT>" ] ) <EOL> self . engine . download_file ( self . urls [ "<STR_LIT>" ] , "<STR_LIT>" ) <EOL> local_zip = zipfile . ZipFile ( self . engine . format_filename ( "<STR_LIT>" ) ) <EOL> filelist = local_zip . namelist ( ) <EOL> local_zip . close ( ) <EOL> self . engine . download_files_from_archive ( self . urls [ "<STR_LIT>" ] , filelist ) <EOL> filelist = [ os . path . basename ( filename ) for filename in filelist ] <EOL> if not self . engine . find_file ( '<STR_LIT>' ) : <EOL> self . engine . download_file ( "<STR_LIT>" , "<STR_LIT>" , clean_line_endings = False ) <EOL> filelist . append ( '<STR_LIT>' ) <EOL> lines = [ ] <EOL> tax = [ ] <EOL> for filename in filelist : <EOL> print "<STR_LIT>" + filename + "<STR_LIT>" <EOL> book = xlrd . open_workbook ( self . engine . format_filename ( filename ) ) <EOL> sh = book . sheet_by_index ( <NUM_LIT:0> ) <EOL> rows = sh . nrows <EOL> cn = { '<STR_LIT>' : [ ] } <EOL> n = <NUM_LIT:0> <EOL> for colnum , c in enumerate ( sh . row ( <NUM_LIT:0> ) ) : <EOL> if not Excel . empty_cell ( c ) : <EOL> cid = Excel . cell_value ( c ) . lower ( ) <EOL> if cid in [ "<STR_LIT>" , "<STR_LIT>" ] : <EOL> cid = "<STR_LIT>" <EOL> if "<STR_LIT>" in cid : <EOL> cid = "<STR_LIT:count>" <EOL> if filename == "<STR_LIT>" and colnum == <NUM_LIT> : <EOL> cid = "<STR_LIT:count>" <EOL> if "<STR_LIT>" in cid or "<STR_LIT>" in cid : <EOL> cn [ "<STR_LIT>" ] . append ( n ) <EOL> else : <EOL> cn [ cid ] = n <EOL> n += <NUM_LIT:1> <EOL> if not "<STR_LIT>" in cn . keys ( ) : <EOL> cn [ "<STR_LIT>" ] = - <NUM_LIT:1> <EOL> if not "<STR_LIT:count>" in cn . keys ( ) : <EOL> cn [ "<STR_LIT:count>" ] = - <NUM_LIT:1> <EOL> for i in range ( <NUM_LIT:1> , rows ) : <EOL> row = sh . row ( i ) <EOL> cellcount = len ( row ) <EOL> if not all ( Excel . empty_cell ( cell ) for cell in row ) : <EOL> try : <EOL> this_line = { } <EOL> def format_value ( s ) : <EOL> s = Excel . cell_value ( s ) <EOL> return str ( s ) . title ( ) . replace ( "<STR_LIT:\\>" , "<STR_LIT:/>" ) . replace ( '<STR_LIT:">' , '<STR_LIT>' ) <EOL> for i in [ "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , <EOL> "<STR_LIT>" , "<STR_LIT:count>" ] : <EOL> if cn [ i ] > - <NUM_LIT:1> : <EOL> this_line [ i ] = format_value ( row [ cn [ i ] ] ) <EOL> if this_line [ i ] == '<STR_LIT>' : <EOL> this_line [ i ] = <NUM_LIT:1> <EOL> this_line [ "<STR_LIT>" ] = [ Excel . cell_value ( row [ c ] ) <EOL> for c in cn [ "<STR_LIT>" ] <EOL> if not Excel . empty_cell ( row [ c ] ) ] <EOL> this_line [ "<STR_LIT>" ] = filename [ <NUM_LIT:0> : - <NUM_LIT:4> ] <EOL> lines . append ( this_line ) <EOL> full_id = <NUM_LIT:0> <EOL> if len ( this_line [ "<STR_LIT>" ] ) < <NUM_LIT:3> : <EOL> if len ( this_line [ "<STR_LIT>" ] ) < <NUM_LIT:3> : <EOL> id_level = "<STR_LIT>" <EOL> else : <EOL> id_level = "<STR_LIT>" <EOL> else : <EOL> id_level = "<STR_LIT>" <EOL> full_id = <NUM_LIT:1> <EOL> tax . append ( ( this_line [ "<STR_LIT>" ] , <EOL> this_line [ "<STR_LIT>" ] , <EOL> this_line [ "<STR_LIT>" ] . lower ( ) . replace ( '<STR_LIT:\\>' , '<STR_LIT>' ) . replace ( '<STR_LIT:">' , '<STR_LIT>' ) , <EOL> id_level , <EOL> str ( full_id ) ) ) <EOL> except : <EOL> raise <EOL> pass <EOL> tax = sorted ( tax , key = lambda group : group [ <NUM_LIT:0> ] + "<STR_LIT:U+0020>" + group [ <NUM_LIT:1> ] + "<STR_LIT:U+0020>" + group [ <NUM_LIT:2> ] ) <EOL> unique_tax = [ ] <EOL> tax_dict = dict ( ) <EOL> tax_count = <NUM_LIT:0> <EOL> for group in tax : <EOL> if not ( group in unique_tax ) : <EOL> unique_tax . append ( group ) <EOL> tax_count += <NUM_LIT:1> <EOL> tax_dict [ group [ <NUM_LIT:0> : <NUM_LIT:3> ] ] = tax_count <EOL> if tax_count % <NUM_LIT:10> == <NUM_LIT:0> : <EOL> msg = "<STR_LIT>" + str ( tax_count ) + "<STR_LIT>" + str ( TAX_GROUPS ) <EOL> sys . stdout . write ( msg + "<STR_LIT>" * len ( msg ) ) <EOL> print "<STR_LIT>" + str ( TAX_GROUPS ) + "<STR_LIT>" + str ( TAX_GROUPS ) <EOL> table = Table ( "<STR_LIT>" , delimiter = "<STR_LIT:U+002C>" ) <EOL> table . columns = [ ( "<STR_LIT>" , ( "<STR_LIT>" , ) ) , <EOL> ( "<STR_LIT>" , ( "<STR_LIT>" , ) ) , <EOL> ( "<STR_LIT>" , ( "<STR_LIT>" , ) ) , <EOL> ( "<STR_LIT>" , ( "<STR_LIT>" , ) ) , <EOL> ( "<STR_LIT>" , ( "<STR_LIT>" , <NUM_LIT:10> ) ) , <EOL> ( "<STR_LIT>" , ( "<STR_LIT:bool>" , ) ) ] <EOL> data = [ '<STR_LIT:U+002C>' . join ( [ str ( tax_dict [ group [ : <NUM_LIT:3> ] ] ) ] + [ '<STR_LIT>' % g for g in group ] ) <EOL> for group in unique_tax ] <EOL> table . pk = '<STR_LIT>' <EOL> table . contains_pk = True <EOL> self . engine . table = table <EOL> self . engine . create_table ( ) <EOL> self . engine . add_to_table ( data ) <EOL> table = Table ( "<STR_LIT>" , delimiter = "<STR_LIT:U+002C>" , contains_pk = False ) <EOL> table . columns = [ ( "<STR_LIT>" , ( "<STR_LIT>" , ) ) , <EOL> ( "<STR_LIT>" , ( "<STR_LIT:int>" , ) ) , <EOL> ( "<STR_LIT>" , ( "<STR_LIT:int>" , ) ) , <EOL> ( "<STR_LIT>" , ( "<STR_LIT>" , <NUM_LIT:12> ) ) , <EOL> ( "<STR_LIT>" , ( "<STR_LIT>" , <NUM_LIT:10> ) ) , <EOL> ( "<STR_LIT>" , ( "<STR_LIT>" , ) ) ] <EOL> stems = [ ] <EOL> counts = [ ] <EOL> for line in lines : <EOL> try : <EOL> liana = line [ "<STR_LIT>" ] <EOL> except KeyError : <EOL> liana = "<STR_LIT>" <EOL> species_info = [ line [ "<STR_LIT>" ] , <EOL> tax_dict [ ( line [ "<STR_LIT>" ] , <EOL> line [ "<STR_LIT>" ] , <EOL> line [ "<STR_LIT>" ] . lower ( ) ) ] , <EOL> line [ "<STR_LIT>" ] , <EOL> liana <EOL> ] <EOL> try : <EOL> counts . append ( [ str ( value ) for value in species_info + [ line [ "<STR_LIT:count>" ] ] ] ) <EOL> except KeyError : <EOL> pass <EOL> for i in line [ "<STR_LIT>" ] : <EOL> stem = species_info + [ i ] <EOL> stems . append ( [ str ( value ) for value in stem ] ) <EOL> data = [ '<STR_LIT:U+002C>' . join ( stem ) for stem in stems ] <EOL> self . engine . table = table <EOL> self . engine . create_table ( ) <EOL> self . engine . add_to_table ( data ) <EOL> table = Table ( "<STR_LIT>" , delimiter = "<STR_LIT:U+002C>" , contains_pk = False ) <EOL> table . columns = [ ( "<STR_LIT>" , ( "<STR_LIT>" , ) ) , <EOL> ( "<STR_LIT>" , ( "<STR_LIT:int>" , ) ) , <EOL> ( "<STR_LIT>" , ( "<STR_LIT:int>" , ) ) , <EOL> ( "<STR_LIT>" , ( "<STR_LIT>" , <NUM_LIT:12> ) ) , <EOL> ( "<STR_LIT>" , ( "<STR_LIT>" , <NUM_LIT:10> ) ) , <EOL> ( "<STR_LIT:count>" , ( "<STR_LIT>" , ) ) ] <EOL> data = [ '<STR_LIT:U+002C>' . join ( count ) for count in counts ] <EOL> self . engine . table = table <EOL> self . engine . create_table ( ) <EOL> self . engine . add_to_table ( data ) <EOL> return self . engine <EOL> SCRIPT = main ( ) </s>
