<s> import datetime <EOL> import itertools <EOL> import logging <EOL> import os <EOL> import sys <EOL> from django . conf import settings <EOL> from worldcat . request . search import SRURequest <EOL> from worldcat . util . extract import extract_elements <EOL> _logger = logging . getLogger ( __name__ ) <EOL> WSKEY = settings . WORLDCAT_KEY <EOL> sortkeys = '<STR_LIT>' <EOL> recordschema = '<STR_LIT>' <EOL> recordpacking = '<STR_LIT>' <EOL> servicelevel = '<STR_LIT>' <EOL> frbrgrouping = '<STR_LIT>' <EOL> MAX_RECORDS = <NUM_LIT:50> <EOL> raw_query = '<STR_LIT>' <EOL> COUNTRIES = ( <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> ) <EOL> OPERATOR_MAP = { <EOL> '<STR_LIT:=>' : '<STR_LIT:e>' , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT:>>' : '<STR_LIT>' , <EOL> '<STR_LIT:<>' : '<STR_LIT>' , <EOL> } <EOL> def str_value ( value ) : <EOL> '''<STR_LIT>''' <EOL> str_value = str ( value ) <EOL> while len ( str_value ) < <NUM_LIT:4> : <EOL> str_value = '<STR_LIT:0>' + str_value <EOL> return str_value <EOL> class TitlePuller ( object ) : <EOL> """<STR_LIT>""" <EOL> year_breaks = [ ] <EOL> def generate_year_dict ( self , start = None , end = None ) : <EOL> '''<STR_LIT>''' <EOL> year_and_action = { } <EOL> if not start or not end : <EOL> special_yr_cases = [ '<STR_LIT>' , '<STR_LIT>' ] <EOL> else : <EOL> special_yr_cases = [ ] <EOL> if start : <EOL> year_and_action [ str ( start ) ] = '<STR_LIT:=>' <EOL> else : <EOL> start = <NUM_LIT> <EOL> year_and_action [ str ( start ) ] = '<STR_LIT>' <EOL> if end : <EOL> year_and_action [ str ( end ) ] = '<STR_LIT:=>' <EOL> else : <EOL> now = datetime . datetime . now ( ) <EOL> end = now . year <EOL> year_and_action [ str ( end ) ] = '<STR_LIT>' <EOL> for year in ( range ( start + <NUM_LIT:1> , end ) + special_yr_cases ) : <EOL> year_and_action [ str ( year ) ] = '<STR_LIT:=>' <EOL> return year_and_action <EOL> def add_to_query ( self , oclc_field , value , relationship , query = None ) : <EOL> '''<STR_LIT>''' <EOL> new_query = '<STR_LIT>' % ( oclc_field , relationship , value ) <EOL> if query : <EOL> new_query = '<STR_LIT>' % ( query , new_query ) <EOL> return new_query <EOL> def generate_srurequest ( self , query ) : <EOL> '''<STR_LIT>''' <EOL> bib_rec = SRURequest ( wskey = WSKEY , <EOL> sortKeys = sortkeys , <EOL> recordSchema = recordschema , <EOL> recordPacking = recordpacking , <EOL> servicelevel = servicelevel , <EOL> query = query , <EOL> startRecord = <NUM_LIT:1> , <EOL> maximumRecords = MAX_RECORDS , <EOL> frbrGrouping = frbrgrouping ) <EOL> return bib_rec <EOL> def generate_requests ( self , lccn = None , oclc = None , raw_query = raw_query , countries = COUNTRIES , <EOL> start = None , end = None , totals_only = False ) : <EOL> '''<STR_LIT>''' <EOL> bibs_to_req = [ ] <EOL> if lccn or oclc : <EOL> if lccn : <EOL> query = self . add_to_query ( '<STR_LIT>' , lccn , '<STR_LIT>' ) <EOL> elif oclc : <EOL> query = self . add_to_query ( '<STR_LIT>' , oclc , '<STR_LIT>' ) <EOL> request = self . generate_srurequest ( query ) <EOL> lccn_count = self . initial_total_count ( request ) <EOL> bibs_to_req . append ( ( request , lccn_count [ <NUM_LIT:0> ] , ( lccn , ) ) ) <EOL> else : <EOL> grand_total = <NUM_LIT:0> <EOL> for country in countries : <EOL> total = <NUM_LIT:0> <EOL> query = self . add_to_query ( '<STR_LIT>' , country , '<STR_LIT>' , raw_query ) <EOL> if start or end : <EOL> request_able = None <EOL> else : <EOL> cntry_req = self . generate_srurequest ( query ) <EOL> cntry_count = self . initial_total_count ( cntry_req ) <EOL> request_able = self . check_for_doable_bulk_request ( cntry_count ) <EOL> _logger . info ( "<STR_LIT>" % ( country . title ( ) , <EOL> cntry_count ) ) <EOL> if request_able == <NUM_LIT:0> : <EOL> _logger . warning ( '<STR_LIT>' % ( country ) ) <EOL> elif request_able : <EOL> bibs_to_req . append ( ( cntry_req , request_able , ( <EOL> country . strip ( '<STR_LIT:*>' ) , '<STR_LIT>' , '<STR_LIT:=>' <EOL> ) ) ) <EOL> total += request_able <EOL> else : <EOL> year_dict = self . generate_year_dict ( start = start , end = end ) <EOL> base_query = query <EOL> for year in sorted ( year_dict . iterkeys ( ) ) : <EOL> operator = year_dict [ year ] <EOL> query = self . add_to_query ( '<STR_LIT>' , year , operator , base_query ) <EOL> yr_request = self . generate_srurequest ( query ) <EOL> yr_count = self . initial_total_count ( yr_request ) <EOL> yr_request_able = self . check_for_doable_bulk_request ( yr_count ) <EOL> _logger . info ( "<STR_LIT>" % ( <EOL> country . title ( ) , year , operator , yr_request_able ) ) <EOL> if yr_request_able or year == str ( datetime . datetime . now ( ) . year ) : <EOL> bibs_to_req . append ( ( yr_request , yr_request_able , ( <EOL> country . strip ( '<STR_LIT:*>' ) , year , operator <EOL> ) ) ) <EOL> else : <EOL> _logger . warning ( "<STR_LIT>" ) <EOL> _logger . warning ( '<STR_LIT>' % yr_request ) <EOL> _logger . warning ( '<STR_LIT>' % yr_request_able ) <EOL> _logger . warning ( '<STR_LIT>' % country . strip ( '<STR_LIT:*>' ) ) <EOL> _logger . warning ( '<STR_LIT>' % year ) <EOL> _logger . warning ( '<STR_LIT>' % operator ) <EOL> continue <EOL> total += yr_request_able <EOL> grand_total += total <EOL> _logger . info ( '<STR_LIT>' % ( grand_total ) ) <EOL> return bibs_to_req <EOL> def grab_content ( self , save_path , bib_requests , search_name = '<STR_LIT>' ) : <EOL> '''<STR_LIT>''' <EOL> if not bib_requests : <EOL> return <EOL> files_saved = <NUM_LIT:0> <EOL> for bib_rec in bib_requests : <EOL> grab_records = True <EOL> counter = <NUM_LIT:0> <EOL> bib_request , total , components = bib_rec <EOL> previous_next = None <EOL> while grab_records : <EOL> bib_resp = next_record = next = end = start = None <EOL> counter += <NUM_LIT:1> <EOL> bib_resp = bib_request . get_response ( ) <EOL> next_record = extract_elements ( bib_request . response , <EOL> element = '<STR_LIT>' ) <EOL> try : <EOL> next = next_record [ <NUM_LIT:0> ] . text <EOL> except IndexError : <EOL> grab_records = False <EOL> try : <EOL> end = int ( next ) - <NUM_LIT:1> <EOL> except TypeError : <EOL> end = total <EOL> try : <EOL> start = int ( next ) - MAX_RECORDS <EOL> except TypeError : <EOL> if counter == <NUM_LIT:1> : <EOL> start = <NUM_LIT:1> <EOL> else : <EOL> start = previous_next <EOL> if start is None : <EOL> grab_records = False <EOL> name_components = [ ] <EOL> for i in components : <EOL> i = i . replace ( '<STR_LIT:U+0020>' , '<STR_LIT:->' ) <EOL> if i in OPERATOR_MAP : <EOL> i = OPERATOR_MAP [ i ] <EOL> name_components . append ( i ) <EOL> batch_name = '<STR_LIT:_>' . join ( name_components ) <EOL> filename = '<STR_LIT:_>' . join ( ( search_name , batch_name , str_value ( start ) , str_value ( end ) ) ) + '<STR_LIT>' <EOL> if counter == <NUM_LIT:1> and len ( bib_requests ) > <NUM_LIT:1> : <EOL> _logger . info ( '<STR_LIT>' % ( filename , total ) ) <EOL> file_location = save_path + '<STR_LIT:/>' + filename <EOL> save_file = open ( file_location , "<STR_LIT:w>" ) <EOL> decoded_data = bib_resp . data . decode ( "<STR_LIT:utf-8>" ) <EOL> save_file . write ( decoded_data . encode ( '<STR_LIT:ascii>' , '<STR_LIT>' ) ) <EOL> save_file . close ( ) <EOL> files_saved += <NUM_LIT:1> <EOL> try : <EOL> previous_next = next <EOL> bib_request . next ( ) <EOL> except StopIteration : <EOL> grab_records = False <EOL> return files_saved <EOL> def initial_total_count ( self , bib_req ) : <EOL> '''<STR_LIT>''' <EOL> totals = [ ] <EOL> for grab in itertools . repeat ( None , <NUM_LIT:3> ) : <EOL> bib_req . get_response ( ) <EOL> _total = extract_elements ( bib_req . response , <EOL> element = '<STR_LIT>' ) <EOL> totals . append ( _total [ <NUM_LIT:0> ] . text ) <EOL> return totals <EOL> def check_for_doable_bulk_request ( self , test_totals ) : <EOL> '''<STR_LIT>''' <EOL> if all ( map ( lambda x : x == test_totals [ <NUM_LIT:0> ] , test_totals ) ) : <EOL> if int ( test_totals [ <NUM_LIT:0> ] ) < <NUM_LIT> : <EOL> return int ( test_totals [ <NUM_LIT:0> ] ) <EOL> return None <EOL> def run ( self , save_path , lccn = None , oclc = None , <EOL> start = None , end = None , countries = COUNTRIES ) : <EOL> '''<STR_LIT>''' <EOL> if not os . path . exists ( save_path ) : <EOL> try : <EOL> os . makedirs ( save_path ) <EOL> except OSError : <EOL> _logger . exception ( '<STR_LIT>' % save_path ) <EOL> sys . exit ( ) <EOL> else : <EOL> if len ( [ f for f in os . listdir ( save_path ) if os . path . isfile ( f ) ] ) : <EOL> _logger . exception ( '<STR_LIT>' % save_path ) <EOL> sys . exit ( ) <EOL> files_saved = <NUM_LIT:0> <EOL> bib_requests = self . generate_requests ( lccn = lccn , oclc = oclc , <EOL> start = start , end = end , countries = countries ) <EOL> files_saved = self . grab_content ( save_path , bib_requests ) <EOL> return files_saved </s>
