<s> import re <EOL> import urlparse <EOL> from time import sleep <EOL> from calaccess_campaign_browser . management . commands import ScrapeCommand <EOL> from calaccess_campaign_browser . models import ( <EOL> Election , <EOL> Office , <EOL> Candidate , <EOL> Filer <EOL> ) <EOL> class Command ( ScrapeCommand ) : <EOL> """<STR_LIT>""" <EOL> help = "<STR_LIT>" <EOL> def build_results ( self ) : <EOL> self . header ( "<STR_LIT>" ) <EOL> url = urlparse . urljoin ( <EOL> self . base_url , <EOL> '<STR_LIT>' <EOL> ) <EOL> soup = self . get ( url ) <EOL> links = soup . findAll ( '<STR_LIT:a>' , href = re . compile ( r'<STR_LIT>' ) ) <EOL> links = [ <EOL> l for l in links <EOL> if l . find_next_sibling ( '<STR_LIT>' ) . text != '<STR_LIT>' <EOL> ] <EOL> results = [ ] <EOL> for i , link in enumerate ( links ) : <EOL> url = urlparse . urljoin ( self . base_url , link [ "<STR_LIT>" ] ) <EOL> data = self . scrape_page ( url ) <EOL> data [ '<STR_LIT>' ] = link . find_next_sibling ( '<STR_LIT>' ) . text . strip ( ) <EOL> data [ '<STR_LIT>' ] = self . parse_election_name ( data [ '<STR_LIT>' ] ) <EOL> data [ '<STR_LIT>' ] = int ( data [ '<STR_LIT>' ] [ : <NUM_LIT:4> ] ) <EOL> data [ '<STR_LIT>' ] = len ( links ) - i <EOL> results . append ( data ) <EOL> sleep ( <NUM_LIT:0.5> ) <EOL> return results <EOL> def scrape_page ( self , url ) : <EOL> """<STR_LIT>""" <EOL> soup = self . get ( url ) <EOL> sections = { } <EOL> for section in soup . findAll ( '<STR_LIT:a>' , { '<STR_LIT:name>' : re . compile ( r'<STR_LIT>' ) } ) : <EOL> section_name_el = section . find ( '<STR_LIT>' , { '<STR_LIT:class>' : '<STR_LIT>' } ) <EOL> if not section_name_el : <EOL> continue <EOL> section_name = section_name_el . text <EOL> sections [ section_name ] = { } <EOL> for office in section . findAll ( '<STR_LIT>' ) : <EOL> title_el = office . find ( '<STR_LIT>' , { '<STR_LIT:class>' : '<STR_LIT>' } ) <EOL> if not title_el : <EOL> continue <EOL> if self . verbosity > <NUM_LIT:2> : <EOL> self . log ( '<STR_LIT>' % title_el . text ) <EOL> people = [ ] <EOL> for p in office . findAll ( '<STR_LIT:a>' , { '<STR_LIT:class>' : '<STR_LIT>' } ) : <EOL> people . append ( { <EOL> '<STR_LIT:name>' : p . text , <EOL> '<STR_LIT:id>' : re . match ( r'<STR_LIT>' , p [ '<STR_LIT>' ] ) . group ( <NUM_LIT:1> ) <EOL> } ) <EOL> for p in office . findAll ( '<STR_LIT>' , { '<STR_LIT:class>' : '<STR_LIT>' } ) : <EOL> people . append ( { <EOL> '<STR_LIT:name>' : p . text , <EOL> '<STR_LIT:id>' : None <EOL> } ) <EOL> sections [ section_name ] [ title_el . text ] = people <EOL> return { <EOL> '<STR_LIT:id>' : int ( re . match ( r'<STR_LIT>' , url ) . group ( <NUM_LIT:1> ) ) , <EOL> '<STR_LIT:data>' : sections , <EOL> } <EOL> def process_results ( self , results ) : <EOL> self . log ( '<STR_LIT>' ) <EOL> self . log ( '<STR_LIT>' % len ( results ) ) <EOL> for d in results : <EOL> self . log ( '<STR_LIT>' % d [ '<STR_LIT>' ] ) <EOL> election , c = Election . objects . get_or_create ( <EOL> year = d [ '<STR_LIT>' ] , <EOL> election_type = d [ '<STR_LIT>' ] , <EOL> id_raw = d [ '<STR_LIT:id>' ] , <EOL> sort_index = d [ '<STR_LIT>' ] <EOL> ) <EOL> if self . verbosity > <NUM_LIT:2> : <EOL> if c : <EOL> self . log ( '<STR_LIT>' % election ) <EOL> for office_dict in d [ '<STR_LIT:data>' ] . values ( ) : <EOL> for office_name , candidates in office_dict . items ( ) : <EOL> office , c = Office . objects . get_or_create ( <EOL> ** self . parse_office_name ( office_name ) <EOL> ) <EOL> if self . verbosity > <NUM_LIT:2> : <EOL> if c : <EOL> self . log ( '<STR_LIT>' % office ) <EOL> for candidate in candidates : <EOL> if not candidate [ '<STR_LIT:id>' ] : <EOL> continue <EOL> try : <EOL> filer = Filer . objects . get ( <EOL> filer_id_raw = int ( candidate [ '<STR_LIT:id>' ] ) <EOL> ) <EOL> candidate , c = Candidate . objects . get_or_create ( <EOL> election = election , <EOL> office = office , <EOL> filer = filer <EOL> ) <EOL> if self . verbosity > <NUM_LIT:2> : <EOL> if c : <EOL> self . log ( '<STR_LIT>' % candidate ) <EOL> except Filer . DoesNotExist : <EOL> pass </s>
