<s> """<STR_LIT>""" <EOL> from __future__ import unicode_literals <EOL> import time <EOL> import logging <EOL> from datetime import date , timedelta <EOL> import xmltodict <EOL> from lxml import etree <EOL> from scrapi import requests <EOL> from scrapi import settings <EOL> from scrapi . base import XMLHarvester <EOL> from scrapi . util import copy_to_unicode <EOL> from scrapi . linter . document import RawDocument <EOL> from scrapi . base . helpers import ( <EOL> compose , <EOL> single_result , <EOL> build_properties , <EOL> datetime_formatter , <EOL> default_name_parser <EOL> ) <EOL> logger = logging . getLogger ( __name__ ) <EOL> element_to_dict = compose ( xmltodict . parse , etree . tostring ) <EOL> def non_string ( item ) : <EOL> return not isinstance ( item , str ) <EOL> class ClinicalTrialsHarvester ( XMLHarvester ) : <EOL> short_name = '<STR_LIT>' <EOL> long_name = '<STR_LIT>' <EOL> url = '<STR_LIT>' <EOL> DEFAULT_ENCODING = '<STR_LIT>' <EOL> record_encoding = None <EOL> schema = { <EOL> "<STR_LIT>" : ( '<STR_LIT>' , default_name_parser ) , <EOL> "<STR_LIT>" : { <EOL> "<STR_LIT>" : ( "<STR_LIT>" , single_result ) <EOL> } , <EOL> "<STR_LIT>" : ( "<STR_LIT>" , compose ( datetime_formatter , single_result ) ) , <EOL> "<STR_LIT:title>" : ( '<STR_LIT>' , '<STR_LIT>' , lambda x , y : single_result ( x ) or single_result ( y ) ) , <EOL> "<STR_LIT:description>" : ( '<STR_LIT>' , '<STR_LIT>' , lambda x , y : single_result ( x ) or single_result ( y ) ) , <EOL> "<STR_LIT>" : ( "<STR_LIT>" , lambda tags : [ tag . lower ( ) for tag in tags ] ) , <EOL> "<STR_LIT>" : [ <EOL> { <EOL> "<STR_LIT>" : { <EOL> "<STR_LIT>" : ( "<STR_LIT>" , single_result ) <EOL> } <EOL> } , <EOL> { <EOL> "<STR_LIT>" : { <EOL> "<STR_LIT>" : ( "<STR_LIT>" , single_result ) <EOL> } <EOL> } <EOL> ] , <EOL> "<STR_LIT>" : build_properties ( <EOL> ( "<STR_LIT>" , "<STR_LIT>" ) , <EOL> ( '<STR_LIT>' , '<STR_LIT>' ) , <EOL> ( '<STR_LIT>' , '<STR_LIT>' ) , <EOL> ( '<STR_LIT>' , '<STR_LIT>' ) , <EOL> ( '<STR_LIT:source>' , '<STR_LIT>' ) , <EOL> ( '<STR_LIT>' , '<STR_LIT>' ) , <EOL> ( '<STR_LIT>' , '<STR_LIT>' ) , <EOL> ( '<STR_LIT>' , '<STR_LIT>' ) , <EOL> ( '<STR_LIT>' , '<STR_LIT>' ) , <EOL> ( '<STR_LIT>' , '<STR_LIT>' ) , <EOL> ( '<STR_LIT:status>' , '<STR_LIT>' ) , <EOL> ( '<STR_LIT>' , '<STR_LIT>' ) , <EOL> ( '<STR_LIT>' , '<STR_LIT>' ) , <EOL> ( '<STR_LIT>' , '<STR_LIT>' ) , <EOL> ( '<STR_LIT>' , '<STR_LIT>' ) , <EOL> ( '<STR_LIT>' , '<STR_LIT>' ) , <EOL> ( '<STR_LIT>' , '<STR_LIT>' ) , <EOL> ( '<STR_LIT>' , '<STR_LIT>' ) , <EOL> ( '<STR_LIT>' , '<STR_LIT>' ) , <EOL> ( '<STR_LIT>' , '<STR_LIT>' ) , <EOL> ( '<STR_LIT>' , '<STR_LIT>' ) , <EOL> ( '<STR_LIT>' , '<STR_LIT>' ) , <EOL> ( '<STR_LIT>' , '<STR_LIT>' ) , <EOL> ( '<STR_LIT>' , '<STR_LIT>' ) , <EOL> ( '<STR_LIT>' , '<STR_LIT>' ) , <EOL> ( '<STR_LIT>' , ( '<STR_LIT>' , compose ( <EOL> lambda x : list ( map ( element_to_dict , x ) ) , <EOL> lambda x : list ( filter ( non_string , x ) ) <EOL> ) ) ) , <EOL> ( '<STR_LIT>' , '<STR_LIT>' ) , <EOL> ( '<STR_LIT>' , '<STR_LIT>' ) <EOL> ) <EOL> } <EOL> @ property <EOL> def namespaces ( self ) : <EOL> return None <EOL> def harvest ( self , start_date = None , end_date = None ) : <EOL> """<STR_LIT>""" <EOL> start_date = start_date or date . today ( ) - timedelta ( settings . DAYS_BACK ) <EOL> end_date = end_date or date . today ( ) <EOL> end_month = end_date . strftime ( '<STR_LIT>' ) <EOL> end_day = end_date . strftime ( '<STR_LIT>' ) <EOL> end_year = end_date . strftime ( '<STR_LIT>' ) <EOL> start_month = start_date . strftime ( '<STR_LIT>' ) <EOL> start_day = start_date . strftime ( '<STR_LIT>' ) <EOL> start_year = start_date . strftime ( '<STR_LIT>' ) <EOL> base_url = '<STR_LIT>' <EOL> url_end = '<STR_LIT>' . format ( start_month , start_day , start_year , end_month , end_day , end_year ) <EOL> url = base_url + url_end <EOL> initial_request = requests . get ( url ) <EOL> record_encoding = initial_request . encoding <EOL> initial_request_xml = etree . XML ( initial_request . content ) <EOL> count = int ( initial_request_xml . xpath ( '<STR_LIT>' ) [ <NUM_LIT:0> ] ) <EOL> xml_list = [ ] <EOL> if int ( count ) > <NUM_LIT:0> : <EOL> url = url + '<STR_LIT>' + str ( count ) <EOL> total_requests = requests . get ( url ) <EOL> initial_doc = etree . XML ( total_requests . content ) <EOL> study_urls = [ ] <EOL> for study in initial_doc . xpath ( '<STR_LIT>' ) : <EOL> study_urls . append ( study . xpath ( '<STR_LIT>' ) [ <NUM_LIT:0> ] + '<STR_LIT>' ) <EOL> logger . info ( "<STR_LIT>" . format ( len ( study_urls ) ) ) <EOL> count = <NUM_LIT:0> <EOL> official_count = <NUM_LIT:0> <EOL> for study_url in study_urls : <EOL> try : <EOL> content = requests . get ( study_url ) <EOL> except requests . exceptions . ConnectionError as e : <EOL> logger . info ( '<STR_LIT>' . format ( e ) ) <EOL> time . sleep ( <NUM_LIT:30> ) <EOL> continue <EOL> doc = etree . XML ( content . content ) <EOL> record = etree . tostring ( doc , encoding = record_encoding ) <EOL> doc_id = doc . xpath ( '<STR_LIT>' ) [ <NUM_LIT:0> ] <EOL> xml_list . append ( RawDocument ( { <EOL> '<STR_LIT>' : record , <EOL> '<STR_LIT:source>' : self . short_name , <EOL> '<STR_LIT>' : copy_to_unicode ( doc_id ) , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> } ) ) <EOL> official_count += <NUM_LIT:1> <EOL> count += <NUM_LIT:1> <EOL> if count % <NUM_LIT:100> == <NUM_LIT:0> : <EOL> logger . info ( "<STR_LIT>" . format ( official_count ) ) <EOL> count = <NUM_LIT:0> <EOL> return xml_list </s>
