<s> """<STR_LIT>""" <EOL> from __future__ import unicode_literals <EOL> import logging <EOL> from datetime import date , timedelta <EOL> from lxml import etree <EOL> from scrapi import requests <EOL> from scrapi import settings <EOL> from scrapi . base import XMLHarvester <EOL> from scrapi . linter . document import RawDocument <EOL> from scrapi . base . helpers import default_name_parser , build_properties , compose , single_result , datetime_formatter <EOL> logger = logging . getLogger ( __name__ ) <EOL> try : <EOL> from scrapi . settings import PLOS_API_KEY <EOL> except ImportError : <EOL> PLOS_API_KEY = None <EOL> logger . error ( '<STR_LIT>' ) <EOL> class PlosHarvester ( XMLHarvester ) : <EOL> short_name = '<STR_LIT>' <EOL> long_name = '<STR_LIT>' <EOL> url = '<STR_LIT>' <EOL> namespaces = { } <EOL> MAX_ROWS_PER_REQUEST = <NUM_LIT> <EOL> BASE_URL = '<STR_LIT>' <EOL> def fetch_rows ( self , start_date , end_date ) : <EOL> query = '<STR_LIT>' . format ( start_date , end_date ) <EOL> resp = requests . get ( self . BASE_URL , params = { <EOL> '<STR_LIT:q>' : query , <EOL> '<STR_LIT>' : '<STR_LIT:0>' , <EOL> '<STR_LIT>' : PLOS_API_KEY , <EOL> } ) <EOL> total_rows = etree . XML ( resp . content ) . xpath ( '<STR_LIT>' ) <EOL> total_rows = int ( total_rows [ <NUM_LIT:0> ] ) if total_rows else <NUM_LIT:0> <EOL> current_row = <NUM_LIT:0> <EOL> while current_row < total_rows : <EOL> response = requests . get ( self . BASE_URL , throttle = <NUM_LIT:5> , params = { <EOL> '<STR_LIT:q>' : query , <EOL> '<STR_LIT:start>' : current_row , <EOL> '<STR_LIT>' : PLOS_API_KEY , <EOL> '<STR_LIT>' : self . MAX_ROWS_PER_REQUEST , <EOL> } ) <EOL> for doc in etree . XML ( response . content ) . xpath ( '<STR_LIT>' ) : <EOL> yield doc <EOL> current_row += self . MAX_ROWS_PER_REQUEST <EOL> def harvest ( self , start_date = None , end_date = None ) : <EOL> start_date = start_date or date . today ( ) - timedelta ( settings . DAYS_BACK ) <EOL> end_date = end_date or date . today ( ) <EOL> if not PLOS_API_KEY : <EOL> return [ ] <EOL> return [ <EOL> RawDocument ( { <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT:source>' : self . short_name , <EOL> '<STR_LIT>' : etree . tostring ( row ) , <EOL> '<STR_LIT>' : row . xpath ( "<STR_LIT>" ) [ <NUM_LIT:0> ] . text , <EOL> } ) <EOL> for row in <EOL> self . fetch_rows ( start_date . isoformat ( ) , end_date . isoformat ( ) ) <EOL> if row . xpath ( "<STR_LIT>" ) <EOL> or row . xpath ( "<STR_LIT>" ) <EOL> ] <EOL> schema = { <EOL> '<STR_LIT>' : { <EOL> '<STR_LIT>' : ( '<STR_LIT>' , compose ( '<STR_LIT>' . format , single_result ) ) , <EOL> } , <EOL> '<STR_LIT>' : ( '<STR_LIT>' , default_name_parser ) , <EOL> '<STR_LIT>' : ( '<STR_LIT>' , compose ( datetime_formatter , single_result ) ) , <EOL> '<STR_LIT:title>' : ( '<STR_LIT>' , single_result ) , <EOL> '<STR_LIT:description>' : ( '<STR_LIT>' , single_result ) , <EOL> '<STR_LIT>' : { <EOL> '<STR_LIT:name>' : ( '<STR_LIT>' , single_result ) <EOL> } , <EOL> '<STR_LIT>' : build_properties ( <EOL> ( '<STR_LIT>' , '<STR_LIT>' ) , <EOL> ( '<STR_LIT>' , '<STR_LIT>' ) , <EOL> ( '<STR_LIT>' , '<STR_LIT>' ) <EOL> ) <EOL> } </s>
