<s> """<STR_LIT>""" <EOL> from __future__ import unicode_literals <EOL> import json <EOL> import logging <EOL> from furl import furl <EOL> from datetime import date , timedelta <EOL> from scrapi import requests <EOL> from scrapi import settings <EOL> from scrapi . base import JSONHarvester <EOL> from scrapi . linter . document import RawDocument <EOL> from scrapi . base . helpers import build_properties , datetime_formatter , compose , default_name_parser <EOL> logger = logging . getLogger ( __name__ ) <EOL> def process_urls ( urls ) : <EOL> all_uris = [ url [ '<STR_LIT:value>' ] for url in urls ] <EOL> doi_index = <NUM_LIT:0> <EOL> for i in range ( <NUM_LIT:0> , len ( all_uris ) ) : <EOL> if '<STR_LIT>' in all_uris [ i ] : <EOL> doi_index = i <EOL> processed_uris = { <EOL> '<STR_LIT>' : all_uris [ doi_index ] , <EOL> '<STR_LIT>' : [ ] , <EOL> '<STR_LIT>' : all_uris <EOL> } <EOL> for url in all_uris : <EOL> if '<STR_LIT>' in url : <EOL> processed_uris [ '<STR_LIT>' ] . append ( url ) <EOL> return processed_uris <EOL> class BiomedCentralHarvester ( JSONHarvester ) : <EOL> short_name = '<STR_LIT>' <EOL> long_name = '<STR_LIT>' <EOL> url = '<STR_LIT>' <EOL> count = <NUM_LIT:0> <EOL> base_url = '<STR_LIT>' <EOL> URL = furl ( base_url ) <EOL> URL . args [ '<STR_LIT>' ] = settings . SPRINGER_API_KEY <EOL> URL . args [ '<STR_LIT:p>' ] = <NUM_LIT:100> <EOL> @ property <EOL> def schema ( self ) : <EOL> return { <EOL> '<STR_LIT>' : ( <EOL> '<STR_LIT>' , <EOL> compose ( <EOL> default_name_parser , <EOL> lambda authors : [ author [ '<STR_LIT>' ] for author in authors ] <EOL> ) <EOL> ) , <EOL> '<STR_LIT>' : ( '<STR_LIT>' , process_urls ) , <EOL> '<STR_LIT:title>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : ( '<STR_LIT>' , datetime_formatter ) , <EOL> '<STR_LIT:description>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : { <EOL> '<STR_LIT>' : ( '<STR_LIT>' , '<STR_LIT>' , lambda x , y : y if x == '<STR_LIT:true>' else None ) <EOL> } , <EOL> '<STR_LIT>' : { <EOL> '<STR_LIT:name>' : '<STR_LIT>' <EOL> } , <EOL> '<STR_LIT>' : ( '<STR_LIT>' , lambda x : [ x ] if x else [ ] ) , <EOL> '<STR_LIT>' : build_properties ( <EOL> ( '<STR_LIT:url>' , '<STR_LIT>' ) , <EOL> ( '<STR_LIT>' , '<STR_LIT>' ) , <EOL> ( '<STR_LIT>' , '<STR_LIT>' ) , <EOL> ( '<STR_LIT>' , '<STR_LIT>' ) , <EOL> ( '<STR_LIT>' , '<STR_LIT>' ) , <EOL> ( '<STR_LIT>' , '<STR_LIT>' ) , <EOL> ( '<STR_LIT>' , '<STR_LIT>' ) , <EOL> ( '<STR_LIT>' , '<STR_LIT>' ) , <EOL> ( '<STR_LIT>' , '<STR_LIT>' ) , <EOL> ( '<STR_LIT>' , '<STR_LIT>' ) <EOL> ) <EOL> } <EOL> def harvest ( self , start_date = None , end_date = None ) : <EOL> start_date = start_date or date . today ( ) - timedelta ( settings . DAYS_BACK ) <EOL> end_date = end_date or date . today ( ) <EOL> delta = end_date - start_date <EOL> date_strings = [ ] <EOL> for i in range ( delta . days + <NUM_LIT:1> ) : <EOL> date_strings . append ( start_date + timedelta ( days = i ) ) <EOL> search_urls = [ ] <EOL> for adate in date_strings : <EOL> self . URL . args [ '<STR_LIT:q>' ] = '<STR_LIT>' . format ( adate ) <EOL> search_urls . append ( self . URL . url ) <EOL> records = self . get_records ( search_urls ) <EOL> records_list = [ ] <EOL> for record in records : <EOL> format_type = record [ '<STR_LIT>' ] <EOL> if format_type . lower ( ) == "<STR_LIT>" : <EOL> records_list . append ( RawDocument ( { <EOL> '<STR_LIT>' : json . dumps ( record ) , <EOL> '<STR_LIT:source>' : self . short_name , <EOL> '<STR_LIT>' : record [ '<STR_LIT>' ] , <EOL> '<STR_LIT>' : '<STR_LIT>' <EOL> } ) ) <EOL> return records_list <EOL> def get_records ( self , search_urls ) : <EOL> all_records_from_all_days = [ ] <EOL> for search_url in search_urls : <EOL> records = requests . get ( search_url ) . json ( ) <EOL> index = <NUM_LIT:1> <EOL> total_records = int ( records [ '<STR_LIT:result>' ] [ <NUM_LIT:0> ] [ '<STR_LIT>' ] ) <EOL> all_records = [ ] <EOL> while len ( all_records ) < total_records : <EOL> record_list = records [ '<STR_LIT>' ] <EOL> all_records += record_list <EOL> index += <NUM_LIT:100> <EOL> records = requests . get ( search_url + '<STR_LIT>' . format ( str ( index ) , throttle = <NUM_LIT:10> ) ) . json ( ) <EOL> all_records_from_all_days = all_records_from_all_days + all_records <EOL> return all_records_from_all_days </s>
