<s> import requests <EOL> import traceback <EOL> from requests . utils import quote <EOL> from datetime import date , datetime , timedelta <EOL> from concurrent . futures import ThreadPoolExecutor <EOL> from collections import defaultdict <EOL> endpoints = { <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> } <EOL> def parse_date ( stringDate ) : <EOL> return datetime . strptime ( stringDate . ljust ( <NUM_LIT:10> , '<STR_LIT:0>' ) , '<STR_LIT>' ) <EOL> def format_date ( d ) : <EOL> return datetime . strftime ( d , '<STR_LIT>' ) <EOL> def timestamps_between ( start , end , increment ) : <EOL> start = datetime ( start . year , start . month , start . day , getattr ( start , '<STR_LIT>' , <NUM_LIT:0> ) ) <EOL> end = datetime ( end . year , end . month , end . day , getattr ( end , '<STR_LIT>' , <NUM_LIT:0> ) ) <EOL> while start <= end : <EOL> yield start <EOL> start += increment <EOL> class PageviewsClient : <EOL> def __init__ ( self , parallelism = <NUM_LIT:10> ) : <EOL> """<STR_LIT>""" <EOL> self . parallelism = parallelism <EOL> def article_views ( <EOL> self , project , articles , <EOL> access = '<STR_LIT>' , agent = '<STR_LIT>' , granularity = '<STR_LIT>' , <EOL> start = None , end = None ) : <EOL> """<STR_LIT>""" <EOL> endDate = end or date . today ( ) <EOL> if type ( endDate ) is not date : <EOL> endDate = parse_date ( end ) <EOL> startDate = start or endDate - timedelta ( <NUM_LIT:30> ) <EOL> if type ( startDate ) is not date : <EOL> startDate = parse_date ( start ) <EOL> articles = [ a . replace ( '<STR_LIT:U+0020>' , '<STR_LIT:_>' ) for a in articles ] <EOL> articlesSafe = [ quote ( a , safe = '<STR_LIT>' ) for a in articles ] <EOL> urls = [ <EOL> '<STR_LIT:/>' . join ( [ <EOL> endpoints [ '<STR_LIT>' ] , project , access , agent , a , granularity , <EOL> format_date ( startDate ) , format_date ( endDate ) , <EOL> ] ) <EOL> for a in articlesSafe <EOL> ] <EOL> outputDays = timestamps_between ( startDate , endDate , timedelta ( days = <NUM_LIT:1> ) ) <EOL> output = defaultdict ( dict , { <EOL> day : { a : None for a in articles } for day in outputDays <EOL> } ) <EOL> try : <EOL> results = self . get_concurrent ( urls ) <EOL> some_data_returned = False <EOL> for result in results : <EOL> if '<STR_LIT>' in result : <EOL> some_data_returned = True <EOL> else : <EOL> continue <EOL> for item in result [ '<STR_LIT>' ] : <EOL> output [ parse_date ( item [ '<STR_LIT>' ] ) ] [ item [ '<STR_LIT>' ] ] = item [ '<STR_LIT>' ] <EOL> if not some_data_returned : <EOL> raise Exception ( <EOL> '<STR_LIT>' . format ( urls ) <EOL> ) <EOL> return output <EOL> except : <EOL> print ( '<STR_LIT>' + str ( urls ) ) <EOL> traceback . print_exc ( ) <EOL> raise <EOL> def project_views ( <EOL> self , projects , <EOL> access = '<STR_LIT>' , agent = '<STR_LIT>' , granularity = '<STR_LIT>' , <EOL> start = None , end = None ) : <EOL> """<STR_LIT>""" <EOL> endDate = end or date . today ( ) <EOL> if type ( endDate ) is not date : <EOL> endDate = parse_date ( end ) <EOL> startDate = start or endDate - timedelta ( <NUM_LIT:30> ) <EOL> if type ( startDate ) is not date : <EOL> startDate = parse_date ( start ) <EOL> urls = [ <EOL> '<STR_LIT:/>' . join ( [ <EOL> endpoints [ '<STR_LIT>' ] , p , access , agent , granularity , <EOL> format_date ( startDate ) , format_date ( endDate ) , <EOL> ] ) <EOL> for p in projects <EOL> ] <EOL> if granularity == '<STR_LIT>' : <EOL> increment = timedelta ( hours = <NUM_LIT:1> ) <EOL> elif granularity == '<STR_LIT>' : <EOL> increment = timedelta ( days = <NUM_LIT:1> ) <EOL> elif granularity == '<STR_LIT>' : <EOL> increment = timedelta ( months = <NUM_LIT:1> ) <EOL> outputDays = timestamps_between ( startDate , endDate , increment ) <EOL> output = defaultdict ( dict , { <EOL> day : { p : None for p in projects } for day in outputDays <EOL> } ) <EOL> try : <EOL> results = self . get_concurrent ( urls ) <EOL> some_data_returned = False <EOL> for result in results : <EOL> if '<STR_LIT>' in result : <EOL> some_data_returned = True <EOL> else : <EOL> continue <EOL> for item in result [ '<STR_LIT>' ] : <EOL> output [ parse_date ( item [ '<STR_LIT>' ] ) ] [ item [ '<STR_LIT>' ] ] = item [ '<STR_LIT>' ] <EOL> if not some_data_returned : <EOL> raise Exception ( <EOL> '<STR_LIT>' . format ( urls ) <EOL> ) <EOL> return output <EOL> except : <EOL> print ( '<STR_LIT>' + str ( urls ) ) <EOL> traceback . print_exc ( ) <EOL> raise <EOL> def top_articles ( <EOL> self , project , access = '<STR_LIT>' , <EOL> year = None , month = None , day = None , limit = <NUM_LIT:1000> ) : <EOL> """<STR_LIT>""" <EOL> yesterday = date . today ( ) - timedelta ( days = <NUM_LIT:1> ) <EOL> year = str ( year or yesterday . year ) <EOL> month = str ( month or yesterday . month ) . rjust ( <NUM_LIT:2> , '<STR_LIT:0>' ) <EOL> day = str ( day or yesterday . day ) . rjust ( <NUM_LIT:2> , '<STR_LIT:0>' ) <EOL> url = '<STR_LIT:/>' . join ( [ endpoints [ '<STR_LIT>' ] , project , access , year , month , day ] ) <EOL> try : <EOL> result = requests . get ( url ) . json ( ) <EOL> if '<STR_LIT>' in result and len ( result [ '<STR_LIT>' ] ) == <NUM_LIT:1> : <EOL> r = result [ '<STR_LIT>' ] [ <NUM_LIT:0> ] [ '<STR_LIT>' ] <EOL> r . sort ( key = lambda x : x [ '<STR_LIT>' ] ) <EOL> return r [ <NUM_LIT:0> : ( limit ) ] <EOL> except : <EOL> print ( '<STR_LIT>' + url ) <EOL> traceback . print_exc ( ) <EOL> raise <EOL> raise Exception ( <EOL> '<STR_LIT>' . format ( url ) <EOL> ) <EOL> def get_concurrent ( self , urls ) : <EOL> with ThreadPoolExecutor ( self . parallelism ) as executor : <EOL> f = lambda url : requests . get ( url ) . json ( ) <EOL> return list ( executor . map ( f , urls ) ) </s>
