<s> from performanceplatform . utils import requests_with_backoff <EOL> from datetime import datetime , timedelta <EOL> import pytz <EOL> from performanceplatform . collector . webtrends . base import ( <EOL> BaseCollector , BaseParser ) <EOL> class Collector ( BaseCollector ) : <EOL> def __init__ ( self , credentials , query , start_at , end_at ) : <EOL> self . api_version = credentials . get ( '<STR_LIT>' ) <EOL> self . base_url = credentials [ '<STR_LIT>' ] <EOL> self . report_id = query . pop ( '<STR_LIT>' ) <EOL> super ( Collector , self ) . __init__ ( credentials , query , start_at , end_at ) <EOL> @ classmethod <EOL> def date_range_for_webtrends ( cls , start_at = None , end_at = None ) : <EOL> """<STR_LIT>""" <EOL> if start_at and end_at : <EOL> start_date = cls . parse_standard_date_string_to_date ( <EOL> start_at ) <EOL> end_date = cls . parse_standard_date_string_to_date ( <EOL> end_at ) <EOL> numdays = ( end_date - start_date ) . days + <NUM_LIT:1> <EOL> start_dates = [ end_date - timedelta ( days = x ) <EOL> for x in reversed ( range ( <NUM_LIT:0> , numdays ) ) ] <EOL> date_range = [ ] <EOL> for i , date in enumerate ( start_dates ) : <EOL> query_date = cls . parse_date_for_query ( date ) <EOL> date_range . append ( ( query_date , query_date ) ) <EOL> return date_range <EOL> else : <EOL> return [ ( "<STR_LIT>" , "<STR_LIT>" ) ] <EOL> def _make_request ( self , start_at_for_webtrends , end_at_for_webtrends ) : <EOL> return requests_with_backoff . get ( <EOL> url = "<STR_LIT>" . format ( <EOL> base_url = self . base_url , <EOL> report_id = self . report_id ) , <EOL> auth = ( self . user , self . password ) , <EOL> params = { <EOL> '<STR_LIT>' : start_at_for_webtrends , <EOL> '<STR_LIT>' : end_at_for_webtrends , <EOL> '<STR_LIT>' : self . query_format <EOL> } <EOL> ) <EOL> def build_parser ( self , data_set_config , options ) : <EOL> if self . api_version == '<STR_LIT>' : <EOL> return V3Parser ( options , data_set_config [ '<STR_LIT>' ] ) <EOL> else : <EOL> return V2Parser ( options , data_set_config [ '<STR_LIT>' ] ) <EOL> class V2Parser ( BaseParser ) : <EOL> def __init__ ( self , options , data_type ) : <EOL> self . row_type_name = options [ '<STR_LIT>' ] <EOL> super ( V2Parser , self ) . __init__ ( options , data_type ) <EOL> def parse_item ( self , item ) : <EOL> initial_data = [ ] <EOL> special_fields = [ ] <EOL> for date , data_point in item . items ( ) : <EOL> start_date = self . __class__ . to_datetime ( date ) <EOL> if data_point [ '<STR_LIT>' ] : <EOL> for row_type , row in data_point [ '<STR_LIT>' ] . items ( ) : <EOL> initial_data . append ( { <EOL> '<STR_LIT>' : start_date , <EOL> } ) <EOL> special_fields . append ( dict ( { <EOL> self . row_type_name : row_type <EOL> } . items ( ) + row [ '<STR_LIT>' ] . items ( ) ) ) <EOL> return initial_data , special_fields <EOL> @ classmethod <EOL> def to_datetime ( cls , webtrends_formatted_date ) : <EOL> start_of_period = webtrends_formatted_date . split ( "<STR_LIT:->" ) [ <NUM_LIT:0> ] <EOL> return datetime . strptime ( <EOL> start_of_period , "<STR_LIT>" ) . replace ( tzinfo = pytz . UTC ) <EOL> class V3Parser ( BaseParser ) : <EOL> def __init__ ( self , options , data_type ) : <EOL> self . row_type_name = options [ '<STR_LIT>' ] <EOL> super ( V3Parser , self ) . __init__ ( options , data_type ) <EOL> def parse_item ( self , items ) : <EOL> initial_data = [ ] <EOL> special_fields = [ ] <EOL> for value in items : <EOL> start_date = self . __class__ . to_datetime ( value ) <EOL> if value [ '<STR_LIT>' ] : <EOL> for row_type , row in value [ '<STR_LIT>' ] [ <NUM_LIT:0> ] . items ( ) : <EOL> initial_data . append ( { <EOL> '<STR_LIT>' : start_date , <EOL> } ) <EOL> special_fields . append ( dict ( { <EOL> self . row_type_name : row_type <EOL> } . items ( ) + row [ '<STR_LIT>' ] . items ( ) ) ) <EOL> return initial_data , special_fields <EOL> @ classmethod <EOL> def to_datetime ( cls , webtrends_formatted_data_point ) : <EOL> start_at = webtrends_formatted_data_point [ '<STR_LIT>' ] <EOL> date_parts = start_at . split ( '<STR_LIT:->' ) <EOL> if len ( date_parts ) == <NUM_LIT:3> : <EOL> return datetime . strptime ( <EOL> start_at , "<STR_LIT>" ) . replace ( tzinfo = pytz . UTC ) <EOL> else : <EOL> return datetime . strptime ( <EOL> start_at , "<STR_LIT>" ) . replace ( tzinfo = pytz . UTC ) <EOL> def main ( credentials , data_set_config , query , options , start_at , end_at ) : <EOL> collector = Collector ( credentials , query , start_at , end_at ) <EOL> collector . collect_parse_and_push ( data_set_config , options ) </s>
