<s> import pymongo <EOL> import os <EOL> import sys <EOL> import timelib <EOL> import argparse <EOL> import json <EOL> script_path = os . path . abspath ( os . path . dirname ( __file__ ) ) <EOL> new_path = os . path . abspath ( ( os . path . join ( script_path , '<STR_LIT:..>' , '<STR_LIT>' ) ) ) <EOL> sys . path . append ( new_path ) <EOL> os . chdir ( new_path ) <EOL> from scrapy . utils . project import get_project_settings <EOL> def total_by_source ( days_back = None ) : <EOL> q = [ <EOL> { "<STR_LIT>" : { '<STR_LIT>' : "<STR_LIT>" , '<STR_LIT>' : { "<STR_LIT>" : <NUM_LIT:1> } } } , <EOL> { "<STR_LIT>" : { '<STR_LIT>' : <NUM_LIT:1> } } <EOL> ] <EOL> if days_back : <EOL> days_back = timelib . strtodatetime ( "<STR_LIT>" % days_back ) <EOL> q . insert ( <NUM_LIT:0> , { "<STR_LIT>" : { '<STR_LIT>' : { "<STR_LIT>" : days_back } } } ) <EOL> totals = conn_coll . aggregate ( q ) <EOL> if not totals : <EOL> return False <EOL> return totals [ '<STR_LIT:result>' ] <EOL> def output_totals ( days_in_past , totals ) : <EOL> sum_all = <NUM_LIT:0> <EOL> if ( days_in_past ) : <EOL> print "<STR_LIT>" % days_in_past <EOL> else : <EOL> print "<STR_LIT>" <EOL> for total in totals : <EOL> print "<STR_LIT>" % ( total [ '<STR_LIT>' ] , total [ '<STR_LIT>' ] ) <EOL> sum_all = sum_all + total [ '<STR_LIT>' ] <EOL> print "<STR_LIT>" % sum_all <EOL> print "<STR_LIT:\n>" <EOL> def format_json ( days_in_past , totals ) : <EOL> sum_all = <NUM_LIT:0> <EOL> key = None <EOL> source_vals = { } <EOL> if ( days_in_past ) : <EOL> key = "<STR_LIT>" % days_in_past <EOL> else : <EOL> key = "<STR_LIT>" <EOL> for total in totals : <EOL> source_vals [ total [ '<STR_LIT>' ] ] = total [ '<STR_LIT>' ] <EOL> sum_all = sum_all + total [ '<STR_LIT>' ] <EOL> source_vals [ '<STR_LIT>' ] = sum_all <EOL> return key , source_vals <EOL> def get_args ( ) : <EOL> parser = argparse . ArgumentParser ( description = '<STR_LIT>' ) <EOL> parser . add_argument ( '<STR_LIT>' , dest = '<STR_LIT>' , action = '<STR_LIT:store_true>' , <EOL> help = '<STR_LIT>' ) <EOL> args = parser . parse_args ( ) <EOL> return args <EOL> JSON_OUTPUT = False <EOL> if __name__ == '<STR_LIT:__main__>' : <EOL> args = get_args ( ) <EOL> JSON_OUTPUT = args . JSON_OUTPUT <EOL> settings = get_project_settings ( ) <EOL> mongo_uri = settings [ '<STR_LIT>' ] <EOL> mongo_db = settings [ '<STR_LIT>' ] <EOL> mongo_coll = settings [ '<STR_LIT>' ] <EOL> conn = pymongo . mongo_client . MongoClient ( mongo_uri ) <EOL> conn_db = conn [ mongo_db ] <EOL> conn_coll = conn_db [ mongo_coll ] <EOL> days = [ <NUM_LIT:1> , <NUM_LIT:7> , <NUM_LIT:30> , <NUM_LIT:0> ] <EOL> json_dict = { } <EOL> for day in days : <EOL> totals = total_by_source ( day ) <EOL> if JSON_OUTPUT : <EOL> source_key , source_vals = format_json ( day , totals ) <EOL> json_dict [ source_key ] = source_vals <EOL> else : <EOL> output_totals ( day , totals ) <EOL> if JSON_OUTPUT : <EOL> print json . dumps ( json_dict , sort_keys = True , indent = <NUM_LIT:4> , <EOL> separators = ( '<STR_LIT:U+002C>' , '<STR_LIT>' ) ) </s>
