<s> import argparse <EOL> import os <EOL> import sys <EOL> import time <EOL> import torndb <EOL> import yaml <EOL> import mimetypes <EOL> import urllib <EOL> from datetime import datetime <EOL> from pymongo import MongoClient , ASCENDING , DESCENDING <EOL> from operator import itemgetter <EOL> from boto . s3 . connection import S3Connection <EOL> from boto . s3 . key import Key <EOL> parser = argparse . ArgumentParser ( ) <EOL> parser . add_argument ( '<STR_LIT>' , default = '<STR_LIT:127.0.0.1>' ) <EOL> parser . add_argument ( '<STR_LIT>' , required = True ) <EOL> parser . add_argument ( '<STR_LIT>' , required = True ) <EOL> parser . add_argument ( '<STR_LIT>' , required = True ) <EOL> parser . add_argument ( '<STR_LIT>' , required = False , default = '<STR_LIT:0>' ) <EOL> parser . add_argument ( '<STR_LIT>' , '<STR_LIT>' , action = '<STR_LIT:count>' , default = <NUM_LIT:0> ) <EOL> args = parser . parse_args ( ) <EOL> reload ( sys ) <EOL> sys . setdefaultencoding ( '<STR_LIT>' ) <EOL> def customers ( ) : <EOL> db = torndb . Connection ( <EOL> host = args . host , <EOL> database = args . database , <EOL> user = args . user , <EOL> password = args . password , <EOL> ) <EOL> sql = """<STR_LIT>""" % args . customergroup <EOL> customers = { } <EOL> for c in db . query ( sql ) : <EOL> if c . property in [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] : <EOL> if not c . value or not c . property : <EOL> continue <EOL> customers . setdefault ( c . entity , { } ) [ c . property ] = c . value <EOL> result = [ ] <EOL> for c , p in customers . iteritems ( ) : <EOL> if not p . get ( '<STR_LIT>' ) or not p . get ( '<STR_LIT>' ) or not p . get ( '<STR_LIT>' ) or not p . get ( '<STR_LIT>' ) : <EOL> continue <EOL> try : <EOL> db = torndb . Connection ( <EOL> host = p . get ( '<STR_LIT>' ) , <EOL> database = p . get ( '<STR_LIT>' ) , <EOL> user = p . get ( '<STR_LIT>' ) , <EOL> password = p . get ( '<STR_LIT>' ) , <EOL> ) <EOL> db . get ( '<STR_LIT>' ) <EOL> except Exception : <EOL> print p <EOL> continue <EOL> result . append ( p ) <EOL> return sorted ( result , key = itemgetter ( '<STR_LIT>' ) ) <EOL> class S3files ( ) : <EOL> def __init__ ( self , db_host , db_name , db_user , db_pass ) : <EOL> self . stats = { } <EOL> self . db_host = db_host <EOL> self . db_name = db_name <EOL> self . db_user = db_user <EOL> self . db_pass = db_pass <EOL> self . db = torndb . Connection ( <EOL> host = db_host , <EOL> database = db_name , <EOL> user = db_user , <EOL> password = db_pass , <EOL> ) <EOL> def files ( self ) : <EOL> sql = """<STR_LIT>""" % self . db_name <EOL> s3_conn = S3Connection ( AWS_ACCESS_KEY , AWS_SECRET_KEY ) <EOL> s3_bucket = s3_conn . get_bucket ( AWS_BUCKET , validate = False ) <EOL> rows = self . db . query ( sql ) <EOL> for r in rows : <EOL> s3_key = s3_bucket . get_key ( r . s3_key ) <EOL> if not s3_key : <EOL> print '<STR_LIT>' % ( r . id , r . s3_key ) <EOL> continue <EOL> if not r . filename : <EOL> continue <EOL> if s3_key . content_disposition != '<STR_LIT>' % urllib . quote ( r . filename . encode ( '<STR_LIT:utf-8>' ) ) : <EOL> print '<STR_LIT>' % ( r . id , r . s3_key , r . filename , s3_key . content_disposition ) <EOL> continue <EOL> print '<STR_LIT>' <EOL> for c in customers ( ) : <EOL> if c . get ( '<STR_LIT>' ) not in [ '<STR_LIT>' ] : <EOL> continue <EOL> print '<STR_LIT>' % ( datetime . now ( ) , c . get ( '<STR_LIT>' ) ) <EOL> m2m = S3files ( <EOL> db_host = c . get ( '<STR_LIT>' ) , <EOL> db_name = c . get ( '<STR_LIT>' ) , <EOL> db_user = c . get ( '<STR_LIT>' ) , <EOL> db_pass = c . get ( '<STR_LIT>' ) <EOL> ) <EOL> m2m . files ( ) <EOL> print '<STR_LIT>' % ( datetime . now ( ) , c . get ( '<STR_LIT>' ) ) <EOL> print '<STR_LIT:%s>' % yaml . safe_dump ( m2m . stats , default_flow_style = False , allow_unicode = True ) </s>
