<s> """<STR_LIT>""" <EOL> import os <EOL> import gc <EOL> import sys <EOL> import json <EOL> import logging <EOL> import functools <EOL> from collections import defaultdict <EOL> import progressbar <EOL> from modularodm import Q <EOL> from framework . celery_tasks import app as celery_app <EOL> from website import mails <EOL> from website . models import User <EOL> from website . app import init_app <EOL> from website . project . model import Node <EOL> from website . files . models import FileVersion <EOL> from website . files . models import OsfStorageFile <EOL> from website . files . models import TrashedFileNode <EOL> from scripts import utils as scripts_utils <EOL> logger = logging . getLogger ( __name__ ) <EOL> logging . basicConfig ( level = logging . INFO ) <EOL> GBs = <NUM_LIT> ** <NUM_LIT> <EOL> USER_LIMIT = <NUM_LIT:5> * GBs <EOL> PROJECT_LIMIT = <NUM_LIT:5> * GBs <EOL> WHITE_LIST_PATH = os . path . join ( os . path . dirname ( __file__ ) , '<STR_LIT>' ) <EOL> try : <EOL> with open ( WHITE_LIST_PATH , '<STR_LIT:r>' ) as fobj : <EOL> WHITE_LIST = set ( json . load ( fobj ) ) <EOL> logger . info ( '<STR_LIT>' . format ( WHITE_LIST_PATH ) ) <EOL> except IOError : <EOL> WHITE_LIST = set ( ) <EOL> logger . warning ( '<STR_LIT>' ) <EOL> def add_to_white_list ( gtg ) : <EOL> gtg = set ( gtg ) . difference ( WHITE_LIST ) <EOL> logger . info ( '<STR_LIT>' . format ( gtg ) ) <EOL> with open ( WHITE_LIST_PATH , '<STR_LIT:w>' ) as fobj : <EOL> json . dump ( list ( WHITE_LIST . union ( gtg ) ) , fobj ) <EOL> logger . info ( '<STR_LIT>' . format ( WHITE_LIST ) ) <EOL> def get_usage ( node ) : <EOL> vids = sum ( [ <EOL> file_node . to_storage ( ) . get ( '<STR_LIT>' , [ ] ) <EOL> for file_node in <EOL> OsfStorageFile . find ( Q ( '<STR_LIT>' , '<STR_LIT>' , node ) ) <EOL> ] , [ ] ) <EOL> t_vids = sum ( [ <EOL> file_node . to_storage ( ) . get ( '<STR_LIT>' , [ ] ) <EOL> for file_node in <EOL> TrashedFileNode . find ( <EOL> Q ( '<STR_LIT>' , '<STR_LIT>' , node ) & <EOL> Q ( '<STR_LIT>' , '<STR_LIT>' , True ) & <EOL> Q ( '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ) <EOL> ) <EOL> ] , [ ] ) <EOL> usage = sum ( [ v . size or <NUM_LIT:0> for v in FileVersion . find ( Q ( '<STR_LIT>' , '<STR_LIT>' , vids ) ) ] ) <EOL> trashed_usage = sum ( [ v . size or <NUM_LIT:0> for v in FileVersion . find ( Q ( '<STR_LIT>' , '<STR_LIT>' , t_vids ) ) ] ) <EOL> return map ( sum , zip ( * ( [ ( usage , trashed_usage ) ] + [ get_usage ( child ) for child in node . nodes_primary ] ) ) ) <EOL> def limit_filter ( limit , ( item , usage ) ) : <EOL> """<STR_LIT>""" <EOL> return item not in WHITE_LIST and sum ( usage ) >= limit <EOL> def main ( send_email = False ) : <EOL> logger . info ( '<STR_LIT>' ) <EOL> init_app ( set_backends = True , routes = False ) <EOL> lines = [ ] <EOL> projects = { } <EOL> users = defaultdict ( lambda : ( <NUM_LIT:0> , <NUM_LIT:0> ) ) <EOL> progress_bar = progressbar . ProgressBar ( maxval = Node . find ( Q ( '<STR_LIT>' , '<STR_LIT>' , None ) ) . count ( ) ) . start ( ) <EOL> for i , node in enumerate ( Node . find ( Q ( '<STR_LIT>' , '<STR_LIT>' , None ) ) ) : <EOL> progress_bar . update ( i + <NUM_LIT:1> ) <EOL> if node . _id in WHITE_LIST : <EOL> continue <EOL> projects [ node . _id ] = get_usage ( node ) <EOL> for contrib in node . contributors : <EOL> if node . can_edit ( user = contrib ) : <EOL> users [ contrib . _id ] = tuple ( map ( sum , zip ( users [ contrib . _id ] , projects [ node . _id ] ) ) ) <EOL> if i % <NUM_LIT> == <NUM_LIT:0> : <EOL> for key in ( '<STR_LIT>' , '<STR_LIT:user>' , '<STR_LIT>' , '<STR_LIT>' ) : <EOL> Node . _cache . data . get ( key , { } ) . clear ( ) <EOL> Node . _object_cache . data . get ( key , { } ) . clear ( ) <EOL> gc . collect ( ) <EOL> progress_bar . finish ( ) <EOL> for model , collection , limit in ( ( User , users , USER_LIMIT ) , ( Node , projects , PROJECT_LIMIT ) ) : <EOL> for item , ( used , deleted ) in filter ( functools . partial ( limit_filter , limit ) , collection . items ( ) ) : <EOL> line = '<STR_LIT>' . format ( model . load ( item ) , limit / GBs , limit , used / GBs , used , deleted / GBs , deleted ) <EOL> logger . info ( line ) <EOL> lines . append ( line ) <EOL> if lines : <EOL> if send_email : <EOL> logger . info ( '<STR_LIT>' ) <EOL> mails . send_mail ( '<STR_LIT>' , mails . EMPTY , body = '<STR_LIT:\n>' . join ( lines ) , subject = '<STR_LIT>' ) <EOL> else : <EOL> logger . info ( '<STR_LIT>' . format ( len ( lines ) ) ) <EOL> logger . info ( '<STR_LIT>' . format ( len ( lines ) ) ) <EOL> else : <EOL> logger . info ( '<STR_LIT>' ) <EOL> @ celery_app . task ( name = '<STR_LIT>' ) <EOL> def run_main ( send_mail = False , white_list = None ) : <EOL> scripts_utils . add_file_logger ( logger , __file__ ) <EOL> if white_list : <EOL> add_to_white_list ( white_list ) <EOL> else : <EOL> main ( send_mail ) </s>
