<s> import time <EOL> import datetime <EOL> import pytz <EOL> from django . utils . timezone import now , make_aware <EOL> from django . conf import settings <EOL> from mturk . main . models import Crawl , HitGroupContent <EOL> def pad_string ( s , l ) : <EOL> return str ( s ) + ( ( l - len ( str ( s ) ) ) * '<STR_LIT:U+0020>' ) <EOL> def print_crawls ( age = datetime . timedelta ( hours = <NUM_LIT:2> ) , sleep = <NUM_LIT:15> , limit = None ) : <EOL> """<STR_LIT>""" <EOL> header = [ <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> ] <EOL> while True : <EOL> crawls = Crawl . objects . filter ( start_time__gt = now ( ) - age <EOL> ) . order_by ( '<STR_LIT>' ) <EOL> if limit : <EOL> crawls = crawls [ : limit ] <EOL> to_print = list ( header ) <EOL> lencrawls = len ( crawls ) <EOL> for i , c in enumerate ( crawls ) : <EOL> dev = '<STR_LIT:error>' <EOL> disp = '<STR_LIT:error>' <EOL> if ( c . groups_available > <NUM_LIT:0> and c . groups_downloaded is not None ) : <EOL> dev = float ( c . groups_downloaded ) / c . groups_available <EOL> th = settings . INCOMPLETE_CRAWL_WARNING_THRESHOLD <EOL> disp = int ( ( <NUM_LIT:1> - dev ) * <NUM_LIT:100> ) <EOL> if c . groups_available == c . groups_downloaded : <EOL> if i != lencrawls - <NUM_LIT:1> : <EOL> disp = '<STR_LIT>' <EOL> else : <EOL> disp = '<STR_LIT>' <EOL> else : <EOL> disp = '<STR_LIT>' . format ( disp ) + ( '<STR_LIT>' if dev < th else '<STR_LIT>' ) <EOL> if c . end_time == c . start_time : <EOL> end_time = now ( ) <EOL> else : <EOL> end_time = c . end_time <EOL> elapsed = round ( <EOL> float ( ( end_time - c . start_time ) . total_seconds ( ) ) / <NUM_LIT> , <NUM_LIT:1> ) <EOL> to_display = [ <EOL> c . start_time . strftime ( '<STR_LIT>' ) , <EOL> pad_string ( elapsed , <NUM_LIT:5> ) , <EOL> pad_string ( c . groups_available , <NUM_LIT:5> ) , <EOL> pad_string ( c . groups_downloaded , <NUM_LIT:5> ) , <EOL> pad_string ( c . hitgroupstatus_set . count ( ) , <NUM_LIT:5> ) , <EOL> disp , <EOL> ] <EOL> to_print . append ( '<STR_LIT>' . join ( map ( str , to_display ) ) ) <EOL> for r in to_print : <EOL> print r <EOL> time . sleep ( sleep ) <EOL> def get_unavailable_group_ids_percent ( period = <NUM_LIT:30> , periods = <NUM_LIT> ) : <EOL> results = [ ] <EOL> today = now ( ) . date ( ) <EOL> today -= datetime . timedelta ( days = today . day ) <EOL> for dt in datetime_range ( today , period = period , periods = periods ) : <EOL> dte = dt + datetime . timedelta ( days = period ) <EOL> items = HitGroupContent . objects . filter ( <EOL> first_crawl__start_time__gt = dt , first_crawl__start_time__lt = dte <EOL> ) . only ( '<STR_LIT>' ) <EOL> hashed = len ( [ i for i in items if i . group_id_hashed ] ) <EOL> percent = float ( hashed ) / len ( items ) * <NUM_LIT:100> if len ( items ) != <NUM_LIT:0> else <NUM_LIT:0> <EOL> results . append ( ( dt , percent ) ) <EOL> print "<STR_LIT>" . format ( * results [ - <NUM_LIT:1> ] ) <EOL> return results <EOL> def get_first_crawl_hits_posted_sums ( days = <NUM_LIT:200> ) : <EOL> results = [ ] <EOL> for i , dt in enumerate ( datetime_range ( now ( ) . date ( ) , periods = days ) , start = <NUM_LIT:1> ) : <EOL> tt = time . time ( ) <EOL> result = sum ( [ <EOL> sum ( [ <EOL> crawl . hitgroupstatus_set . get ( hit_group_content__id = content . id <EOL> ) . hits_available for content in crawl . hitgroupcontent_set . all ( ) <EOL> ] ) for crawl in <EOL> Crawl . objects . filter ( <EOL> start_time__gt = dt , <EOL> start_time__lt = dt + datetime . timedelta ( days = <NUM_LIT:1> ) ) <EOL> ] ) <EOL> results . append ( ( dt , result ) ) <EOL> print '<STR_LIT>' . format ( <EOL> i , dt , result , time . time ( ) - tt ) <EOL> return results <EOL> def convert_date_to_aware_utc_datetime ( dt ) : <EOL> return make_aware ( datetime . datetime . combine ( dt , datetime . time ( <NUM_LIT:0> ) ) , pytz . UTC ) <EOL> def datetime_range ( start = now ( ) , period = <NUM_LIT:1> , periods = <NUM_LIT:30> ) : <EOL> return [ convert_date_to_aware_utc_datetime ( <EOL> start - datetime . timedelta ( days = period * x ) ) for x in range ( <NUM_LIT:0> , periods ) ] </s>
