<s> import sys <EOL> import time <EOL> import logging <EOL> from optparse import make_option <EOL> from django . db import transaction <EOL> from django . core . management . base import BaseCommand <EOL> from django . conf import settings <EOL> from utils . pid import Pid <EOL> from utils . sql import query_to_tuples , execute_sql <EOL> log = logging . getLogger ( '<STR_LIT>' ) <EOL> class Command ( BaseCommand ) : <EOL> """<STR_LIT>""" <EOL> option_list = BaseCommand . option_list + ( <EOL> make_option ( "<STR_LIT>" , dest = "<STR_LIT>" , type = "<STR_LIT:int>" , <EOL> help = "<STR_LIT>" ) , <EOL> make_option ( "<STR_LIT>" , dest = "<STR_LIT>" , <EOL> default = False , action = "<STR_LIT:store_true>" , <EOL> help = "<STR_LIT>" <EOL> "<STR_LIT>" ) , <EOL> make_option ( "<STR_LIT>" , dest = "<STR_LIT>" , type = "<STR_LIT:int>" , <EOL> default = <NUM_LIT> , help = "<STR_LIT>" <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" ) , <EOL> make_option ( "<STR_LIT>" , dest = "<STR_LIT>" , <EOL> default = False , action = "<STR_LIT:store_true>" , <EOL> help = "<STR_LIT>" ) , <EOL> make_option ( "<STR_LIT>" , dest = "<STR_LIT:all>" , default = False , action = "<STR_LIT:store_true>" , <EOL> help = "<STR_LIT>" <EOL> "<STR_LIT>" ) , <EOL> make_option ( "<STR_LIT>" , dest = "<STR_LIT>" , default = False , <EOL> action = "<STR_LIT:store_true>" , <EOL> help = "<STR_LIT>" <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" ) , <EOL> ) <EOL> help = "<STR_LIT>" <EOL> def handle ( self , ** options ) : <EOL> self . start_time = time . time ( ) <EOL> log . info ( '<STR_LIT>' ) <EOL> pid = Pid ( '<STR_LIT>' , True ) <EOL> self . having_hits_mv = not options . get ( '<STR_LIT:all>' ) <EOL> self . chunk_size = options . get ( '<STR_LIT>' ) <EOL> self . chunked = not options . get ( '<STR_LIT>' ) <EOL> self . limit = options . get ( '<STR_LIT>' ) <EOL> self . fix_interrupted = options . get ( '<STR_LIT>' ) <EOL> self . fix_interrupted and self . update_interrupted_crawl_stats ( ) <EOL> self . crawl_count = self . get_crawls_count ( ) <EOL> if self . crawl_count == <NUM_LIT:0> or options . get ( '<STR_LIT>' ) : <EOL> self . handle_count_only ( ) <EOL> log . info ( '<STR_LIT>' <EOL> '<STR_LIT>' . format ( <EOL> '<STR_LIT>' . format ( self . limit ) if self . limit else '<STR_LIT>' , <EOL> self . crawl_count ) ) <EOL> ids = self . get_crawl_ids ( ) <EOL> deleted = self . do_deletes ( ids ) <EOL> log . info ( '<STR_LIT>' . format ( <EOL> self . time_elapsed ( ) , deleted ) ) <EOL> pid . remove_pid ( ) <EOL> def time_elapsed ( self ) : <EOL> return time . time ( ) - self . start_time <EOL> def handle_count_only ( self ) : <EOL> """<STR_LIT>""" <EOL> log . info ( '<STR_LIT>' . format ( self . crawl_count ) ) <EOL> sys . exit ( <NUM_LIT:0> ) <EOL> def get_crawl_ids ( self ) : <EOL> """<STR_LIT>""" <EOL> return [ a for ( a , ) in query_to_tuples ( self . __get_crawls_query ( ) ) ] <EOL> def get_crawls_count ( self ) : <EOL> """<STR_LIT>""" <EOL> return [ a for a in query_to_tuples ( self . __get_crawls_query ( <EOL> count_only = True ) ) ] [ <NUM_LIT:0> ] [ <NUM_LIT:0> ] <EOL> crawls_query = """<STR_LIT>""" <EOL> def __get_crawls_query ( self , what = '<STR_LIT:id>' , ordered = True , count_only = False ) : <EOL> if count_only : <EOL> ordered = False <EOL> what = '<STR_LIT>' <EOL> ordering = "<STR_LIT>" if ordered else "<STR_LIT>" <EOL> hmv = '<STR_LIT>' if self . having_hits_mv else '<STR_LIT>' <EOL> return self . crawls_query . format ( what = what , ordering = ordering , <EOL> having_hits_mv = hmv , <EOL> crawl_threshold = settings . INCOMPLETE_CRAWL_THRESHOLD ) <EOL> def do_deletes ( self , * args , ** kwargs ) : <EOL> """<STR_LIT>""" <EOL> log . info ( '<STR_LIT>' ) <EOL> self . delete_crawl_agregates ( * args , ** kwargs ) <EOL> log . info ( '<STR_LIT>' ) <EOL> if self . chunked : <EOL> return self . do_deletes_chunked ( * args , ** kwargs ) <EOL> else : <EOL> return self . do_deletes_simple ( * args , ** kwargs ) <EOL> def delete_crawl_agregates ( self , ids ) : <EOL> """<STR_LIT>""" <EOL> qq = self . __get_delete_queries ( [ '<STR_LIT>' ] , '<STR_LIT>' ) . next ( ) <EOL> execute_sql ( qq . format ( self . __chunk_str ( ids ) ) ) <EOL> transaction . commit_unless_managed ( ) <EOL> def do_deletes_simple ( self , ids ) : <EOL> """<STR_LIT>""" <EOL> qs = list ( self . __get_delete_queries ( [ '<STR_LIT>' , '<STR_LIT>' ] , '<STR_LIT:=>' ) ) <EOL> for i , crawl_id in enumerate ( ids , start = <NUM_LIT:1> ) : <EOL> if self . limit and i > self . limit : <EOL> break <EOL> for q in qs : <EOL> execute_sql ( q . format ( crawl_id ) ) <EOL> if i % <NUM_LIT:10> == <NUM_LIT:0> : <EOL> log . info ( ( "<STR_LIT>" <EOL> ) . format ( i , self . crawl_count , self . time_elapsed ( ) ) ) <EOL> transaction . commit_unless_managed ( ) <EOL> execute_sql ( ( "<STR_LIT>" <EOL> "<STR_LIT>" ) . format ( crawl_id ) ) <EOL> transaction . commit_unless_managed ( ) <EOL> return i <EOL> def read_chunks ( self , iterator , chunk_size = <NUM_LIT:10> , limit = None ) : <EOL> """<STR_LIT>""" <EOL> items = list ( ) <EOL> for i , crawl_id in enumerate ( iterator , start = <NUM_LIT:1> ) : <EOL> if limit and i > limit : <EOL> break <EOL> items . append ( crawl_id ) <EOL> if len ( items ) == chunk_size : <EOL> yield items <EOL> items = list ( ) <EOL> if items : <EOL> yield items <EOL> def __chunk_str ( self , ids ) : <EOL> return "<STR_LIT>" . format ( "<STR_LIT:U+002C>" . join ( [ str ( a ) for a in ids ] ) ) <EOL> def do_deletes_chunked ( self , ids ) : <EOL> """<STR_LIT>""" <EOL> processed = <NUM_LIT:0> <EOL> qs = list ( self . __get_delete_queries ( [ '<STR_LIT>' , '<STR_LIT>' ] , '<STR_LIT>' ) ) <EOL> for chunk in self . read_chunks ( <EOL> ids , limit = self . limit , chunk_size = self . chunk_size ) : <EOL> chunk_str = self . __chunk_str ( chunk ) <EOL> for q in qs : <EOL> execute_sql ( q . format ( chunk_str ) ) <EOL> execute_sql ( ( "<STR_LIT>" <EOL> "<STR_LIT>" ) . format ( chunk_str ) ) <EOL> processed += len ( chunk ) <EOL> log . info ( '<STR_LIT>' . format ( <EOL> chunk , processed , self . crawl_count , self . time_elapsed ( ) ) ) <EOL> transaction . commit_unless_managed ( ) <EOL> return processed <EOL> def __get_delete_queries ( self , tables , comparator = '<STR_LIT:=>' ) : <EOL> """<STR_LIT>""" <EOL> for t in tables : <EOL> yield "<STR_LIT>" . format ( <EOL> t , comparator ) <EOL> recount_query = """<STR_LIT>""" <EOL> def update_interrupted_crawl_stats ( self ) : <EOL> """<STR_LIT>""" <EOL> log . info ( '<STR_LIT>' <EOL> '<STR_LIT>' ) <EOL> execute_sql ( self . recount_query ) <EOL> transaction . commit_unless_managed ( ) </s>
