<s> import time <EOL> import logging <EOL> from utils . sql import query_to_dicts , execute_sql , query_to_tuples <EOL> log = logging . getLogger ( __name__ ) <EOL> def clean_duplicates ( ) : <EOL> ids = query_to_dicts ( "<STR_LIT>" ) <EOL> for id in ids : <EOL> log . info ( "<STR_LIT>" % id [ '<STR_LIT>' ] ) <EOL> execute_sql ( """<STR_LIT>""" % ( id [ '<STR_LIT>' ] , id [ '<STR_LIT>' ] ) ) <EOL> execute_sql ( """<STR_LIT>""" % ( id [ '<STR_LIT>' ] , id [ '<STR_LIT>' ] ) ) <EOL> execute_sql ( '<STR_LIT>' ) <EOL> def calculate_first_crawl_id ( ) : <EOL> progress = <NUM_LIT:10> <EOL> results = query_to_dicts ( "<STR_LIT>" ) <EOL> log . info ( '<STR_LIT>' ) <EOL> for i , r in enumerate ( results ) : <EOL> log . info ( "<STR_LIT>" % r [ '<STR_LIT:id>' ] ) <EOL> execute_sql ( """<STR_LIT>""" % r [ '<STR_LIT:id>' ] ) <EOL> if i % progress == <NUM_LIT:0> : <EOL> execute_sql ( '<STR_LIT>' ) <EOL> log . info ( "<STR_LIT>" % i ) <EOL> execute_sql ( '<STR_LIT>' ) <EOL> def update_diffs ( limit = <NUM_LIT:100> ) : <EOL> start_time = time . time ( ) <EOL> execute_sql ( """<STR_LIT>""" , <EOL> ( int ( limit ) , ) ) <EOL> execute_sql ( '<STR_LIT>' ) <EOL> log . debug ( '<STR_LIT>' , limit , ( time . time ( ) - start_time ) ) <EOL> def update_first_occured_agregates ( ) : <EOL> missing_crawls = query_to_tuples ( """<STR_LIT>""" ) <EOL> for row in missing_crawls : <EOL> crawl_id = row [ <NUM_LIT:0> ] <EOL> log . info ( "<STR_LIT>" % crawl_id ) <EOL> execute_sql ( """<STR_LIT>""" % crawl_id ) <EOL> execute_sql ( '<STR_LIT>' ) </s>
