<s> import pymongo <EOL> import os <EOL> import sys <EOL> import logging <EOL> import argparse <EOL> script_path = os . path . abspath ( os . path . dirname ( __file__ ) ) <EOL> new_path = os . path . abspath ( ( os . path . join ( script_path , '<STR_LIT:..>' , '<STR_LIT>' ) ) ) <EOL> sys . path . append ( new_path ) <EOL> os . chdir ( new_path ) <EOL> from scrapy . utils . project import get_project_settings <EOL> from scrapy . exceptions import DropItem <EOL> from openrecipes . items import RecipeItem , RecipeItemLoader <EOL> from openrecipes . pipelines import RejectinvalidPipeline <EOL> def clean_item ( old_dict ) : <EOL> source_dict = dict ( old_dict ) <EOL> del source_dict [ '<STR_LIT>' ] <EOL> del source_dict [ '<STR_LIT>' ] <EOL> if VERBOSE : <EOL> print "<STR_LIT>" % ( old_dict [ '<STR_LIT:name>' ] , <EOL> old_dict [ '<STR_LIT:source>' ] , <EOL> old_dict [ '<STR_LIT>' ] ) <EOL> loader = RecipeItemLoader ( RecipeItem ( ) ) <EOL> for k , v in source_dict . iteritems ( ) : <EOL> loader = set_value ( loader , k , v ) <EOL> new_item = loader . load_item ( ) <EOL> return new_item , source_dict <EOL> def set_value ( loader , k , v ) : <EOL> try : <EOL> loader . replace_value ( k , v ) <EOL> except Exception , e : <EOL> logging . exception ( e ) <EOL> print "<STR_LIT>" % ( k , None ) <EOL> loader . replace_value ( k , None ) <EOL> return loader <EOL> def update_record ( conn_coll , new_dict , old_dict ) : <EOL> new_dict [ '<STR_LIT>' ] = old_dict [ '<STR_LIT>' ] <EOL> new_dict [ '<STR_LIT>' ] = old_dict [ '<STR_LIT>' ] <EOL> if VERBOSE : <EOL> print "<STR_LIT>" % ( old_dict [ '<STR_LIT:name>' ] , <EOL> old_dict [ '<STR_LIT:source>' ] , <EOL> old_dict [ '<STR_LIT>' ] ) <EOL> conn_coll . update ( { '<STR_LIT>' : new_dict [ '<STR_LIT>' ] } , new_dict ) <EOL> def output_changes ( new_item , old_dict ) : <EOL> print "<STR_LIT>" % ( old_dict [ '<STR_LIT:name>' ] , old_dict [ '<STR_LIT:source>' ] ) <EOL> for k , v in new_item . iteritems ( ) : <EOL> new_val = v <EOL> old_val = old_dict . get ( k , None ) <EOL> if new_val != old_val : <EOL> print "<STR_LIT>" % ( k , new_val ) <EOL> def get_args ( ) : <EOL> parser = argparse . ArgumentParser ( description = '<STR_LIT>' ) <EOL> parser . add_argument ( '<STR_LIT>' , dest = '<STR_LIT>' , action = '<STR_LIT:store_true>' , <EOL> help = '<STR_LIT>' ) <EOL> parser . add_argument ( '<STR_LIT>' , dest = '<STR_LIT>' , action = '<STR_LIT:store_true>' , <EOL> help = '<STR_LIT>' ) <EOL> args = parser . parse_args ( ) <EOL> return args <EOL> VERBOSE = False <EOL> UPDATE = False <EOL> if __name__ == '<STR_LIT:__main__>' : <EOL> args = get_args ( ) <EOL> VERBOSE = args . VERBOSE <EOL> UPDATE = args . UPDATE <EOL> settings = get_project_settings ( ) <EOL> mongo_uri = settings [ '<STR_LIT>' ] <EOL> mongo_db = settings [ '<STR_LIT>' ] <EOL> mongo_coll = settings [ '<STR_LIT>' ] <EOL> conn = pymongo . mongo_client . MongoClient ( mongo_uri ) <EOL> conn_db = conn [ mongo_db ] <EOL> conn_coll = conn_db [ mongo_coll ] <EOL> rip = RejectinvalidPipeline ( ) <EOL> for r_dict in conn_coll . find ( ) : <EOL> try : <EOL> new_item , source_dict = clean_item ( r_dict ) <EOL> if new_item != source_dict : <EOL> output_changes ( new_item , r_dict ) <EOL> try : <EOL> new_item = rip . process_item ( new_item , object ( ) ) <EOL> if UPDATE : <EOL> update_record ( conn_coll , dict ( new_item ) , r_dict ) <EOL> except DropItem , e : <EOL> print "<STR_LIT>" % ( r_dict [ "<STR_LIT>" ] , e . message ) <EOL> except Exception , e : <EOL> logging . exception ( e ) <EOL> print "<STR_LIT>" % ( r_dict [ "<STR_LIT>" ] , e . message ) </s>
