<s> from crawler import WebCrawler , log <EOL> import logging <EOL> import os <EOL> import hashlib <EOL> from miner . text . util import strip_control_characters <EOL> import MySQLdb <EOL> from MySQLdb import cursors <EOL> class FeedItemWebCrawler ( WebCrawler ) : <EOL> temp_outfile_path = '<STR_LIT>' <EOL> outfile_base_path = '<STR_LIT>' <EOL> hdfs_path = '<STR_LIT>' <EOL> def fetch_url ( self , line ) : <EOL> issue_id , url , data = line . split ( '<STR_LIT:\t>' ) <EOL> url = url . strip ( '<STR_LIT:">' ) <EOL> outfile = os . sep . join ( [ self . outfile_base_path , hashlib . md5 ( '<STR_LIT>' . join ( [ issue_id , url or data ] ) ) . hexdigest ( ) ] ) + '<STR_LIT>' <EOL> if url and not os . path . exists ( outfile ) : <EOL> new_data = super ( FeedItemWebCrawler , self ) . fetch_url ( url ) <EOL> if new_data : <EOL> data = new_data <EOL> if not os . path . exists ( outfile ) : <EOL> with open ( outfile , '<STR_LIT:w>' ) as f : <EOL> f . write ( '<STR_LIT:\t>' . join ( [ issue_id , strip_control_characters ( data ) ] ) ) <EOL> return '<STR_LIT>' <EOL> else : <EOL> return '<STR_LIT>' <EOL> @ classmethod <EOL> def write_to_temp ( cls , temp_file , host , user , password , db ) : <EOL> conn = MySQLdb . connect ( host = host , <EOL> user = user , <EOL> passwd = password , <EOL> db = db ) <EOL> feed_item_query = """<STR_LIT>""" % temp_file <EOL> cur = conn . cursor ( ) <EOL> cur . execute ( feed_item_query ) <EOL> if __name__ == '<STR_LIT:__main__>' : <EOL> from fabric . api import local <EOL> import time <EOL> from optparse import OptionParser <EOL> log . setLevel ( logging . DEBUG ) <EOL> parser = OptionParser ( ) <EOL> parser . add_option ( '<STR_LIT>' , '<STR_LIT>' , dest = '<STR_LIT>' ) <EOL> parser . add_option ( '<STR_LIT>' , '<STR_LIT>' , dest = '<STR_LIT>' ) <EOL> parser . add_option ( '<STR_LIT>' , '<STR_LIT>' , dest = '<STR_LIT>' , default = FeedItemWebCrawler . temp_outfile_path ) <EOL> parser . add_option ( '<STR_LIT>' , '<STR_LIT>' , dest = '<STR_LIT>' , default = '<STR_LIT:localhost>' ) <EOL> parser . add_option ( '<STR_LIT>' , '<STR_LIT>' , dest = '<STR_LIT>' , default = '<STR_LIT:root>' ) <EOL> parser . add_option ( '<STR_LIT>' , '<STR_LIT>' , dest = '<STR_LIT>' , default = '<STR_LIT>' ) <EOL> parser . add_option ( '<STR_LIT>' , '<STR_LIT>' , dest = '<STR_LIT>' , default = '<STR_LIT>' ) <EOL> options , args = parser . parse_args ( ) <EOL> crawler = FeedItemWebCrawler ( ) <EOL> local ( '<STR_LIT>' % options . tempfile_path ) <EOL> crawler . write_to_temp ( options . tempfile_path , options . mysql_host , options . mysql_user , options . mysql_password , options . mysql_database ) <EOL> log . info ( '<STR_LIT>' ) <EOL> t1 = time . time ( ) <EOL> for result in crawler . crawl ( open ( options . tempfile_path , '<STR_LIT:r>' ) ) : <EOL> log . info ( result ) <EOL> t2 = time . time ( ) <EOL> log . info ( '<STR_LIT>' % ( t2 - t1 ) ) <EOL> local ( '<STR_LIT>' % options . tempfile_path ) <EOL> local ( '<STR_LIT>' % { '<STR_LIT>' : options . tempfile_path , <EOL> '<STR_LIT>' : FeedItemWebCrawler . outfile_base_path } ) <EOL> local ( '<STR_LIT>' % ( options . hadoop_path , options . tempfile_path , options . hdfs_path ) ) </s>
