<s> import MySQLdb <EOL> from bs4 import BeautifulSoup <EOL> import json <EOL> import re <EOL> import time <EOL> from math import ceil <EOL> import logging <EOL> import threading <EOL> import Queue <EOL> import ConfigParser <EOL> from util import get_content <EOL> class UpdateOneTopic ( threading . Thread ) : <EOL> def __init__ ( self , queue ) : <EOL> self . queue = queue <EOL> threading . Thread . __init__ ( self ) <EOL> cf = ConfigParser . ConfigParser ( ) <EOL> cf . read ( "<STR_LIT>" ) <EOL> host = cf . get ( "<STR_LIT>" , "<STR_LIT:host>" ) <EOL> port = int ( cf . get ( "<STR_LIT>" , "<STR_LIT:port>" ) ) <EOL> user = cf . get ( "<STR_LIT>" , "<STR_LIT:user>" ) <EOL> passwd = cf . get ( "<STR_LIT>" , "<STR_LIT>" ) <EOL> db_name = cf . get ( "<STR_LIT>" , "<STR_LIT>" ) <EOL> charset = cf . get ( "<STR_LIT>" , "<STR_LIT>" ) <EOL> use_unicode = cf . get ( "<STR_LIT>" , "<STR_LIT>" ) <EOL> self . db = MySQLdb . connect ( host = host , port = port , user = user , passwd = passwd , db = db_name , charset = charset , use_unicode = use_unicode ) <EOL> self . cursor = self . db . cursor ( ) <EOL> def run ( self ) : <EOL> while not self . queue . empty ( ) : <EOL> t = self . queue . get ( ) <EOL> link_id = t [ <NUM_LIT:0> ] <EOL> count_id = t [ <NUM_LIT:1> ] <EOL> self . find_new_question_by_topic ( link_id , count_id ) <EOL> def find_question_by_link ( self , topic_url , count_id ) : <EOL> content = get_content ( topic_url , count_id ) <EOL> if content == "<STR_LIT>" : <EOL> return <NUM_LIT:0> <EOL> soup = BeautifulSoup ( content ) <EOL> questions = soup . findAll ( '<STR_LIT:a>' , attrs = { '<STR_LIT:class>' : '<STR_LIT>' } ) <EOL> i = <NUM_LIT:0> <EOL> p_str = '<STR_LIT>' <EOL> anser_list = [ ] <EOL> time_now = int ( time . time ( ) ) <EOL> for question in questions : <EOL> tem_text = question . get_text ( ) <EOL> tem_id = question . get ( '<STR_LIT>' ) <EOL> tem_id = tem_id . replace ( '<STR_LIT>' , '<STR_LIT>' ) <EOL> anser_list = anser_list + [ ( tem_text , int ( tem_id ) , <NUM_LIT:0> , <NUM_LIT:0> , <NUM_LIT:0> , time_now , <NUM_LIT:0> ) ] <EOL> self . cursor . executemany ( p_str , anser_list ) <EOL> return self . cursor . rowcount <EOL> def find_new_question_by_topic ( self , link_id , count_id ) : <EOL> new_question_amount_total = <NUM_LIT:0> <EOL> for i in range ( <NUM_LIT:1> , <NUM_LIT:7> ) : <EOL> topic_url = '<STR_LIT>' + link_id + '<STR_LIT>' + str ( i ) <EOL> new_question_amount_one_page = self . find_question_by_link ( topic_url , count_id ) <EOL> new_question_amount_total = new_question_amount_total + new_question_amount_one_page <EOL> if new_question_amount_one_page <= <NUM_LIT:2> : <EOL> break <EOL> if count_id % <NUM_LIT:2> == <NUM_LIT:0> : <EOL> print str ( count_id ) + "<STR_LIT>" + self . getName ( ) + "<STR_LIT>" + link_id + "<STR_LIT>" + str ( i ) + "<STR_LIT>" + str ( new_question_amount_total ) + "<STR_LIT>" <EOL> time_now = int ( time . time ( ) ) <EOL> sql = "<STR_LIT>" <EOL> self . cursor . execute ( sql , ( time_now , link_id ) ) <EOL> class UpdateTopics : <EOL> def __init__ ( self ) : <EOL> cf = ConfigParser . ConfigParser ( ) <EOL> cf . read ( "<STR_LIT>" ) <EOL> host = cf . get ( "<STR_LIT>" , "<STR_LIT:host>" ) <EOL> port = int ( cf . get ( "<STR_LIT>" , "<STR_LIT:port>" ) ) <EOL> user = cf . get ( "<STR_LIT>" , "<STR_LIT:user>" ) <EOL> passwd = cf . get ( "<STR_LIT>" , "<STR_LIT>" ) <EOL> db_name = cf . get ( "<STR_LIT>" , "<STR_LIT>" ) <EOL> charset = cf . get ( "<STR_LIT>" , "<STR_LIT>" ) <EOL> use_unicode = cf . get ( "<STR_LIT>" , "<STR_LIT>" ) <EOL> self . topic_thread_amount = int ( cf . get ( "<STR_LIT>" , "<STR_LIT>" ) ) <EOL> self . db = MySQLdb . connect ( host = host , port = port , user = user , passwd = passwd , db = db_name , charset = charset , use_unicode = use_unicode ) <EOL> self . cursor = self . db . cursor ( ) <EOL> def run ( self ) : <EOL> time_now = int ( time . time ( ) ) <EOL> before_last_vist_time = time_now - <NUM_LIT:10> <EOL> queue = Queue . Queue ( ) <EOL> threads = [ ] <EOL> i = <NUM_LIT:0> <EOL> sql = "<STR_LIT>" <EOL> self . cursor . execute ( sql , ( before_last_vist_time , ) ) <EOL> results = self . cursor . fetchall ( ) <EOL> for row in results : <EOL> link_id = str ( row [ <NUM_LIT:0> ] ) <EOL> queue . put ( [ link_id , i ] ) <EOL> i = i + <NUM_LIT:1> <EOL> for i in range ( self . topic_thread_amount ) : <EOL> threads . append ( UpdateOneTopic ( queue ) ) <EOL> for i in range ( self . topic_thread_amount ) : <EOL> threads [ i ] . start ( ) <EOL> for i in range ( self . topic_thread_amount ) : <EOL> threads [ i ] . join ( ) <EOL> self . db . close ( ) <EOL> print '<STR_LIT>' <EOL> if __name__ == '<STR_LIT:__main__>' : <EOL> topic_spider = UpdateTopics ( ) <EOL> topic_spider . run ( ) </s>
