<s> __author__ = '<STR_LIT>' <EOL> from twisted . internet import reactor <EOL> import engine <EOL> from ants . crawl import scheduler <EOL> '''<STR_LIT>''' <EOL> class CrawlServer ( ) : <EOL> '''<STR_LIT>''' <EOL> STATUS_RUNNING = <NUM_LIT:1> <EOL> STATUS_STOP = <NUM_LIT:2> <EOL> def __init__ ( self , cluster_manager ) : <EOL> self . settings = cluster_manager . settings <EOL> self . cluster_manager = cluster_manager <EOL> self . spider_manager = cluster_manager . node_manager . spider_manager <EOL> self . status = self . STATUS_STOP <EOL> self . scheduler = scheduler . SchedulerServer ( cluster_manager . settings ) <EOL> self . running_spider_dict = dict ( ) <EOL> self . init_spider_dict = dict ( ) <EOL> self . stop_spider_dict = dict ( ) <EOL> self . distribute_index = <NUM_LIT:0> <EOL> self . distribute_node_list = list ( self . cluster_manager . cluster_info . node_list ) <EOL> def init_spider_job ( self , spider_name ) : <EOL> if spider_name in self . running_spider_dict : <EOL> raise RunningJobException ( ) <EOL> spider = self . spider_manager . create ( spider_name , settings = self . settings ) <EOL> crawl_engine = engine . EngineServer ( spider , self . cluster_manager , self . scheduler ) <EOL> self . running_spider_dict [ spider_name ] = crawl_engine <EOL> def run_engine ( self , spider_name ) : <EOL> del self . init_spider_dict [ spider_name ] <EOL> self . running_spider_dict [ spider_name ] . run ( ) <EOL> if self . status == self . STATUS_STOP : <EOL> self . status = self . STATUS_RUNNING <EOL> self . distribute ( ) <EOL> def init_init_job_dict ( self , spider_name , node_list ) : <EOL> self . init_spider_dict [ spider_name ] = list ( node_list ) <EOL> def remove_init_node ( self , spider_name , node_info ) : <EOL> self . init_spider_dict [ spider_name ] . remove ( node_info ) <EOL> def init_stop_job_dict ( self , spider_name , node_list ) : <EOL> self . stop_spider_dict [ spider_name ] = list ( node_list ) <EOL> def remove_stop_job_dict ( self , spider_name , node_info ) : <EOL> self . stop_spider_dict [ spider_name ] . remove ( node_info ) <EOL> def stop_engine ( self , spider_name ) : <EOL> self . running_spider_dict [ spider_name ] . stop ( ) <EOL> del self . running_spider_dict [ spider_name ] <EOL> if not self . running_spider_dict : <EOL> self . status = self . STATUS_STOP <EOL> def distribute ( self ) : <EOL> if self . status == self . STATUS_STOP : <EOL> return <EOL> request = self . scheduler . next_request ( ) <EOL> if request : <EOL> if '<STR_LIT>' in request . meta : <EOL> node_info = self . cluster_manager . cluster_info . get_node_by_name ( request . node_name ) <EOL> try : <EOL> self . distribute_node_list . remove ( node_info ) <EOL> except : <EOL> pass <EOL> else : <EOL> if not self . distribute_node_list : <EOL> self . distribute_node_list = list ( self . cluster_manager . cluster_info . node_list ) <EOL> node_info = self . distribute_node_list . pop ( ) <EOL> self . cluster_manager . node_manager . send_request_to_client ( request , node_info ) <EOL> self . running_spider_dict [ request . spider_name ] . log_distributed_request ( request , node_info ) <EOL> reactor . callLater ( <NUM_LIT:0> , self ) <EOL> def __call__ ( self , * args , ** kwargs ) : <EOL> self . distribute ( ) <EOL> def accept_request ( self , spider_name , request ) : <EOL> self . running_spider_dict [ spider_name ] . add_request ( request ) <EOL> def accept_crawl_result ( self , spider_name , node_name , request_hash_code , msg ) : <EOL> self . running_spider_dict [ spider_name ] . accept_request_result ( node_name , request_hash_code , msg ) <EOL> class CrawlClient ( ) : <EOL> '''<STR_LIT>''' <EOL> STATUS_RUNNING = <NUM_LIT:1> <EOL> STATUS_STOP = <NUM_LIT:2> <EOL> def __init__ ( self , node_manager ) : <EOL> self . settings = node_manager . settings <EOL> self . status = self . STATUS_STOP <EOL> self . node_manager = node_manager <EOL> self . spider_manager = self . node_manager . spider_manager <EOL> self . scheduler = scheduler . SchedulerClient ( node_manager . settings ) <EOL> self . running_spider_dict = dict ( ) <EOL> def spider_list ( self ) : <EOL> return self . spider_manager . list ( ) <EOL> def init_engine ( self , spider_name ) : <EOL> if spider_name in self . running_spider_dict : <EOL> raise RunningJobException ( ) <EOL> spider = self . spider_manager . create ( spider_name , settings = self . settings ) <EOL> crawl_engine = engine . EngineClient ( spider , self . node_manager , self . scheduler ) <EOL> self . running_spider_dict [ spider_name ] = crawl_engine <EOL> self . run ( ) <EOL> def stop_engine ( self , spider_name ) : <EOL> self . running_spider_dict [ spider_name ] . stop ( ) <EOL> del self . running_spider_dict [ spider_name ] <EOL> def run ( self ) : <EOL> if self . status == self . STATUS_STOP : <EOL> self . status = self . STATUS_RUNNING <EOL> self . distribute ( ) <EOL> def __call__ ( self , * args , ** kwargs ) : <EOL> self . distribute ( ) <EOL> def distribute ( self ) : <EOL> request = self . scheduler . next_request ( ) <EOL> if request : <EOL> self . running_spider_dict [ request . spider_name ] . crawl_request ( request ) <EOL> reactor . callLater ( <NUM_LIT:0> , self ) <EOL> def accept_request ( self , spider_name , request ) : <EOL> self . running_spider_dict [ spider_name ] . accept_request ( request ) <EOL> class RunningJobException ( Exception ) : <EOL> '''<STR_LIT>''' </s>
