﻿# -*- coding:utf-8 -*-

from sailing.core import Sailor, new_task, FileTask
#from sailing.core.web.website import WebSite
from sailing.conf import settings
from sailing.common.common import join_path, trackable, is_file
from sailing.common.utils import import_class
from sailing.core.concurrency import TaskScheduler
import logging
from web_client import WebClient
from robot_api import Robot
import thread

class Webrobot(Sailor):
    
    def ready(self):
        #self.http = HTTPClient()
        #self.http.set_proxy(settings.PROXY)
        self.logger = logging.getLogger("webrobot")
        
        self._web_clients = {}
        self._load_actions()
        self.robot_api = Robot()
        #thread.start_new_thread(self.robot_task_tacker, ())
        self.task_pool = None
        
        if settings.WORK_THREAD_COUNT > 1:
            self.task_pool = TaskScheduler(settings.WORK_THREAD_COUNT, self.logger)
        
        
        
    def _load_actions(self, ):
        self.actions = settings.ACTIONS
        for k, v in self.actions.iteritems():
            self.actions[k] = self._init_actions(*v)  

            
    def _init_actions(self, cls, param={}):
        cls = import_class(cls)
        
        if cls is None: raise RuntimeError, "Could not loading crawler '%s'." % cls
        
        return cls(**param)            

    def start(self, t):
        
        client_id = t.header('Client_id')
                
        for l in t.list_actions():
            client_id, action_id, action, args = self._parse_action(l, None)
            if self.task_pool is None:
                self._process_task(client_id, action_id, action, args)
            else:
                self.task_pool.add_task(self._process_task, (client_id, action_id, action, args), client_id)
        
        if self.task_pool is not None:
            self.logger.info("waiting all task done")
            while self.task_pool.pending_count() > 0:
                import time
                time.sleep(3)
            self.logger.info("all task done")
        
                
    def _process_task(self, client_id, action_id, action, args):
        self.logger.info("start to process task for id:%s" % client_id)        
        
        client = self._web_clients.get(client_id, None) 
        if client is None:
            client = WebClient(client_id, "", "clients")
            client.start_client()
            self._web_clients[client_id] = client

        try:
            handler = self.actions.get(action, None)
            self.logger.debug("[%s] %s, args:%s" % (client_id, action, str(args)))
            if handler is not None:
                result = handler(client, self.robot_api, **args)
                self.action_done(action_id, client, result, self.robot_api)
            else:
                self.logger.error("not found sipder action:'%s'", action)
                                    
        except Exception, e:
            self.logger.exception(e)

                
    def action_done(self, action_id, client, result, api):
        self.logger.debug("done [%s] %s on %s" % (result, action_id, client))
        api.action_done(task_id=action_id, result=result)

        
    def _parse_action(self, l, site):
        '''
        return [action, url, local_path, args]
        
        81711368->xxx@passwod=11,bb
        '''
        action_id = 0
        client_id, action = l.split("-->", 1)
        #url, save_as = url.strip(), save_as.strip()  
        
        if ':' in client_id:
            action_id, client_id = client_id.split(":", 1)
        
        if '@' in action:
            action_name, args = action.split("@", 1) 
        else:
            action_name = action
        
        kw = {}    
        for a in args.split(","):
            k, v = a.split("=", 1)
            kw[str(k.strip())] = v.strip()
        
                    
        return (client_id, action_id, action_name, kw)
    
    def robot_task_tacker(self):
        
        self.logger.info("start task tacker...")            
        
        from captcha.http_client import HTTPClient
        import time
        client = HTTPClient()
        while True:
            try:
                data = client.get("http://emopad.sinaapp.com/robot/app/task_list")
                
                self.logger.info("tracker new task:%s" % data)                
                
                if data.strip():
                    t = new_task("webrobot")
                    for line in data.split("\n"):
                        t.add_action(line.strip())
                    t.close()
                    t.status = 'waiting'
                time.sleep(30)
                
                from sailing.conf import settings as CONFIG
                task_list = FileTask.search(CONFIG.APP_NAME, "done", len=15)
                
                for t in task_list:
                    self.logger.info("remove done task:%s" % t._id)       
                    #t.remove()
                
            except:
                pass

    
﻿# -*- coding:utf-8 -*-

from sailing.core import Sailor, new_task, FileTask
#from sailing.core.web.website import WebSite
from sailing.conf import settings
from sailing.common.common import join_path, trackable, is_file
from sailing.common.utils import import_class
from sailing.core.concurrency import TaskScheduler
import logging
from web_client import WebClient
from robot_api import Robot
import thread

class Webrobot(Sailor):
    
    def ready(self):
        #self.http = HTTPClient()
        #self.http.set_proxy(settings.PROXY)
        self.logger = logging.getLogger("webrobot")
        
        self._web_clients = {}
        self._load_actions()
        self.robot_api = Robot()
        #thread.start_new_thread(self.robot_task_tacker, ())
        self.task_pool = None
        
        if settings.WORK_THREAD_COUNT > 1:
            self.task_pool = TaskScheduler(settings.WORK_THREAD_COUNT, self.logger)
        
        
        
    def _load_actions(self, ):
        self.actions = settings.ACTIONS
        for k, v in self.actions.iteritems():
            self.actions[k] = self._init_actions(*v)  

            
    def _init_actions(self, cls, param={}):
        cls = import_class(cls)
        
        if cls is None: raise RuntimeError, "Could not loading crawler '%s'." % cls
        
        return cls(**param)            

    def start(self, t):
        
        client_id = t.header('Client_id')
                
        for l in t.list_actions():
            client_id, action_id, action, args = self._parse_action(l, None)
            if self.task_pool is None:
                self._process_task(client_id, action_id, action, args)
            else:
                self.task_pool.add_task(self._process_task, (client_id, action_id, action, args), client_id)
        
        if self.task_pool is not None:
            self.logger.info("waiting all task done")
            while self.task_pool.pending_count() > 0:
                import time
                time.sleep(3)
            self.logger.info("all task done")
        
                
    def _process_task(self, client_id, action_id, action, args):
        self.logger.info("start to process task for id:%s" % client_id)        
        
        client = self._web_clients.get(client_id, None) 
        if client is None:
            client = WebClient(client_id, "", "clients")
            client.start_client()
            self._web_clients[client_id] = client

        try:
            handler = self.actions.get(action, None)
            self.logger.debug("[%s] %s, args:%s" % (client_id, action, str(args)))
            if handler is not None:
                result = handler(client, self.robot_api, **args)
                self.action_done(action_id, client, result, self.robot_api)
            else:
                self.logger.error("not found sipder action:'%s'", action)
                                    
        except Exception, e:
            self.logger.exception(e)

                
    def action_done(self, action_id, client, result, api):
        self.logger.debug("done [%s] %s on %s" % (result, action_id, client))
        api.action_done(task_id=action_id, result=result)

        
    def _parse_action(self, l, site):
        '''
        return [action, url, local_path, args]
        
        81711368->xxx@passwod=11,bb
        '''
        action_id = 0
        client_id, action = l.split("-->", 1)
        #url, save_as = url.strip(), save_as.strip()  
        
        if ':' in client_id:
            action_id, client_id = client_id.split(":", 1)
        
        if '@' in action:
            action_name, args = action.split("@", 1) 
        else:
            action_name = action
        
        kw = {}    
        for a in args.split(","):
            k, v = a.split("=", 1)
            kw[str(k.strip())] = v.strip()
        
                    
        return (client_id, action_id, action_name, kw)
    
    def robot_task_tacker(self):
        
        self.logger.info("start task tacker...")            
        
        from captcha.http_client import HTTPClient
        import time
        client = HTTPClient()
        while True:
            try:
                data = client.get("http://emopad.sinaapp.com/robot/app/task_list")
                
                self.logger.info("tracker new task:%s" % data)                
                
                if data.strip():
                    t = new_task("webrobot")
                    for line in data.split("\n"):
                        t.add_action(line.strip())
                    t.close()
                    t.status = 'waiting'
                time.sleep(30)
                
                from sailing.conf import settings as CONFIG
                task_list = FileTask.search(CONFIG.APP_NAME, "done", len=15)
                
                for t in task_list:
                    self.logger.info("remove done task:%s" % t._id)       
                    #t.remove()
                
            except:
                pass

    
# -*- coding:utf-8 -*- #from sailing.core.web.website import WebSite #self.http = HTTPClient() #self.http.set_proxy(settings.PROXY) #thread.start_new_thread(self.robot_task_tacker, ()) return [action, url, local_path, args]                  81711368->xxx@passwod=11,bb #url, save_as = url.strip(), save_as.strip() #t.remove()