class Config(MutableMapping):
    def __init__(self, __values=None, **kwargs):
        self.attrs = {}
        self.update(__values, **kwargs)

    def __getitem__(self, name):
        if name not in self:
            return None
        return self.attrs[name]

    def __contains__(self, name):
        return name in self.attrs

    def get(self, name, default=None):
        return self[name] if self[name] is not None else default

    def getbool(self, name, default=None):
        v = self.get(name, default)
        return getbool(v)

    def getint(self, name, default=None):
        v = self.get(name, default)
        return getint(v)

    def getfloat(self, name, default=None):
        v = self.get(name, default)
        return getfloat(v)

    def getlist(self, name, default=None):
        v = self.get(name, default)
        return getlist(v)

    def __setitem__(self, name, value):
        self.attrs[name] = value

    def set(self, name, value):
        self[name] = value

    def setdefault(self, k, default=None):
        if k not in self:
            self[k] = default

    def update(self, __values=None, **kwargs):
        if __values is not None:
            if isinstance(__values, Config):
                for name in __values:
                    self[name] = __values[name]
            else:
                for name, value in __values.items():
                    self[name] = value
        for k, v in kwargs.items():
            self[k] = v

    def delete(self, name):
        del self.attrs[name]

    def __delitem__(self, name):
        del self.attrs[name]

    def copy(self):
        return copy.deepcopy(self)

    def __iter__(self):
        return iter(self.attrs)

    def __len__(self):
        return len(self.attrs) DEFAULT_CONFIG = {
    'daemon': False,
    'log_level': 'info',
    'log_format': '%(asctime)s %(name)s [%(levelname)s] %(message)s',
    'log_dateformat': '[%Y-%m-%d %H:%M:%S %z]',
    'downloader': 'xpaw.downloader.Downloader',
    'user_agent': ':desktop',
    'random_user_agent': False,
    'retry_enabled': True,
    'stats_collector': 'xpaw.stats.StatsCollector',
    'queue': 'xpaw.queue.PriorityQueue',
    'dupe_filter': 'xpaw.dupefilter.HashDupeFilter',
    'default_extensions': [
        'xpaw.extensions.DefaultHeadersMiddleware',
        'xpaw.extensions.UserAgentMiddleware',
        'xpaw.extensions.RetryMiddleware',
        'xpaw.extensions.ProxyMiddleware',
        'xpaw.extensions.SpeedLimitMiddleware',
        'xpaw.extensions.DepthMiddleware',
    ]
} class CrawlerRunner:
    def __init__(self, crawler):
        self.crawler = crawler

        self._workers = None
        self._workers_done = None
        self._req_in_worker = None
        self._start_future = None
        self._supervisor_future = None
        self._is_running = False
        self._run_lock = None

    async def run(self):
        if self._is_running:
            return
        self._is_running = True
        await self._init()
        self._run_lock = asyncio.Future()
        await self._run_lock
        self._run_lock = None

    async def _init(self):
        await self.crawler.event_bus.send(events.crawler_start)
        self._supervisor_future = asyncio.ensure_future(self._supervisor())
        self._start_future = asyncio.ensure_future(self._generate_start_requests())
        downloader_clients = self.crawler.downloader.max_clients
        log.info("The maximum number of simultaneous clients: %s", downloader_clients)
        self._workers = []
        for i in range(downloader_clients):
            f = asyncio.ensure_future(self._download(i))
            self._workers.append(f)
        self._workers_done = set()
        self._req_in_worker = [None] * downloader_clients
        log.info('Crawler is initialized')

    def stop(self):
        if not self._is_running:
            return
        self._is_running = False
        asyncio.ensure_future(self._shutdown())

    async def _shutdown(self):
        log.info("Shutdown now")
        cancelled_futures = []
        if self._workers:
            for f in self._workers:
                f.cancel()
                cancelled_futures.append(f)
            self._workers = None
            self._workers_done = None
        if self._start_future:
            self._start_future.cancel()
            cancelled_futures.append(self._start_future)
            self._start_future = None
        if self._supervisor_future:
            self._supervisor_future.cancel()
            cancelled_futures.append(self._supervisor_future)
            self._supervisor_future = None
        # put back the unfinished requests
        if self._req_in_worker:
            for r in self._req_in_worker:
                if r:
                    await self.crawler.schedule(r)
            self._req_in_worker = None
        await self.crawler.event_bus.send(events.crawler_shutdown)
        # wait cancelled futures
        await asyncio.wait(cancelled_futures)
        log.info('Crawler is stopped')
        if self._run_lock:
            self._run_lock.set_result(True)

    async def _supervisor(self):
        while True:
            await asyncio.sleep(5)
            for i in range(len(self._workers)):
                f = self._workers[i]
                if f.done():
                    if i not in self._workers_done:
                        self._workers_done.add(i)
                        reason = "This future is cancelled" if f.cancelled() else str(f.exception())
                        log.error("Worker[%s] is shutdown: %s", i, reason)
                        self._req_in_worker[i] = None
            if self._all_done():
                break
        self.stop()

    def _all_done(self):
        if self._start_future.done() and len(self.crawler.queue) <= 0:
            no_active = True
            for i in range(len(self._workers)):
                if self._req_in_worker[i]:
                    no_active = False
                    break
            return no_active
        elif len(self._workers_done) == len(self._workers):
            log.error('No alive worker')
            return True
        return False

    async def _generate_start_requests(self):
        if hasattr(self.crawler.spider.start_requests, "cron_job"):
            tick = self.crawler.spider.start_requests.cron_tick
        else:
            tick = 0
        while True:
            t = time.time()
            reqs = await self.crawler.start_requests()
            for r in reqs:
                await self.crawler.schedule(r)
            if tick <= 0:
                break
            t = time.time() - t
            if t < tick:
                await asyncio.sleep(tick - t)

    async def _download(self, coro_id):
        while True:
            req = await self.crawler.next_request()
            log.debug("%s -> worker[%s]", req, coro_id)
            self._req_in_worker[coro_id] = req
            try:
                await self.crawler.fetch(req)
            except StopCrawler:
                self.stop()
                continue
            self._req_in_worker[coro_id] = None
            # check if it's all done
            if self._all_done():
                self.stop() class Crawler:
    def __init__(self, config):
        self.config = config
        self.event_bus = EventBus()
        self.stats_collector = self._instance_from_crawler(self.config.get('stats_collector'))
        self.queue = self._instance_from_crawler(self.config.get('queue'))
        self.dupe_filter = self._instance_from_crawler(self.config.get('dupe_filter'))
        self.downloader = self._instance_from_crawler(self.config.get('downloader'))
        self.spider = self._instance_from_crawler(self.config.get('spider'))
        assert isinstance(self.spider, Spider), 'spider must inherit from the Spider class'
        log.info('Spider: %s', str(self.spider))
        self.extension = ExtensionManager.from_crawler(self)
        log.info('Extensions: %s', self._log_objects(self.extension.components))

    async def start_requests(self):
        try:
            res = await self._start_requests()
        except CancelledError:
            raise
        except Exception:
            log.warning("Failed to get start requests", exc_info=True)
        else:
            return [r for r in res if isinstance(r, HttpRequest)]

    async def _start_requests(self):
        res = self.spider.start_requests()
        if inspect.iscoroutine(res):
            res = await res
        assert res is None or isiterable(res), \
            "Start requests must be None or an iterable object, got {}".format(type(res).__name__)
        result = await iterable_to_list(res)
        if result:
            res = await self.extension.handle_start_requests(result)
            result = await iterable_to_list(res)
        return result

    async def schedule(self, request):
        try:
            res = self.dupe_filter.is_duplicated(request)
            if inspect.iscoroutine(res):
                res = await res
            if not res:
                await self.event_bus.send(events.request_scheduled, request=request)
                await self.queue.push(request)
        except Exception:
            log.warning('Failed to schedule %s', request, exc_info=True)

    async def next_request(self):
        req = await self.queue.pop()
        return req

    async def fetch(self, req):

        try:
            resp = await self._fetch(req)
        except CancelledError:
            raise
        except Exception as e:
            if isinstance(e, IgnoreRequest):
                await self.event_bus.send(events.request_ignored, request=req, error=e)
            elif isinstance(e, (ClientError, HttpError)):
                log.info('Failed to make %s: %s', req, e)
            else:
                log.warning("Failed to request %s", req, exc_info=True)
            await self.spider.request_error(req, e)
        else:
            await self._handle_response(resp)

    async def _fetch(self, req):
        try:
            res = await self.extension.handle_request(req)
            if isinstance(res, HttpRequest):
                return res
            if res is None:
                res = await self.downloader.fetch(req)
        except CancelledError:
            raise
        except Exception as e:
            res = await self.extension.handle_error(req, e)
            if isinstance(res, Exception):
                raise res
        if isinstance(res, HttpResponse):
            _res = await self.extension.handle_response(req, res)
            if _res:
                res = _res
            # bind request
            res.request = req
        return res

    async def _handle_response(self, resp):
        if isinstance(resp, HttpRequest):
            await self.schedule(resp)
        elif isinstance(resp, HttpResponse):
            await self.event_bus.send(events.response_received, response=resp)
            try:
                result = await self._parse(resp)
            except CancelledError:
                raise
            except Exception as e:
                if isinstance(e, StopCrawler):
                    log.info('Request to stop crawler: %s', e)
                    raise
                elif isinstance(e, IgnoreRequest):
                    await self.event_bus.send(events.request_ignored, request=resp.request, error=e)
                else:
                    log.warning("Failed to parse %s", resp, exc_info=True)
            else:
                for r in result:
                    await self._handle_parsing_result(r)

    async def _parse(self, response):
        request = response.request
        try:
            try:
                await self.extension.handle_spider_input(response)
            except CancelledError:
                raise
            except Exception as e:
                await self.spider.request_error(request, e)
                raise e
            res = await self.spider.request_success(response)
            assert res is None or isiterable(res), \
                "Parsing result must be None or an iterable object, got {}".format(type(res).__name__)
            result = await iterable_to_list(res)
        except CancelledError:
            raise
        except Exception as e:
            res = await self.extension.handle_spider_error(response, e)
            if isinstance(res, Exception):
                raise res
            result = await iterable_to_list(res)
        if result:
            res = await self.extension.handle_spider_output(response, result)
            result = await iterable_to_list(res)
        return result

    async def _handle_parsing_result(self, result):
        if isinstance(result, HttpRequest):
            await self.schedule(result)
        elif isinstance(result, (BaseItem, dict)):
            try:
                await self.extension.handle_item(result)
            except CancelledError:
                raise
            except Exception as e:
                if isinstance(e, IgnoreItem):
                    await self.event_bus.send(events.item_ignored, item=result, error=e)
                else:
                    log.warning("Failed to handle %s", result, exc_info=True)
            else:
                await self.event_bus.send(events.item_scraped, item=result)

    def _instance_from_crawler(self, cls_path):
        obj_cls = load_object(cls_path)
        if inspect.isclass(obj_cls):
            if hasattr(obj_cls, "from_crawler"):
                obj = obj_cls.from_crawler(self)
            else:
                obj = obj_cls()
        else:
            obj = obj_cls
        return obj

    @staticmethod
    def _log_objects(objects):
        if objects:
            return ''.join(['\n\t({}/{}) {}'.format(i + 1, len(objects), o) for i, o in enumerate(objects)])
        return '' def configure_logger(name, config):
    log_level = config.get('log_level').upper()
    log_format = config.get('log_format')
    log_dateformat = config.get('log_dateformat')
    logger = logging.getLogger(name)
    logger.setLevel(log_level)
    log_file = config.get('log_file')
    if log_file:
        handler = logging.FileHandler(log_file)
    else:
        handler = logging.StreamHandler()
    handler.setLevel(log_level)
    formatter = logging.Formatter(log_format, log_dateformat)
    handler.setFormatter(formatter)
    logger.handlers.clear()
    logger.addHandler(handler)
    return logger def configure_tornado_logger(handlers):
    log = logging.getLogger('tornado')
    if log.handlers:
        return
    log.handlers = handlers
    log.setLevel('WARNING') def daemonize():
    if os.fork():
        os._exit(0)
    os.setsid()
    if os.fork():
        os._exit(0)
    os.umask(0o22)
    os.closerange(0, 3)
    fd_null = os.open(os.devnull, os.O_RDWR)
    if fd_null != 0:
        os.dup2(fd_null, 0)
    os.dup2(fd_null, 1)
    os.dup2(fd_null, 2) def load_config(fname):
    if fname is None or not isfile(fname):
        raise ValueError('{} is not a file'.format(fname))
    code = compile(open(fname, 'rb').read(), fname, 'exec')
    cfg = {
        "__builtins__": __builtins__,
        "__name__": "__config__",
        "__file__": fname,
        "__doc__": None,
        "__package__": None
    }
    exec(code, cfg, cfg)
    return cfg def iter_settings(config):
    for key, value in config.items():
        if not key.startswith('_'):
            yield key, value class RequestsSpider(Spider):
    def start_requests(self):
        requests = self.config.get('start_requests')
        i = 0
        for r in requests:
            if isinstance(r, str):
                r = HttpRequest(r)
            if isinstance(r, HttpRequest):
                r.meta['request_index'] = i
                r.dont_filter = True
                r.callback = self.parse
                r.errback = self.handle_error
                yield r
            else:
                self.logger.warning('Requests must be str or HttpRequest, got %s', type(r).__name__)
            i += 1

    def parse(self, response):
        results = self.config.get('results')
        results[response.meta['request_index']] = response

    def handle_error(self, request, err):
        results = self.config.get('results')
        results[request.meta['request_index']] = err
import logging
import os
import sys
import signal
from os.path import join, isfile
from tornado.ioloop import IOLoop
from .config import Config, DEFAULT_CONFIG
from .crawler import CrawlerRunner, Crawler
from .utils import configure_logger, configure_tornado_logger, daemonize, load_config, iter_settings
from .spider import RequestsSpider
# coding=utf-8




log = logging.getLogger(__name__)


def run_spider_project(proj_dir, **kwargs):
    run_crawler(proj_dir=proj_dir, config=kwargs)


def run_spider(spider, **kwargs):
    kwargs['spider'] = spider
    run_crawler(config=kwargs)


def run_crawler(proj_dir=None, config=None):
    _c = Config(DEFAULT_CONFIG)
    _c.update(load_project_config(proj_dir))
    _c.update(config)
    config = _c

    logger = configure_logger('xpaw', config)
    configure_tornado_logger(logger.handlers)
    if config.getbool('daemon'):
        daemonize()
    pid_file = config.get('pid_file')
    _write_pid_file(pid_file)
    try:

        crawler_runner = CrawlerRunner(Crawler(config))