ruia/spider.py
async def multiple_request(self, urls, is_gather=False, **kwargs):
        """For crawling multiple urls"""
        if is_gather:
            resp_results = await asyncio.gather(
                *[
                    self.handle_request(self.request(url=url, **kwargs))
                    for url in urls
                ],
                return_exceptions=True)
            for index, task_result in enumerate(resp_results):
                if not isinstance(task_result, RuntimeError) and task_result:
                    _, response = task_result
                    response.index = index
                    yield response
        else:
            for index, url in enumerate(urls):
                _, response = await self.handle_request(
                    self.request(url=url, **kwargs))
                response.index = index
                yield response
async def multiple_request(self, urls, is_gather=False, **kwargs):
        if is_gather:
            resp_results = await asyncio.gather(
                *[
                    self.handle_request(self.request(url=url, **kwargs))
                    for url in urls
                ],
                return_exceptions=True)
            for index, task_result in enumerate(resp_results):
                if not isinstance(task_result, RuntimeError) and task_result:
                    _, response = task_result
                    response.index = index
                    yield response
        else:
            for index, url in enumerate(urls):
                _, response = await self.handle_request(
                    self.request(url=url, **kwargs))
                response.index = index
                yield response
async def multiple_request ( self , urls , is_gather = False , * * kwargs ) : if is_gather : resp_results = await asyncio . gather ( * [ self . handle_request ( self . request ( url = url , * * kwargs ) ) for url in urls ] , return_exceptions = True ) for index , task_result in enumerate ( resp_results ) : if not isinstance ( task_result , RuntimeError ) and task_result : _ , response = task_result response . index = index yield response else : for index , url in enumerate ( urls ) : _ , response = await self . handle_request ( self . request ( url = url , * * kwargs ) ) response . index = index yield response
For crawling multiple urls
For crawling multiple urls