File size: 571 Bytes
3c1be09 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 |
import os
import asyncio
from .asyncioPoliciesFix import decorator_asyncio_fix
from constants.headers import HEADER_AIO
HTTP_PROXY = os.environ.get("HTTP_PROXY", None)
class Scraper:
@decorator_asyncio_fix
async def _get_html(self, session, url):
try:
async with session.get(url, headers=HEADER_AIO, proxy=HTTP_PROXY) as r:
return await r.text()
except:
return None
async def get_all_results(self, session, url):
return await asyncio.gather(asyncio.create_task(self._get_html(session, url))) |