<filename>scrapper.py<gh_stars>0
"""
Scrapper implementation
"""

from datetime import datetime
import json
from pathlib import Path
import shutil


from bs4 import BeautifulSoup, element
import requests

from core_utils.article import Article
from constants import HEADERS, CRAWLER_CONFIG_PATH, ASSETS_PATH, HTTP_MAIN


class IncorrectURLError(Exception):
    """
    Seed URL does not match standard pattern
    """

class NumberOfArticlesOutOfRangeError(Exception):
    """
    Total number of articles to parse is too big
    """

class IncorrectNumberOfArticlesError(Exception):
    """
    Total number of articles to parse in not integer
    """


class Crawler:
    """
    Crawler implementation
    """
    def __init__(self, seed_urls, max_articles: int):
        self.seed_urls = seed_urls
        self.max_articles = max_articles
        self.urls = []

    def _extract_url(self, article_bs):
        main_bs = article_bs.find('div', {'class': 'article-tile-news'})
        if main_bs is None:
            main_bs = article_bs.find('div', {'class': 'wrap preview-section__rows preview-section__rows_3'})
        for url_bs in main_bs:
            if len(self.urls) < self.max_articles:
                if isinstance(url_bs, element.NavigableString):
                    break
                link = url_bs.find('a')
                full_link = link['href']
                pattern_and_link = HTTP_MAIN + full_link
                if pattern_and_link not in self.urls:
                    self.urls.append(pattern_and_link)

    def find_articles(self):
        """
        Finds articles
        """
        for seed_url in self.seed_urls:
            response = requests.get(url=seed_url, headers=HEADERS)
            if not response.ok:
                continue
            soup = BeautifulSoup(response.text, 'lxml')
            self._extract_url(soup)

    def get_search_urls(self):
        """
        Returns seed_urls param
        """
        pass

class HTMLParser:
    def __init__(self, article_url, article_id):
        self.article_url = article_url
        self.article_id = article_id
        self.article = Article(self.article_url, self.article_id)

    def _fill_article_with_text(self, article_bs):
        text_bs = article_bs.find('div', {'class': 'article-detail__content'})
        if text_bs is not None:
            self.article.text = text_bs.text

    def _fill_article_with_meta_information(self, article_bs):
        date = article_bs.find('div', {'class': 'breadcrumbs__date'})
        if date is not None:
            date_test = date.text.strip()
            date_for_meta = datetime.strptime(date_test, "%d.%m.%Y, %H:%M")
            self.article.date = date_for_meta

        author = article_bs.find('div', {'class': 'author-wrapper article-detail__author'})
        author_name = author.find('span', {'field': 'name'})
        if not author_name:
            author_name = 'Not Found'
        self.article.author = author_name.text.strip()

        title = article_bs.find('h1', {'class': 'article-detail__title'})
        if title is not None:
            self.article.title = title.text

        topics = article_bs.find_all('div', {'class': 'tags__item'})
        topic_list = []
        if topics is not None:
            for topic in topics:
                topic_list.append(topic.find('a').text.strip())
            self.article.topics = ','.join(topic_list)

        tags = article_bs.find_all('div', {'class': 'breadcrumbs__item'})
        tag_list = []
        if tags is not None:
            for tag in tags:
                tag_list.append(tag.find('a').text.strip())
            self.article.tags = ','.join(tag_list)

    def parse(self):
        response = requests.get(url=self.article_url, headers=HEADERS)
        article_bs = BeautifulSoup(response.text, 'lxml')

        self._fill_article_with_text(article_bs)
        self._fill_article_with_meta_information(article_bs)
        return self.article


def prepare_environment(base_path):
    """
    Creates ASSETS_PATH folder if not created and removes existing folder
    """
    path = Path(base_path)
    if path.exists():
        shutil.rmtree(base_path)
    path.mkdir(parents=True, exist_ok=True)


def validate_config(crawler_path):
    """
    Validates given config
    """
    with open(crawler_path) as file:
        config = json.load(file)
    seed_urls = config['seed_urls']
    total_articles = config["total_articles_to_find_and_parse"]

    if not seed_urls:
        raise IncorrectURLError

    for seed_url in seed_urls:
        pattern = 'https://esquire.ru/'
        if pattern not in seed_url:
            raise IncorrectURLError

    if not isinstance(seed_urls, list):
        raise IncorrectURLError

    if not isinstance(total_articles, int):
        raise IncorrectNumberOfArticlesError

    if total_articles <= 0:
        raise IncorrectNumberOfArticlesError

    if total_articles > 180:
        raise NumberOfArticlesOutOfRangeError

    return seed_urls, total_articles


if __name__ == '__main__':
    # YOUR CODE HERE
    given_seed_urls, given_max_articles = validate_config(CRAWLER_CONFIG_PATH)
    prepare_environment(ASSETS_PATH)

    crawler = Crawler(given_seed_urls, given_max_articles)
    crawler.find_articles()

    for id_of_article, url in enumerate(crawler.urls):
        article_parser = HTMLParser(url, id_of_article + 1)
        article = article_parser.parse()
        article.save_raw()

<filename>scrapper.py<gh_stars>0
"""
Scrapper implementation
"""

from datetime import datetime
import json
from pathlib import Path
import shutil


from bs4 import BeautifulSoup, element
import requests

from core_utils.article import Article
from constants import HEADERS, CRAWLER_CONFIG_PATH, ASSETS_PATH, HTTP_MAIN


class IncorrectURLError(Exception):
    """
    Seed URL does not match standard pattern
    """

class NumberOfArticlesOutOfRangeError(Exception):
    """
    Total number of articles to parse is too big
    """

class IncorrectNumberOfArticlesError(Exception):
    """
    Total number of articles to parse in not integer
    """


class Crawler:
    """
    Crawler implementation
    """
    def __init__(self, seed_urls, max_articles: int):
        self.seed_urls = seed_urls
        self.max_articles = max_articles
        self.urls = []

    def _extract_url(self, article_bs):
        main_bs = article_bs.find('div', {'class': 'article-tile-news'})
        if main_bs is None:
            main_bs = article_bs.find('div', {'class': 'wrap preview-section__rows preview-section__rows_3'})
        for url_bs in main_bs:
            if len(self.urls) < self.max_articles:
                if isinstance(url_bs, element.NavigableString):
                    break
                link = url_bs.find('a')
                full_link = link['href']
                pattern_and_link = HTTP_MAIN + full_link
                if pattern_and_link not in self.urls:
                    self.urls.append(pattern_and_link)

    def find_articles(self):
        """
        Finds articles
        """
        for seed_url in self.seed_urls:
            response = requests.get(url=seed_url, headers=HEADERS)
            if not response.ok:
                continue
            soup = BeautifulSoup(response.text, 'lxml')
            self._extract_url(soup)

    def get_search_urls(self):
        """
        Returns seed_urls param
        """
        pass

class HTMLParser:
    def __init__(self, article_url, article_id):
        self.article_url = article_url
        self.article_id = article_id
        self.article = Article(self.article_url, self.article_id)

    def _fill_article_with_text(self, article_bs):
        text_bs = article_bs.find('div', {'class': 'article-detail__content'})
        if text_bs is not None:
            self.article.text = text_bs.text

    def _fill_article_with_meta_information(self, article_bs):
        date = article_bs.find('div', {'class': 'breadcrumbs__date'})
        if date is not None:
            date_test = date.text.strip()
            date_for_meta = datetime.strptime(date_test, "%d.%m.%Y, %H:%M")
            self.article.date = date_for_meta

        author = article_bs.find('div', {'class': 'author-wrapper article-detail__author'})
        author_name = author.find('span', {'field': 'name'})
        if not author_name:
            author_name = 'Not Found'
        self.article.author = author_name.text.strip()

        title = article_bs.find('h1', {'class': 'article-detail__title'})
        if title is not None:
            self.article.title = title.text

        topics = article_bs.find_all('div', {'class': 'tags__item'})
        topic_list = []
        if topics is not None:
            for topic in topics:
                topic_list.append(topic.find('a').text.strip())
            self.article.topics = ','.join(topic_list)

        tags = article_bs.find_all('div', {'class': 'breadcrumbs__item'})
        tag_list = []
        if tags is not None:
            for tag in tags:
                tag_list.append(tag.find('a').text.strip())
            self.article.tags = ','.join(tag_list)

    def parse(self):
        response = requests.get(url=self.article_url, headers=HEADERS)
        article_bs = BeautifulSoup(response.text, 'lxml')

        self._fill_article_with_text(article_bs)
        self._fill_article_with_meta_information(article_bs)
        return self.article


def prepare_environment(base_path):
    """
    Creates ASSETS_PATH folder if not created and removes existing folder
    """
    path = Path(base_path)
    if path.exists():
        shutil.rmtree(base_path)
    path.mkdir(parents=True, exist_ok=True)


def validate_config(crawler_path):
    """
    Validates given config
    """
    with open(crawler_path) as file:
        config = json.load(file)
    seed_urls = config['seed_urls']
    total_articles = config["total_articles_to_find_and_parse"]

    if not seed_urls:
        raise IncorrectURLError

    for seed_url in seed_urls:
        pattern = 'https://esquire.ru/'
        if pattern not in seed_url:
            raise IncorrectURLError

    if not isinstance(seed_urls, list):
        raise IncorrectURLError

    if not isinstance(total_articles, int):
        raise IncorrectNumberOfArticlesError

    if total_articles <= 0:
        raise IncorrectNumberOfArticlesError

    if total_articles > 180:
        raise NumberOfArticlesOutOfRangeError

    return seed_urls, total_articles


if __name__ == '__main__':
    # YOUR CODE HERE
    given_seed_urls, given_max_articles = validate_config(CRAWLER_CONFIG_PATH)
    prepare_environment(ASSETS_PATH)

    crawler = Crawler(given_seed_urls, given_max_articles)
    crawler.find_articles()

    for id_of_article, url in enumerate(crawler.urls):
        article_parser = HTMLParser(url, id_of_article + 1)
        article = article_parser.parse()
        article.save_raw()

Scrapper implementation Seed URL does not match standard pattern Total number of articles to parse is too big Total number of articles to parse in not integer Crawler implementation Finds articles Returns seed_urls param Creates ASSETS_PATH folder if not created and removes existing folder Validates given config # YOUR CODE HERE