<filename>corpus_builder/spiders/newspaper/prothom_alo.py
# -*- coding: utf-8 -*-
import datetime

import scrapy

from corpus_builder.templates.spider import CommonSpider


class ProthomAloSpider(CommonSpider):
    name = "prothom_alo"
    allowed_domains = ["prothomalo.com"]
    base_url = "https://www.prothomalo.com"
    allowed_configurations = [
        ['start_page', 'end_page'],
        ['start_page'],
        ['archive', 'start_date'],
        ['archive', 'start_date', 'end_date'],
        ['category', 'start_page'],
        ['category', 'start_page', 'end_page']
    ]

    start_request_url = base_url

    content_body = {
        'xpath': '//article//p//text()'
    }

    def request_index(self, response):
        if not self.archive:
            categories = response.xpath('.//ul[@id=1]/li/a/@href').re('^[a-z0-9-]*$')
            categories.remove('todays-paper')

            # implement someday
            # subcategories = response.xpath('.//ul[@id=1]/li/a[@href="{0}"]/../ul/li/a/@href').re('^[a-z0-9-]*$'.format(
            #             category)

            if self.category:
                if self.category in categories:
                    categories = [self.category]
                else:
                    raise ValueError('invalid category slug. available slugs: \'%s\'' % "', '".join(categories))

            if self.start_page:
                for category in categories:
                    # http://www.prothom-alo.com/opinion/article?page=2
                    for page_number in range(self.start_page, self.end_page + 1):
                        url = self.base_url + '/{0}/article?page={1}'.format(category, page_number)
                        yield scrapy.Request(url, callback=self.extract_news_category)
        else:
            date_processing = self.start_date
            while date_processing <= self.end_date:
                url = self.base_url + '/archive/{0}'.format(date_processing.strftime('%Y-%m-%d'))
                yield scrapy.Request(url, callback=self.extract_news_archive)

                date_processing += datetime.timedelta(days=1)

    def extract_news_category(self, response):
        news_links = list(set(response.css('.blog_archive_widget a::attr("href")').extract()))

        for link in news_links:
            if not link[:4] == 'http':
                link = self.base_url + link
            yield scrapy.Request(link, callback=self.parse_content)

    def extract_news_archive(self, response):

        next_page_links = set(response.css('div.pagination a.next_page::attr("href")').extract())

        for link in next_page_links:
            yield scrapy.Request(link, callback=self.extract_news_archive)

        news_links = set(response.css('.contents_listing a::attr("href")').extract())

        for link in news_links:
            if not link[:4] == 'http':
                link = self.base_url + link
            yield scrapy.Request(link, callback=self.parse_content)

<filename>corpus_builder/spiders/newspaper/prothom_alo.py
# -*- coding: utf-8 -*-
import datetime

import scrapy

from corpus_builder.templates.spider import CommonSpider


class ProthomAloSpider(CommonSpider):
    name = "prothom_alo"
    allowed_domains = ["prothomalo.com"]
    base_url = "https://www.prothomalo.com"
    allowed_configurations = [
        ['start_page', 'end_page'],
        ['start_page'],
        ['archive', 'start_date'],
        ['archive', 'start_date', 'end_date'],
        ['category', 'start_page'],
        ['category', 'start_page', 'end_page']
    ]

    start_request_url = base_url

    content_body = {
        'xpath': '//article//p//text()'
    }

    def request_index(self, response):
        if not self.archive:
            categories = response.xpath('.//ul[@id=1]/li/a/@href').re('^[a-z0-9-]*$')
            categories.remove('todays-paper')

            # implement someday
            # subcategories = response.xpath('.//ul[@id=1]/li/a[@href="{0}"]/../ul/li/a/@href').re('^[a-z0-9-]*$'.format(
            #             category)

            if self.category:
                if self.category in categories:
                    categories = [self.category]
                else:
                    raise ValueError('invalid category slug. available slugs: \'%s\'' % "', '".join(categories))

            if self.start_page:
                for category in categories:
                    # http://www.prothom-alo.com/opinion/article?page=2
                    for page_number in range(self.start_page, self.end_page + 1):
                        url = self.base_url + '/{0}/article?page={1}'.format(category, page_number)
                        yield scrapy.Request(url, callback=self.extract_news_category)
        else:
            date_processing = self.start_date
            while date_processing <= self.end_date:
                url = self.base_url + '/archive/{0}'.format(date_processing.strftime('%Y-%m-%d'))
                yield scrapy.Request(url, callback=self.extract_news_archive)

                date_processing += datetime.timedelta(days=1)

    def extract_news_category(self, response):
        news_links = list(set(response.css('.blog_archive_widget a::attr("href")').extract()))

        for link in news_links:
            if not link[:4] == 'http':
                link = self.base_url + link
            yield scrapy.Request(link, callback=self.parse_content)

    def extract_news_archive(self, response):

        next_page_links = set(response.css('div.pagination a.next_page::attr("href")').extract())

        for link in next_page_links:
            yield scrapy.Request(link, callback=self.extract_news_archive)

        news_links = set(response.css('.contents_listing a::attr("href")').extract())

        for link in news_links:
            if not link[:4] == 'http':
                link = self.base_url + link
            yield scrapy.Request(link, callback=self.parse_content)

# -*- coding: utf-8 -*- # implement someday # subcategories = response.xpath('.//ul[@id=1]/li/a[@href="{0}"]/../ul/li/a/@href').re('^[a-z0-9-]*$'.format( #             category) # http://www.prothom-alo.com/opinion/article?page=2