<filename>sources/_down/9kqw.py<gh_stars>10-100
# -*- coding: utf-8 -*-
import logging
import re
from urllib.parse import parse_qsl, urlparse

from lncrawl.core.crawler import Crawler

logger = logging.getLogger(__name__)

chapter_details_url = 'https://9kqw.com/book/ajaxchap'


class TikNovelCrawler(Crawler):
    base_url = [
        'https://9kqw.com/',
        'http://www.tiknovel.com/',
        'https://www.tiknovel.com/',
    ]

    def initialize(self):
        self.base_url = 'https://9kqw.com/'
    # end def

    def read_novel_info(self):
        logger.debug('Visiting %s', self.novel_url)
        soup = self.get_soup(self.novel_url)

        possible_title = soup.select_one('#content .detail-wrap h1.detail-tit')
        assert possible_title, 'No novel title'
        self.novel_title = possible_title.text
        logger.info('Novel title: %s', self.novel_title)

        possible_authors = soup.select('#content table.detail-profile td')
        for td in possible_authors:
            if '作者' in td.find('strong').text:
                td.find('strong').extract()
                self.novel_author = td.text.strip()
                break
            # end if
        # end for
        logger.info('Novel author: %s', self.novel_author)

        possible_image = soup.select_one('#content .detail-thumb-box img')
        if possible_image:
            self.novel_cover = self.absolute_url(possible_image['src'])
        logger.info('Novel cover: %s', self.novel_cover)

        volumes = set()
        for a in soup.select('#content .contents-lst li a'):
            ch_id = int(a.find('span').text.strip())
            vol_id = 1 + (ch_id - 1) // 100
            volumes.add(vol_id)
            self.chapters.append({
                'id': ch_id,
                'volume': vol_id,
                'title': a['title'],
                'url':  self.absolute_url(a['href']),
            })
        # end for

        self.volumes = [{'id': x} for x in volumes]
    # end def

    def download_chapter_body(self, chapter):
        query_str = urlparse(chapter['url']).query
        data_params = {x[0]: int(x[1]) for x in parse_qsl(query_str)}
        logging.debug("Requesting body with: %s", data_params)
        response = self.submit_form(chapter_details_url, data=data_params)
        data = response.json()
        chap_desc = data['data']['chap']['desc']
        chap_desc = re.sub(r'((<br\/>)|\n)+', '\n\n', chap_desc, flags=re.I)
        contents = chap_desc.split('\n\n')
        contents = [p for p in contents if p and p.strip()]
        return '<p>' + '</p><p>'.join(contents) + '</p>'
    # end def
# end class

<filename>sources/_down/9kqw.py<gh_stars>10-100
# -*- coding: utf-8 -*-
import logging
import re
from urllib.parse import parse_qsl, urlparse

from lncrawl.core.crawler import Crawler

logger = logging.getLogger(__name__)

chapter_details_url = 'https://9kqw.com/book/ajaxchap'


class TikNovelCrawler(Crawler):
    base_url = [
        'https://9kqw.com/',
        'http://www.tiknovel.com/',
        'https://www.tiknovel.com/',
    ]

    def initialize(self):
        self.base_url = 'https://9kqw.com/'
    # end def

    def read_novel_info(self):
        logger.debug('Visiting %s', self.novel_url)
        soup = self.get_soup(self.novel_url)

        possible_title = soup.select_one('#content .detail-wrap h1.detail-tit')
        assert possible_title, 'No novel title'
        self.novel_title = possible_title.text
        logger.info('Novel title: %s', self.novel_title)

        possible_authors = soup.select('#content table.detail-profile td')
        for td in possible_authors:
            if '作者' in td.find('strong').text:
                td.find('strong').extract()
                self.novel_author = td.text.strip()
                break
            # end if
        # end for
        logger.info('Novel author: %s', self.novel_author)

        possible_image = soup.select_one('#content .detail-thumb-box img')
        if possible_image:
            self.novel_cover = self.absolute_url(possible_image['src'])
        logger.info('Novel cover: %s', self.novel_cover)

        volumes = set()
        for a in soup.select('#content .contents-lst li a'):
            ch_id = int(a.find('span').text.strip())
            vol_id = 1 + (ch_id - 1) // 100
            volumes.add(vol_id)
            self.chapters.append({
                'id': ch_id,
                'volume': vol_id,
                'title': a['title'],
                'url':  self.absolute_url(a['href']),
            })
        # end for

        self.volumes = [{'id': x} for x in volumes]
    # end def

    def download_chapter_body(self, chapter):
        query_str = urlparse(chapter['url']).query
        data_params = {x[0]: int(x[1]) for x in parse_qsl(query_str)}
        logging.debug("Requesting body with: %s", data_params)
        response = self.submit_form(chapter_details_url, data=data_params)
        data = response.json()
        chap_desc = data['data']['chap']['desc']
        chap_desc = re.sub(r'((<br\/>)|\n)+', '\n\n', chap_desc, flags=re.I)
        contents = chap_desc.split('\n\n')
        contents = [p for p in contents if p and p.strip()]
        return '<p>' + '</p><p>'.join(contents) + '</p>'
    # end def
# end class

# -*- coding: utf-8 -*- # end def # end if # end for # end for # end def # end def # end class