<filename>scrapydm/scrapydm/spiders/douban.py
import scrapy

headers = """
Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9
Accept-Encoding: gzip, deflate, br
Accept-Language: zh-CN,zh;q=0.9,en;q=0.8
Cache-Control: max-age=0
Connection: keep-alive
Host: movie.douban.com
sec-ch-ua: " Not;A Brand";v="99", "Google Chrome";v="91", "Chromium";v="91"
sec-ch-ua-mobile: ?0
Sec-Fetch-Dest: document
Sec-Fetch-Mode: navigate
Sec-Fetch-Site: cross-site
Sec-Fetch-User: ?1
Upgrade-Insecure-Requests: 1
User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.114 Safari/537.36
"""

cookies = 'bid=HR1Um1_4ZfM; __gads=ID=aa1834a3d7528478-229b09520dca00a3:T=1624935267:RT=1624935267:S=ALNI_MYw6JcKQijDMwQcJYfK8eJb6vJwIw; ll="118282"; __utmc=30149280; __utmc=223695111; _vwo_uuid_v2=D0DA96F48BFA77A258909F59E9D98C2F0|b8229286f92259123676cfad79419e73; dbcl2="222545695:INXeCgog32A"; ck=e4C-; _pk_ref.100001.4cf6=%5B%22%22%2C%22%22%2C1625213875%2C%22https%3A%2F%2Fopen.weixin.qq.com%2F%22%5D; __utmz=30149280.1625213875.7.3.utmcsr=open.weixin.qq.com|utmccn=(referral)|utmcmd=referral|utmcct=/; __utma=223695111.1414003225.1624935266.1625207580.1625213875.6; __utmz=223695111.1625213875.6.2.utmcsr=open.weixin.qq.com|utmccn=(referral)|utmcmd=referral|utmcct=/; push_noty_num=0; push_doumail_num=0; _pk_id.100001.4cf6=bd99338ac7d3c0cc.1624935266.6.1625214424.1625209046.; __utma=30149280.479020663.1624346418.1625213875.1625217144.8; __utmt=1; __utmv=30149280.22254; __utmb=30149280.2.10.1625217144'


class DoubanSpider(scrapy.Spider):
    name = 'douban'

    def start_requests(self):
        _ = 'https://movie.douban.com/top250?start={}'

        for i in range(10):
            url = _.format(i * 25)
            yield scrapy.Request(url, callback=self.parser, headers=headers, cookies=cookies)

    def parser(self, response):

        parse_map = {
            'title': 'div.hd a span::text',
            'comm': 'div.info > div.bd p span.inq::text',
            'detail': 'div.hd a::attr(href)'
        }

        for i in response.css('div.item'):
            data = {}
            for k, v in parse_map.items():
                data[k] = i.css(v).extract()

            if data.get('detail'):
                # 高级特性： 直接使用 a 标签里面的 href
                yield response.follow(data.get('detail')[0], callback=self.parse_detail, meta=data, headers=headers)

        yield from response.follow_all(response.css('div.pic a'), callback=self.get_img, headers=headers)

    def parse_detail(self, response):

        short = response.xpath(
            '//*[@id="link-report"]/span[1]/span/text()[1] | //*[@id="link-report"]/span[1]/text()[1]').extract()

        data = response.meta
        data['introduction'] = short

        print(data)

    def get_img(self, response):
        print(response.url)

<filename>scrapydm/scrapydm/spiders/douban.py
import scrapy

headers = """
Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9
Accept-Encoding: gzip, deflate, br
Accept-Language: zh-CN,zh;q=0.9,en;q=0.8
Cache-Control: max-age=0
Connection: keep-alive
Host: movie.douban.com
sec-ch-ua: " Not;A Brand";v="99", "Google Chrome";v="91", "Chromium";v="91"
sec-ch-ua-mobile: ?0
Sec-Fetch-Dest: document
Sec-Fetch-Mode: navigate
Sec-Fetch-Site: cross-site
Sec-Fetch-User: ?1
Upgrade-Insecure-Requests: 1
User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.114 Safari/537.36
"""

cookies = 'bid=HR1Um1_4ZfM; __gads=ID=aa1834a3d7528478-229b09520dca00a3:T=1624935267:RT=1624935267:S=ALNI_MYw6JcKQijDMwQcJYfK8eJb6vJwIw; ll="118282"; __utmc=30149280; __utmc=223695111; _vwo_uuid_v2=D0DA96F48BFA77A258909F59E9D98C2F0|b8229286f92259123676cfad79419e73; dbcl2="222545695:INXeCgog32A"; ck=e4C-; _pk_ref.100001.4cf6=%5B%22%22%2C%22%22%2C1625213875%2C%22https%3A%2F%2Fopen.weixin.qq.com%2F%22%5D; __utmz=30149280.1625213875.7.3.utmcsr=open.weixin.qq.com|utmccn=(referral)|utmcmd=referral|utmcct=/; __utma=223695111.1414003225.1624935266.1625207580.1625213875.6; __utmz=223695111.1625213875.6.2.utmcsr=open.weixin.qq.com|utmccn=(referral)|utmcmd=referral|utmcct=/; push_noty_num=0; push_doumail_num=0; _pk_id.100001.4cf6=bd99338ac7d3c0cc.1624935266.6.1625214424.1625209046.; __utma=30149280.479020663.1624346418.1625213875.1625217144.8; __utmt=1; __utmv=30149280.22254; __utmb=30149280.2.10.1625217144'


class DoubanSpider(scrapy.Spider):
    name = 'douban'

    def start_requests(self):
        _ = 'https://movie.douban.com/top250?start={}'

        for i in range(10):
            url = _.format(i * 25)
            yield scrapy.Request(url, callback=self.parser, headers=headers, cookies=cookies)

    def parser(self, response):

        parse_map = {
            'title': 'div.hd a span::text',
            'comm': 'div.info > div.bd p span.inq::text',
            'detail': 'div.hd a::attr(href)'
        }

        for i in response.css('div.item'):
            data = {}
            for k, v in parse_map.items():
                data[k] = i.css(v).extract()

            if data.get('detail'):
                # 高级特性： 直接使用 a 标签里面的 href
                yield response.follow(data.get('detail')[0], callback=self.parse_detail, meta=data, headers=headers)

        yield from response.follow_all(response.css('div.pic a'), callback=self.get_img, headers=headers)

    def parse_detail(self, response):

        short = response.xpath(
            '//*[@id="link-report"]/span[1]/span/text()[1] | //*[@id="link-report"]/span[1]/text()[1]').extract()

        data = response.meta
        data['introduction'] = short

        print(data)

    def get_img(self, response):
        print(response.url)

Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9 Accept-Encoding: gzip, deflate, br Accept-Language: zh-CN,zh;q=0.9,en;q=0.8 Cache-Control: max-age=0 Connection: keep-alive Host: movie.douban.com sec-ch-ua: " Not;A Brand";v="99", "Google Chrome";v="91", "Chromium";v="91" sec-ch-ua-mobile: ?0 Sec-Fetch-Dest: document Sec-Fetch-Mode: navigate Sec-Fetch-Site: cross-site Sec-Fetch-User: ?1 Upgrade-Insecure-Requests: 1 User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.114 Safari/537.36 # 高级特性： 直接使用 a 标签里面的 href