def parse(self, response):
        """Parse pagenated list of products"""

        # Check if page is out of range
        no_more_products = re.search(
            r'No matching products were found',
            response.css('div.paged-results').extract_first(),
            flags=re.I
        )
        if no_more_products:
            pass  # no more pages to populate, stop scraping

        else:
            # Scrape products list
            for product in response.css('article.product-list__item'):
                product_url = product.css('a::attr("href")').extract_first()
                yield response.follow(product_url, self.parse_detail)

            (base, params) = split_url(response.url)
            params.update({'page': int(params.get('page', '1')) + 1})
            next_page_url = join_url(base, params)
            self.logger.info(next_page_url)
            yield response.follow(next_page_url, self.parse)
def parse ( self , response ) : no_more_products = re . search ( r'No matching products were found' , response . css ( 'div.paged-results' ) . extract_first ( ) , flags = re . I ) if no_more_products : pass else : for product in response . css ( 'article.product-list__item' ) : product_url = product . css ( 'a::attr("href")' ) . extract_first ( ) yield response . follow ( product_url , self . parse_detail ) ( base , params ) = split_url ( response . url ) params . update ( { 'page' : int ( params . get ( 'page' , '1' ) ) + 1 } ) next_page_url = join_url ( base , params ) self . logger . info ( next_page_url ) yield response . follow ( next_page_url , self . parse )
Parse pagenated list of products