<filename>bedbathandbeyond/spiders/bbbspidy.py
import scrapy
from ..items import BedbathandbeyondItem
from datetime import datetime  

class BbbspidySpider(scrapy.Spider):
	name = 'bbbspidy'
	allowed_domains = ['www.bedbathandbeyond.com.mx']
	headers = {
	  'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:98.0) Gecko/20100101 Firefox/98.0',
	  'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8',
	  'Accept-Language': 'en-CA,en-US;q=0.7,en;q=0.3',
	  'Accept-Encoding': 'gzip, deflate, br',
	  'Referer': 'https://www.bedbathandbeyond.com.mx/productos.html',
	  'Connection': 'keep-alive',
	  'Upgrade-Insecure-Requests': '1',
	  'Sec-Fetch-Dest': 'document',
	  'Sec-Fetch-Mode': 'navigate',
	  'Sec-Fetch-Site': 'same-origin',
	  'Sec-Fetch-User': '?1',
	  'Pragma': 'no-cache',
	  'Cache-Control': 'no-cache'
	}

	def start_requests(self):
		url = "https://www.bedbathandbeyond.com.mx/productos.html"
		yield scrapy.Request(url,headers=self.headers,callback=self.grab_menu)
		
	def grab_menu(self,response):
		categories = response.xpath("//div[contains(@class,'mainmenuwrap')]//ul[contains(@class,'clearfix')]/li")
		for category in categories:
			cat = category.xpath("./a/text()").get()
			sub_categories = category.xpath(".//div[@class='tabimgtext']")
			for sub_category in sub_categories:
				sub_cat = sub_category.xpath("./text()").get()
				sub_categories2 = sub_category.xpath("./../following-sibling::ul")
				if sub_categories2 == []: #subcategory2 doesnot exist 
					url = sub_category.xpath("./../@href").get() #we take the url of subcategory
					yield scrapy.Request(url,headers=self.headers,cb_kwargs={'category':cat,'sub_category':sub_cat,'sub_category2':""},callback=self.grab_product_urls)
				else:
					for sub_category2 in sub_categories2:
						for url,sub_cat2 in zip(sub_category2.xpath(".//a/@href"),sub_category2.xpath(".//a//text()")):
							yield scrapy.Request(url.get(),headers=self.headers,cb_kwargs={'category':cat,'sub_category':sub_cat,'sub_category2':sub_cat2.get()},callback=self.grab_product_urls)
							
		
	def grab_product_urls(self, response,category,sub_category,sub_category2):
		prod_urls = response.xpath("//div[@class='product-item-info']/a/@href").getall()
		for prod_url in prod_urls:
			yield scrapy.Request(prod_url,headers=self.headers,cb_kwargs={'category':category,'sub_category':sub_category,'sub_category2':sub_category2},callback=self.parse_product_page)
		
		#for pagination
		next_page = response.xpath("//a[@class='action  next']/@href").getall()
		if next_page != []:
			yield scrapy.Request(next_page[0],headers=self.headers,cb_kwargs={'category':category,'sub_category':sub_category,'sub_category2':sub_category2},callback=self.grab_product_urls)
		
	def parse_product_page(self,response,category,sub_category,sub_category2):
		price = response.xpath("//div[@class='product-info-main']//span[text()='Antes']/following-sibling::span/@data-price-amount").get()
		sale_price = response.xpath("//div[@class='product-info-main']//span[contains(@id,'product-price')]/@data-price-amount").get()
			
		if price != None and sale_price != None:
			final_price = min(float(price),float(sale_price))
		else:
			final_price = sale_price if sale_price != None else price
			if sale_price != None and price == None:
				price = sale_price 
				sale_price = None
				
		
		item = BedbathandbeyondItem(
			date = datetime.now(),
			canal = "Bed Bath and Beyond",
			category = category.capitalize(),
			subcategory = sub_category.capitalize(),
			subcategory2 = sub_category2.capitalize(),
			subcategory3 ="",
			marca = response.xpath("//td[@data-th='Marca']/text()").get(),
			modelo = "",
			sku = response.xpath("//div[@itemprop='sku']/text()").get(),
			upc = response.xpath("//td[@data-th='UPC']/text()").get(),
			item = response.xpath("//span[@itemprop='name']/text()").get(),
			item_characteristics = " | ".join(response.xpath("//div[@class='product attribute description']//text()").getall()),
			url_sku = response.url, 
			image =  response.xpath("//meta[@property='og:image:url']/@content").getall(),
			price = price,
			sale_price = sale_price,
			shipment_cost = "",
			sales_flag = response.xpath("//img[@class='label-image']/@src").getall(),
			store_id = "",
			store_name = "",
			store_address = "",
			stock = "",
			upc_wm = response.xpath("//div[@itemprop='sku']/text()").get().zfill(16),
			final_price  = final_price
		)
		
		yield item
		

<filename>bedbathandbeyond/spiders/bbbspidy.py
import scrapy
from ..items import BedbathandbeyondItem
from datetime import datetime  

class BbbspidySpider(scrapy.Spider):
	name = 'bbbspidy'
	allowed_domains = ['www.bedbathandbeyond.com.mx']
	headers = {
	  'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:98.0) Gecko/20100101 Firefox/98.0',
	  'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8',
	  'Accept-Language': 'en-CA,en-US;q=0.7,en;q=0.3',
	  'Accept-Encoding': 'gzip, deflate, br',
	  'Referer': 'https://www.bedbathandbeyond.com.mx/productos.html',
	  'Connection': 'keep-alive',
	  'Upgrade-Insecure-Requests': '1',
	  'Sec-Fetch-Dest': 'document',
	  'Sec-Fetch-Mode': 'navigate',
	  'Sec-Fetch-Site': 'same-origin',
	  'Sec-Fetch-User': '?1',
	  'Pragma': 'no-cache',
	  'Cache-Control': 'no-cache'
	}

	def start_requests(self):
		url = "https://www.bedbathandbeyond.com.mx/productos.html"
		yield scrapy.Request(url,headers=self.headers,callback=self.grab_menu)
		
	def grab_menu(self,response):
		categories = response.xpath("//div[contains(@class,'mainmenuwrap')]//ul[contains(@class,'clearfix')]/li")
		for category in categories:
			cat = category.xpath("./a/text()").get()
			sub_categories = category.xpath(".//div[@class='tabimgtext']")
			for sub_category in sub_categories:
				sub_cat = sub_category.xpath("./text()").get()
				sub_categories2 = sub_category.xpath("./../following-sibling::ul")
				if sub_categories2 == []: #subcategory2 doesnot exist 
					url = sub_category.xpath("./../@href").get() #we take the url of subcategory
					yield scrapy.Request(url,headers=self.headers,cb_kwargs={'category':cat,'sub_category':sub_cat,'sub_category2':""},callback=self.grab_product_urls)
				else:
					for sub_category2 in sub_categories2:
						for url,sub_cat2 in zip(sub_category2.xpath(".//a/@href"),sub_category2.xpath(".//a//text()")):
							yield scrapy.Request(url.get(),headers=self.headers,cb_kwargs={'category':cat,'sub_category':sub_cat,'sub_category2':sub_cat2.get()},callback=self.grab_product_urls)
							
		
	def grab_product_urls(self, response,category,sub_category,sub_category2):
		prod_urls = response.xpath("//div[@class='product-item-info']/a/@href").getall()
		for prod_url in prod_urls:
			yield scrapy.Request(prod_url,headers=self.headers,cb_kwargs={'category':category,'sub_category':sub_category,'sub_category2':sub_category2},callback=self.parse_product_page)
		
		#for pagination
		next_page = response.xpath("//a[@class='action  next']/@href").getall()
		if next_page != []:
			yield scrapy.Request(next_page[0],headers=self.headers,cb_kwargs={'category':category,'sub_category':sub_category,'sub_category2':sub_category2},callback=self.grab_product_urls)
		
	def parse_product_page(self,response,category,sub_category,sub_category2):
		price = response.xpath("//div[@class='product-info-main']//span[text()='Antes']/following-sibling::span/@data-price-amount").get()
		sale_price = response.xpath("//div[@class='product-info-main']//span[contains(@id,'product-price')]/@data-price-amount").get()
			
		if price != None and sale_price != None:
			final_price = min(float(price),float(sale_price))
		else:
			final_price = sale_price if sale_price != None else price
			if sale_price != None and price == None:
				price = sale_price 
				sale_price = None
				
		
		item = BedbathandbeyondItem(
			date = datetime.now(),
			canal = "Bed Bath and Beyond",
			category = category.capitalize(),
			subcategory = sub_category.capitalize(),
			subcategory2 = sub_category2.capitalize(),
			subcategory3 ="",
			marca = response.xpath("//td[@data-th='Marca']/text()").get(),
			modelo = "",
			sku = response.xpath("//div[@itemprop='sku']/text()").get(),
			upc = response.xpath("//td[@data-th='UPC']/text()").get(),
			item = response.xpath("//span[@itemprop='name']/text()").get(),
			item_characteristics = " | ".join(response.xpath("//div[@class='product attribute description']//text()").getall()),
			url_sku = response.url, 
			image =  response.xpath("//meta[@property='og:image:url']/@content").getall(),
			price = price,
			sale_price = sale_price,
			shipment_cost = "",
			sales_flag = response.xpath("//img[@class='label-image']/@src").getall(),
			store_id = "",
			store_name = "",
			store_address = "",
			stock = "",
			upc_wm = response.xpath("//div[@itemprop='sku']/text()").get().zfill(16),
			final_price  = final_price
		)
		
		yield item
		

#subcategory2 doesnot exist #we take the url of subcategory #for pagination