<reponame>thismakessand/alltheplaces<filename>locations/spiders/finishline.py<gh_stars>1-10
import scrapy
import re
from locations.items import GeojsonPointItem

class FinishlineSpider(scrapy.Spider):

    name = "finishline"
    allowed_domains = ["stores.finishline.com"]
    download_delay = 0.5
    start_urls = (
        'https://stores.finishline.com/',
    )

    def parse_stores(self, response):
        ref = re.findall(r"[^(\/)]+$", response.url)
        if (len(ref) > 0):
            ref = ref[0].split('.')[0]
        properties = {
            'addr_full': " ".join(response.xpath('//span[@itemprop="streetAddress"]/span/text()').extract()),
            'phone': response.xpath(
                'normalize-space(//span[@itemprop="telephone"]/text())').extract_first(),
            'city': response.xpath('normalize-space(//span[@itemprop="addressLocality"]/text())').extract_first(),
            'state': response.xpath('normalize-space(//abbr[@itemprop="addressRegion"]/text())').extract_first(),
            'postcode': response.xpath('normalize-space(//span[@itemprop="postalCode"]/text())').extract_first(),
            'ref': ref,
            'website': response.url,
            'lat': float(response.xpath('normalize-space(//meta[@itemprop="latitude"]/@content)').extract_first()),
            'lon': float(response.xpath('normalize-space(//meta[@itemprop="longitude"]/@content)').extract_first()),
        }
        hours = response.xpath('//tr[@itemprop="openingHours"]/@content').extract()
        if hours != []:
            hours = "; ".join(hours)
            properties['opening_hours'] = hours
        yield GeojsonPointItem(**properties)

    def parse_city_stores(self, response):
        stores = response.xpath('//a[contains(text(),"Shoe Store Details")]/@href').extract()
        for store in stores:
            yield scrapy.Request(response.urljoin(store), callback=self.parse_stores)

    def parse_state(self, response):
        urls = response.xpath('//div[@class="c-directory-list-content-wrapper"]/ul/li/a/@href').extract()
        for path in urls:
            path_split = len(path.split('/'))
            if (path_split == 2):
                yield scrapy.Request(response.urljoin(path), callback=self.parse_city_stores)
            else:
                yield scrapy.Request(response.urljoin(path), callback=self.parse_stores)
    def parse(self, response):
        urls = response.xpath('//div[@class="c-directory-list-content-wrapper"]/ul/li/a/@href').extract()
        for path in urls:
            path_split = len(path.split('/'))
            if (path_split==1):
                yield scrapy.Request(response.urljoin(path), callback=self.parse_state)
            elif (path_split==2):
                yield scrapy.Request(response.urljoin(path), callback=self.parse_city_stores)
            else:
                yield scrapy.Request(response.urljoin(path), callback=self.parse_stores)
import scrapy
import re
from locations.items import GeojsonPointItem

class FinishlineSpider(scrapy.Spider):

    name = "finishline"
    allowed_domains = ["stores.finishline.com"]
    download_delay = 0.5
    start_urls = (
        'https://stores.finishline.com/',
    )

    def parse_stores(self, response):
        ref = re.findall(r"[^(\/)]+$", response.url)
        if (len(ref) > 0):
            ref = ref[0].split('.')[0]
        properties = {
            'addr_full': " ".join(response.xpath('//span[@itemprop="streetAddress"]/span/text()').extract()),
            'phone': response.xpath(
                'normalize-space(//span[@itemprop="telephone"]/text())').extract_first(),
            'city': response.xpath('normalize-space(//span[@itemprop="addressLocality"]/text())').extract_first(),
            'state': response.xpath('normalize-space(//abbr[@itemprop="addressRegion"]/text())').extract_first(),
            'postcode': response.xpath('normalize-space(//span[@itemprop="postalCode"]/text())').extract_first(),
            'ref': ref,
            'website': response.url,
            'lat': float(response.xpath('normalize-space(//meta[@itemprop="latitude"]/@content)').extract_first()),
            'lon': float(response.xpath('normalize-space(//meta[@itemprop="longitude"]/@content)').extract_first()),
        }
        hours = response.xpath('//tr[@itemprop="openingHours"]/@content').extract()
        if hours != []:
            hours = "; ".join(hours)
            properties['opening_hours'] = hours
        yield GeojsonPointItem(**properties)

    def parse_city_stores(self, response):
        stores = response.xpath('//a[contains(text(),"Shoe Store Details")]/@href').extract()
        for store in stores:
            yield scrapy.Request(response.urljoin(store), callback=self.parse_stores)

    def parse_state(self, response):
        urls = response.xpath('//div[@class="c-directory-list-content-wrapper"]/ul/li/a/@href').extract()
        for path in urls:
            path_split = len(path.split('/'))
            if (path_split == 2):
                yield scrapy.Request(response.urljoin(path), callback=self.parse_city_stores)
            else:
                yield scrapy.Request(response.urljoin(path), callback=self.parse_stores)
    def parse(self, response):
        urls = response.xpath('//div[@class="c-directory-list-content-wrapper"]/ul/li/a/@href').extract()
        for path in urls:
            path_split = len(path.split('/'))
            if (path_split==1):
                yield scrapy.Request(response.urljoin(path), callback=self.parse_state)
            elif (path_split==2):
                yield scrapy.Request(response.urljoin(path), callback=self.parse_city_stores)
            else:
                yield scrapy.Request(response.urljoin(path), callback=self.parse_stores)