Manages recursive requests.
        Determines urls to recursivly crawl if they do not match certain file
        extensions and do not match a certain regex set in the config file.

        :param obj response: the response to extract any urls from
        :param obj spider: the crawler the callback should be called on
        :param str ignore_regex: a regex that should that any extracted url
                                 shouldn't match
        :param str ignore_file_extensions: a regex of file extensions that the
                                           end of any url may not match
        :return list: Scrapy Requests

def recursive_requests(response, spider, ignore_regex='',
                           ignore_file_extensions='pdf'):
        """
        Manages recursive requests.
        Determines urls to recursivly crawl if they do not match certain file
        extensions and do not match a certain regex set in the config file.

        :param obj response: the response to extract any urls from
        :param obj spider: the crawler the callback should be called on
        :param str ignore_regex: a regex that should that any extracted url
                                 shouldn't match
        :param str ignore_file_extensions: a regex of file extensions that the
                                           end of any url may not match
        :return list: Scrapy Requests
        """
        # Recursivly crawl all URLs on the current page
        # that do not point to irrelevant file types
        # or contain any of the given ignore_regex regexes
        return [
            scrapy.Request(response.urljoin(href), callback=spider.parse)
            for href in response.css("a::attr('href')").extract() if re.match(
                r'.*\.' + ignore_file_extensions +
                r'$', response.urljoin(href), re.IGNORECASE
            ) is None
            and len(re.match(ignore_regex, response.urljoin(href)).group(0)) == 0
        ]