This function finds URLs inside of valid HTML bytes.

def _html_find_urls(bytes, mimetype, base_url=None):
    """ This function finds URLs inside of valid HTML bytes. """

    def _recursive_tag_values(tag, values=[]):
        """ This sub-function recursively loops through all of the tags to get all of the attribute values. """

        if hasattr(tag, 'children'):
            for child in tag.children:
                if hasattr(child, 'attrs'):
                    for key in child.attrs:
                        if isinstance(child.attrs[key], list):
                            for value in child.attrs[key]:
                                values.append(value)
                        elif isinstance(child.attrs[key], str):
                            values.append(child.attrs[key])

                    values = _recursive_tag_values(child, values)

        return values

    # Only convert the ASCII bytes to HTML.
    ascii_bytes = b''.join(re.compile(b'[\x00\x09\x0A\x0D\x20-\x7E]{8,}').findall(bytes))
    ascii_bytes = ascii_bytes.replace(b'\x00', b'')

    # Store all of the URLs we find.
    urls = []

    # Convert the bytes into soup. Also try url decoding the bytes to bypass some obfuscation.
    soups = []
    soups.append(BeautifulSoup(ascii_bytes, 'lxml'))
    try:
        soups.append(BeautifulSoup(urllib.parse.unquote(str(ascii_bytes)), 'lxml'))
    except:
        pass

    # Loop over both soups.
    for soup in soups:

        # Find any URLs embedded in script + document.write JavaScript.
        # \r\n\r\ndocument.write(unescape('<meta HTTP-EQUIV="REFRESH" content="0; url=http://somebadsite.com/catalog/index.php">'));\r\n\r\n
        script_urls = []
        script_tags = soup.find_all('script')
        for script_tag in script_tags:
            for tag_content in script_tag.contents:
                if 'document.write' in tag_content.lower():
                    # Find the last position of the ( character, which should denote where the
                    # code that they are writing to the page begins. Also find the position of
                    # the first ) character which should denote where the code ends.
                    code_begin = tag_content.rfind('(')
                    code_end = tag_content.find(')')
                    code = tag_content[code_begin+1:code_end]
                    # Strip off any ' or " quotes.
                    if code.startswith('\'') and code.endswith('\''):
                        code = code[1:-1]
                    if code.startswith('"') and code.endswith('"'):
                        code = code[1:-1]
                    # Turn the string into bytes and feed it back into the _html_find_urls function.
                    code = code.encode()
                    script_urls += _html_find_urls(code, mimetype)

        # Find any URLs embedded in <script> tags. Use the _ascii_find_urls function to try and catch everything.
        for script_tag in script_tags:
            script_urls += _ascii_find_urls(str(script_tag).encode('utf-8'), mimetype)

        # Find any meta-refresh URLs.
        # <meta HTTP-Equiv="refresh" content="0; URL=UntitledNotebook1.html">
        meta_urls = []
        meta_tags = soup.find_all('meta')
        for meta_tag in meta_tags:
            for key in meta_tag.attrs:
                if key.lower() == 'content':
                    value = meta_tag.attrs[key]
                    if 'url=' in value.lower():
                        split_value = value.split('=', 1)
                        url = split_value[1]
                        # Remove any quotes around the URL.
                        if url.startswith('"') and url.endswith('"'):
                            url = url[1:-1]
                        if url.startswith("'") and url.endswith("'"):
                            url = url[1:-1]
                        meta_urls.append(url)

        # Hacky way to find URLs in the CSS.
        css_urls = re.compile(r'url\((.*?)\)').findall(str(soup))
        
        # Make sure none of the CSS URLs are in quotes.
        for u in css_urls[:]:
            if u.startswith('\'') and u.endswith('\''):
                css_urls.remove(u)
                css_urls.append(u[1:-1])
            if u.startswith('\"') and u.endswith('\"'):
                css_urls.remove(u)
                css_urls.append(u[1:-1])

        # Look to see if there is a "base" HTML tag specified. This is the "baseStriker" method.
        for tag in soup.find_all('base'):
            try:
                if tag['href']:
                    base_url = tag['href'].replace('\\\\', '//')
            except:
                pass

        # If we were given a base URL, only extract specific tag values that are likely
        # to be URLs. Otherwise, we would end up with joined URLs for every single tag
        # attribute value that exists in the HTML, which is not the correct behavior.
        if base_url:

            # Join any of the CSS URLs we found.
            for css_url in css_urls:
                urls.append(urljoin(base_url, css_url))

            # Join any of the script URLs we found.
            for script_url in script_urls:
                urls.append(urljoin(base_url, script_url))

            # Join any of the meta-refresh URLs we found.
            for meta_url in meta_urls:
                urls.append(urljoin(base_url, meta_url))

            # Get all of the action URLs.
            for tag in soup.find_all(action=True):
                urls.append(urljoin(base_url, tag['action']))

            # Get all of the background URLs.
            for tag in soup.find_all(background=True):
                urls.append(urljoin(base_url, tag['background']))

            # Get all of the href URLs.
            for tag in soup.find_all(href=True):
                urls.append(urljoin(base_url, tag['href']))

            # Get all of the src URLs.
            for tag in soup.find_all(src=True):
                urls.append(urljoin(base_url, tag['src']))

            # Get all of the xmlns URLs.
            for tag in soup.find_all(xmlns=True):
                urls.append(urljoin(base_url, tag['xmlns']))

        # We weren't given a base URL, so just search for URLs by getting every single
        # tag attribute value. That way we will catch everything regardless of the attribute name.
        else:
            urls = _recursive_tag_values(soup)
            urls += css_urls
            urls += meta_urls
            urls += script_urls

        # As a last-ditch effort, find URLs in the visible text of the HTML. However,
        # we only want to add strings that are valid URLs as they are. What we do not
        # want is to add every string as a potential URL, since if a base_url was given,
        # we will likely end up with a joined URL for every string found in the HTML.
        for s in soup.stripped_strings:
            if is_valid(s):
                urls.append(s)

        # Remove any newlines from the potential URLs. Some HTML has newlines
        # after the href=" part before the actual URL begins. This renders
        # correctly in web browsers but is otherwise considered an invalid
        # URL by the is_valid function.
        urls = [u.strip() for u in urls]

        # Remove any leading //'s from the URLs.
        urls = [u[2:] if u.startswith('//') else u for u in urls]

    return urls