This function uses several methods to extract URLs from 'thing', which can be a string or raw bytes.
        If you supply the base URL, it will attempt to use it with urljoin to create full URLs from relative paths.

def find_urls(thing, base_url=None, mimetype=None, log=False):
    """ This function uses several methods to extract URLs from 'thing', which can be a string or raw bytes.
        If you supply the base URL, it will attempt to use it with urljoin to create full URLs from relative paths. """

    if log:
        logging.basicConfig(level=logging.INFO,
                            format='%(asctime)s %(name)s.%(funcName)s +%(lineno)s: %(levelname)-8s %(message)s')
        logger = logging.getLogger(__name__)

    # Convert "thing" to bytes if it is a string.
    try:
        if isinstance(thing, str):
            thing = thing.encode(encoding='ascii', errors='ignore')
    except:
        if log:
            logger.exception('Unable to convert thing to bytes.')
        return []

    # Store any URLs we find in the bytes.
    all_urls = []

    # Continue if we have bytes.
    if isinstance(thing, bytes):

        # Return an empty list if we failed to get the mimetype.
        try:
            if not mimetype:
                mimetype = magic.from_buffer(thing)
        except:
            if log:
                logger.exception('Unable to get mimetype from the bytes buffer.')
            return []

        mimetype = mimetype.lower()

        # If the bytes are HTML...
        if 'html' in mimetype:
            try:
                all_urls += _html_find_urls(thing, mimetype, base_url)
            except:
                if log:
                    logger.exception('Error when finding HTML URLs.')

        # If the bytes are a PDF...
        elif 'pdf' in mimetype:
            try:
                all_urls += _pdf_find_urls(thing, mimetype)
            except:
                if log:
                    logger.exception('Error when finding PDF URLs.')

        # If the bytes are an RFC 822 e-mail...
        elif 'rfc 822' in mimetype:
            return []

        # If the bytes are ASCII or Unicode text...
        elif 'ascii' in mimetype or 'text' in mimetype:
            try:
                all_urls += _html_find_urls(thing, mimetype, base_url)
            except:
                if log:
                    logger.exception('Error when finding ASCII/HTML URLs.')

            try:
                all_urls += _ascii_find_urls(thing, mimetype)
            except:
                if log:
                    logger.exception('Error when finding ASCII URLs.')

            try:
                all_urls += _pdf_find_urls(thing, mimetype)
            except:
                if log:
                    logger.exception('Error when finding ASCII/PDF URLs.')

        # If the bytes are anything else...
        else:

            # Try to treat the bytes as a PDF and find URLs.
            try:
                all_urls += _pdf_find_urls(thing, mimetype)
            except:
                if log:
                    logger.exception('Error when finding unknown/PDF URLs.')

            # If we don't know how to handle this mimetype, the bytes are likely just "data".
            # In that case, we don't want to find all possible ASCII URLs, as it will result
            # in a lot of bad URLs. Try to treat the bytes as ASCII and find URLs.
            try:
                all_urls += _ascii_find_urls(thing, mimetype, extra_tokens=False)
            except:
                if log:
                    logger.exception('Error when finding unknown/ASCII URLs.')

    # Make sure we only have valid URLs.
    valid_urls = []
    for url in list(set(all_urls)):
        try:
            # If the URL is valid as-is, just add it to the list.
            if is_valid(url):
                valid_urls.append(url)

            # The URL is not valid. If we were given a base URL, try joining them and checking if the result is valid.
            elif base_url:
                joined_url = urljoin(base_url, url)
                if is_valid(joined_url):
                    valid_urls.append(joined_url)
        except:
            pass

    # For the edge cases of HTML files where we didn't find any URLs, treat it as an ASCII file
    # and re-find any URLs that way.
    if not valid_urls and 'html' in mimetype:
        try:
            for url in _ascii_find_urls(thing, mimetype):
                # If the URL is valid as-is, just add it to the list.
                if is_valid(url):
                    valid_urls.append(url)

                # The URL is not valid. If we were given a base URL, try joining them and checking if the result is valid.
                elif base_url:
                    joined_url = urljoin(base_url, url)
                    if is_valid(joined_url):
                        valid_urls.append(joined_url)
        except:
            pass

    # Return the valid URLs in ASCII form.
    ascii_urls = []
    for url in valid_urls:
        try:
            if isinstance(url, str):
                ascii_urls.append(url)

            if isinstance(url, bytes):
                ascii_urls.append(url.decode('ascii', errors='ignore'))
        except:
            pass

    # Check if any of the URLs are Proofpoint URLs and try to decode them.
    for url in ascii_urls[:]:
        if 'urldefense.proofpoint.com/v2/url' in url:
            try:
                query_u=parse_qs(urlparse(url).query)['u'][0]
                decoded_url = query_u.replace('-3A', ':').replace('_', '/').replace('-2D', '-')
                if is_valid(decoded_url):
                    ascii_urls.append(decoded_url)
            except:
                if log:
                    logger.exception('Error decoding Proofpoint URL: {}'.format(url))

    # Check if any of the URLs are Outlook safelinks and try to decode them.
    for url in ascii_urls[:]:
        if 'safelinks.protection.outlook.com' in url:
            try:
                query_url=parse_qs(urlparse(url).query)['url'][0]
                decoded_url = urllib.parse.unquote(query_url)
                if is_valid(decoded_url):
                    ascii_urls.append(decoded_url)
            except:
                if log:
                    logger.exception('Error decoding Outlook safelinks URL: {}'.format(url))

    # Check if any of the URLs are Google redirection URLs and try to decode them.
    for url in ascii_urls[:]:
        if 'www.google.com/url?' in url:
            try:
                query_url=parse_qs(urlparse(url).query)['q'][0]
                decoded_url = urllib.parse.unquote(query_url)
                if is_valid(decoded_url):
                    ascii_urls.append(decoded_url)
            except:
                if log:
                    logger.exception('Error decoding Google redirection URL: {}'.format(url))

    # Add an unquoted version of each URL to the list.
    for url in ascii_urls[:]:
        ascii_urls.append(urllib.parse.unquote(url))

    # Add http:// to the beginning of each URL if it isn't there already. This lets us properly
    # catch URLs that may not have the scheme on the front of them.
    ascii_urls = ['http://' + u if not u.lower().startswith('http') and not u.lower().startswith('ftp') else u for u in ascii_urls]

    # Remove any trailing "/" from the URLs so that they are consistent with how they go in CRITS.
    ascii_urls = [u[:-1] if u.endswith('/') else u for u in ascii_urls]

    return sorted(list(set(ascii_urls)))