This function tokenizes the input bytes based on some common characters. It returns
        the tokens in ASCII format.

def _tokenize(bytes, mimetype, extra_tokens=True):
    """ This function tokenizes the input bytes based on some common characters. It returns
        the tokens in ASCII format. """

    # Start with only the ASCII bytes. Limit it to 8+ character strings.
    try:
        ascii_bytes = b' '.join(re.compile(b'[\x00\x09\x0A\x0D\x20-\x7E]{8,}').findall(bytes))
        ascii_bytes = ascii_bytes.replace(b'\x00', b'')
    except:
        return []

    tokens = []

    # Find anything sandwiched between ( )
    results = re.compile(b'\(([^\)]+)\)').findall(ascii_bytes)
    #results = re.compile(b'\(([^\(\)]+)\)').findall(ascii_bytes)
    tokens += results

    # Find anything sandwiched between < >
    results = re.compile(b'\<([^\>]+)\>').findall(ascii_bytes)
    #results = re.compile(b'\<([^\<\>]+)\>').findall(ascii_bytes)
    tokens += results

    # Find anything sandwiched between [ ]
    results = re.compile(b'\[([^\]]+)\]').findall(ascii_bytes)
    #results = re.compile(b'\[([^\[\]]+)\]').findall(ascii_bytes)
    tokens += results

    # Find anything sandwiched between { }
    results = re.compile(b'\{([^\}]+)\}').findall(ascii_bytes)
    #results = re.compile(b'\{([^\{\}]+)\}').findall(ascii_bytes)
    tokens += results

    if not mimetype == 'data':

        # Find anything sandwiched between ' '
        results = re.compile(b'\'\s*(.*?)\s*\'').findall(ascii_bytes)
        tokens += results

        # Find anything sandwiched between " "
        results = re.compile(b'\"\s*(.*?)\s*\"').findall(ascii_bytes)
        tokens += results

    # Find anything sandwiched between spaces
    results = re.compile(b'\ ([^\ ]+)\ ').findall(ascii_bytes)
    tokens += results

    # Break the bytes apart if we want to generate even more tokens.
    if extra_tokens:

        # Add each newline as a token.
        lines = ascii_bytes.decode('ascii', errors='ignore').splitlines()
        tokens += [l.encode('ascii') for l in lines]

        # Now replace these characters in the ASCII bytes with spaces to create more tokens.
        replace_these = [b'(', b')', b'<', b'>', b'[', b']', b'{', b'}', b"'", b'"', b'\n', b'\r', b'\t', b'\\', b'`', b';']
        for char in replace_these:
            ascii_bytes = ascii_bytes.replace(char, b' ')

        # Split the bytes on spaces and use the results as tokens.
        tokens += ascii_bytes.split(b' ')

    # Decode the tokens as ASCII.
    tokens = [token.decode('ascii', errors='ignore') for token in tokens]

    # Since we want this to find full URLs and not just potential domain names, we want
    # to ensure that the tokens have at least "http" or "/" in them.
    tokens = list(set([t for t in tokens if ('http' in t or 'ftp' in t or ('/' in t and '.' in t)) and len(t) >= 8]))

    # Remove any tokens that look like they're probably bad based on the regex statements above.
    tokens = [t for t in tokens if not all(x in t for x in ['(', ')'])]
    tokens = [t for t in tokens if not all(x in t for x in ['<', '>'])]
    tokens = [t for t in tokens if not all(x in t for x in ['[', ']'])]
    tokens = [t for t in tokens if not all(x in t for x in ['{', '}'])]
    tokens = [t for t in tokens if not any(x in t for x in ['\t', '\n', '\r'])]
    tokens = [t for t in tokens if t[:1].isalpha() or t[:1].isdigit()]
    tokens = [t for t in tokens if not t.count('\'') > 1]
    tokens = [t for t in tokens if not t.count('\"') > 1]

    # Return a list of the tokens in ASCII format.
    return tokens