parsedmarc/utils.py
def get_base_domain(domain, use_fresh_psl=False):
    """
    Gets the base domain name for the given domain

    .. note::
        Results are based on a list of public domain suffixes at
        https://publicsuffix.org/list/public_suffix_list.dat.

    Args:
        domain (str): A domain or subdomain
        use_fresh_psl (bool): Download a fresh Public Suffix List

    Returns:
        str: The base domain of the given domain

    """
    psl_path = os.path.join(tempdir, "public_suffix_list.dat")

    def download_psl():
        url = "https://publicsuffix.org/list/public_suffix_list.dat"
        # Use a browser-like user agent string to bypass some proxy blocks
        headers = {"User-Agent": USER_AGENT}
        fresh_psl = requests.get(url, headers=headers).text
        with open(psl_path, "w", encoding="utf-8") as fresh_psl_file:
            fresh_psl_file.write(fresh_psl)

    if use_fresh_psl:
        if not os.path.exists(psl_path):
            download_psl()
        else:
            psl_age = datetime.now() - datetime.fromtimestamp(
                os.stat(psl_path).st_mtime)
            if psl_age > timedelta(hours=24):
                try:
                    download_psl()
                except Exception as error:
                    logger.warning(
                        "Failed to download an updated PSL {0}".format(error))
        with open(psl_path, encoding="utf-8") as psl_file:
            psl = publicsuffix2.PublicSuffixList(psl_file)

        return psl.get_public_suffix(domain)
    else:
        return publicsuffix2.get_public_suffix(domain)
def get_base_domain(domain, use_fresh_psl=False):
    psl_path = os.path.join(tempdir, "public_suffix_list.dat")

    def download_psl():
        url = "https://publicsuffix.org/list/public_suffix_list.dat"
        # Use a browser-like user agent string to bypass some proxy blocks
        headers = {"User-Agent": USER_AGENT}
        fresh_psl = requests.get(url, headers=headers).text
        with open(psl_path, "w", encoding="utf-8") as fresh_psl_file:
            fresh_psl_file.write(fresh_psl)

    if use_fresh_psl:
        if not os.path.exists(psl_path):
            download_psl()
        else:
            psl_age = datetime.now() - datetime.fromtimestamp(
                os.stat(psl_path).st_mtime)
            if psl_age > timedelta(hours=24):
                try:
                    download_psl()
                except Exception as error:
                    logger.warning(
                        "Failed to download an updated PSL {0}".format(error))
        with open(psl_path, encoding="utf-8") as psl_file:
            psl = publicsuffix2.PublicSuffixList(psl_file)

        return psl.get_public_suffix(domain)
    else:
        return publicsuffix2.get_public_suffix(domain)
def get_base_domain ( domain , use_fresh_psl = False ) : psl_path = os . path . join ( tempdir , "public_suffix_list.dat" ) def download_psl ( ) : url = "https://publicsuffix.org/list/public_suffix_list.dat" # Use a browser-like user agent string to bypass some proxy blocks headers = { "User-Agent" : USER_AGENT } fresh_psl = requests . get ( url , headers = headers ) . text with open ( psl_path , "w" , encoding = "utf-8" ) as fresh_psl_file : fresh_psl_file . write ( fresh_psl ) if use_fresh_psl : if not os . path . exists ( psl_path ) : download_psl ( ) else : psl_age = datetime . now ( ) - datetime . fromtimestamp ( os . stat ( psl_path ) . st_mtime ) if psl_age > timedelta ( hours = 24 ) : try : download_psl ( ) except Exception as error : logger . warning ( "Failed to download an updated PSL {0}" . format ( error ) ) with open ( psl_path , encoding = "utf-8" ) as psl_file : psl = publicsuffix2 . PublicSuffixList ( psl_file ) return psl . get_public_suffix ( domain ) else : return publicsuffix2 . get_public_suffix ( domain )
Gets the base domain name for the given domain

    .. note::
        Results are based on a list of public domain suffixes at
        https://publicsuffix.org/list/public_suffix_list.dat.

    Args:
        domain (str): A domain or subdomain
        use_fresh_psl (bool): Download a fresh Public Suffix List

    Returns:
        str: The base domain of the given domain
Gets the base domain name for the given domain