def attrs(self):
        """Attributes used when constructing url parameters."""
        data = {
            'client_id': self.client_id,
            'redirect_uri': quote(self.redirect_uri),
        }

        if self.scope is not None:
            data['scope'] = quote(self.scope)

        if self.state is not None:
            data['state'] = self.state

        return data

def attrs ( self ) : data = { 'client_id' : self . client_id , 'redirect_uri' : quote ( self . redirect_uri ) , } if self . scope is not None : data [ 'scope' ] = quote ( self . scope ) if self . state is not None : data [ 'state' ] = self . state return data

Attributes used when constructing url parameters .


def parameters(self) -> str:
        """URL parameters used."""
        return '&'.join('{0}={1}'.format(*item) for item in self.attrs.items())

def parameters ( self ) -> str : return '&' . join ( '{0}={1}' . format ( * item ) for item in self . attrs . items ( ) )


async def build(self):
        """get the track object for each link in the partial tracks data

        Returns
        -------
        tracks : List[Track]
            The tracks
        """
        data = await self.__func()
        return list(PlaylistTrack(self.__client, track) for track in data['items'])

async def build ( self ) : data = await self . __func ( ) return list ( PlaylistTrack ( self . __client , track ) for track in data [ 'items' ] )


async def get_all_tracks(self) -> List[PlaylistTrack]:
        """Get all playlist tracks from the playlist.

        Returns
        -------
        tracks : List[PlaylistTrack]
            The playlists tracks.
        """
        if isinstance(self._tracks, PartialTracks):
            return await self._tracks.build()

        _tracks = []
        offset = 0
        while len(self.tracks) < self.total_tracks:
            data = await self.__client.http.get_playlist_tracks(self.owner.id, self.id, limit=50, offset=offset)

            _tracks += [PlaylistTrack(self.__client, item) for item in data['items']]
            offset += 50

        self.total_tracks = len(self._tracks)
        return list(self._tracks)

async def get_all_tracks ( self ) -> List [ PlaylistTrack ] : if isinstance ( self . _tracks , PartialTracks ) : return await self . _tracks . build ( ) _tracks = [ ] offset = 0 while len ( self . tracks ) < self . total_tracks : data = await self . __client . http . get_playlist_tracks ( self . owner . id , self . id , limit = 50 , offset = offset ) _tracks += [ PlaylistTrack ( self . __client , item ) for item in data [ 'items' ] ] offset += 50 self . total_tracks = len ( self . _tracks ) return list ( self . _tracks )

Get all playlist tracks from the playlist .


async def resume(self, *, device: Optional[SomeDevice] = None):
        """Resume playback on the user's account.

        Parameters
        ----------
        device : Optional[:obj:`SomeDevice`]
            The Device object or id of the device this command is targeting.
            If not supplied, the user’s currently active device is the target.
        """
        await self._user.http.play_playback(None, device_id=str(device))

async def resume ( self , * , device : Optional [ SomeDevice ] = None ) : await self . _user . http . play_playback ( None , device_id = str ( device ) )


async def transfer(self, device: SomeDevice, ensure_playback: bool = False):
        """Transfer playback to a new device and determine if it should start playing.

        Parameters
        ----------
        device : :obj:`SomeDevice`
            The device on which playback should be started/transferred.
        ensure_playback : bool
            if `True` ensure playback happens on new device.
            else keep the current playback state.
        """
        await self._user.http.transfer_player(str(device), play=ensure_playback)

async def transfer ( self , device : SomeDevice , ensure_playback : bool = False ) : await self . _user . http . transfer_player ( str ( device ) , play = ensure_playback )

Transfer playback to a new device and determine if it should start playing .


async def from_href(self):
        """Get the full object from spotify with a `href` attribute."""
        if not hasattr(self, 'href'):
            raise TypeError('Spotify object has no `href` attribute, therefore cannot be retrived')

        elif hasattr(self, 'http'):
            return await self.http.request(('GET', self.href))

        else:
            cls = type(self)

        try:
            client = getattr(self, '_{0}__client'.format(cls.__name__))
        except AttributeError:
            raise TypeError('Spotify object has no way to access a HTTPClient.')
        else:
            http = client.http

        data = await http.request(('GET', self.href))

        return cls(client, data)

Get the full object from spotify with a href attribute .


def get(self):  # pragma: no cover
        """
        Execute the logic behind the meaning of ExpirationDate + return the matched status.

        :return:
            The status of the tested domain.
            Can be one of the official status.
        :rtype: str
        """

        # We get the status of the domain validation.
        domain_validation = self.checker.is_domain_valid()
        # We get the status of the IPv4 validation.
        ip_validation = self.checker.is_ip_valid()

        if "current_test_data" in PyFunceble.INTERN:
            # The end-user want more information whith his test.

            # We update some index.
            PyFunceble.INTERN["current_test_data"].update(
                {
                    "domain_syntax_validation": domain_validation,
                    "ip4_syntax_validation": ip_validation,
                }
            )

        if (
            domain_validation
            and not ip_validation
            or domain_validation
            or PyFunceble.CONFIGURATION["local"]
        ):
            # * The element is a valid domain.
            # and
            # * The element is not ahe valid IPv4.
            # or
            # * The element is a valid domain.

            # * We get the HTTP status code of the currently tested element.
            # and
            # * We try to get the element status from the IANA database.
            PyFunceble.INTERN.update(
                {"http_code": HTTPCode().get(), "referer": Referer().get()}
            )

            if not PyFunceble.INTERN["referer"]:
                # We could not get the referer.

                # We parse the referer status into the upstream call.
                return PyFunceble.INTERN["referer"]

            # The WHOIS record status is not into our list of official status.

            if PyFunceble.INTERN["referer"] and not self.checker.is_subdomain():
                # * The iana database comparison status is not None.
                # and
                # * The domain we are testing is not a subdomain.

                # We try to extract the expiration date from the WHOIS record.
                # And we return the matched status.
                return self._extract()

            # The iana database comparison status is None.

            # We log our whois record if the debug mode is activated.
            Logs().whois(self.whois_record)

            # And we return None, we could not extract the expiration date.
            return None

        if (
            ip_validation
            and not domain_validation
            or ip_validation
            or PyFunceble.CONFIGURATION["local"]
        ):
            # * The element is a valid IPv4.
            # and
            # * The element is not a valid domain.
            # or
            # * The element is a valid IPv4.

            # We get the HTTP status code.
            PyFunceble.INTERN["http_code"] = HTTPCode().get()

            # We log our whois record if the debug mode is activated.
            Logs().whois(self.whois_record)

            # And we return None, there is no expiration date to look for.
            return None

        # The validation was not passed.

        # We log our whois record if the debug mode is activated.
        Logs().whois(self.whois_record)

        # And we return False, the domain could not pass the IP and domains syntax validation.
        return False

Execute the logic behind the meaning of ExpirationDate + return the matched status .


def _convert_or_shorten_month(cls, data):
        """
        Convert a given month into our unified format.

        :param data: The month to convert or shorten.
        :type data: str

        :return: The unified month name.
        :rtype: str
        """

        # We map the different month and their possible representation.
        short_month = {
            "jan": [str(1), "01", "Jan", "January"],
            "feb": [str(2), "02", "Feb", "February"],
            "mar": [str(3), "03", "Mar", "March"],
            "apr": [str(4), "04", "Apr", "April"],
            "may": [str(5), "05", "May"],
            "jun": [str(6), "06", "Jun", "June"],
            "jul": [str(7), "07", "Jul", "July"],
            "aug": [str(8), "08", "Aug", "August"],
            "sep": [str(9), "09", "Sep", "September"],
            "oct": [str(10), "Oct", "October"],
            "nov": [str(11), "Nov", "November"],
            "dec": [str(12), "Dec", "December"],
        }

        for month in short_month:
            # We loop through our map.

            if data in short_month[month]:
                # If the parsed data (or month if you prefer) is into our map.

                # We return the element (or key if you prefer) assigned to
                # the month.
                return month

        # The element is not into our map.

        # We return the parsed element (or month if you prefer).
        return data

Convert a given month into our unified format .


def _update_code_urls(self):
        """
        Read the code and update all links.
        """

        to_ignore = [".gitignore", ".keep"]

        for root, _, files in PyFunceble.walk(
            PyFunceble.CURRENT_DIRECTORY
            + PyFunceble.directory_separator
            + "PyFunceble"
            + PyFunceble.directory_separator
        ):
            # We loop through every directories and files in the `PyFunceble` directory.

            for file in files:
                # We loop through the list of files of the currently read directory.

                if file not in to_ignore and "__pycache__" not in root:
                    # * The filename is not into the list of file to ignore.
                    # and
                    # * The directory we are reading is not `__pycache__`.

                    if root.endswith(PyFunceble.directory_separator):
                        # The root directory ends with the directory separator.

                        # We fix the path in the currently read file.
                        self._update_docs(root + file)
                    else:
                        # The root directory does not ends with the directory separator.

                        # We fix the path in the currently read file.
                        # (after appending the directory separator between the root and file)
                        self._update_docs(root + PyFunceble.directory_separator + file)

        for root, _, files in PyFunceble.walk(
            PyFunceble.CURRENT_DIRECTORY
            + PyFunceble.directory_separator
            + "tests"
            + PyFunceble.directory_separator
        ):
            # We loop through every directories and files in the `tests` directory.
            for file in files:
                # We loop through the list of files of the currently read directory.

                if file not in to_ignore and "__pycache__" not in root:
                    # * The filename is not into the list of file to ignore.
                    # and
                    # * The directory we are reading is not `__pycache__`.

                    if root.endswith(PyFunceble.directory_separator):
                        # The root directory ends with the directory separator.

                        # We fix the path in the currently read file.
                        self._update_docs(root + file)
                    else:
                        # The root directory does not ends with the directory separator.

                        # We fix the path in the currently read file.
                        # (after appending the directory separator between the root and file)
                        self._update_docs(root + PyFunceble.directory_separator + file)


def _is_version_greater(self):
        """
        Check if the current version is greater as the older older one.
        """

        # we compare the 2 versions.
        checked = Version(True).check_versions(
            self.current_version[0], self.version_yaml
        )

        if checked is not None and not checked:
            # The current version is greater as the older one.

            # We return True.
            return True

        # We return False
        return False

Check if the current version is greater as the older older one .


def is_dev_version(cls):
        """
        Check if the current branch is `dev`.
        """

        # We initiate the command we have to run in order to
        # get the branch we are currently working with.
        command = "git branch"

        # We execute and get the command output.
        command_result = Command(command).execute()

        for branch in command_result.split("\n"):
            # We loop through each line of the command output.

            if branch.startswith("*") and "dev" in branch:
                # The current branch is `dev`.

                # We return True.
                return True

        # The current branch is not `dev`.

        # We return False.
        return False


def _does_require_deprecation(self):
        """
        Check if we have to put the previous version into the deprecated list.
        """

        for index, version_number in enumerate(self.current_version[0][:2]):
            # We loop through the 2 last elements of the version.

            if version_number > self.version_yaml[index]:
                # The currently read version number is greater than the one we have in
                # the version.yaml.

                # We return True.
                return True

        # We return False, we do not need to deprecate anything.
        return False

Check if we have to put the previous version into the deprecated list .


def backup(self):
        """
        Backup the current execution state.
        """

        if PyFunceble.CONFIGURATION["auto_continue"]:
            # The auto_continue subsystem is activated.

            # We initiate the location where we are going to save the data to backup.
            data_to_backup = {}
            # We get the current counter states.
            configuration_counter = PyFunceble.INTERN["counter"]["number"]

            # We initiate the data we have to backup.
            data_to_backup[PyFunceble.INTERN["file_to_test"]] = {
                # We backup the number of tested.
                "tested": configuration_counter["tested"],
                # We backup the number of up.
                "up": configuration_counter["up"],
                # We backup the number of down.
                "down": configuration_counter["down"],
                # We backup the number of invalid.
                "invalid": configuration_counter["invalid"],
            }

            # We initiate the final data we have to save.
            # We initiate this variable instead of updating backup_content because
            # we do not want to touch the backup_content.
            to_save = {}

            # We add the backup_content into to_save.
            to_save.update(self.backup_content)
            # And we overwrite with the newly data to backup.
            to_save.update(data_to_backup)

            # Finaly, we save our informations into the log file.
            Dict(to_save).to_json(self.autocontinue_log_file)


def restore(self):
        """
        Restore data from the given path.
        """

        if PyFunceble.CONFIGURATION["auto_continue"] and self.backup_content:
            # The auto_continue subsystem is activated and the backup_content
            # is not empty.

            # We get the file we have to restore.
            file_to_restore = PyFunceble.INTERN["file_to_test"]

            if file_to_restore in self.backup_content:
                # The file we are working with is already into the backup content.

                # We initiate the different status to set.
                to_initiate = ["up", "down", "invalid", "tested"]

                # Because at some time it was not the current status, we have to map
                # the new with the old. This way, if someone is running the latest
                # version but with old data, we still continue like nothing happend.
                alternatives = {
                    "up": "number_of_up",
                    "down": "number_of_down",
                    "invalid": "number_of_invalid",
                    "tested": "number_of_tested",
                }

                for string in to_initiate:
                    # We loop over the status we have to initiate.

                    try:
                        # We try to update the counters by using the currently read status.
                        PyFunceble.INTERN["counter"]["number"].update(
                            {string: self.backup_content[file_to_restore][string]}
                        )
                    except KeyError:
                        # But if the status is not present, we try with the older index
                        # we mapped previously.
                        PyFunceble.INTERN["counter"]["number"].update(
                            {
                                string: self.backup_content[file_to_restore][
                                    alternatives[string]
                                ]
                            }
                        )


def _is_to_ignore(cls, line):
        """
        Check if we have to ignore the given line.

        :param line: The line from the file.
        :type line: str
        """

        # We set the list of regex to match to be
        # considered as ignored.
        to_ignore = [r"(^!|^@@|^\/|^\[|^\.|^-|^_|^\?|^&)"]  # , r"(\$|,)(image)"]

        for element in to_ignore:
            # We loop through the list of regex.

            if Regex(line, element, return_data=False).match():
                # The currently read line match the currently read
                # regex.

                # We return true, it has to be ignored.
                return True

        # Wer return False, it does not has to be ignored.
        return False

Check if we have to ignore the given line .


def _handle_options(self, options):
        """
        Handle the data from the options.

        :param options: The list of options from the rule.
        :type options: list

        :return: The list of domains to return globally.
        :rtype: list
        """

        # We initiate a variable which will save our result
        result = []

        # We initiate the regex which will be used to extract the domain listed
        # under the option domain=
        regex_domain_option = r"domain=(.*)"

        for option in options:
            # We loop through the list of option.
            try:
                # We try to extract the list of domains from the currently read
                # option.
                domains = Regex(
                    option, regex_domain_option, return_data=True, rematch=True, group=0
                ).match()[-1]

                if domains:
                    # We could extract something.

                    if self.aggressive:  # pragma: no cover
                        result.extend(
                            [
                                x
                                for x in domains.split("|")
                                if x and not x.startswith("~")
                            ]
                        )
                    else:
                        # We return True.
                        return True
            except TypeError:
                pass

        # We return the result.
        return result


def _extract_base(self, element):
        """
        Extract the base of the given element.

        .. example:
            given "hello/world?world=beautiful" return "hello"

        :param element: The element we are working with.
        :type element: str|list
        """

        if isinstance(element, list):
            # The given element is a list.

            # We get the base of each element of the list.
            return [self._extract_base(x) for x in element]

        # We get the base if it is an URL.
        base = self.checker.is_url_valid(url=element, return_base=True)

        if base:
            # It is an URL.

            # We return the extracted base.
            return base

        if "/" in element:
            # / is in the given element.

            # We return the first element before the
            # first /
            return element.split("/")[0]

        # / is not in the given element.

        # We return the given element.
        return element


def _format_decoded(self, to_format, result=None):  # pragma: no cover
        """
        Format the exctracted adblock line before passing it to the system.

        :param to_format: The extracted line from the file.
        :type to_format: str

        :param result: A list of the result of this method.
        :type result: list

        :return: The list of domains or IP to test.
        :rtype: list
        """

        if not result:
            # The result is not given.

            # We set the result as an empty list.
            result = []

        for data in List(to_format).format():
            # We loop through the different lines to format.

            if data:
                # The currently read line is not empty.

                if "^" in data:
                    # There is an accent in the currently read line.

                    # We recall this method but with the current result state
                    # and splited data.
                    return self._format_decoded(data.split("^"), result)

                if "#" in data:
                    # There is a dash in the currently read line.

                    # We recall this method but with the current result state
                    # and splited data.
                    return self._format_decoded(data.split("#"), result)

                if "," in data:
                    # There is a comma in the currently read line.

                    # We recall this method but with the current result state
                    # and splited data.
                    return self._format_decoded(data.split(","), result)

                if "!" in data:
                    # There is an exclamation mark in the currently read line.

                    # We recall this method but with the current result state
                    # and splited data.
                    return self._format_decoded(data.split("!"), result)

                if "|" in data:
                    # There is a vertival bar in the currently read line.

                    # We recall this method but with the current result state
                    # and splited data.
                    return self._format_decoded(data.split("|"), result)

                if data:
                    # The currently read line is not empty.

                    data = self._extract_base(data)

                    if data and (
                        self.checker.is_domain_valid(data)
                        or self.checker.is_ip_valid(data)
                    ):
                        # The extraced base is not empty.
                        # and
                        # * The currently read line is a valid domain.
                        # or
                        # * The currently read line is a valid IP.

                        # We append the currently read line to the result.
                        result.append(data)
                    elif data:
                        # * The currently read line is not a valid domain.
                        # or
                        # * The currently read line is not a valid IP.

                        # We try to get the url base.
                        url_base = self.checker.is_url_valid(data, return_base=True)

                        if url_base:
                            # The url_base is not empty or equal to False or None.

                            # We append the url base to the result.
                            result.append(url_base)

        # We return the result element.
        return result

Format the exctracted adblock line before passing it to the system .


def _access(self):  # pragma: no cover
        """
        Get the HTTP code status.

        :return: The matched HTTP status code.
        :rtype: int|None
        """

        try:
            # We try to get the HTTP status code.

            if PyFunceble.INTERN["to_test_type"] == "url":
                # We are globally testing a URL.

                # We get the head of the URL.
                req = PyFunceble.requests.head(
                    self.to_get,
                    timeout=PyFunceble.CONFIGURATION["seconds_before_http_timeout"],
                    headers=self.headers,
                    verify=PyFunceble.CONFIGURATION["verify_ssl_certificate"],
                )
            else:
                # We are not globally testing a URL.

                # We get the head of the constructed URL.
                req = PyFunceble.requests.head(
                    self.to_get,
                    timeout=PyFunceble.CONFIGURATION["seconds_before_http_timeout"],
                    headers=self.headers,
                )

            # And we try to get the status code.
            return req.status_code

        except (
            PyFunceble.requests.exceptions.InvalidURL,
            PyFunceble.socket.timeout,
            PyFunceble.requests.exceptions.Timeout,
            PyFunceble.requests.ConnectionError,
            urllib3_exceptions.InvalidHeader,
            UnicodeDecodeError,  # The probability that this happend in production is minimal.
        ):
            # If one of the listed exception is matched, that means that something
            # went wrong and we were unable to extract the status code.

            # We return None.
            return None


def get(self):
        """
        Return the HTTP code status.

        :return: The matched and formatted status code.
        :rtype: str|int|None
        """
        if PyFunceble.HTTP_CODE["active"]:
            # The http status code extraction is activated.

            # We get the http status code.
            http_code = self._access()

            # We initiate a variable which will save the list of allowed
            # http status code.
            list_of_valid_http_code = []

            for codes in [
                PyFunceble.HTTP_CODE["list"]["up"],
                PyFunceble.HTTP_CODE["list"]["potentially_down"],
                PyFunceble.HTTP_CODE["list"]["potentially_up"],
            ]:
                # We loop throught the list of http status code.

                # We extend the list of valid with the currently read
                # codes.
                list_of_valid_http_code.extend(codes)

            if http_code not in list_of_valid_http_code or http_code is None:
                # * The extracted http code is not in the list of valid http code.
                # or
                # * The extracted http code is equal to `None`.

                # We return 3 star in order to mention that we were not eable to extract
                # the http status code.
                return "*" * 3

            # * The extracted http code is in the list of valid http code.
            # or
            # * The extracted http code is not equal to `None`.

            # We return the extracted http status code.
            return http_code

        # The http status code extraction is activated.

        # We return None.
        return None


def syntax_check(domain):  # pragma: no cover
    """
    Check the syntax of the given domain.

    :param domain: The domain to check the syntax for.
    :type domain: str

    :return: The syntax validity.
    :rtype: bool

    .. warning::
        If an empty or a non-string :code:`domain` is given, we return :code:`None`.
    """

    if domain and isinstance(domain, str):
        # * The given domain is not empty nor None.
        # and
        # * The given domain is a string.

        # We silently load the configuration.
        load_config(True)

        return Check(domain).is_domain_valid()

    # We return None, there is nothing to check.
    return None


def is_subdomain(domain):  # pragma: no cover
    """
    Check if the given domain is a subdomain.

    :param domain: The domain we are checking.
    :type domain: str

    :return: The subdomain state.
    :rtype: bool

    .. warning::
        If an empty or a non-string :code:`domain` is given, we return :code:`None`.
    """

    if domain and isinstance(domain, str):
        # * The given domain is not empty nor None.
        # and
        # * The given domain is a string.

        # We silently load the configuration.
        load_config(True)

        return Check(domain).is_subdomain()

    # We return None, there is nothing to check.
    return None

Check if the given domain is a subdomain .


def ipv4_syntax_check(ip):  # pragma: no cover
    """
    Check the syntax of the given IPv4.

    :param ip: The IPv4 to check the syntax for.
    :type ip: str

    :return: The syntax validity.
    :rtype: bool

    .. warning::
        If an empty or a non-string :code:`ip` is given, we return :code:`None`.
    """

    if ip and isinstance(ip, str):
        # The given IP is not empty nor None.
        # and
        # * The given IP is a string.

        # We silently load the configuration.
        load_config(True)

        return Check(ip).is_ip_valid()

    # We return None, there is nothing to check.
    return None


def is_ipv4_range(ip):  # pragma: no cover
    """
    Check if the given IP is an IP range.

    :param ip: The IP we are checking.
    :type ip: str

    :return: The IPv4 range state.
    :rtype: bool

    .. warning::
        If an empty or a non-string :code:`ip` is given, we return :code:`None`.
    """

    if ip and isinstance(ip, str):
        # The given IP is not empty nor None.
        # and
        # * The given IP is a string.

        # We silently load the configuration.
        load_config(True)

        return Check(ip).is_ip_range()

    # We return None, there is nothing to check.
    return None


def url_syntax_check(url):  # pragma: no cover
    """
    Check the syntax of the given URL.

    :param url: The URL to check the syntax for.
    :type url: str

    :return: The syntax validity.
    :rtype: bool

    .. warning::
        If an empty or a non-string :code:`url` is given, we return :code:`None`.
    """

    if url and isinstance(url, str):
        # The given URL is not empty nor None.
        # and
        # * The given URL is a string.

        # We silently load the configuration.
        load_config(True)

        return Check(url).is_url_valid()

    # We return None, there is nothing to check.
    return None


def load_config(under_test=False, custom=None):  # pragma: no cover
    """
    Load the configuration.

    :param under_test:
        Tell us if we only have to load the configuration file (True)
        or load the configuration file and initate the output directory
        if it does not exist (False).
    :type under_test: bool

    :param custom:
        A dict with the configuration index (from .PyFunceble.yaml) to update.
    :type custom: dict

    .. warning::
        If :code:`custom` is given, the given :code:`dict` overwrite
        the last value of the given configuration indexes.
    """

    if "config_loaded" not in INTERN:
        # The configuration was not already loaded.

        # We load and download the different configuration file if they are non
        # existant.
        Load(CURRENT_DIRECTORY)

        if not under_test:
            # If we are not under test which means that we want to save informations,
            # we initiate the directory structure.
            DirectoryStructure()

        # We save that the configuration was loaded.
        INTERN.update({"config_loaded": True})

        if custom and isinstance(custom, dict):
            # The given configuration is not None or empty.
            # and
            # It is a dict.

            # We update the configuration index.
            CONFIGURATION.update(custom)


def stay_safe():  # pragma: no cover
    """
    Print a friendly message.
    """

    random = int(choice(str(int(time()))))

    if not CONFIGURATION["quiet"] and random % 3 == 0:
        print("\n" + Fore.GREEN + Style.BRIGHT + "Thanks for using PyFunceble!")
        print(
            Fore.YELLOW
            + Style.BRIGHT
            + "Share your experience on "
            + Fore.CYAN
            + "Twitter"
            + Fore.YELLOW
            + " with "
            + Fore.CYAN
            + "#PyFunceble"
            + Fore.YELLOW
            + "!"
        )
        print(
            Fore.GREEN
            + Style.BRIGHT
            + "Have a feedback, an issue or an improvement idea ?"
        )
        print(
            Fore.YELLOW
            + Style.BRIGHT
            + "Let us know on "
            + Fore.CYAN
            + "GitHub"
            + Fore.YELLOW
            + "!"
        )


def _entry_management_url_download(self, passed):
        """
        Check if the given information is a URL.
        If it is the case, it download and update the location of file to test.

        :param passed: The url passed to the system.
        :type passed: str

        :return: The state of the check.
        :rtype: bool
        """

        if passed and self.checker.is_url_valid(passed):
            # The passed string is an URL.

            # We get the file name based on the URL.
            # We actually just get the  string after the last `/` in the URL.
            file_to_test = passed.split("/")[-1]

            if (
                not PyFunceble.path.isfile(file_to_test)
                or PyFunceble.INTERN["counter"]["number"]["tested"] == 0
            ):
                # The filename does not exist in the current directory
                # or the currently number of tested is equal to 0.

                # We download the content of the link.
                Download(passed, file_to_test).text()

            # The files does exist or the currently number of tested is greater than
            # 0.

            # We initiate the file we have to test.
            PyFunceble.INTERN["file_to_test"] = file_to_test

            # We return true to say that everything goes right.
            return True

        # The passed string is not an URL.

        # We do not need to do anything else.
        return False

Check if the given information is a URL . If it is the case it download and update the location of file to test .


def _entry_management_url(self):
        """
        Manage the loading of the url system.
        """

        if (
            self.url_file  # pylint: disable=no-member
            and not self._entry_management_url_download(
                self.url_file  # pylint: disable=no-member
            )
        ):  # pylint: disable=no-member
            # The current url_file is not a URL.

            # We initiate the filename as the file we have to test.
            PyFunceble.INTERN[
                "file_to_test"
            ] = self.url_file


def _print_header(cls):
        """
        Decide if we print or not the header.
        """

        if (
            not PyFunceble.CONFIGURATION["quiet"]
            and not PyFunceble.CONFIGURATION["header_printed"]
        ):
            # * The quiet mode is not activated.
            # and
            # * The header has not been already printed.

            # We print a new line.
            print("\n")

            if PyFunceble.CONFIGURATION["less"]:
                # We have to show less informations on screen.

                # We print the `Less` header.
                Prints(None, "Less").header()
            else:
                # We have to show every informations on screen.

                # We print the `Generic` header.
                Prints(None, "Generic").header()

            # The header was printed.

            # We initiate the variable which say that the header has been printed to True.
            PyFunceble.CONFIGURATION["header_printed"] = True


def _file_decision(self, current, last, status=None):
        """
        Manage the database, autosave and autocontinue systems for the case that we are reading
        a file.

        :param current: The currently tested element.
        :type current: str

        :param last: The last element of the list.
        :type last: str

        :param status: The status of the currently tested element.
        :type status: str
        """

        if (
            status
            and not PyFunceble.CONFIGURATION["simple"]
            and PyFunceble.INTERN["file_to_test"]
        ):
            # * The status is given.
            # and
            # * The simple mode is deactivated.
            # and
            # * A file to test is set.

            # We run the mining logic.
            self.mining.process()

            # We delete the currently tested element from the mining
            # database.
            # Indeed, as it is tested, it is already in our
            # testing process which means that we don't need it into
            # the mining database.
            self.mining.remove()

            if (
                status.lower() in PyFunceble.STATUS["list"]["up"]
                or status.lower() in PyFunceble.STATUS["list"]["valid"]
            ):
                # The status is in the list of up status.

                if self.inactive_database.is_present():
                    # The currently tested element is in the database.

                    # We generate the suspicious file(s).
                    Generate(PyFunceble.STATUS["official"]["up"]).analytic_file(
                        "suspicious"
                    )

                    # We remove the currently tested element from the
                    # database.
                    self.inactive_database.remove()

            else:
                # The status is not in the list of up status.

                # We add the currently tested element to the
                # database.
                self.inactive_database.add()

            # We backup the current state of the file reading
            # for the case that we need to continue later.
            self.auto_continue.backup()

            if current != last:
                # The current element is not the last one.

                # We run the autosave logic.
                AutoSave()
            else:
                # The current element is the last one.

                # We stop and log the execution time.
                ExecutionTime("stop", last=True)

                # We show/log the percentage.
                self.percentage.log()

                # We reset the counters as we end the process.
                self.reset_counters()

                # We backup the current state of the file reading
                # for the case that we need to continue later.
                self.auto_continue.backup()

                # We show the colored logo.
                self.colorify_logo()

                # We save and stop the script if we are under
                # Travis CI.
                AutoSave(True)

        for index in ["http_code", "referer"]:
            # We loop through some configuration index we have to empty.

            if index in PyFunceble.INTERN:
                # The index is in the configuration.

                # We empty the configuration index.
                PyFunceble.INTERN[index] = ""

Manage the database autosave and autocontinue systems for the case that we are reading a file .


def domain(self, domain=None, last_domain=None):
        """
        Manage the case that we want to test only a domain.

        :param domain: The domain or IP to test.
        :type domain: str

        :param last_domain:
            The last domain to test if we are testing a file.
        :type last_domain: str

        :param return_status: Tell us if we need to return the status.
        :type return_status: bool
        """

        # We print the header.
        self._print_header()

        if domain:
            # A domain is given.

            # We format and set the domain we are testing and treating.
            PyFunceble.INTERN["to_test"] = self._format_domain(domain)
        else:
            # A domain is not given.

            # We set the domain we are testing and treating to None.
            PyFunceble.INTERN["to_test"] = None

        if PyFunceble.INTERN["to_test"]:
            # The domain is given (Not None).

            if PyFunceble.CONFIGURATION["syntax"]:
                # The syntax mode is activated.

                # We get the status from Syntax.
                status = self.syntax_status.get()
            else:
                # We test and get the status of the domain.
                status, _ = self.status.get()

            # We run the file decision logic.
            self._file_decision(PyFunceble.INTERN["to_test"], last_domain, status)

            if PyFunceble.CONFIGURATION["simple"]:
                # The simple mode is activated.

                # We print the domain and the status.
                print(PyFunceble.INTERN["to_test"], status)

            # We return the tested domain and its status.
            return PyFunceble.INTERN["to_test"], status

        # We return None, there is nothing to test.
        return None

Manage the case that we want to test only a domain .


def url(self, url_to_test=None, last_url=None):
        """
        Manage the case that we want to test only a given url.

        :param url_to_test: The url to test.
        :type url_to_test: str

        :param last_url:
            The last url of the file we are testing
            (if exist)
        :type last_url: str
        """

        # We print the header.
        self._print_header()

        if url_to_test:
            # An url to test is given.

            # We set the url we are going to test.
            PyFunceble.INTERN["to_test"] = url_to_test
        else:
            # An URL to test is not given.

            # We set the url we are going to test to None.
            PyFunceble.INTERN["to_test"] = None

        if PyFunceble.INTERN["to_test"]:
            # An URL to test is given.

            if PyFunceble.CONFIGURATION["syntax"]:
                # The syntax mode is activated.

                # We get the status from Syntax.
                status = self.syntax_status.get()
            else:
                # The syntax mode is not activated.

                # We get the status from URL.
                status = self.url_status.get()

            # We run the file decision logic.
            self._file_decision(PyFunceble.INTERN["to_test"], last_url, status)

            if PyFunceble.CONFIGURATION["simple"]:
                # The simple mode is activated.

                # We print the URL informations.
                print(PyFunceble.INTERN["to_test"], status)

            # We return the URL we tested and its status.
            return PyFunceble.INTERN["to_test"], status

        # We return None, there is nothing to test.
        return None

Manage the case that we want to test only a given url .


def colorify_logo(cls, home=False):
        """
        Print the colored logo based on global results.

        :param home: Tell us if we have to print the initial coloration.
        :type home: bool
        """

        if not PyFunceble.CONFIGURATION["quiet"]:
            # The quiet mode is not activated.

            to_print = []

            if home:
                # We have to print the initial logo.

                for line in PyFunceble.ASCII_PYFUNCEBLE.split("\n"):
                    # We loop through each lines of the ASCII representation
                    # of PyFunceble.

                    # And we append to the data to print the currently read
                    # line with the right coloration.
                    to_print.append(
                        PyFunceble.Fore.YELLOW + line + PyFunceble.Fore.RESET
                    )

            elif PyFunceble.INTERN["counter"]["percentage"]["up"] >= 50:
                # The percentage of up is greater or equal to 50%.

                for line in PyFunceble.ASCII_PYFUNCEBLE.split("\n"):
                    # We loop through each lines of the ASCII representation
                    # of PyFunceble.

                    # And we append to the data to print the currently read
                    # line with the right coloration.
                    to_print.append(
                        PyFunceble.Fore.GREEN + line + PyFunceble.Fore.RESET
                    )
            else:
                # The percentage of up is less than 50%.

                for line in PyFunceble.ASCII_PYFUNCEBLE.split("\n"):
                    # We loop through each lines of the ASCII representation
                    # of PyFunceble.

                    # And we append to the data to print the currently read
                    # line with the right coloration.
                    to_print.append(PyFunceble.Fore.RED + line + PyFunceble.Fore.RESET)

            print("\n".join(to_print))

Print the colored logo based on global results .


def _format_domain(cls, extracted_domain):
        """
        Format the extracted domain before passing it to the system.

        :param extracted_domain: The extracted domain.
        :type extracted_domain: str

        :return: The formatted domain or IP to test.
        :rtype: str

        .. note:
            Understand by formating the fact that we get rid
            of all the noises around the domain we want to test.
        """

        if not extracted_domain.startswith("#"):
            # The line is not a commented line.

            if "#" in extracted_domain:
                # There is a comment at the end of the line.

                # We delete the comment from the line.
                extracted_domain = extracted_domain[
                    : extracted_domain.find("#")
                ].strip()

            if " " in extracted_domain or "\t" in extracted_domain:
                # A space or a tabs is in the line.

                # We remove all whitestring from the extracted line.
                splited_line = extracted_domain.split()

                # As there was a space or a tab in the string, we consider
                # that we are working with the hosts file format which means
                # that the domain we have to test is after the first string.
                # So we set the index to 1.
                index = 1

                while index < len(splited_line):
                    # We loop until the index is greater than the length of
                    #  the splited line.

                    if splited_line[index]:
                        # The element at the current index is not an empty string.

                        # We break the loop.
                        break

                    # The element at the current index is an empty string.

                    # We increase the index number.
                    index += 1

                # We return the last read element.
                return splited_line[index]

            # We return the extracted line.
            return extracted_domain

        # The extracted line is a comment line.

        # We return an empty string as we do not want to work with commented line.
        return ""

Format the extracted domain before passing it to the system .


def _extract_domain_from_file(cls):
        """
        Extract all non commented lines from the file we are testing.

        :return: The elements to test.
        :rtype: list
        """

        # We initiate the variable which will save what we are going to return.
        result = []

        if PyFunceble.path.isfile(PyFunceble.INTERN["file_to_test"]):
            # The give file to test exist.

            try:
                with open(PyFunceble.INTERN["file_to_test"]) as file:
                    # We open and read the file.

                    for line in file:
                        # We loop through each lines.

                        if not line.startswith("#"):
                            # The currently read line is not a commented line.

                            # We append the current read line to the result.
                            result.append(line.rstrip("\n").strip())
            except UnicodeDecodeError:
                with open(PyFunceble.INTERN["file_to_test"], encoding="utf-8") as file:
                    # We open and read the file.

                    for line in file:
                        # We loop through each lines.

                        if not line.startswith("#"):
                            # The currently read line is not a commented line.

                            # We append the current read line to the result.
                            result.append(line.rstrip("\n").strip())

        else:
            # The given file to test does not exist.

            # We raise a FileNotFoundError exception.
            raise FileNotFoundError(PyFunceble.INTERN["file_to_test"])

        # We return the result.
        return result

Extract all non commented lines from the file we are testing .


def file(self):
        """
        Manage the case that need to test each domain of a given file path.

        .. note::
            1 domain per line.
        """

        # We get, format, filter, clean the list to test.
        list_to_test = self._file_list_to_test_filtering()

        if PyFunceble.CONFIGURATION["idna_conversion"]:
            # We have to convert domains to idna.

            # We convert if we need to convert.
            list_to_test = domain2idna(list_to_test)

            if PyFunceble.CONFIGURATION["hierarchical_sorting"]:
                # The hierarchical sorting is desired by the user.

                # We format the list.
                list_to_test = List(list_to_test).custom_format(Sort.hierarchical)
            else:
                # The hierarchical sorting is not desired by the user.

                # We format the list.
                list_to_test = List(list_to_test).custom_format(Sort.standard)

        # We initiate a local variable which will save the current state of the list.
        not_filtered = list_to_test

        try:
            # We remove the element which are in the database from the
            # current list to test.
            list_to_test = List(
                list(
                    set(
                        list_to_test[PyFunceble.INTERN["counter"]["number"]["tested"] :]
                    )
                    - set(PyFunceble.INTERN["flatten_inactive_db"])
                )
            ).format()
            _ = list_to_test[-1]
        except IndexError:
            # Our list to test is the one with the element from the database.
            list_to_test = not_filtered[
                PyFunceble.INTERN["counter"]["number"]["tested"] :
            ]

            # We delete the undesired variable.
            del not_filtered

        if PyFunceble.CONFIGURATION["hierarchical_sorting"]:
            # The hierarchical sorting is desired by the user.

            # We format the list.
            list_to_test = List(list(list_to_test)).custom_format(Sort.hierarchical)

        try:
            # We test each element of the list to test.
            return [self.domain(x, list_to_test[-1]) for x in list_to_test if x]
        except IndexError:
            # We print a message on screen.
            print(PyFunceble.Fore.CYAN + PyFunceble.Style.BRIGHT + "Nothing to test.")

Manage the case that need to test each domain of a given file path .


def file_url(self):
        """
        Manage the case that we have to test a file

        .. note::
            1 URL per line.
        """

        # We get, format, clean the list of URL to test.
        list_to_test = self._file_list_to_test_filtering()

        # We initiate a local variable which will save the current state of the list.
        not_filtered = list_to_test

        try:
            # We remove the element which are in the database from the
            # current list to test.
            list_to_test = List(
                list(
                    set(
                        list_to_test[PyFunceble.INTERN["counter"]["number"]["tested"] :]
                    )
                    - set(PyFunceble.INTERN["flatten_inactive_db"])
                )
            ).format()
            _ = list_to_test[-1]
        except IndexError:
            # Our list to test is the one with the element from the database.
            list_to_test = not_filtered[
                PyFunceble.INTERN["counter"]["number"]["tested"] :
            ]

            # We delete the undesired variable.
            del not_filtered

        if PyFunceble.CONFIGURATION["hierarchical_sorting"]:
            # The hierarchical sorting is desired by the user.

            # We format the list.
            list_to_test = List(list(list_to_test)).custom_format(Sort.hierarchical)

        try:
            # We test each URL from the list to test.
            return [self.url(x, list_to_test[-1]) for x in list_to_test if x]
        except IndexError:
            # We print a message on screen.
            print(PyFunceble.Fore.CYAN + PyFunceble.Style.BRIGHT + "Nothing to test.")


def switch(
        cls, variable, custom=False
    ):  # pylint: disable=inconsistent-return-statements
        """
        Switch PyFunceble.CONFIGURATION variables to their opposite.

        :param variable:
            The variable name to switch.
            The variable should be an index our configuration system.
            If we want to switch a bool variable, we should parse
            it here.
        :type variable: str|bool

        :param custom:
            Let us know if have to switch the parsed variable instead
            of our configuration index.
        :type custom: bool

        :return:
            The opposite of the configuration index or the given variable.
        :rtype: bool

        :raises:
            :code:`Exception`
                When the configuration is not valid. In other words,
                if the PyFunceble.CONFIGURATION[variable_name] is not a bool.
        """

        if not custom:
            # We are not working with custom variable which is not into
            # the configuration.

            # We get the current state.
            current_state = dict.get(PyFunceble.CONFIGURATION, variable)
        else:
            # We are working with a custom variable which is not into the
            # configuration
            current_state = variable

        if isinstance(current_state, bool):
            # The current state is a boolean.

            if current_state:
                # The current state is equal to True.

                # We return False.
                return False

            # The current state is equal to False.

            # We return True.
            return True

        # The current state is not a boolean.

        # We set the message to raise.
        to_print = "Impossible to switch %s. Please post an issue to %s"

        # We raise an exception inviting the user to report an issue.
        raise Exception(
            to_print % (repr(variable), PyFunceble.LINKS["repo"] + "/issues.")
        )

Switch PyFunceble . CONFIGURATION variables to their opposite .


def get(cls):
        """
        Get the status while testing for an IP or domain.

        .. note::
            We consider that the domain or IP we are currently testing
            is into :code:`PyFunceble.INTERN["to_test"]`.
        """

        if "to_test" in PyFunceble.INTERN and PyFunceble.INTERN["to_test"]:
            expiration_date = ExpirationDate().get()

            if expiration_date is False:
                return cls.handle(status="invalid")

            if expiration_date == PyFunceble.STATUS["official"]["up"]:
                return expiration_date, "WHOIS"

            return cls.handle(status="inactive")

        raise NotImplementedError("We expect `INTERN['to_test']` to be set.")

Get the status while testing for an IP or domain .


def handle(self):
        """
        Handle the backend of the given status.
        """

        # We initiate the source we are going to parse to the Generate class.
        source = "URL"

        if self.catched.lower() not in PyFunceble.STATUS["list"]["invalid"]:
            # The parsed status is not in the list of invalid.

            # We generate the status file with the catched status.
            Generate(self.catched, source).status_file()
        else:
            # The parsed status is in the list of invalid.

            # We generate the status file with the parsed status.
            Generate(self.catched, "SYNTAX").status_file()

        # We return the parsed status.
        return self.catched

Handle the backend of the given status .


def _get_structure(self):
        """
        Get the structure we are going to work with.

        :return: The structure we have to work with.
        :rtype: dict
        """

        # We initiate an empty variable which is going to save the location of
        # file we are going to download.
        structure_file = ""

        # We initiate the variable which will save the request instance.
        req = ""

        if PyFunceble.path.isfile(self.structure):
            # The structure path file exist.

            # We set it as the destination file.
            structure_file = self.structure
        elif PyFunceble.path.isfile(self.base + "dir_structure_production.json"):
            # * The structure path file does not exist.
            # but
            # * The production structure path file exist.

            # We set it as the destination file
            structure_file = self.base + "dir_structure_production.json"
        else:
            # * The structure path file does not exist.
            # and
            # * The production structure path file does not exist.

            if "dev" not in PyFunceble.VERSION:
                # `dev` is not into the local version name.

                # We get the production file from the master branch.
                req = PyFunceble.requests.get(
                    PyFunceble.LINKS["dir_structure"].replace("dev", "master")
                )
            else:
                # `dev` is into the local version name.

                # We get the production file from the dev branch.
                req = PyFunceble.requests.get(
                    PyFunceble.LINKS["dir_structure"].replace("master", "dev")
                )

        if structure_file.endswith("_production.json"):
            # The destination is the production file.

            # And we return the updated the structure from the last read file.
            # (with the names from the configuration file).
            return self._update_structure_from_config(
                Dict().from_json(File(structure_file).read())
            )

        # The destination is not the production file.

        if structure_file.endswith(".json"):
            # The destination ends with `.json`.

            # And we return the updated the structure from the given file.
            # (with the names from the configuration file).
            return self._update_structure_from_config(
                Dict().from_json(File(structure_file).read())
            )

        # The destination does not ends with `.json`.

        # We return the updated the structure from the link we previously got.
        # (with the names from the configuration file).
        return self._update_structure_from_config(Dict().from_json(req.text))

Get the structure we are going to work with .


def _create_directory(cls, directory, loop=False):
        """
        Creates the given directory if it does not exists.

        :param directory: The directory to create.
        :type directory: str

        :param loop: Tell us if we are in the creation loop or not.
        :type loop: bool
        """

        if not loop and PyFunceble.directory_separator in directory:
            # * We are not in the loop.
            # and
            # * The directory separator in the given directory.

            # We split the directories separator.
            splited_directory = directory.split(PyFunceble.directory_separator)

            # We initiate a variable which will save the full path to create.
            full_path_to_create = ""

            for single_directory in splited_directory:
                # We loop through each directory.

                # We append the currently read directory to the full path.
                full_path_to_create += single_directory + PyFunceble.directory_separator

                # And we create the directory if it does not exist.
                cls._create_directory(full_path_to_create, True)

        if not PyFunceble.path.isdir(directory):
            # The given directory does not exist.

            # We update the permission.
            # (Only if we are under Travis CI.)
            AutoSave.travis_permissions()

            # We create the directory.
            PyFunceble.mkdir(directory)

            # We update the permission.
            # (Only if we are under Travis CI.)
            AutoSave.travis_permissions()

Creates the given directory if it does not exists .


def delete_uneeded(self):
        """
        Delete the directory which are not registered into our structure.
        """

        # We get the structure we have to apply.
        structure = self._get_structure()

        # We get the list of key which is implicitly the list of directory we do not bave to delete.
        list_of_key = list(structure.keys())

        # We move to the content of the parent as we know that we are creating only one directory.
        # Note: if one day we will have to create multiple directory, we will have to change
        # the following.
        structure = structure[list_of_key[0]]

        # We also set the parent directory as we are going to construct its childen.
        parent_path = list_of_key[0]

        if not parent_path.endswith(PyFunceble.directory_separator):
            parent_path += PyFunceble.directory_separator

        for root, _, _ in PyFunceble.walk(parent_path):
            # We loop through each directories of the parent path.

            # We fix the path in order to avoid issues.
            root = Directory(root).fix_path()

            if root.replace(parent_path, "") not in structure:
                # The currently read directory is not in our structure.

                # We delete it.
                PyFunceble.rmtree(root)

Delete the directory which are not registered into our structure .


def _set_path_to_configs(cls, path_to_config):
        """
        Set the paths to the configuration files.

        :param path_to_config: The possible path to the config to load.
        :type path_to_config: str

        :return:
            The path to the config to read (0), the path to the default
            configuration to read as fallback.(1)
        :rtype: tuple
        """

        if not path_to_config.endswith(PyFunceble.directory_separator):
            # The path to the config does not ends with the directory separator.

            # We initiate the default and the parsed variable with the directory separator.
            default = parsed = path_to_config + PyFunceble.directory_separator
        else:
            # The path to the config does ends with the directory separator.

            # We initiate the default and the parsed variable.
            default = parsed = path_to_config

        # We append the `CONFIGURATION_FILENAME` to the parsed variable.
        parsed += PyFunceble.CONFIGURATION_FILENAME
        # And we append the `DEFAULT_CONFIGURATION_FILENAME` to the default variable.
        default += PyFunceble.DEFAULT_CONFIGURATION_FILENAME

        # We finaly return a tuple which contain both informations.
        return (parsed, default)

Set the paths to the configuration files .


def _load_config_file(self):
        """
        Load .PyFunceble.yaml into the system.
        """

        try:
            # We try to load the configuration file.

            PyFunceble.CONFIGURATION.update(
                Dict.from_yaml(File(self.path_to_config).read())
            )

            # We install the latest iana configuration file.
            self._install_iana_config()

            # We install the latest public suffix configuration file.
            self._install_psl_config()

            # We install the latest directory structure file.
            self._install_directory_structure_file()
        except FileNotFoundError as exception:
            # But if the configuration file is not found.

            if PyFunceble.path.isfile(self.path_to_default_config):
                # The `DEFAULT_CONFIGURATION_FILENAME` file exists.

                # We copy it as the configuration file.
                File(self.path_to_default_config).copy(self.path_to_config)

                # And we load the configuration file as it does exist (yet).
                self._load_config_file()
            else:
                # The `DEFAULT_CONFIGURATION_FILENAME` file does not exists.

                # We raise the exception we were handling.
                raise exception

Load . PyFunceble . yaml into the system .


def _install_production_config(self):
        """
        Download the production configuration and install it in the
        current directory.
        """

        # We initiate the link to the production configuration.
        # It is not hard coded because this method is called only if we
        # are sure that the configuration file exist.
        production_config_link = "https://raw.githubusercontent.com/funilrys/PyFunceble/master/.PyFunceble_production.yaml"  # pylint: disable=line-too-long

        # We update the link according to our current version.
        production_config_link = Version(True).right_url_from_version(
            production_config_link
        )

        if not Version(True).is_cloned():
            # The current version is not the cloned one.

            # We download the link content and save it inside the default location.
            #
            # Note: We add this one in order to allow the enduser to always have
            # a copy of our upstream configuration file.
            Download(production_config_link, self.path_to_default_config).text()

        # And we download the link content and return the download status.
        return Download(production_config_link, self.path_to_config).text()

Download the production configuration and install it in the current directory .


def _install_iana_config(cls):
        """
        Download `iana-domains-db.json` if not present.
        """

        # We initiate the link to the iana configuration.
        # It is not hard coded because this method is called only if we
        # are sure that the configuration file exist.
        iana_link = PyFunceble.CONFIGURATION["links"]["iana"]

        # We update the link according to our current version.
        iana_link = Version(True).right_url_from_version(iana_link)

        # We set the destination of the downloaded file.
        destination = PyFunceble.CURRENT_DIRECTORY + "iana-domains-db.json"

        if not Version(True).is_cloned() or not PyFunceble.path.isfile(destination):
            # The current version is not the cloned version.

            # We Download the link content and return the download status.
            return Download(iana_link, destination).text()

        # We are in the cloned version.

        # We do not need to download the file, so we are returning None.
        return None

Download iana - domains - db . json if not present .


def _install_psl_config(cls):
        """
        Download `public-suffix.json` if not present.
        """

        # We initiate the link to the public suffix configuration.
        # It is not hard coded because this method is called only if we
        # are sure that the configuration file exist.
        psl_link = PyFunceble.CONFIGURATION["links"]["psl"]

        # We update the link according to our current version.
        psl_link = Version(True).right_url_from_version(psl_link)

        # We set the destination of the downloaded file.
        destination = (
            PyFunceble.CURRENT_DIRECTORY
            + PyFunceble.CONFIGURATION["outputs"]["default_files"]["public_suffix"]
        )

        if not Version(True).is_cloned() or not PyFunceble.path.isfile(destination):
            # The current version is not the cloned version.

            # We Download the link content and return the download status.
            return Download(psl_link, destination).text()

        # We are in the cloned version.

        # We do not need to download the file, so we are returning None.
        return None

Download public - suffix . json if not present .


def _install_directory_structure_file(cls):
        """
        Download the latest version of `dir_structure_production.json`.
        """

        # We initiate the link to the public suffix configuration.
        # It is not hard coded because this method is called only if we
        # are sure that the configuration file exist.
        dir_structure_link = PyFunceble.CONFIGURATION["links"]["dir_structure"]

        # We update the link according to our current version.
        dir_structure_link = Version(True).right_url_from_version(dir_structure_link)

        # We set the destination of the downloaded file.
        destination = (
            PyFunceble.CURRENT_DIRECTORY
            + PyFunceble.CONFIGURATION["outputs"]["default_files"]["dir_structure"]
        )

        if not Version(True).is_cloned() or not PyFunceble.path.isfile(destination):
            # The current version is not the cloned version.

            # We Download the link content and return the download status.
            data = Download(dir_structure_link, destination, return_data=True).text()

            File(destination).write(data, overwrite=True)
            return True

        # We are in the cloned version.

        # We do not need to download the file, so we are returning None.
        return None

Download the latest version of dir_structure_production . json .


def _merge_values(self):
        """
        Simply merge the older into the new one.
        """

        to_remove = []

        self.new_config = Dict(
            Dict(self.upstream_config).merge(PyFunceble.CONFIGURATION)
        ).remove_key(to_remove)

def _merge_values ( self ) : to_remove = [ ] self . new_config = Dict ( Dict ( self . upstream_config ) . merge ( PyFunceble . CONFIGURATION ) ) . remove_key ( to_remove )

Simply merge the older into the new one .


def _load(self):
        """
        Execute the logic behind the merging.
        """

        if "PYFUNCEBLE_AUTO_CONFIGURATION" not in PyFunceble.environ:
            # The auto configuration environment variable is not set.

            while True:
                # We infinitly loop until we get a reponse which is `y|Y` or `n|N`.

                # We ask the user if we should install and load the default configuration.
                response = input(
                    PyFunceble.Style.BRIGHT
                    + PyFunceble.Fore.RED
                    + "A configuration key is missing.\n"
                    + PyFunceble.Fore.RESET
                    + "Try to merge upstream configuration file into %s ? [y/n] "
                    % (
                        PyFunceble.Style.BRIGHT
                        + self.path_to_config
                        + PyFunceble.Style.RESET_ALL
                    )
                )

                if isinstance(response, str):
                    # The response is a string

                    if response.lower() == "y":
                        # The response is a `y` or `Y`.

                        # We merge the old values inside the new one.
                        self._merge_values()

                        # And we save.
                        self._save()

                        print(
                            PyFunceble.Style.BRIGHT + PyFunceble.Fore.GREEN + "Done!\n"
                            "Please try again, if it happens again,"
                            " please fill a new issue."
                        )

                        # And we break the loop as we got a satisfied response.
                        break

                    elif response.lower() == "n":
                        # The response is a `n` or `N`.

                        # We inform the user that something went wrong.
                        raise Exception("Configuration key still missing.")
        else:
            # The auto configuration environment variable is set.

            # We merge the old values inside the new one.
            self._merge_values()

            # And we save.
            self._save()


def split_versions(cls, version, return_non_digits=False):
        """
        Convert the versions to a shorter one.

        :param version: The version to split.
        :type version: str

        :param return_non_digits:
            Activate the return of the non-digits parts of the splitted
            version.
        :type return_non_digits: bool

        :return: The splitted version name/numbers.
        :rtype: list
        """

        # We split the version.
        splited_version = version.split(".")

        # We split the parsed version and keep the digits.
        digits = [x for x in splited_version if x.isdigit()]

        if not return_non_digits:
            # We do not have to return the non digits part of the version.

            # We return the digits part of the version.
            return digits

        # We have to return the non digit parts of the version.

        # We split the parsed version and keep the non digits.
        non_digits = [x for x in splited_version if not x.isdigit()]

        # We return a tuple with first the digits part and finally the non digit parts.
        return (digits, non_digits[0])


def check_versions(cls, local, upstream):
        """
        Compare the given versions.

        :param local: The local version converted by split_versions().
        :type local: list

        :param upstream: The upstream version converted by split_versions().
        :type upstream: list

        :return:
            - True: local < upstream
            - None: local == upstream
            - False: local > upstream
        :rtype: bool|None
        """

        # A version should be in format [1,2,3] which is actually the version `1.2.3`
        # So as we only have 3 elements in the versioning,
        # we initiate the following variable in order to get the status of each parts.
        status = [None, None, None]

        for index, version_number in enumerate(local):
            # We loop through the local version.

            if int(version_number) < int(upstream[index]):
                # The local version is less than the upstream version.

                # We initiate its status to True which means that we are in
                # an old version (for the current version part).
                status[index] = True
            elif int(version_number) > int(upstream[index]):
                # The local version is greater then the upstream version.

                # We initiate its status to False which means that we are in
                # a more recent version (for the current version part).
                status[index] = False

            # Otherwise the status stay None which means that there is no change
            # between both local and upstream.

        if False in status:
            # There is a False in the status.

            # We return False which means that we are in a more recent version.
            return False

        if True in status:
            # There is a True in the status.

            # We return True which means that we are in a older version.
            return True

        # There is no True or False in the status.

        # We return None which means that we are in the same version as upstream.
        return None


def is_cloned(cls):
        """
        Let us know if we are currently in the cloned version of
        PyFunceble which implicitly mean that we are in developement mode.
        """

        if not PyFunceble.path.isdir(".git"):
            # The git directory does not exist.

            # We return False, the current version is not the cloned version.
            return False

        # We list the list of file which can be found only in a cloned version.
        list_of_file = [
            ".coveragerc",
            ".coveralls.yml",
            ".gitignore",
            ".PyFunceble_production.yaml",
            ".travis.yml",
            "CODE_OF_CONDUCT.md",
            "CONTRIBUTING.md",
            "dir_structure_production.json",
            "MANIFEST.in",
            "README.rst",
            "requirements.txt",
            "setup.py",
            "version.yaml",
        ]

        # We list the list of directory which can be found only in a cloned
        # version.
        list_of_dir = ["docs", "PyFunceble", "tests"]

        for file in list_of_file:
            # We loop through the list of file.

            if not PyFunceble.path.isfile(file):
                # The file does not exist in the current directory.

                # We return False, the current version is not the cloned version.
                return False

        # All required files exist in the current directory.

        for directory in list_of_dir:
            # We loop through the list of directory.

            if not PyFunceble.path.isdir(directory):
                # The directory does not exist in the current directory.

                # We return False, the current version is not the cloned version.
                return False

        # All required directories exist in the current directory.

        # We return True, the current version is a cloned version.
        return True

Let us know if we are currently in the cloned version of PyFunceble which implicitly mean that we are in developement mode .


def _handle_non_existant_index(cls):
        """
        Handle and check that some configuration index exists.
        """

        try:
            # We try to call the http code.
            PyFunceble.INTERN["http_code"]
        except KeyError:
            # If it is not found.

            # We initiate an empty http code.
            PyFunceble.INTERN["http_code"] = "*" * 3

        try:
            # We try to call the referer.
            PyFunceble.INTERN["referer"]
        except KeyError:
            # If it is not found.

            # We initate an `Unknown` referer.
            PyFunceble.INTERN["referer"] = "Unknown"

def
_handle_non_existant_index
(
cls
)
:
try
:
# We try to call the http code.
PyFunceble
.
INTERN
[
"http_code"
]
except
KeyError
:
# If it is not found.
# We initiate an empty http code.
PyFunceble
.
INTERN
[
"http_code"
]
=
"*"
*
3
try
:
# We try to call the referer.
PyFunceble
.
INTERN
[
"referer"
]
except
KeyError
:
# If it is not found.
# We initate an `Unknown` referer.
PyFunceble
.
INTERN
[
"referer"
]
=
"Unknown"

Handle and check that some configuration index exists .


def _analytic_host_file_directory(self):
        """
        Return the analytic directory to write depending of the matched
        status.
        """

        # We construct the path to the analytic directory.
        output_dir = (
            self.output_parent_dir
            + PyFunceble.OUTPUTS["analytic"]["directories"]["parent"]
        )

        if self.domain_status.lower() in PyFunceble.STATUS["list"]["potentially_up"]:
            # The status is in the list of analytic up status.

            # We complete the output directory.
            output_dir += PyFunceble.OUTPUTS["analytic"]["directories"][
                "potentially_up"
            ]
        elif (
            self.domain_status.lower() in PyFunceble.STATUS["list"]["potentially_down"]
        ):
            # The status is in the list of analytic down status.

            # We complete the output directory.
            output_dir += PyFunceble.OUTPUTS["analytic"]["directories"][
                "potentially_down"
            ]
        elif self.domain_status.lower() in PyFunceble.STATUS["list"]["suspicious"]:
            # The status is in the list of analytic suspicious status.

            # We complete the output directory.
            output_dir += PyFunceble.OUTPUTS["analytic"]["directories"]["suspicious"]
        else:
            # The status is not in the list of analytic down or up status.

            # We complete the output directory.
            output_dir += PyFunceble.OUTPUTS["analytic"]["directories"]["up"]

        return output_dir

Return the analytic directory to write depending of the matched status .


def unified_file(self):
        """
        Generate unified file. Understand by that that we use an unified table
        instead of a separate table for each status which could result into a
        misunderstanding.
        """

        if (
            "file_to_test" in PyFunceble.INTERN
            and PyFunceble.INTERN["file_to_test"]
            and PyFunceble.CONFIGURATION["unified"]
        ):
            # * We are not testing as an imported module.
            # and
            # * The unified file generation is activated.

            # We construct the path of the unified file.
            output = (
                self.output_parent_dir + PyFunceble.OUTPUTS["default_files"]["results"]
            )

            if PyFunceble.CONFIGURATION["less"]:
                # We have to print less information.

                if PyFunceble.HTTP_CODE["active"]:
                    # The http status code request is activated.

                    # We construct what we have to print.
                    to_print = [
                        self.tested,
                        self.domain_status,
                        PyFunceble.INTERN["http_code"],
                    ]
                else:
                    # The http status code request is not activated.

                    # We construct what we have to print.
                    to_print = [self.tested, self.domain_status, self.source]

                # And we print the informations on file.
                Prints(to_print, "Less", output, True).data()
            else:
                # The unified file generation is not activated.

                # We construct what we have to print.
                to_print = [
                    self.tested,
                    self.domain_status,
                    self.expiration_date,
                    self.source,
                    PyFunceble.INTERN["http_code"],
                    PyFunceble.CURRENT_TIME,
                ]

                # And we print the information on file.
                Prints(to_print, "Generic_File", output, True).data()

Generate unified file . Understand by that that we use an unified table instead of a separate table for each status which could result into a misunderstanding .


def status_file(self):  # pylint: disable=inconsistent-return-statements
        """
        Generate a file according to the domain status.
        """

        if "file_to_test" in PyFunceble.INTERN:
            # We are not testing as an imported module.

            # We generate the hosts file.
            Generate(self.domain_status, self.source, self.expiration_date).info_files()

            # We are testing a file content.

            # We increase the percentage count.
            Percentage(self.domain_status).count()

            # We print on screen if needed.
            self._prints_status_screen()

            if self._do_not_produce_file():
                return None

            if (
                not PyFunceble.CONFIGURATION["no_files"]
                and PyFunceble.CONFIGURATION["split"]
            ):
                # * The file non-generation of file is globaly deactivated.
                # and
                # * We have to split the outputs.

                # We print or generate the files.
                self._prints_status_file()
            else:
                # * The file non-generation of file is globaly activated.
                # or
                # * We do not have to split the outputs.

                # We print or generate the unified files.
                self.unified_file()

Generate a file according to the domain status .


def _do_not_produce_file(self):
        """
        Check if we are allowed to produce a file based from the given
        information.

        :return:
            The state of the production.
            True: We do not produce file.
            False: We do produce file.
        :rtype: bool
        """

        if (
            Inactive().is_present()
            and self.domain_status
            in [
                PyFunceble.STATUS["official"]["down"],
                PyFunceble.STATUS["official"]["invalid"],
            ]
            and PyFunceble.INTERN["to_test"]
            not in PyFunceble.INTERN["extracted_list_to_test"]
        ):
            return True
        return False

def _do_not_produce_file ( self ) : if ( Inactive ( ) . is_present ( ) and self . domain_status in [ PyFunceble . STATUS [ "official" ] [ "down" ] , PyFunceble . STATUS [ "official" ] [ "invalid" ] , ] and PyFunceble . INTERN [ "to_test" ] not in PyFunceble . INTERN [ "extracted_list_to_test" ] ) : return True return False

Check if we are allowed to produce a file based from the given information .


def _extensions(self, line):
        """
        Extract the extension from the given line.

        :param line: The line from the official public suffix repository.
        :type line: str
        """

        # We strip the parsed line.
        line = line.strip()

        if not line.startswith("//") and "." in line:
            # * The parsed line is not a commented line.
            # and
            # * There is a point in the parsed line.
            line = line.encode("idna").decode("utf-8")

            if line.startswith("*."):
                # The parsed line start with `*.`.

                # We remove the first two characters.
                line = line[2:]

            # We we split the points and we get the last element.
            # Explanation: The idea behind this action is to
            # always get the extension.
            extension = line.split(".")[-1]

            if extension in self.public_suffix_db:
                # The extension is alrady in our database.

                # We update the content of the 1st level TDL with
                # the content of the suffix.
                # In between, we format so that we ensure that there is no
                # duplicate in the database index content.
                self.public_suffix_db[extension] = List(
                    self.public_suffix_db[extension] + [line]
                ).format()
            else:
                # The extension is not already in our database.

                # We append the currently formatted extension and the line content.
                self.public_suffix_db.update({extension: [line]})

Extract the extension from the given line .


def load(self):
        """
        Load the public suffix database into the system.
        """

        if not PyFunceble.INTERN["psl_db"]:
            # The public database was not already loaded.

            # * We read, convert to dict and return the file content.
            # and
            # * We fill/create the database.
            PyFunceble.INTERN["psl_db"] = Dict().from_json(
                File(self.destination).read()
            )

Load the public suffix database into the system .


def standard(cls, element):
        """
        Implement the standard and alphabetical sorting.

        :param element: The element we are currently reading.
        :type element: str

        :return: The formatted element.
        :rtype: str
        """

        # We remove all special characters and return the formatted string.
        return (
            Regex(element, cls.regex_replace, replace_with="@funilrys")
            .replace()
            .replace("@funilrys", "")
        )

Implement the standard and alphabetical sorting .


def hierarchical(cls, element):
        """
        The idea behind this method is to sort a list of domain hierarchicaly.

        :param element: The element we are currently reading.
        :type element: str

        :return: The formatted element.
        :rtype: str

        .. note::
            For a domain like :code:`aaa.bbb.ccc.tdl`.

            A normal sorting is done in the following order:
                1. :code:`aaa`
                2. :code:`bbb`
                3. :code:`ccc`
                4. :code:`tdl`

            This method allow the sorting to be done in the following order:
                1. :code:`tdl`
                2. :code:`ccc`
                3. :code:`bbb`
                4. :code:`aaa`

        """

        # We initiate a variable which will save the element to sort without
        # the extension.
        to_sort = ""

        # We initiate a variable which will save the full extension.
        full_extension = ""

        # We convert the parsed element to lower case.
        element = element.lower()

        # We try to get the url base.
        url_base = Check().is_url_valid(element, return_base=True)

        if not isinstance(url_base, str):
            # The url base is not found.

            if "." in element:
                # There is point in the parsed element.

                # We get the position of the first letter of the extension.
                extension_index = element.rindex(".") + 1

                # We get the extension from the position of the first letter
                # of the extension.
                extension = element[extension_index:]

                if extension in PyFunceble.INTERN["psl_db"]:
                    # The extension is in the public suffix database.

                    for suffix in PyFunceble.INTERN["psl_db"][extension]:
                        # We loop through the list of suffix of the extracted extension.

                        # We suffix the sufix with a point.
                        formatted_suffix = "." + suffix

                        if element.endswith(formatted_suffix):
                            # The elements ends with the suffix.

                            # We get the position of the first character of the suffix in
                            # the parsed element.
                            suffix_index = element.rindex(formatted_suffix)

                            # We update the to_sort variable with the element without the suffix.
                            to_sort = element[:suffix_index]

                            # We replace the full extension with the currently read suffix.
                            full_extension = suffix

                            # We break the loop, we got what we wanted.
                            break

                if not full_extension:
                    # The full extension is empty.

                    # We initiate it with the extension.
                    full_extension = element[extension_index:]

                    # We update the to_sort variable with the element without the extension.
                    to_sort = element[: extension_index - 1]

                # We append a point to the full extension because the point has to be
                # at the end and not at the begining of the extension.
                # To understand: Imagine a miror.
                full_extension += "."

                # We reverse the to_sort string.
                tros_ot = to_sort[::-1]

                if "." in tros_ot:
                    # There is a point in the reversed string.

                    # We prefix the full extension with the top level
                    # domain name.
                    full_extension = (
                        tros_ot[: tros_ot.index(".")][::-1] + "." + full_extension
                    )

                    # We remove the tor level domain from the rest of
                    # the reversed string.
                    tros_ot = tros_ot[tros_ot.index(".") + 1 :]

                    # * We reverse each level of the parsed element.
                    # and
                    # * We glue each level of the parsed element with each other.
                    #
                    # Note: after this, there is no point anymore.
                    reversion = full_extension + ".".join(
                        [x[::-1] for x in tros_ot.split(".")]
                    )

                    # We remove all special characters and return the formatted string.
                    return (
                        Regex(reversion, cls.regex_replace, replace_with="@funilrys")
                        .replace()
                        .replace("@funilrys", "")
                    )

                # We remove all special characters and return the formatted string.
                return (
                    Regex(
                        to_sort + full_extension,
                        cls.regex_replace,
                        replace_with="@funilrys",
                    )
                    .replace()
                    .replace("@funilrys", "")
                )

            # There is no point in the parsed element.

            # We return the parsed element.
            return element

        # The url base is found.

        # We get the position of the element.
        protocol_position = element.rindex(url_base)

        # We extract the protocol from the element position.
        protocol = element[:protocol_position]

        # We return the output of this method but with the url base instead of the full url.
        return protocol + cls.hierarchical(url_base)

The idea behind this method is to sort a list of domain hierarchicaly .


def load(self):
        """
        Initiate the IANA database if it is not the case.
        """

        if "iana_db" not in PyFunceble.INTERN or not PyFunceble.INTERN["iana_db"]:
            # The global database is empty, None or does not exist.

            # We update it with the database content.
            PyFunceble.INTERN["iana_db"] = self.iana_db

Initiate the IANA database if it is not the case .


def _referer(self, extension):
        """
        Return the referer for the given extension.

        :param extension: A valid domain extension.
        :type extension: str

        :return: The whois server to use to get the WHOIS record.
        :rtype: str
        """

        # We get the a copy of the page.
        iana_record = self.lookup.whois(
            PyFunceble.CONFIGURATION["iana_whois_server"], "hello.%s" % extension
        )

        if iana_record and "refer" in iana_record:
            # The record is not empty.

            # We initiate a regex which will extract the referer.
            regex_referer = r"(?s)refer\:\s+([a-zA-Z0-9._-]+)\n"

            # We try to extract the referer.
            matched = Regex(
                iana_record, regex_referer, return_data=True, group=1
            ).match()

            if matched:
                # The referer was extracted successfully.

                # We return the matched referer.
                return matched

        # * The referer was not extracted successfully.
        # or
        # * The iana record is empty.

        if extension in self.manual_server:
            # The extension is in the list of manual entries.

            # We return the server which we set manually.
            return self.manual_server[extension]

        # We return None because we weren't able to get the server to call for
        # the given extension.
        return None

Return the referer for the given extension .


def _extensions(self):
        """
        Extract the extention from the given block.
        Plus get its referer.
        """

        upstream_lines = (
            Download(self.iana_url, return_data=True)
            .text()
            .split('<span class="domain tld">')
        )

        # We extract the different extension from the currently readed line.
        regex_valid_extension = r"(/domains/root/db/)(.*)(\.html)"

        for block in upstream_lines:
            if "/domains/root/db/" in block:
                # The link is in the line.

                # We try to extract the extension.
                matched = Regex(
                    block, regex_valid_extension, return_data=True, rematch=True
                ).match()[1]

                if matched:
                    # The extraction is not empty or None.

                    # We get the referer.
                    referer = self._referer(matched)

                    # We yield the matched extension and its referer.
                    yield (matched, referer)

Extract the extention from the given block . Plus get its referer .


def update(self):
        """
        Update the content of the `iana-domains-db` file.
        """

        if not PyFunceble.CONFIGURATION["quiet"]:
            # * The quiet mode is not activated.

            # We print on screen what we are doing.
            print("Update of iana-domains-db", end=" ")

        # We loop through the line of the iana website.
        for extension, referer in self._extensions():

            if extension not in self.iana_db or self.iana_db[extension] != referer:
                # We add the extension to the databae.
                self.iana_db[extension] = referer

                # We save the content of the constructed database.
                Dict(self.iana_db).to_json(self.destination)

        if not PyFunceble.CONFIGURATION["quiet"]:
            # The quiet mode is not activated.

            # We indicate that the work is done without any issue.
            print(PyFunceble.INTERN["done"])

Update the content of the iana - domains - db file .