@classmethod
    def _fetch_remote_html(cls, uri, follow_meta_refresh=True, follow_meta_refresh_max=5, bs4features="html5lib"):

        document = None
        follow_meta_refresh_depth = 1

        while True:
            (response, uri) = cls._fetch_remote(uri, follow_meta_refresh_depth)

            if not response:
                break

            document = BeautifulSoup(response, features=bs4features)

            if not follow_meta_refresh:
                break

            refresh_uri = cls.__html_find_meta_refresh(document)
            if not refresh_uri:
                break
            refresh_uri = urljoin(uri, refresh_uri)

            if refresh_uri == uri:
                break

            cls._log("debug", "fetch remote HTML(%d): %s meta-refreshes to %s" %
                     (follow_meta_refresh_depth, uri, refresh_uri))
            uri = refresh_uri

            follow_meta_refresh_depth += 1
            if follow_meta_refresh_depth >= follow_meta_refresh_max:
                break

        if not document:
            cls._log("debug", "fetch remote HTML: %s is empty" % uri)

        base = uri

        doc_base = cls.__html_find_base(document)
        if doc_base:
            base = urljoin(base, doc_base)

        return document, base, uri

    ## general functions
returns remote HTML document, actual remote uri and base uri
dataset/ETHPy150Open k4cg/nichtparasoup/crawler/__init__.py/Crawler._fetch_remote_html