text = self._clean_data(f.read())
                yield Document(page_content=text, metadata={"source": str(p)})
[docs]    def load(self) -> List[Document]:
        """Load documents."""
        return list(self.lazy_load())
    def _clean_data(self, data: str) -> str:
        from bs4 import BeautifulSoup
        soup = BeautifulSoup(data, "html.parser", **self.bs_kwargs)
        # default tags
        html_tags = [
            ("div", {"role": "main"}),
            ("main", {"id": "main-content"}),
        ]
        if self.custom_html_tag is not None:
            html_tags.append(self.custom_html_tag)
        element = None
        # reversed order. check the custom one first
        for tag, attrs in html_tags[::-1]:
            element = soup.find(tag, attrs)
            # if found, break
            if element is not None:
                break
        if element is not None and _get_link_ratio(element) <= self.exclude_links_ratio:
            text = _get_clean_text(element)
        else:
            text = ""
        # trim empty lines
        return "\n".join([t for t in text.split("\n") if t])
def _get_clean_text(element: Tag) -> str:
    """Returns cleaned text with newlines preserved and irrelevant elements removed."""
    elements_to_skip = [
        "script",
        "noscript",
        "canvas",
        "meta",
        "svg",
        "map",
        "area",
        "audio",
        "source",
        "track",
        "video",
        "embed",
        "object",
        "param",
        "picture",
        "iframe",
        "frame",
        "frameset",