# Extract all navigable strings recursively from this element.
                text_parts += get_navigable_strings(element)
                # To avoid duplicate text, remove all descendants from the soup.
                element.decompose()
        return " ".join(text_parts)
[docs]    @staticmethod
    def remove_unnecessary_lines(content: str) -> str:
        """
        Clean up the content by removing unnecessary lines.
        Args:
            content: A string, which may contain unnecessary lines or spaces.
        Returns:
            A cleaned string with unnecessary lines removed.
        """
        lines = content.split("\n")
        stripped_lines = [line.strip() for line in lines]
        non_empty_lines = [line for line in stripped_lines if line]
        cleaned_content = " ".join(non_empty_lines)
        return cleaned_content
[docs]    async def atransform_documents(
        self,
        documents: Sequence[Document],
        **kwargs: Any,
    ) -> Sequence[Document]:
        raise NotImplementedError
[docs]def get_navigable_strings(element: Any) -> Iterator[str]:
    from bs4 import NavigableString, Tag
    for child in cast(Tag, element).children:
        if isinstance(child, Tag):
            yield from get_navigable_strings(child)
        elif isinstance(child, NavigableString):
            if (element.name == "a") and (href := element.get("href")):
                yield f"{child.strip()} ({href})"
            else:
                yield child.strip()