Source code for langchain_community.document_loaders.docusaurus
"""Load Documents from Docusarus Documentation"""
from typing import Any, List, Optional
from langchain_community.document_loaders.sitemap import SitemapLoader
[docs]class DocusaurusLoader(SitemapLoader):
    """
    Loader that leverages the SitemapLoader to loop through the generated pages of a
    Docusaurus Documentation website and extracts the content by looking for specific
    HTML tags. By default, the parser searches for the main content of the Docusaurus
    page, which is normally the <article>. You also have the option to define your own
    custom HTML tags by providing them as a list, for example: ["div", ".main", "a"].
    """
[docs]    def __init__(
        self,
        url: str,
        custom_html_tags: Optional[List[str]] = None,
        **kwargs: Any,
    ):
        """
        Initialize DocusaurusLoader
        Args:
            url: The base URL of the Docusaurus website.
            custom_html_tags: Optional custom html tags to extract content from pages.
            kwargs: Additional args to extend the underlying SitemapLoader, for example:
                filter_urls, blocksize, meta_function, is_local, continue_on_failure
        """
        if not kwargs.get("is_local"):
            url = f"{url}/sitemap.xml"
        self.custom_html_tags = custom_html_tags or ["main article"]
        super().__init__(
            url,
            parsing_function=kwargs.get("parsing_function") or self._parsing_function,
            **kwargs,
        )
    def _parsing_function(self, content: Any) -> str:
        """Parses specific elements from a Docusarus page."""