Source code for langchain_community.document_loaders.sitemap
import itertools
import re
from typing import Any, Callable, Generator, Iterable, List, Optional, Tuple
from urllib.parse import urlparse
from langchain_core.documents import Document
from langchain_community.document_loaders.web_base import WebBaseLoader
def _default_parsing_function(content: Any) -> str:
    return str(content.get_text())
def _default_meta_function(meta: dict, _content: Any) -> dict:
    return {"source": meta["loc"], **meta}
def _batch_block(iterable: Iterable, size: int) -> Generator[List[dict], None, None]:
    it = iter(iterable)
    while item := list(itertools.islice(it, size)):
        yield item
def _extract_scheme_and_domain(url: str) -> Tuple[str, str]:
    """Extract the scheme + domain from a given URL.
    Args:
        url (str): The input URL.
    Returns:
        return a 2-tuple of scheme and domain
    """
    parsed_uri = urlparse(url)
    return parsed_uri.scheme, parsed_uri.netloc
[docs]class SitemapLoader(WebBaseLoader):
    """Load a sitemap and its URLs.
    **Security Note**: This loader can be used to load all URLs specified in a sitemap.
        If a malicious actor gets access to the sitemap, they could force
        the server to load URLs from other domains by modifying the sitemap.
        This could lead to server-side request forgery (SSRF) attacks; e.g.,
        with the attacker forcing the server to load URLs from internal
        service endpoints that are not publicly accessible. While the attacker
        may not immediately gain access to this data, this data could leak