Source code for langchain.document_loaders.recursive_url_loader
import asyncio
import re
from typing import Callable, Iterator, List, Optional, Set, Union
from urllib.parse import urljoin, urlparse
import requests
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
[docs]class RecursiveUrlLoader(BaseLoader):
    """Loads all child links from a given url."""
[docs]    def __init__(
        self,
        url: str,
        max_depth: Optional[int] = None,
        use_async: Optional[bool] = None,
        extractor: Optional[Callable[[str], str]] = None,
        exclude_dirs: Optional[str] = None,
        timeout: Optional[int] = None,
        prevent_outside: Optional[bool] = None,
    ) -> None:
        """Initialize with URL to crawl and any subdirectories to exclude.
        Args:
            url: The URL to crawl.
            exclude_dirs: A list of subdirectories to exclude.
            use_async: Whether to use asynchronous loading,
            if use_async is true, this function will not be lazy,
            but it will still work in the expected way, just not lazy.
            extractor: A function to extract the text from the html,
            when extract function returns empty string, the document will be ignored.
            max_depth: The max depth of the recursive loading.
            timeout: The timeout for the requests, in the unit of seconds.
        """
        self.url = url
        self.exclude_dirs = exclude_dirs
        self.use_async = use_async if use_async is not None else False
        self.extractor = extractor if extractor is not None else lambda x: x
        self.max_depth = max_depth if max_depth is not None else 2