if link.startswith("/") and not link.startswith("//"):
                absolute_paths.append(urljoin(base_url, link))
                continue
            # Some may have omitted the protocol like //to/path
            if link.startswith("//"):
                absolute_paths.append(f"{urlparse(base_url).scheme}:{link}")
                continue
        # Remove duplicates
        # also do another filter to prevent outside links
        absolute_paths = list(
            set(
                [
                    path
                    for path in absolute_paths
                    if not self.prevent_outside
                    or path.startswith(base_url)
                    and path != base_url
                ]
            )
        )
        return absolute_paths
    def _gen_metadata(self, raw_html: str, url: str) -> dict:
        """Build metadata from BeautifulSoup output."""
        try:
            from bs4 import BeautifulSoup
        except ImportError:
            print("The bs4 package is required for the RecursiveUrlLoader.")
            print("Please install it with `pip install bs4`.")
        metadata = {"source": url}
        soup = BeautifulSoup(raw_html, "html.parser")
        if title := soup.find("title"):
            metadata["title"] = title.get_text()
        if description := soup.find("meta", attrs={"name": "description"}):
            metadata["description"] = description.get("content", None)
        if html := soup.find("html"):
            metadata["language"] = html.get("lang", None)
        return metadata
    def _get_child_links_recursive(
        self, url: str, visited: Optional[Set[str]] = None, depth: int = 0
    ) -> Iterator[Document]:
        """Recursively get all child links starting with the path of the input URL.
        Args:
            url: The URL to crawl.