Args:
            url: The URL to crawl.
            visited: A set of visited URLs.
        """
        if depth > self.max_depth:
            return []
        # Add a trailing slash if not present
        if not url.endswith("/"):
            url += "/"
        # Exclude the root and parent from a list
        visited = set() if visited is None else visited
        # Exclude the links that start with any of the excluded directories
        if self.exclude_dirs and any(
            url.startswith(exclude_dir) for exclude_dir in self.exclude_dirs
        ):
            return []
        # Get all links that can be accessed from the current URL
        try:
            response = requests.get(url, timeout=self.timeout)
        except Exception:
            return []
        absolute_paths = self._get_sub_links(response.text, url)
        # Store the visited links and recursively visit the children
        for link in absolute_paths:
            # Check all unvisited links
            if link not in visited:
                visited.add(link)
                try:
                    response = requests.get(link)
                    text = response.text
                except Exception:
                    # unreachable link, so just ignore it
                    continue
                loaded_link = Document(
                    page_content=self.extractor(text),
                    metadata=self._gen_metadata(text, link),
                )
                yield loaded_link
                # If the link is a directory (w/ children) then visit it
                if link.endswith("/"):
                    yield from self._get_child_links_recursive(link, visited, depth + 1)
        return []
    async def _async_get_child_links_recursive(
        self, url: str, visited: Optional[Set[str]] = None, depth: int = 0
    ) -> List[Document]: