sub_tasks = []
            for link in absolute_paths:
                sub_tasks.append(
                    self._async_get_child_links_recursive(link, visited, depth + 1)
                )
            # sub_tasks returns coroutines of list,
            # so we need to flatten the list await asyncio.gather(*sub_tasks)
            flattened = []
            next_results = await asyncio.gather(*sub_tasks)
            for sub_result in next_results:
                if isinstance(sub_result, Exception):
                    # We don't want to stop the whole process, so just ignore it
                    # Not standard html format or invalid url or 404 may cause this
                    # But we can't do anything about it.
                    continue
                if sub_result is not None:
                    flattened += sub_result
            results += flattened
            return list(filter(lambda x: x is not None, results))
[docs]    def lazy_load(self) -> Iterator[Document]:
        """Lazy load web pages.
        When use_async is True, this function will not be lazy,
        but it will still work in the expected way, just not lazy."""
        if self.use_async:
            results = asyncio.run(self._async_get_child_links_recursive(self.url))
            if results is None:
                return iter([])
            else:
                return iter(results)
        else:
            return self._get_child_links_recursive(self.url)
[docs]    def load(self) -> List[Document]:
        """Load web pages."""
        return list(self.lazy_load())