return []
            absolute_paths = self._get_sub_links(text, url)
            # Worker will be only called within the current function
            # Worker function will process the link
            # then recursively call get_child_links_recursive to process the children
            async def worker(link: str) -> Union[Document, None]:
                try:
                    async with aiohttp.ClientSession(
                        connector=aiohttp.TCPConnector(ssl=False),
                        timeout=aiohttp.ClientTimeout(self.timeout),
                    ) as session:
                        response = await session.get(link)
                        text = await response.text()
                        extracted = self.extractor(text)
                        if len(extracted) > 0:
                            return Document(
                                page_content=extracted,
                                metadata=self._gen_metadata(text, link),
                            )
                        else:
                            return None
                # Despite the fact that we have filtered some links,
                # there may still be some invalid links, so catch the exception
                except aiohttp.client_exceptions.InvalidURL:
                    return None
                # There may be some other exceptions, so catch them,
                # we don't want to stop the whole process
                except Exception:
                    # print(e)
                    return None
            # The coroutines that will be executed
            tasks = []
            # Generate the tasks
            for link in absolute_paths:
                # Check all unvisited links
                if link not in visited:
                    visited.add(link)
                    tasks.append(worker(link))
            # Get the not None results
            results = list(
                filter(lambda x: x is not None, await asyncio.gather(*tasks))
            )
            # Recursively call the function to get the children of the children
            sub_tasks = []
            for link in absolute_paths:
                sub_tasks.append(