Debug - Recurision Error

main.py

@@ -474,18 +474,71 @@ async def crawl_sync(request: CrawlRequest) -> Dict[str, Any]:
     raise HTTPException(status_code=408, detail="Task timed out")
 
 
+# @app.post(
+#     "/crawl_direct", dependencies=[secure_endpoint()] if CRAWL4AI_API_TOKEN else []
+# )
+# async def crawl_direct(request: CrawlRequest) -> Dict[str, Any]:
+#     try:
+#         crawler = await crawler_service.crawler_pool.acquire(**request.crawler_params)
+#         extraction_strategy = crawler_service._create_extraction_strategy(
+#             request.extraction_config
+#         )
+
+#         try:
+#             if isinstance(request.urls, list):
+#                 results = await crawler.arun_many(
+#                     urls=[str(url) for url in request.urls],
+#                     extraction_strategy=extraction_strategy,
+#                     js_code=request.js_code,
+#                     wait_for=request.wait_for,
+#                     css_selector=request.css_selector,
+#                     screenshot=request.screenshot,
+#                     magic=request.magic,
+#                     cache_mode=request.cache_mode,
+#                     session_id=request.session_id,
+#                     **request.extra,
+#                 )
+#                 return {"results": [result.dict() for result in results]}
+#             else:
+#                 result = await crawler.arun(
+#                     url=str(request.urls),
+#                     extraction_strategy=extraction_strategy,
+#                     js_code=request.js_code,
+#                     wait_for=request.wait_for,
+#                     css_selector=request.css_selector,
+#                     screenshot=request.screenshot,
+#                     magic=request.magic,
+#                     cache_mode=request.cache_mode,
+#                     session_id=request.session_id,
+#                     **request.extra,
+#                 )
+#                 return {"result": result.dict()}
+#         finally:
+#             await crawler_service.crawler_pool.release(crawler)
+#     except Exception as e:
+#         logger.error(f"Error in direct crawl: {str(e)}")
+#         raise HTTPException(status_code=500, detail=str(e))
+
+
 @app.post(
     "/crawl_direct", dependencies=[secure_endpoint()] if CRAWL4AI_API_TOKEN else []
 )
 async def crawl_direct(request: CrawlRequest) -> Dict[str, Any]:
+    logger.info("Received request to crawl directly.")
     try:
+        logger.debug("Acquiring crawler from the crawler pool.")
         crawler = await crawler_service.crawler_pool.acquire(**request.crawler_params)
+        logger.debug("Crawler acquired successfully.")
+
+        logger.debug("Creating extraction strategy based on the request configuration.")
         extraction_strategy = crawler_service._create_extraction_strategy(
             request.extraction_config
         )
+        logger.debug("Extraction strategy created successfully.")
 
         try:
             if isinstance(request.urls, list):
+                logger.info("Processing multiple URLs.")
                 results = await crawler.arun_many(
                     urls=[str(url) for url in request.urls],
                     extraction_strategy=extraction_strategy,
@@ -498,8 +551,10 @@ async def crawl_direct(request: CrawlRequest) -> Dict[str, Any]:
                     session_id=request.session_id,
                     **request.extra,
                 )
+                logger.info("Crawling completed for multiple URLs.")
                 return {"results": [result.dict() for result in results]}
             else:
+                logger.info("Processing a single URL.")
                 result = await crawler.arun(
                     url=str(request.urls),
                     extraction_strategy=extraction_strategy,
@@ -512,9 +567,12 @@ async def crawl_direct(request: CrawlRequest) -> Dict[str, Any]:
                     session_id=request.session_id,
                     **request.extra,
                 )
+                logger.info("Crawling completed for a single URL.")
                 return {"result": result.dict()}
         finally:
+            logger.debug("Releasing crawler back to the pool.")
             await crawler_service.crawler_pool.release(crawler)
+            logger.debug("Crawler released successfully.")
     except Exception as e:
         logger.error(f"Error in direct crawl: {str(e)}")
         raise HTTPException(status_code=500, detail=str(e))