feat: add logging mechanism

src/deepengineer/deepsearch/analyse_markdown_agent.py

@@ -5,7 +5,7 @@ Simple agent to analyse a markdown, just to test some ideas.
 from enum import Enum
 
 from mistralai import OCRResponse
-from smolagents import CodeAgent, LiteLLMModel, Tool
+from smolagents import CodeAgent, LiteLLMModel
 
 from deepengineer.webcrawler.pdf_utils import (
     convert_ocr_response_to_markdown,
@@ -13,6 +13,7 @@ from deepengineer.webcrawler.pdf_utils import (
     get_markdown_by_page_numbers,
     get_table_of_contents_per_page_markdown,
 )
+from deepengineer.logging_tools import LoggingTool
 
 
 class ToolNames(Enum):
@@ -22,7 +23,7 @@ class ToolNames(Enum):
     FIND_IN_MARKDOWN = "find_in_markdown"
 
 
-class GetTableOfContentsTool(Tool):
+class GetTableOfContentsTool(LoggingTool):
     name = ToolNames.GET_TABLE_OF_CONTENTS.value
     description = "Returns all of the titles in the document along with the page number they are on."
     inputs = {}
@@ -39,7 +40,7 @@ class GetTableOfContentsTool(Tool):
         return self.table_of_contents
 
 
-class GetMarkdownTool(Tool):
+class GetMarkdownTool(LoggingTool):
     name = ToolNames.GET_MARKDOWN.value
     description = f"Returns the markdown entire content of the document. Beware this might be too long to be useful, except for small documents, use {ToolNames.GET_PAGES_CONTENT.value} instead. You can use {ToolNames.GET_TABLE_OF_CONTENTS.value} to get the table of contents of the document including the number of pages."
     inputs = {}
@@ -54,7 +55,7 @@ class GetMarkdownTool(Tool):
         return self.markdown_content
 
 
-class GetPagesContentTool(Tool):
+class GetPagesContentTool(LoggingTool):
     name = ToolNames.GET_PAGES_CONTENT.value
     description = f"Returns the content of the pages. You can use {ToolNames.GET_TABLE_OF_CONTENTS.value} to get the table of contents of the document including the number of pages. Expects a list of page numbers as integers as input."
     inputs = {
@@ -73,7 +74,7 @@ class GetPagesContentTool(Tool):
         return get_markdown_by_page_numbers(self.markdown, page_numbers)
 
 
-class FindInMarkdownTool(Tool):
+class FindInMarkdownTool(LoggingTool):
     name = ToolNames.FIND_IN_MARKDOWN.value
     description = f"Finds the page numbers of the document that contain the search queries. If you are looking for a specific information, you can use this tool to find the page numbers of the document that contain the information and then use {ToolNames.GET_PAGES_CONTENT.value} to get the content of the pages."
     inputs = {

src/deepengineer/deepsearch/draw_agent.py

@@ -12,8 +12,8 @@ from PIL import Image
 
 from smolagents import CodeAgent, LiteLLMModel
 from smolagents.agents import ActionStep
-from smolagents import Tool
 from deepengineer.webcrawler.crawl_database import DataBase
+from deepengineer.logging_tools import LoggingTool
 
 
 def _find_and_save_matplotlib_figure(image_path: Path = Path("figure.png")) -> str:
@@ -28,7 +28,7 @@ def _find_and_save_matplotlib_figure(image_path: Path = Path("figure.png")) -> s
     return f"Figure saved to {image_path}."
 
 
-class SaveMatplotlibFigTool(Tool):
+class SaveMatplotlibFigTool(LoggingTool):
     name = "save_matplotlib_fig"
     description = """Save the current matplotlib figure to the current directory. Then plt.close() is called to clear the figure. The image is returned as a markdown string, use this markdown inside the final answer to include the image.
     """
@@ -148,7 +148,7 @@ def draw_matplotlib_image_from_prompt(
     return image_path
 
 
-class DrawImageTool(Tool):
+class DrawImageTool(LoggingTool):
     name = "draw_image"
     description = f"Draw an image based on a prompt. The image is saved in the current directory. The image is returned as a markdown image, use this markdown inside the final answer to include the image. You must be very specific in your prompt."
     inputs = {

src/deepengineer/deepsearch/main_agent.py

@@ -15,6 +15,7 @@ from deepengineer.common_path import DATA_DIR
 from smolagents import CodeAgent, LiteLLMModel
 import random
 from pathlib import Path
+import queue
 
 
 def _create_output_image_path():
@@ -97,7 +98,7 @@ Run verification steps if that's needed, you must make sure you find the correct
 
 
 def create_main_search_agent(
-    model_id="deepseek/deepseek-reasoner", database: DataBase | None = None
+    model_id="deepseek/deepseek-reasoner", database: DataBase | None = None, log_queue: queue.Queue | None = None
 ):
     """
     Simple agent that can search the web and answer the question. This is much faster and better for simple questions that do not require deep research.
@@ -111,14 +112,14 @@ def create_main_search_agent(
 
     # Web search and crawling tools
     WEB_SEARCH_TOOLS = [
-        SearchTool(),
-        ArxivSearchTool(),
-        ScientificSearchTool(),
-        GetTableOfContentsTool(database),
-        GetMarkdownTool(database),
-        GetPagesContentTool(database),
-        FindInMarkdownTool(database),
-        SaveMatplotlibFigTool(output_dir=output_image_path),
+        SearchTool(log_queue=log_queue,),
+        ArxivSearchTool(log_queue=log_queue,),
+        ScientificSearchTool(log_queue=log_queue,),
+        GetTableOfContentsTool(log_queue=log_queue, database=database),
+        GetMarkdownTool(log_queue=log_queue, database=database),
+        GetPagesContentTool(log_queue=log_queue, database=database),
+        FindInMarkdownTool(log_queue=log_queue, database=database),
+        SaveMatplotlibFigTool(log_queue=log_queue,output_dir=output_image_path),
     ]
 
     search_agent = CodeAgent(
@@ -138,7 +139,7 @@ def create_main_search_agent(
     return search_agent
 
 
-def main_search(task: str):
+def main_search(task: str, log_queue: queue.Queue | None = None):
     MAIN_PROMPT = """
 You are DeepDraft, an advanced research and analysis agent specialized in deep technical research, data visualization, and comprehensive information synthesis. You have access to powerful tools for web search, document analysis, and data visualization.
 
@@ -164,7 +165,7 @@ Failure or 'I cannot answer' or 'None found' will not be tolerated, success will
 Run verification steps if that's needed, you must make sure you find the correct answer! Here is the task:
 {task}
 """
-    agent = create_main_search_agent(model_id="mistral/mistral-medium-latest")
+    agent = create_main_search_agent(model_id="mistral/mistral-medium-latest", log_queue=log_queue)
     answer = agent.run(MAIN_PROMPT.format(task=task))
     return answer
 

src/deepengineer/deepsearch/scawl_web_agent.py

@@ -1,7 +1,7 @@
 import asyncio
 from enum import Enum
 
-from smolagents import CodeAgent, LiteLLMModel, Tool
+from smolagents import CodeAgent, LiteLLMModel
 
 from deepengineer.webcrawler.async_search import (
     SearchResponse,
@@ -17,6 +17,7 @@ from deepengineer.webcrawler.pdf_utils import (
     get_markdown_by_page_numbers,
     get_table_of_contents_per_page_markdown,
 )
+from deepengineer.logging_tools import LoggingTool
 
 
 class ToolNames(Enum):
@@ -41,7 +42,7 @@ def filter_search_results(
     return search_response.to_string()
 
 
-class SearchTool(Tool):
+class SearchTool(LoggingTool):
     name = ToolNames.SEARCH_TOOL.value
     description = """Search the web using Linkup API. Good for deep research with sourced answers.
     Linkup also provides an answer. This answer is not always correct, so you might want to check the sources.
@@ -63,7 +64,7 @@ class SearchTool(Tool):
         return filter_search_results(result)
 
 
-class ArxivSearchTool(Tool):
+class ArxivSearchTool(LoggingTool):
     name = ToolNames.ARXIV_SEARCH.value
     description = """Search arXiv for academic papers and preprints with Linkup API.
     Linkup also provides an answer. This answer is not always correct, so you might want to check the sources.
@@ -81,7 +82,7 @@ class ArxivSearchTool(Tool):
         return filter_search_results(result)
 
 
-class PubmedSearchTool(Tool):
+class PubmedSearchTool(LoggingTool):
     name = ToolNames.PUBMED_SEARCH.value
     description = """Search PubMed for medical and scientific literature with Linkup API.
     Linkup also provides an answer. This answer is not always correct, so you might want to check the sources.
@@ -99,7 +100,7 @@ class PubmedSearchTool(Tool):
         return filter_search_results(result)
 
 
-class ScientificSearchTool(Tool):
+class ScientificSearchTool(LoggingTool):
     name = ToolNames.SCIENTIFIC_SEARCH.value
     description = """Search across multiple scientific domains: Wikipedia, arXiv, PubMed, and ScienceDirect.
     Linkup also provides an answer. This answer is not always correct, so you might want to check the sources.
@@ -120,7 +121,7 @@ class ScientificSearchTool(Tool):
 URL_EXPLAINATION = """The URL can be be converted to a markdown. If the URL points to a PDF, the pdf is converted to markdown, otherwise the URL is crawled and the markdown is extracted. This markdown is split into pages that are numbered. You can use the page numbers to get the content of the pages."""
 
 
-class GetTableOfContentsTool(Tool):
+class GetTableOfContentsTool(LoggingTool):
     name = ToolNames.GET_TABLE_OF_CONTENTS.value
     description = f"""Returns all of the titles in the document along with the page number they are on.
     {URL_EXPLAINATION}
@@ -143,7 +144,7 @@ class GetTableOfContentsTool(Tool):
         return table_of_contents
 
 
-class GetMarkdownTool(Tool):
+class GetMarkdownTool(LoggingTool):
     name = ToolNames.GET_MARKDOWN.value
     description = f"Returns in markdown entire content of the url. Beware this might be too long to be useful, except for small documents, use {ToolNames.GET_PAGES_CONTENT.value} instead. You can also use {ToolNames.GET_TABLE_OF_CONTENTS.value} first to get the table of contents of the document including the number of pages."
     inputs = {
@@ -161,7 +162,7 @@ class GetMarkdownTool(Tool):
         return markdown_content
 
 
-class GetPagesContentTool(Tool):
+class GetPagesContentTool(LoggingTool):
     name = ToolNames.GET_PAGES_CONTENT.value
     description = f"Returns the content of the pages. You can use {ToolNames.GET_TABLE_OF_CONTENTS.value} to get the table of contents of the url including the number of pages. Expects a list of page numbers as integers as input. {URL_EXPLAINATION}"
     inputs = {
@@ -182,7 +183,7 @@ class GetPagesContentTool(Tool):
         return get_markdown_by_page_numbers(markdown, page_numbers)
 
 
-class FindInMarkdownTool(Tool):
+class FindInMarkdownTool(LoggingTool):
     name = ToolNames.FIND_IN_MARKDOWN.value
     description = f"Finds the page numbers of the url that contain the search queries. If you are looking for a specific information, you can use this tool to find the page numbers of the url that contain the information and then use {ToolNames.GET_PAGES_CONTENT.value} to get the content of the pages. {URL_EXPLAINATION}"
     inputs = {

src/deepengineer/logging_tools.py

@@ -0,0 +1,15 @@
+import queue
+from typing import Any
+from smolagents import Tool
+
+class LoggingTool(Tool):
+    """
+    Base class for tools that can push logs to a queue.
+    """
+    def __init__(self, log_queue: queue.Queue | None = None):
+        super().__init__()
+        self.log_queue = log_queue
+
+    def push_log(self, msg: str):
+        if self.log_queue:
+            self.log_queue.put(msg)