feat: add logging mechanism

gradio_app.py

@@ -1,41 +1,7 @@
 import gradio as gr
-import threading, time, queue, logging
+import threading, time, queue
+from deepengineer.deepsearch.main_agent import main_search
 
-# ---------- 1.  A thread‑safe queue for log messages ----------
-
-    
-class _BaseTool:
-    def __init__(self, log_queue: queue.Queue | None = None):
-        self.log_queue = log_queue
-
-    def push_log(self, msg: str):
-        if self.log_queue:
-            self.log_queue.put(msg)
-
-    def forward(self, input: str) -> str:
-        raise NotImplementedError("Subclasses must implement forward method")
-
-# ---------- 2.  Tools that report progress ----------
-class SearchTool(_BaseTool):
-    def forward(self, query: str) -> str:
-        self.push_log(f"🔍 SearchTool → {query}")
-        time.sleep(2)                      # expensive work…
-        out = f"Found {query}"
-        self.push_log(f"✅ SearchTool done")
-        return out
-
-class DrawTool(_BaseTool):
-    def forward(self, prompt: str) -> str:
-        self.push_log(f"🎨 DrawTool → {prompt}")
-        time.sleep(2)
-        out = f"Drawing {prompt}"
-        self.push_log(f"✅ DrawTool done")
-        return out
-
-# ---------- 3.  Your unchanged agent -------------------------
-def agent(user_input: str, log_queue: queue.Queue) -> str:
-    st, dt = SearchTool(log_queue), DrawTool(log_queue)
-    return st.forward(user_input) + dt.forward(user_input) + st.forward(user_input) + dt.forward(user_input)
 
 # ---------- 4.  Wrapper that streams -------------------------
 def run_agent_stream(user_input: str):
@@ -48,6 +14,7 @@ def run_agent_stream(user_input: str):
     Yields tuples: (agent_output, log_output)
     """
     log_queue = queue.Queue()
+    
     # empty queue before each run
     while not log_queue.empty():
         log_queue.get_nowait()
@@ -56,7 +23,7 @@ def run_agent_stream(user_input: str):
     done = threading.Event()
 
     def _worker():
-        answer_container["text"] = agent(user_input, log_queue)
+        answer_container["text"] = main_search(user_input, log_queue)
         done.set()
 
     threading.Thread(target=_worker, daemon=True).start()
@@ -81,17 +48,17 @@ def run_agent_stream(user_input: str):
 
 # ---------- 5.  Gradio UI ------------------------------------
 with gr.Blocks() as demo:
-    gr.Markdown("## Agent Interface with Real‑Time Tool Logging")
+    gr.Markdown("# Agent Interface with Real‑Time Tool Logging")
     user_input  = gr.Textbox(label="User Message")
-    agent_output = gr.Textbox(label="Agent Response")
-    log_output   = gr.Textbox(label="Tool Invocation Log", interactive=False)
+    agent_output = gr.Markdown(label="Agent Response")
+    log_output = gr.Textbox(label="Tool Invocation Log", interactive=False)
 
     send = gr.Button("Send")
     send.click(
         fn=run_agent_stream,
         inputs=[user_input],
-        outputs=[agent_output, log_output],
-        concurrency_limit=4,
+        outputs=[log_output, agent_output],
+        concurrency_limit=2,
     )
 
 if __name__ == "__main__":

src/deepengineer/deepsearch/draw_agent.py

@@ -14,6 +14,7 @@ from smolagents import CodeAgent, LiteLLMModel
 from smolagents.agents import ActionStep
 from deepengineer.webcrawler.crawl_database import DataBase
 from deepengineer.logging_tools import LoggingTool
+import queue
 
 
 def _find_and_save_matplotlib_figure(image_path: Path = Path("figure.png")) -> str:
@@ -40,11 +41,12 @@ class SaveMatplotlibFigTool(LoggingTool):
     }
     output_type = "string"
 
-    def __init__(self, output_dir: Path):
-        super().__init__()
+    def __init__(self, output_dir: Path, log_queue: queue.Queue | None = None):
+        super().__init__(log_queue=log_queue)
         self.output_dir: Path = output_dir
 
     def forward(self, image_name: str) -> str:
+        self.push_log(f"🖼️ Saving matplotlib figure to {image_name}")
         if not image_name.endswith(".png"):
             image_name = image_name + ".png"
         output_path = self.output_dir / image_name
@@ -173,6 +175,7 @@ class DrawImageTool(LoggingTool):
         self.output_dir: Path = output_dir
 
     def forward(self, prompt: str, image_name: str) -> str:
+        self.push_log(f"🖊️ Drawing image from prompt: {prompt}")
         if not image_name.endswith(".png"):
             image_name = image_name + ".png"
         output_path = draw_matplotlib_image_from_prompt(

src/deepengineer/deepsearch/scawl_web_agent.py

@@ -18,7 +18,7 @@ from deepengineer.webcrawler.pdf_utils import (
     get_table_of_contents_per_page_markdown,
 )
 from deepengineer.logging_tools import LoggingTool
-
+import queue
 
 class ToolNames(Enum):
     # Search tools
@@ -54,8 +54,12 @@ class SearchTool(LoggingTool):
         },
     }
     output_type = "object"
+    
+    def __init__(self, log_queue: queue.Queue | None = None):
+        super().__init__(log_queue=log_queue)
 
     def forward(self, search_query: str) -> str:
+        self.push_log(f"🔍 Searching web for: {search_query}")
         result = asyncio.run(
             linkup_search_async(
                 search_query=search_query,
@@ -76,8 +80,12 @@ class ArxivSearchTool(LoggingTool):
         }
     }
     output_type = "object"
+    
+    def __init__(self, log_queue: queue.Queue | None = None):
+        super().__init__(log_queue=log_queue)
 
     def forward(self, search_query: str) -> str:
+        self.push_log(f"🔍 Searching arXiv for: {search_query}")
         result = asyncio.run(arxiv_search_async(search_query))
         return filter_search_results(result)
 
@@ -94,8 +102,12 @@ class PubmedSearchTool(LoggingTool):
         }
     }
     output_type = "object"
+    
+    def __init__(self, log_queue: queue.Queue | None = None):
+        super().__init__(log_queue=log_queue)
 
     def forward(self, search_query: str) -> str:
+        self.push_log(f"🔍 Searching PubMed for: {search_query}")
         result = asyncio.run(pubmed_search_async(search_query))
         return filter_search_results(result)
 
@@ -113,7 +125,11 @@ class ScientificSearchTool(LoggingTool):
     }
     output_type = "object"
 
+    def __init__(self, log_queue: queue.Queue | None = None):
+        super().__init__(log_queue=log_queue)
+
     def forward(self, search_query: str) -> dict:
+        self.push_log(f"🔍 Searching scientific domains for: {search_query}")
         result = asyncio.run(scientific_search_async(search_query))
         return filter_search_results(result)
 
@@ -123,7 +139,7 @@ URL_EXPLAINATION = """The URL can be be converted to a markdown. If the URL poin
 
 class GetTableOfContentsTool(LoggingTool):
     name = ToolNames.GET_TABLE_OF_CONTENTS.value
-    description = f"""Returns all of the titles in the document along with the page number they are on.
+    description = f"""Returns all of the titles in the url along with the page number they are on.
     {URL_EXPLAINATION}
     """
     inputs = {
@@ -134,11 +150,12 @@ class GetTableOfContentsTool(LoggingTool):
     }
     output_type = "string"
 
-    def __init__(self, database: DataBase):
-        super().__init__()
+    def __init__(self, database: DataBase, log_queue: queue.Queue | None = None):
+        super().__init__(log_queue=log_queue)
         self.database: DataBase = database
 
     def forward(self, url: str) -> str:
+        self.push_log(f"🔍 Getting table of contents for url: {url}")
         markdown = self.database.get_markdown_of_url(url)
         table_of_contents: str = get_table_of_contents_per_page_markdown(markdown)
         return table_of_contents
@@ -152,11 +169,12 @@ class GetMarkdownTool(LoggingTool):
     }
     output_type = "string"
 
-    def __init__(self, database: DataBase):
-        super().__init__()
+    def __init__(self, database: DataBase, log_queue: queue.Queue | None = None):
+        super().__init__(log_queue=log_queue)
         self.database: DataBase = database
 
     def forward(self, url: str) -> str:
+        self.push_log(f"🔍 Getting markdown for url: {url}")
         markdown = self.database.get_markdown_of_url(url)
         markdown_content: str = convert_ocr_response_to_markdown(markdown)
         return markdown_content
@@ -174,11 +192,12 @@ class GetPagesContentTool(LoggingTool):
     }
     output_type = "string"
 
-    def __init__(self, database: DataBase):
-        super().__init__()
+    def __init__(self, database: DataBase, log_queue: queue.Queue | None = None):
+        super().__init__(log_queue=log_queue)
         self.database: DataBase = database
 
     def forward(self, url: str, page_numbers: list[int]) -> str:
+        self.push_log(f"🔍 Getting content of pages {page_numbers} for url: {url}")
         markdown = self.database.get_markdown_of_url(url)
         return get_markdown_by_page_numbers(markdown, page_numbers)
 
@@ -195,17 +214,18 @@ class FindInMarkdownTool(LoggingTool):
     }
     output_type = "array"
 
-    def __init__(self, database: DataBase):
-        super().__init__()
+    def __init__(self, database: DataBase, log_queue: queue.Queue | None = None):
+        super().__init__(log_queue=log_queue)
         self.database: DataBase = database
 
     def forward(self, url: str, search_queries: list[str]) -> list[int]:
+        self.push_log(f"🔍 Finding {search_queries} in url: {url}")
         markdown = self.database.get_markdown_of_url(url)
         return find_in_markdown(markdown, search_queries)
 
 
 def create_web_search_agent(
-    model_id="deepseek/deepseek-reasoner", database: DataBase | None = None
+    model_id="deepseek/deepseek-reasoner", database: DataBase | None = None, log_queue: queue.Queue | None = None
 ):
     """Create a web search agent with search, crawling, and PDF analysis capabilities."""
 
@@ -215,14 +235,14 @@ def create_web_search_agent(
 
     # Web search and crawling tools
     WEB_SEARCH_TOOLS = [
-        SearchTool(),
-        ArxivSearchTool(),
-        PubmedSearchTool(),
-        ScientificSearchTool(),
-        GetTableOfContentsTool(database),
-        GetMarkdownTool(database),
-        GetPagesContentTool(database),
-        FindInMarkdownTool(database),
+        SearchTool(log_queue=log_queue),
+        ArxivSearchTool(log_queue=log_queue),
+        PubmedSearchTool(log_queue=log_queue),
+        ScientificSearchTool(log_queue=log_queue),
+        GetTableOfContentsTool(database=database, log_queue=log_queue),
+        GetMarkdownTool(database=database, log_queue=log_queue),
+        GetPagesContentTool(database=database, log_queue=log_queue),
+        FindInMarkdownTool(database=database, log_queue=log_queue),
     ]
 
     web_search_agent = CodeAgent(

tests/deepsearch/test_main_agent.py

@@ -0,0 +1,12 @@
+from deepengineer.deepsearch.main_agent import main_search
+import queue
+
+
+def test_main_agent():
+    log_queue = queue.Queue()
+
+    main_search(
+        task="""
+    Search a paper called "High Energy Physics Opportunities Using Reactor Antineutrinos" on arXiv, download it and extract the table of contents
+    """, log_queue=log_queue
+    )

tests/deepsearch/test_pdf_agent.py

@@ -1,6 +1,6 @@
 import pytest
 from deepengineer.common_path import DATA_DIR
-from deepengineer.deepsearch.analyse_markdown_agent import (
+from deepengineer.deepsearch.main_pdf_agent import (
     FindInMarkdownTool,
     GetMarkdownTool,
     GetPagesContentTool,

tests/deepsearch/test_web_agent.py

@@ -16,5 +16,3 @@ def test_run_web_search_agent():
         is not None
     )
 
-
-test_run_web_search_agent()