Added a tool to do web search and rerank results

app.py

@@ -3,20 +3,51 @@ import datetime
 import requests
 import pytz
 import yaml
+from rank_bm25 import BM25Okapi
 from tools.final_answer import FinalAnswerTool
+from tools.visit_webpage import VisitWebpageTool
+from tools.web_search import DuckDuckGoSearchTool
 
 from Gradio_UI import GradioUI
 
-# Below is an example of a tool that does nothing. Amaze us with your creativity !
+def rank_urls_by_relevance(search_results, query):
+    """Ranks URLs based on BM25 similarity between query and article titles/snippets."""
+    docs = []
+    for res in search_results:
+        docs.append(res[0] + " " + res[2])
+    
+    tokenized_docs = [doc.lower().split() for doc in docs]
+    
+    bm25 = BM25Okapi(tokenized_docs)
+    query_tokens = query.lower().split()
+    scores = bm25.get_scores(query_tokens)
+
+    # Sort results by BM25 score (higher is better)
+    ranked_results = sorted(zip(search_results, scores), key=lambda x: x[1], reverse=True)
+    return ranked_results
+
+# The tool below uses the DuckDuckGoSearchTool to fetch data from the web that is relevant to the query.
+# The fetched results are reranked using BM25 and data from the url with the highest score is fetched.
 @tool
-def my_custom_tool(arg1:str, arg2:int)-> str: #it's import to specify the return type
-    #Keep this format for the description / args / args description but feel free to modify the tool
-    """A tool that does nothing yet 
+def get_answer_from_web(query:str)-> str: #it's import to specify the return type
+    """A tool that fetches results from the web. This function first calls DuckDuckGoSearchTool and the url received is passed to VisitWebpageTool to receive the final results.
     Args:
-        arg1: the first argument
-        arg2: the second argument
+        query: A string that represents the query to search the web for.
     """
-    return "What magic will you build ?"
+    try:
+        web_search_tool = DuckDuckGoSearchTool()
+        results = web_search_tool.forward(query)
+        
+        visit_webpage_tool = VisitWebpageTool()
+
+        ranked_results = rank_urls_by_relevance(results, query)
+
+        page_content = visit_webpage_tool.forward(ranked_results[0][0][1])
+
+        return page_content
+        
+    except Exception as e:
+        return f"Error fetching results"
 
 @tool
 def get_current_time_in_timezone(timezone: str) -> str:
@@ -34,6 +65,7 @@ def get_current_time_in_timezone(timezone: str) -> str:
         return f"Error fetching time for timezone '{timezone}': {str(e)}"
 
 
+
 final_answer = FinalAnswerTool()
 
 # If the agent does not answer, the model is overloaded, please use another model or the following Hugging Face Endpoint that also contains qwen2.5 coder:
@@ -55,7 +87,7 @@ with open("prompts.yaml", 'r') as stream:
     
 agent = CodeAgent(
     model=model,
-    tools=[final_answer], ## add your tools here (don't remove final answer)
+    tools=[final_answer, image_generation_tool, get_current_time_in_timezone, get_answer_from_web], ## add your tools here (don't remove final answer)
     max_steps=6,
     verbosity_level=1,
     grammar=None,

requirements.txt

@@ -1,5 +1,64 @@
-markdownify
-smolagents
-requests
-duckduckgo_search
-pandas
+aiofiles==23.2.1
+annotated-types==0.7.0
+anyio==4.8.0
+beautifulsoup4==4.13.3
+Brotli==1.1.0
+certifi==2025.1.31
+charset-normalizer==3.4.1
+click==8.1.8
+duckduckgo_search==7.4.2
+fastapi==0.115.8
+ffmpy==0.5.0
+filelock==3.17.0
+fsspec==2025.2.0
+gradio==5.16.1
+gradio_client==1.7.0
+h11==0.14.0
+h2==4.2.0
+hpack==4.1.0
+httpcore==1.0.7
+httpx==0.28.1
+huggingface-hub==0.28.1
+hyperframe==6.1.0
+idna==3.10
+Jinja2==3.1.5
+lxml==5.3.1
+markdown-it-py==3.0.0
+markdownify==0.14.1
+MarkupSafe==2.1.5
+mdurl==0.1.2
+numpy==2.2.3
+orjson==3.10.15
+packaging==24.2
+pandas==2.2.3
+pillow==11.1.0
+pydantic==2.10.6
+pydantic_core==2.27.2
+pydub==0.25.1
+Pygments==2.19.1
+python-dateutil==2.9.0.post0
+python-dotenv==1.0.1
+python-multipart==0.0.20
+pytz==2025.1
+PyYAML==6.0.2
+rank-bm25==0.2.2
+requests==2.32.3
+rich==13.9.4
+ruff==0.9.6
+safehttpx==0.1.6
+semantic-version==2.10.0
+shellingham==1.5.4
+six==1.17.0
+smolagents==1.9.2
+sniffio==1.3.1
+socksio==1.0.0
+soupsieve==2.6
+starlette==0.45.3
+tomlkit==0.13.2
+tqdm==4.67.1
+typer==0.15.1
+typing_extensions==4.12.2
+tzdata==2025.1
+urllib3==2.3.0
+uvicorn==0.34.0
+websockets==14.2

tools/visit_webpage.py

@@ -1,6 +1,7 @@
 from typing import Any, Optional
 from smolagents.tools import Tool
 import requests
+import re
 import markdownify
 import smolagents
 

tools/web_search.py

@@ -23,5 +23,6 @@ class DuckDuckGoSearchTool(Tool):
         results = self.ddgs.text(query, max_results=self.max_results)
         if len(results) == 0:
             raise Exception("No results found! Try a less restrictive/shorter query.")
-        postprocessed_results = [f"[{result['title']}]({result['href']})\n{result['body']}" for result in results]
-        return "## Search Results\n\n" + "\n\n".join(postprocessed_results)
+        # postprocessed_results = [f"[{result['title']}]({result['href']})\n{result['body']}" for result in results]
+        postprocessed_results = [(result['title'], result['href'], result['body']) for result in results]
+        return postprocessed_results