Upload 61 files

.dockerignore

@@ -0,0 +1,29 @@
+venv/
+__pycache__/
+*.pyc
+*.pyo
+*.pyd
+.Python
+env/
+pip-log.txt
+pip-delete-this-directory.txt
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.log
+.git/
+.mypy_cache/
+.pytest_cache/
+.hypothesis/
+downloads_folder/
+reports/
+uploads/
+*.html
+*.pdf
+*.docx
+*.pptx
+*.xlsx 

.env.template

@@ -0,0 +1,18 @@
+# Copy this file to .env and uncomment/modify the variables you need.
+
+# 1. HUGGING FACE CONFIGURATION (RECOMMENDED)
+# For Hugging Face usage: Only HF_TOKEN is required or set it in UI
+HF_TOKEN=your_huggingface_token_here
+MODEL_ID=Qwen/Qwen3-Coder-480B-A35B-Instruct
+
+# MODEL_ID=Qwen/Qwen3-Coder-30B-A3B-Instruct
+# MODEL_ID=Qwen/Qwen2.5-Coder-32B-Instruct
+
+# 2. OLLAMA CONFIGURATION (LOCAL MODELS)
+# For Ollama usage: Uncomment MODEL_ID, OPENAI_API_BASE, and OPENAI_API_KEY
+# This project uses OpenAI-compatible variables for Ollama integration
+ 
+# MODEL_ID=qwen2.5-coder:7b # replace with any model (qwen2.5-coder:32b is recommended)
+# OPENAI_API_BASE=http://localhost:11434/v1
+# OPENAI_API_KEY=ollama
+

.gitattributes

@@ -0,0 +1,36 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+assets/images/image_logo.png filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,35 @@
+# Python virtual environment
+venv/
+env/
+ENV/
+
+# Python cache files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# Distribution / packaging
+dist/
+build/
+*.egg-info/
+
+# Environment variables
+.env
+
+# IDE specific files
+.idea/
+.vscode/
+*.swp
+*.swo
+
+# Project specific
+downloads_folder/
+*.log
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# Local development settings
+*.local
+
+.gradio

LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

README-hf.md

@@ -0,0 +1,14 @@
+---
+title: Hugging Research
+emoji: 🔎
+colorFrom: yellow
+colorTo: purple
+sdk: gradio
+sdk_version: 5.42.0
+app_file: app.py
+pinned: true
+license: apache-2.0
+short_description: CodeAgent-based research assistant for the Hugging Face Hub
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

README.md

@@ -0,0 +1,52 @@
+
+<p align="center">
+  <img src="assets/images/image_logo.png" alt="Hugging Research logo" width="120" />
+</p>
+<h1 align="center">Hugging Research</h1>
+
+Hugging Research is a lightweight CodeAgent‑based research assistant for the Hugging Face Hub (models, datasets, Spaces, users, collections, papers). It gathers links via dedicated tools and organizes them for easy review.
+
+![screenshot](assets/images/ss1.png)
+
+## What it does
+- Finds relevant models/datasets/Spaces/papers on the Hub
+- Uses domain‑restricted search for tutorials and docs
+- Avoids hallucinated links (only cites tool‑returned URLs)
+- Organizes the found links into a simple, categorized view in the Report view
+
+## Quick start
+1) Clone and install
+```bash
+git clone https://github.com/mcdaqc/hugging-research
+cd hugging-research
+python -m venv venv
+venv\Scripts\activate  # Windows
+pip install -r requirements.txt
+```
+
+2) Configure your environment
+```bash
+cp .env.template .env
+# Edit .env and set:
+# HF_TOKEN=hf_xxx                # only for the inference model
+# MODEL_ID=Qwen/Qwen3-Coder-480B-A35B-Instruct  # optional
+```
+
+3) Run the app
+```bash
+python app.py
+# open http://localhost:7860
+```
+
+4) Use the app
+- Enter your Hugging Face API key in the sidebar
+- Click a Basic/Medium/Advanced example, or type your query in natural language
+- Review the organized links in the Report view
+
+## Configuration
+- `HF_TOKEN`: used for the inference model (agent). Tools are anonymous/read‑only.
+- `MODEL_ID`: default `Qwen/Qwen3-Coder-480B-A35B-Instruct`.
+
+
+
+

app.py

@@ -0,0 +1,1197 @@
+import json
+import mimetypes
+import os
+import re
+import shutil
+import threading
+import uuid
+from typing import Optional
+from loguru import logger
+from datetime import datetime
+
+import gradio as gr
+from dotenv import load_dotenv
+from huggingface_hub import login, HfApi
+from smolagents import (
+    CodeAgent,
+    InferenceClientModel,
+    Tool,
+    DuckDuckGoSearchTool,
+)
+from smolagents.agent_types import (
+    AgentAudio,
+    AgentImage,
+    AgentText,
+    handle_agent_output_types,
+)
+from smolagents.gradio_ui import stream_to_gradio
+
+from scripts.text_inspector_tool import TextInspectorTool
+from scripts.text_web_browser import (
+    ArchiveSearchTool,
+    FinderTool,
+    FindNextTool,
+    PageDownTool,
+    PageUpTool,
+    SimpleTextBrowser,
+    VisitTool,
+)
+from scripts.visual_qa import visualizer
+from scripts.report_generator import HFLinkReportTool
+from scripts.hf_tools import (
+    HFModelsSearchTool,
+    HFModelInfoTool,
+    HFDatasetsSearchTool,
+    HFDatasetInfoTool,
+    HFSpacesSearchTool,
+    HFSpaceInfoTool,
+    HFUserInfoTool,
+    HFCollectionsListTool,
+    HFCollectionGetTool,
+    HFPaperInfoTool,
+    HFPaperReposTool,
+    HFDailyPapersTool,
+    HFRepoInfoTool,
+    HFSiteSearchTool,
+)
+
+# web_search = GoogleSearchTool(provider="serper")
+web_search = DuckDuckGoSearchTool()
+
+AUTHORIZED_IMPORTS = [
+    "requests",
+    "zipfile",
+    "pandas",
+    "numpy",
+    "sympy",
+    "json",
+    "bs4",
+    "pubchempy",
+    "xml",
+    "yahoo_finance",
+    "Bio",
+    "sklearn",
+    "scipy",
+    "pydub",
+    "PIL",
+    "chess",
+    "PyPDF2",
+    "pptx",
+    "torch",
+    "datetime",
+    "fractions",
+    "csv",
+    "plotly",
+    "plotly.express",
+    "plotly.graph_objects",
+    "jinja2",
+]
+
+load_dotenv(override=True)
+
+# Only login if HF_TOKEN is available and valid in environment
+if os.getenv("HF_TOKEN"):
+    try:
+        login(os.getenv("HF_TOKEN"))
+        logger.info("Successfully logged in with HF_TOKEN from environment")
+    except Exception as e:
+        logger.warning(f"Failed to login with HF_TOKEN from environment: {e}")
+        logger.info("You can still use the application by providing a valid API key in the interface")
+
+# Global session storage for independent user sessions
+user_sessions = {}
+session_lock = threading.Lock()
+
+append_answer_lock = threading.Lock()
+
+# Initialize browser
+browser = SimpleTextBrowser(request_kwargs={})
+
+def validate_hf_api_key(api_key: str) -> tuple[bool, str]:
+    """Validate Hugging Face API key by making a test request."""
+    if not api_key or not api_key.strip():
+        return False, "❌ API key cannot be empty"
+    
+    api_key = api_key.strip()
+    
+    # Basic format validation
+    if not api_key.startswith("hf_"):
+        return False, "❌ Invalid API key format. Hugging Face API keys start with 'hf_'"
+    
+    try:
+        # Test the API key by making a simple request
+        api = HfApi(token=api_key)
+        # Try to get user info to validate the token
+        user_info = api.whoami()
+        return True, f"✅ API key validated successfully! Welcome, {user_info.get('name', 'User')}!"
+    except Exception as e:
+        return False, f"❌ Invalid API key: {str(e)}"
+
+def create_model_with_api_key(hf_token: str, model_id: str = None) -> InferenceClientModel:
+    """Create a model instance with the provided API key."""
+    if not model_id:
+        model_id = "Qwen/Qwen2.5-Coder-32B-Instruct"
+    
+    # Store original token
+    original_token = os.environ.get("HF_TOKEN")
+    
+    try:
+        # Set the token in environment for this session
+        os.environ["HF_TOKEN"] = hf_token
+        
+        # Create model without explicit token parameter
+        model = InferenceClientModel(
+            model_id=model_id,
+        )
+        
+        return model
+    finally:
+        # Restore original token
+        if original_token:
+            os.environ["HF_TOKEN"] = original_token
+        elif "HF_TOKEN" in os.environ:
+            del os.environ["HF_TOKEN"]
+
+def create_tools_with_model(model: InferenceClientModel):
+    """Create tools with the provided model."""
+    # Verify the model was created correctly
+    if model is None:
+        raise ValueError("Model is None, cannot create TextInspectorTool")
+    
+    # Text inspector tool disabled for now (inspect_file_as_text)
+    # Reason: model attempted to use it with remote URLs; keep only for local uploads when re-enabled.
+    # ti_tool = TextInspectorTool(model, 20000)
+    
+    # Hugging Face tools (public-only, anonymous)
+    hf_tools = [
+        HFModelsSearchTool(),
+        HFModelInfoTool(),
+        HFDatasetsSearchTool(),
+        HFDatasetInfoTool(),
+        HFSpacesSearchTool(),
+        HFSpaceInfoTool(),
+        HFUserInfoTool(),
+        HFCollectionsListTool(),
+        HFCollectionGetTool(),
+        HFPaperInfoTool(),
+        HFPaperReposTool(),
+        HFDailyPapersTool(),
+        HFRepoInfoTool(),
+        HFSiteSearchTool(),
+    ]
+
+    tools = hf_tools + [
+        web_search,  # duckduckgo
+        VisitTool(browser),
+        PageUpTool(browser),
+        PageDownTool(browser),
+        FinderTool(browser),
+        FindNextTool(browser),
+        ArchiveSearchTool(browser),
+        # ti_tool,  # TextInspectorTool (disabled) — only for uploaded local files; do not use with URLs
+    ]
+    
+    return tools
+
+# Agent creation in a factory function
+def create_agent(hf_token: str = None, model_id: str = None, max_steps: int = 10):
+    """Creates a fresh agent instance for each session"""
+    if not hf_token:
+        raise ValueError("A valid Hugging Face API key is required to create an agent.")
+    
+    logger.info(f"Creating agent with token: {hf_token[:10]}...")
+    
+    # Use session-specific model with HF_TOKEN
+    model = create_model_with_api_key(hf_token, model_id)
+    tools = create_tools_with_model(model)
+    
+    # TextInspectorTool temporarily disabled; skip presence check
+    # Previous enforcement kept for reference:
+    # has_text_inspector = any(getattr(tool, 'name', '') == 'inspect_file_as_text' for tool in tools)
+    # if not has_text_inspector:
+    #     raise ValueError("TextInspectorTool not found in tools list")
+    
+    agent = CodeAgent(
+        model=model,
+        tools=[visualizer] + tools,
+        max_steps=max_steps,
+        verbosity_level=1,
+        additional_authorized_imports=AUTHORIZED_IMPORTS,
+        planning_interval=4,
+    )
+    
+    logger.info("Agent created successfully")
+    return agent
+
+def get_user_session(request: gr.Request) -> str:
+    """Get or create a unique session ID for the user."""
+    if not request:
+        logger.warning("No request object, using random session ID")
+        return str(uuid.uuid4())
+    
+    # Try to get session from headers or create new one
+    session_id = request.headers.get("x-session-id")
+    if not session_id:
+        # Use client IP and user agent as a more stable identifier
+        client_ip = request.client.host if hasattr(request, 'client') and request.client else "unknown"
+        user_agent = request.headers.get("user-agent", "unknown")
+        # Create a hash-based session ID for more stability
+        import hashlib
+        session_hash = hashlib.md5(f"{client_ip}:{user_agent}".encode()).hexdigest()
+        session_id = f"session_{session_hash[:8]}"
+        logger.info(f"Created stable session ID {session_id} for client {client_ip}")
+    
+    return session_id
+
+def get_stable_session_id(request: gr.Request) -> str:
+    """Get a stable session ID that persists across requests."""
+    if not request:
+        logger.warning("No request object, using random session ID")
+        return f"random_{str(uuid.uuid4())[:8]}"
+    
+    # Use a combination of client info for more stable sessions
+    client_ip = getattr(request.client, 'host', 'unknown') if request.client else 'unknown'
+    user_agent = request.headers.get("user-agent", "unknown")
+    
+    # Add additional uniqueness factors
+    accept_language = request.headers.get("accept-language", "unknown")
+    accept_encoding = request.headers.get("accept-encoding", "unknown")
+    
+    # Create a more unique session ID
+    import hashlib
+    session_data = f"{client_ip}:{user_agent}:{accept_language}:{accept_encoding}"
+    session_hash = hashlib.md5(session_data.encode()).hexdigest()
+    session_id = f"user_{session_hash[:16]}"
+    
+    logger.info(f"Generated session ID: {session_id}")
+    logger.info(f"Session data: {session_data}")
+    
+    return session_id
+
+def get_unique_session_id(request: gr.Request) -> str:
+    """Get a truly unique session ID for each request."""
+    if not request:
+        return f"unique_{str(uuid.uuid4())[:8]}"
+    
+    # Use timestamp + client info for uniqueness
+    import time
+    timestamp = int(time.time() * 1000)  # milliseconds
+    client_ip = getattr(request.client, 'host', 'unknown') if request.client else 'unknown'
+    user_agent = request.headers.get("user-agent", "unknown")
+    
+    # Create a unique session ID
+    import hashlib
+    session_data = f"{timestamp}:{client_ip}:{user_agent}"
+    session_hash = hashlib.md5(session_data.encode()).hexdigest()
+    session_id = f"unique_{session_hash[:16]}"
+    
+    logger.info(f"Generated unique session ID: {session_id}")
+    
+    return session_id
+
+def get_persistent_session_id(request: gr.Request) -> str:
+    """Get a persistent session ID that stays the same for the same client."""
+    if not request:
+        return f"persistent_{str(uuid.uuid4())[:8]}"
+    
+    # Use only client info for persistence (no timestamp)
+    client_ip = getattr(request.client, 'host', 'unknown') if request.client else 'unknown'
+    user_agent = request.headers.get("user-agent", "unknown")
+    accept_language = request.headers.get("accept-language", "unknown")
+    
+    # Create a persistent session ID
+    import hashlib
+    session_data = f"{client_ip}:{user_agent}:{accept_language}"
+    session_hash = hashlib.md5(session_data.encode()).hexdigest()
+    session_id = f"persistent_{session_hash[:16]}"
+    
+    logger.info(f"Generated persistent session ID: {session_id}")
+    logger.info(f"Session data: {session_data}")
+    
+    return session_id
+
+def get_session_data(session_id: str) -> dict:
+    """Get session data for a specific user."""
+    with session_lock:
+        if session_id not in user_sessions:
+            user_sessions[session_id] = {
+                "hf_token": None,
+                "agent": None,
+                "max_steps": 10,
+                "created_at": datetime.now()
+            }
+        return user_sessions[session_id]
+
+def clear_session_data(session_id: str):
+    """Clear session data for a specific user."""
+    with session_lock:
+        if session_id in user_sessions:
+            # Clear sensitive data
+            user_sessions[session_id]["hf_token"] = None
+            user_sessions[session_id]["agent"] = None
+            logger.info(f"Session {session_id[:8]}... cleared")
+
+def clear_agent_only(session_id: str):
+    """Clear only the agent, keeping the API key for convenience."""
+    with session_lock:
+        if session_id in user_sessions:
+            if "agent" in user_sessions[session_id]:
+                del user_sessions[session_id]["agent"]
+            logger.info(f"Session {session_id[:8]}... agent cleared")
+
+
+
+class GradioUI:
+    """A one-line interface to launch your agent in Gradio"""
+
+    def __init__(self, file_upload_folder: str | None = None):
+        self.file_upload_folder = file_upload_folder
+        if self.file_upload_folder is not None:
+            if not os.path.exists(file_upload_folder):
+                os.mkdir(file_upload_folder)
+        # No on-disk report saving; reports are rendered in-app only
+
+    def validate_api_key(self, api_key: str) -> tuple[str, str]:
+        """Validate API key and return status message."""
+        is_valid, message = validate_hf_api_key(api_key)
+        if is_valid:
+            return message, "success"
+        else:
+            return message, "error"
+
+    def interact_with_agent(self, prompt, messages, request: gr.Request):
+        """Handle agent interaction with proper session management."""
+        # Get unique session ID for this user
+        session_id = get_persistent_session_id(request)
+        session_data = get_session_data(session_id)
+        
+        logger.info(f"Processing request for session {session_id}...")
+        logger.info(f"Request client: {request.client.host if request and request.client else 'unknown'}")
+        logger.info(f"Request user-agent: {request.headers.get('user-agent', 'unknown')[:50] if request else 'unknown'}")
+        logger.info(f"All active sessions: {list(user_sessions.keys())}")
+        logger.info(f"Session data for {session_id}: {session_data}")
+        
+        # Check if we have a valid agent for this session
+        if not session_data.get("agent"):
+            # Check if we have a valid HF_TOKEN in session
+            hf_token = session_data.get("hf_token")
+            
+            # If no token in session, try to get it from .env file
+            if not hf_token:
+                env_token = os.getenv("HF_TOKEN")
+                if env_token:
+                    hf_token = env_token
+                    session_data["hf_token"] = env_token
+                    session_data["max_steps"] = 10  # Default max_steps
+                    logger.info(f"Using HF_TOKEN from .env file for session {session_id[:8]}...")
+                else:
+                    logger.warning(f"No API key found for session {session_id[:8]}...")
+                    error_msg = "❌ No API key configured for your session. Please enter your Hugging Face API key in the API Configuration section above and click 'Setup API Key'."
+                    messages.append(gr.ChatMessage(role="assistant", content=error_msg))
+                    yield messages, "", ""
+                    return
+            
+            logger.info(f"Creating agent for session {session_id[:8]}...")
+            
+            if hf_token:
+                try:
+                    max_steps = session_data.get("max_steps", 10)
+                    session_data["agent"] = create_agent(hf_token, model_id=os.getenv("MODEL_ID"), max_steps=max_steps)
+                    logger.info(f"Agent created successfully for session {session_id[:8]}...")
+                except Exception as e:
+                    logger.error(f"Failed to create agent for session {session_id[:8]}: {e}")
+                    error_msg = f"❌ Failed to create agent with provided API key: {str(e)}"
+                    messages.append(gr.ChatMessage(role="assistant", content=error_msg))
+                    yield messages, "", ""
+                    return
+        else:
+            logger.info(f"Agent already exists for session {session_id[:8]}...")
+        
+        # Adding monitoring
+        try:
+            # log the existence of agent memory
+            has_memory = hasattr(session_data["agent"], "memory")
+            print(f"Agent has memory: {has_memory}")
+            if has_memory:
+                print(f"Memory type: {type(session_data['agent'].memory)}")
+
+            # Get current date for the prompt
+            from datetime import datetime
+            current_date = datetime.now().strftime("%Y-%m-%d")
+            
+             # Prepare the system prompt (Hugging Search)
+            system_prompt = f"""You are Hugging Research, an assistant focused on Hugging Face content (models, datasets, Spaces, users, collections, papers) and related learning/blog/news.
+
+TODAY'S DATE: {current_date}
+
+STYLE
+- Warm, collaborative, concise. use second person (you)
+
+ACCESS BOUNDARIES
+- Read‑only. Use only public information.
+- If a tool indicates 401/403/private/gated, state "no access" and continue with other public sources.
+
+AVAILABLE TOOLS
+- web_search, visit, page_up, page_down, find, find_next, archive_search, visualizer
+- hf_models_search, hf_model_info, hf_datasets_search, hf_dataset_info, hf_spaces_search, hf_space_info
+- hf_user_info, hf_collections_list, hf_collection_get, hf_paper_info, hf_paper_repos, hf_daily_papers
+- hf_repo_info, hf_site_search
+
+LINK POLICY (anti‑hallucination)
+- Only cite URLs that come directly from tool outputs. Never invent or guess links.
+- Prefer official huggingface.co URLs for models/datasets/Spaces/papers.
+- For tutorials/blogs/news, prefer huggingface.co when the same content exists there.
+- If you need a URL that isn't present, first use a tool (web_search or hf_site_search) to retrieve it, then cite it.
+
+TOOL USAGE POLICY
+- You can write compact Python to orchestrate multiple tool calls in one block.
+- Never dump large/raw JSON. If using python_interpreter, ensure visible output by printing a short structured summary (<=20 lines) or leaving a final expression; otherwise summarize in natural language.
+- Keep parameters minimal: include query and limit; add owner only if asked; use a single pipeline_tag or tags only if explicitly implied; use sort/direction when asked or implied (default downloads/descending; 'trending' allowed).
+- Default to limit=10 for searches unless the user explicitly asks for more.
+- Use web_search to capture fresh/trending context; use hf_site_search for tutorials/blog/Learn.
+- Use only the listed tools; do not call undefined helpers (e.g., visit_page).
+ - web_search returns plain text; never json.load or index it. Use it only for keywords or discovering links.
+ - hf_* tools return JSON serialized as string; always json.loads(...) before indexing keys like 'results' or 'item'.
+
+STARTING MOVE
+- Begin with multiple web_search to capture today‑relevant terms (include "Hugging Face" in the query when helpful). Derive 3–5 keywords and reuse them across hf_* calls.
+
+DECISION RULES
+- Prefer hf_* tools for official Hub content. Use derived keywords; do not rely only on date sort.
+- Stop calling tools once you have enough signal for a confident, useful answer.
+
+FINAL STEP GUIDANCE
+ - Do not call any dashboard/report tool. The app will automatically generate a dashboard from your final answer text for the Report tab. Focus on writing a clean Final Answer with accurate inline links derived from tool outputs.
+
+OUTPUT REQUIREMENTS
+- Provide a conversational summary tailored to the user’s goal.
+- Structure: brief opening (what we looked for and why), key findings woven into short prose.
+- Use inline links to official HF pages for repos and to reputable external sources for tutorials/news.
+- Briefly mention at least one relevant item with inline links across these categories when available: models, datasets, Spaces, papers, blogs/docs, repositories, videos, news.
+
+EXAMPLES (GOOD)
+# Derive keywords then orchestrate searches
+results_web = web_search(query="diffusion models Hugging Face latest")
+import json
+models = json.loads(hf_models_search(query="semantic search", limit=5)).get("results", [])
+ds = json.loads(hf_datasets_search(query="semantic search", limit=5)).get("results", [])
+repo = json.loads(hf_model_info(repo_id="sentence-transformers/all-MiniLM-L6-v2")).get("item")
+spaces = json.loads(hf_spaces_search(query="whisper transcription", limit=5)).get("results", [])
+learn = json.loads(hf_site_search(query="fine-tuning tutorial Hugging Face course", limit=5)).get("results", [])
+# Final step: compose the final answer in natural language with inline links.
+# The app will build a dashboard automatically from your final answer (no extra tool call needed).
+final_answer_text = "We looked at semantic search models and datasets, including https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2 ..."
+ 
+ Now is your turn to answer the user query.
+ 
+User Query: """
+
+            # Combine system prompt with user message
+            full_prompt = system_prompt + prompt
+
+            # Extract clean message for display (remove internal context)
+            display_message = prompt
+            if "[INTERNAL CONTEXT:" in prompt:
+                display_message = prompt.split("[INTERNAL CONTEXT:")[0].strip()
+
+            messages.append(gr.ChatMessage(role="user", content=display_message))
+            yield messages, "", ""
+
+            logger.info(f"Starting agent interaction for session {session_id[:8]}...")
+            latest_assistant_text = ""
+            for msg in stream_to_gradio(
+                session_data["agent"], task=full_prompt, reset_agent_memory=False
+            ):
+                # If the message contains an HTML report, just pass it through (no on-disk saving)
+                # We render the dashboard in the Report tab below.
+                # (Intentionally no file saving)
+                messages.append(msg)
+                if getattr(msg, "role", None) == "assistant" and isinstance(msg.content, str):
+                    latest_assistant_text = msg.content
+                yield messages, "", ""
+            
+            # Clear sensitive data from session after interaction (AUTOMATIC)
+            # Note: We clear the agent but keep the API key for convenience
+            if "agent" in session_data:
+                del session_data["agent"]
+            logger.info(f"Session {session_id[:8]}... agent cleared after interaction")
+            
+            # Build Report tab content
+            last_answer = latest_assistant_text or ""
+            report_md = ""
+            if display_message or last_answer:
+                report_md = f"### Prompt\n{display_message}\n\n{last_answer}"
+            # Generate report HTML from the final answer
+            dashboard_html = ""
+            try:
+                dashboard_html = HFLinkReportTool().forward(final_answer=last_answer, query=display_message)
+            except Exception:
+                dashboard_html = ""
+            yield messages, report_md, dashboard_html
+        except Exception as e:
+            logger.error(f"Error in interaction for session {session_id[:8]}: {str(e)}")
+            print(f"Error in interaction: {str(e)}")
+            error_msg = f"❌ Error during interaction: {str(e)}"
+            messages.append(gr.ChatMessage(role="assistant", content=error_msg))
+            yield messages, "", ""
+
+    def setup_api_key(self, api_key: str, request: gr.Request) -> str:
+        """Setup API key for the user's session."""
+        # Get unique session ID for this user
+        session_id = get_persistent_session_id(request)
+        session_data = get_session_data(session_id)
+        
+        logger.info(f"Setting up API key for session {session_id}...")
+        logger.info(f"Setup request client: {request.client.host if request and request.client else 'unknown'}")
+        logger.info(f"Setup request user-agent: {request.headers.get('user-agent', 'unknown')[:50] if request else 'unknown'}")
+        logger.info(f"All active sessions before setup: {list(user_sessions.keys())}")
+        logger.info(f"Session data before setup: {session_data}")
+        
+        # Check if API key is provided from interface
+        if api_key and api_key.strip():
+            # Use the API key from interface
+            token_to_use = api_key.strip()
+            source = "interface"
+        else:
+            # Try to use token from .env file
+            env_token = os.getenv("HF_TOKEN")
+            if env_token:
+                token_to_use = env_token
+                source = ".env file"
+            else:
+                return "❌ No API key provided. Please enter your Hugging Face API key or set HF_TOKEN in your .env file."
+        
+        # Validate the token
+        is_valid, message = validate_hf_api_key(token_to_use)
+        
+        if is_valid:
+            # Store HF_TOKEN in session data
+            session_data["hf_token"] = token_to_use
+            session_data["max_steps"] = 10
+            logger.info(f"API key stored in session {session_id[:8]}... from {source}")
+            logger.info(f"Max steps set to fixed value: 10")
+            
+            # Create new agent with the HF_TOKEN and max_steps
+            try:
+                session_data["agent"] = create_agent(token_to_use, model_id=os.getenv("MODEL_ID"), max_steps=10)
+                logger.info(f"Agent created successfully for session {session_id[:8]}...")
+                return f"✅ API key from {source} validated and agent created successfully! {message.split('!')[1] if '!' in message else ''}"
+            except Exception as e:
+                logger.error(f"Failed to create agent for session {session_id[:8]}: {e}")
+                return f"❌ Failed to create agent with API key from {source}: {str(e)}"
+        else:
+            logger.warning(f"Invalid API key for session {session_id[:8]}... from {source}")
+            return f"❌ Invalid API key from {source}: {message}"
+
+    def upload_file(
+        self,
+        file,
+        file_uploads_log,
+        allowed_file_types=[
+            "application/pdf",
+            "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+            "text/plain",
+        ],
+    ):
+        """
+        Handle file uploads, default allowed types are .pdf, .docx, and .txt
+        """
+        if file is None:
+            return gr.Textbox("No file uploaded", visible=True), file_uploads_log
+
+        try:
+            mime_type, _ = mimetypes.guess_type(file.name)
+        except Exception as e:
+            return gr.Textbox(f"Error: {e}", visible=True), file_uploads_log
+
+        if mime_type not in allowed_file_types:
+            return gr.Textbox("File type disallowed", visible=True), file_uploads_log
+
+        # Sanitize file name
+        original_name = os.path.basename(file.name)
+        sanitized_name = re.sub(
+            r"[^\w\-.]", "_", original_name
+        )  # Replace any non-alphanumeric, non-dash, or non-dot characters with underscores
+
+        type_to_ext = {}
+        for ext, t in mimetypes.types_map.items():
+            if t not in type_to_ext:
+                type_to_ext[t] = ext
+
+        # Ensure the extension correlates to the mime type
+        sanitized_name = sanitized_name.split(".")[:-1]
+        sanitized_name.append("" + type_to_ext[mime_type])
+        sanitized_name = "".join(sanitized_name)
+
+        # Save the uploaded file to the specified folder
+        file_path = os.path.join(
+            self.file_upload_folder, os.path.basename(sanitized_name)
+        )
+        shutil.copy(file.name, file_path)
+
+        return gr.Textbox(
+            f"File uploaded: {file_path}", visible=True
+        ), file_uploads_log + [file_path]
+
+    def log_user_message(self, text_input, file_uploads_log):
+        # Create the user message for display (clean, without file info)
+        display_message = text_input
+        
+        # Create the internal message for the agent (with file context)
+        internal_message = text_input
+        if len(file_uploads_log) > 0:
+            file_names = [os.path.basename(f) for f in file_uploads_log]
+            file_paths = [f for f in file_uploads_log]  # Full paths
+            # Note: inspect_file_as_text is currently disabled (only for local uploads when re-enabled)
+            internal_message += f"\n\n[Uploaded files available: {', '.join(file_names)}. You can reference their content if needed (plain text).]"
+        
+        return (
+            internal_message,  # This goes to the agent (with file context)
+            gr.Textbox(
+                value="",
+                interactive=False,
+                placeholder="Please wait while Steps are getting populated",
+            ),
+            gr.Button(interactive=False),
+        )
+
+    def detect_device(self, request: gr.Request):
+        # Check whether the user device is a mobile or a computer
+
+        if not request:
+            return "Desktop"  # Default to desktop if no request info
+        
+        # Method 1: Check sec-ch-ua-mobile header (most reliable)
+        is_mobile_header = request.headers.get("sec-ch-ua-mobile")
+        if is_mobile_header:
+            return "Mobile" if "?1" in is_mobile_header else "Desktop"
+
+        # Method 2: Check user-agent string
+        user_agent = request.headers.get("user-agent", "").lower()
+        mobile_keywords = ["android", "iphone", "ipad", "mobile", "phone", "tablet"]
+        
+        # More comprehensive mobile detection
+        if any(keyword in user_agent for keyword in mobile_keywords):
+            return "Mobile"
+        
+        # Check for mobile-specific patterns
+        if "mobile" in user_agent or "android" in user_agent or "iphone" in user_agent:
+            return "Mobile"
+
+        # Method 3: Check platform
+        platform = request.headers.get("sec-ch-ua-platform", "").lower()
+        if platform:
+            if platform in ['"android"', '"ios"']:
+                return "Mobile"
+            elif platform in ['"windows"', '"macos"', '"linux"']:
+                return "Desktop"
+
+        # Method 4: Check viewport width (if available)
+        viewport_width = request.headers.get("viewport-width")
+        if viewport_width:
+            try:
+                width = int(viewport_width)
+                return "Mobile" if width <= 768 else "Desktop"
+            except ValueError:
+                pass
+
+        # Default case if no clear indicators
+        return "Desktop"
+
+    def launch(self, **kwargs):
+        # Custom CSS for mobile optimization
+        custom_css = """
+        @media (max-width: 768px) {
+            .gradio-container {
+                max-width: 100% !important;
+                padding: 10px !important;
+            }
+            .main {
+                padding: 10px !important;
+            }
+            .chatbot {
+                max-height: 60vh !important;
+            }
+            .textbox {
+                font-size: 16px !important; /* Prevents zoom on iOS */
+            }
+            .button {
+                min-height: 44px !important; /* Better touch targets */
+            }
+        }
+        """
+        
+        with gr.Blocks(theme="ocean", fill_height=True, css=custom_css) as demo:
+            # Different layouts for mobile and computer devices
+            @gr.render()
+            def layout(request: gr.Request):
+                device = self.detect_device(request)
+                print(f"device - {device}")
+                # Render layout with sidebar
+                # Prepare logo as data URI for reliable rendering
+                try:
+                    import base64
+                    _logo_src = ""
+                    _used = ""
+                    for _p in ("assets/images/@image_logo.png", "assets/images/image_logo.png"):
+                        if os.path.exists(_p):
+                            with open(_p, "rb") as _lf:
+                                _b64 = base64.b64encode(_lf.read()).decode("ascii")
+                                _logo_src = f"data:image/png;base64,{_b64}"
+                                _used = _p
+                            break
+                    print(f"Logo path used: {_used or 'none'}")
+                except Exception as _e:
+                    print(f"Logo load error: {_e}")
+                    _logo_src = ""
+                _logo_img_html = (
+                    f'<img src="{_logo_src}" alt="App logo" style="vertical-align: middle; margin-right: 10px; height: 40px; display:inline-block;">'
+                    if _logo_src else ""
+                )
+                if device == "Desktop":
+                    with gr.Blocks(
+                        fill_height=True,
+                    ):
+                        file_uploads_log = gr.State([])
+                        with gr.Sidebar():
+                            # Project title and repository link at the top
+                            gr.Markdown(value=f"<h1 style=\"display:flex; align-items:center; gap:10px; margin:0; text-align:left;\">{_logo_img_html}Hugging Research</h1>")
+                            gr.Markdown("""<img src=\"https://github.githubassets.com/images/modules/logos_page/GitHub-Mark.png\" width=\"20\" height=\"20\" style=\"display: inline-block; vertical-align: middle; margin-right: 8px;\"> <a href=\"https://github.com/mcdaqc/hugging-research\" target=\"_blank\">Github Repository</a>""")
+                            
+                            # About section
+                            with gr.Accordion("ℹ️ About", open=False):
+                                gr.Markdown("""**What it does:**
+Hugging Research finds Hugging Face models, datasets, and Spaces with direct links and short summaries.
+
+**Available tools:**
+- Hugging Face Hub API endpoints (via hf_* tools) — see [Hub API](https://huggingface.co/docs/hub/en/api)
+- Web search + basic navigation (DuckDuckGo `web_search`, `visit`, `page_up/down`, `find`, `archive_search`)
+
+**Model configuration:**
+- Default: Qwen/Qwen3-Coder-480B-A35B-Instruct (HF Inference API)
+- Optional: Ollama/local via `.env`
+
+**How to use:**
+- Enter your Hugging Face API key
+- Ask in natural language (e.g., models/datasets/spaces by topic or owner; or “I’m new to LLMs/fine‑tuning—where to start?”)
+- Get concise, linked results""")
+                            
+                            with gr.Group():
+                                gr.Markdown("**Your request**", container=True)
+                                text_input = gr.Textbox(
+                                    lines=3,
+                                    label="Your request",
+                                    container=False,
+                                    placeholder="Enter your prompt here and press Shift+Enter or press the button",
+                                )
+                                launch_research_btn = gr.Button(
+                                    "Run", variant="primary"
+                                )
+
+                            # Examples Section
+                            with gr.Accordion("💡 Example Prompts", open=False):
+                                gr.Markdown("**Click any example below to populate your request field:**")
+                                
+                                example_btn_1 = gr.Button("Basic: Tiny chatbot", size="sm", variant="secondary")
+                                example_btn_2 = gr.Button("Medium: RAG Q&A", size="sm", variant="secondary")
+                                example_btn_3 = gr.Button("Advanced: Instr. tuning", size="sm", variant="secondary")
+                                
+                                # Example button events
+                                example_btn_1.click(
+                                    lambda: "I want a small chatbot I can run on my laptop. Recommend a few lightweight chat models, a small dialogue dataset to fine‑tune, and a beginner‑friendly finetuning guide. Include a Space I can duplicate or clear steps to run locally.",
+                                    None,
+                                    [text_input]
+                                )
+                                example_btn_2.click(
+                                    lambda: "I'm building a document Q&A RAG app. Recommend CPU‑friendly embedding models and an optional reranker, give sensible chunk size and overlap defaults, suggest a small starter dataset, link a Space I can duplicate or a repo for an end‑to‑end pipeline, and provide a short guide to evaluate answer quality.",
+                                    None,
+                                    [text_input]
+                                )
+                                example_btn_3.click(
+                                    lambda: "I'm exploring instruction‑tuning and preference optimization for code LLMs. Surface recent papers from 2024 and 2025 on SFT, DPO, ORPO, and GRPO, link relevant datasets, models and repos that implement these methods, outline typical training and evaluation setups, and highlight open challenges and safety notes.",
+                                    None,
+                                    [text_input]
+                                )
+
+                            # API Key Configuration Section
+                            with gr.Accordion("🔑 API Configuration", open=False):
+                                gr.Markdown("**Configure your Hugging Face API Key**")
+                                gr.Markdown("🔒 **Security**: Your API key is only kept during this session.")
+                                gr.Markdown("Get your API key from: https://huggingface.co/settings/tokens")
+                                
+                                api_key_input = gr.Textbox(
+                                    label="Hugging Face API Key",
+                                    placeholder="hf_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx",
+                                    type="password",
+                                    lines=1
+                                )
+                                api_key_status = gr.Textbox(
+                                    label="Status",
+                                    value="✅ HF_TOKEN found in .env file. To use a different key, enter it above and click 'Setup API Key'." if os.getenv("HF_TOKEN") else "⚠️ Please enter your Hugging Face API key above and click 'Setup API Key' to start using the application.",
+                                    interactive=False
+                                )
+                                
+                                # Agent configuration (fixed steps)
+                                gr.Markdown("**Agent Configuration** — steps are fixed for stability.")
+                                
+                                setup_api_btn = gr.Button("Setup API Key", variant="secondary")
+
+                            # If an upload folder is provided, enable the upload feature
+                            # COMMENTED: File upload feature temporarily disabled - works but consumes too many steps for parsing
+                            # TODO: Re-enable after optimizing TextInspectorTool to use fewer steps
+                            # if self.file_upload_folder is not None:
+                            #     upload_file = gr.File(label="Upload a file")
+                            #     upload_status = gr.Textbox(
+                            #         label="Upload Status",
+                            #         interactive=False,
+                            #         visible=False,
+                            #     )
+                            #     upload_file.change(
+                            #         self.upload_file,
+                            #         [upload_file, file_uploads_log],
+                            #         [upload_status, file_uploads_log],
+                            #     )
+
+                            # Powered by smolagents
+                            with gr.Row():
+                                gr.HTML("""<div style="display: flex; align-items: center; gap: 8px; font-family: system-ui, -apple-system, sans-serif;">Powered by
+                        <img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/smolagents/mascot_smol.png" style="width: 32px; height: 32px; object-fit: contain;" alt="logo">
+                        <a target="_blank" href="https://github.com/huggingface/smolagents"><b>hf/smolagents</b></a>
+                        </div>""")
+
+                        # Chat interface
+                        stored_messages = gr.State([])
+                        chatbot = gr.Chatbot(
+                            label="open-Deep-Research",
+                            type="messages",
+                            avatar_images=(
+                                None,
+                                "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/smolagents/mascot_smol.png",
+                            ),
+                            resizeable=False,
+                            scale=1,
+                            elem_id="my-chatbot",
+                        )
+
+                        # Tabs for Research and Report
+                        with gr.Tabs():
+                            with gr.Tab("Report"):
+                                report_markdown = gr.Markdown(value="")
+                                report_dashboard = gr.HTML(value="")
+
+                        # API Key setup event
+                        setup_api_btn.click(
+                            self.setup_api_key,
+                            [api_key_input],
+                            [api_key_status]
+                        )
+                        
+
+
+                        text_input.submit(
+                            self.log_user_message,
+                            [text_input, file_uploads_log],
+                            [stored_messages, text_input, launch_research_btn],
+                        ).then(
+                            self.interact_with_agent,
+                            [stored_messages, chatbot],
+                            [chatbot, report_markdown, report_dashboard],
+                        ).then(
+                            lambda: (
+                                gr.Textbox(
+                                    interactive=True,
+                                    placeholder="Enter your prompt here and press the button",
+                                ),
+                                gr.Button(interactive=True),
+                            ),
+                            None,
+                            [text_input, launch_research_btn],
+                        )
+                        launch_research_btn.click(
+                            self.log_user_message,
+                            [text_input, file_uploads_log],
+                            [stored_messages, text_input, launch_research_btn],
+                        ).then(
+                            self.interact_with_agent,
+                            [stored_messages, chatbot],
+                            [chatbot, report_markdown, report_dashboard],
+                        ).then(
+                            lambda: (
+                                gr.Textbox(
+                                    interactive=True,
+                                    placeholder="Enter your prompt here and press the button",
+                                ),
+                                gr.Button(interactive=True),
+                            ),
+                            None,
+                            [text_input, launch_research_btn],
+                        )
+
+                # Render simple layout for mobile
+                else:
+                    try:
+                        with gr.Blocks(
+                            fill_height=True,
+                        ):
+                            # Project title and repository link at the top
+                            gr.Markdown(value=f"<h1 style=\"display:flex; align-items:center; gap:10px; margin:0; text-align:left;\">{_logo_img_html}Hugging Research</h1>")
+                            gr.Markdown("""<img src=\"https://github.githubassets.com/images/modules/logos_page/GitHub-Mark.png\" width=\"20\" height=\"20\" style=\"display: inline-block; vertical-align: middle; margin-right: 8px;\"> <a href=\"https://github.com/mcdaqc/hugging-research\" target=\"_blank\">Github Repository</a>""")
+                            
+                            # About section for mobile
+                            with gr.Accordion("ℹ️ About", open=False):
+                                gr.Markdown("""**What it does:**
+Hugging Research finds Hugging Face models, datasets, and Spaces with direct links and short summaries.
+
+**Available tools:**
+- Hugging Face Hub API endpoints (via hf_* tools) — see [Hub API](https://huggingface.co/docs/hub/en/api)
+- Web search + basic navigation (DuckDuckGo `web_search`, `visit`, `page_up/down`, `find`, `archive_search`)
+
+**Model configuration:**
+- Default: Qwen/Qwen3-Coder-480B-A35B-Instruct (HF Inference API)
+- Optional: Ollama/local via `.env`
+
+**How to use:**
+- Enter your Hugging Face API key
+- Ask in natural language (e.g., models/datasets/spaces by topic or owner; or “I’m new to LLMs/fine‑tuning—where to start?”)
+- Get concise, linked results""")
+
+                        # API Key Configuration Section for Mobile
+                        with gr.Accordion("🔑 API Configuration", open=False):
+                            gr.Markdown("**Configure your Hugging Face API Key**")
+                            gr.Markdown("🔒 **Security**: Your API key is only kept during this session.")
+                            gr.Markdown("Get your API key from: https://huggingface.co/settings/tokens")
+                            
+                            mobile_api_key_input = gr.Textbox(
+                                label="Hugging Face API Key",
+                                placeholder="hf_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx",
+                                type="password",
+                                lines=1
+                            )
+                            mobile_api_key_status = gr.Textbox(
+                                label="Status",
+                                value="✅ HF_TOKEN found in .env file. To use a different key, enter it above and click 'Setup API Key'." if os.getenv("HF_TOKEN") else "⚠️ Please enter your Hugging Face API key above and click 'Setup API Key' to start using the application.",
+                                interactive=False
+                            )
+                            
+                            # Agent configuration for mobile
+                            gr.Markdown("**Agent Configuration**")
+                            mobile_max_steps_slider = gr.Slider(
+                                minimum=5,
+                                maximum=30,
+                                value=10,
+                                step=1,
+                                label="Maximum Steps",
+                                info="Number of steps the agent can take per session (higher = more detailed but slower)"
+                            )
+                            
+                            mobile_setup_api_btn = gr.Button("Setup API Key", variant="secondary")
+                        
+                        # Chat interface for mobile
+                        stored_messages = gr.State([])
+                        file_uploads_log = gr.State([])
+                        chatbot = gr.Chatbot(
+                            label="open-Deep-Research",
+                            type="messages",
+                            avatar_images=(
+                                None,
+                                "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/smolagents/mascot_smol.png",
+                            ),
+                            resizeable=True,
+                            scale=1,
+                        )
+                        
+                        # Input section for mobile
+                        text_input = gr.Textbox(
+                            lines=1,
+                            label="Your request",
+                            placeholder="Enter your prompt here and press the button",
+                        )
+                        launch_research_btn = gr.Button(
+                            "Run",
+                            variant="primary",
+                        )
+
+                        # File upload section for mobile (simple)
+                        # COMMENTED: File upload feature temporarily disabled - works but consumes too many steps for parsing
+                        # TODO: Re-enable after optimizing
+                        # if self.file_upload_folder is not None:
+                        #     mobile_upload_file = gr.File(label="📎 Upload PDF/TXT file (optional)")
+                        #     mobile_upload_status = gr.Textbox(
+                        #         label="Upload Status",
+                        #         interactive=False,
+                        #         visible=False,
+                        #     )
+                        #     mobile_upload_file.change(
+                        #         self.upload_file,
+                        #         [mobile_upload_file, file_uploads_log],
+                        #         [mobile_upload_status, file_uploads_log],
+                        #     )
+
+                        # Examples Section for Mobile
+                        with gr.Accordion("💡 Example Prompts", open=False):
+                            gr.Markdown("**Click any example below to populate your request field:**")
+                            
+                            mobile_example_btn_1 = gr.Button("Basic: Tiny chatbot", size="sm", variant="secondary")
+                            mobile_example_btn_2 = gr.Button("Medium: RAG Q&A", size="sm", variant="secondary")
+                            mobile_example_btn_3 = gr.Button("Advanced: Instr. tuning", size="sm", variant="secondary")
+                        
+                        # Powered by smolagents for mobile
+                        with gr.Row():
+                            gr.HTML("""<div style="display: flex; align-items: center; gap: 8px; font-family: system-ui, -apple-system, sans-serif;">Powered by
+                        <img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/smolagents/mascot_smol.png" style="width: 32px; height: 32px; object-fit: contain;" alt="logo">
+                        <a target="_blank" href="https://github.com/huggingface/smolagents"><b>hf/smolagents</b></a>
+                        </div>""")
+
+                        # Mobile API Key setup event
+                        mobile_setup_api_btn.click(
+                            self.setup_api_key,
+                            [mobile_api_key_input],
+                            [mobile_api_key_status]
+                        )
+                        
+                        # Mobile Example button events
+                        mobile_example_btn_1.click(
+                            lambda: "I want a small chatbot I can run on my laptop. Recommend a few lightweight chat models, a small dialogue dataset to fine‑tune, and a beginner‑friendly finetuning guide. Include a Space I can duplicate or clear steps to run locally.",
+                            None,
+                            [text_input]
+                        )
+                        mobile_example_btn_2.click(
+                            lambda: "I'm building a document Q&A RAG app. Recommend CPU‑friendly embedding models and an optional reranker, give sensible chunk size and overlap defaults, suggest a small starter dataset, link a Space I can duplicate or a repo for an end‑to‑end pipeline, and provide a short guide to evaluate answer quality.",
+                            None,
+                            [text_input]
+                        )
+                        mobile_example_btn_3.click(
+                            lambda: "I'm exploring instruction‑tuning and preference optimization for code LLMs. Surface recent papers from 2024 and 2025 on SFT, DPO, ORPO, and GRPO, link relevant datasets, models and repos that implement these methods, outline typical training and evaluation setups, and highlight open challenges and safety notes.",
+                            None,
+                            [text_input]
+                        )
+
+                        # Research and Report panels for mobile
+                        with gr.Tabs():
+                            with gr.Tab("Report"):
+                                m_report_markdown = gr.Markdown(value="")
+                                m_report_dashboard = gr.HTML(value="")
+
+                        # Mobile chat events
+                        text_input.submit(
+                            self.log_user_message,
+                            [text_input, file_uploads_log],
+                            [stored_messages, text_input, launch_research_btn],
+                        ).then(
+                            self.interact_with_agent,
+                            [stored_messages, chatbot],
+                            [chatbot, m_report_markdown, m_report_dashboard],
+                        ).then(
+                            lambda: (
+                                gr.Textbox(
+                                    interactive=True,
+                                    placeholder="Enter your prompt here and press the button",
+                                ),
+                                gr.Button(interactive=True),
+                            ),
+                            None,
+                            [text_input, launch_research_btn],
+                        )
+                        launch_research_btn.click(
+                            self.log_user_message,
+                            [text_input, file_uploads_log],
+                            [stored_messages, text_input, launch_research_btn],
+                        ).then(
+                            self.interact_with_agent,
+                            [stored_messages, chatbot],
+                            [chatbot, m_report_markdown, m_report_dashboard],
+                        ).then(
+                            lambda: (
+                                gr.Textbox(
+                                    interactive=True,
+                                    placeholder="Enter your prompt here and press the button",
+                                ),
+                                gr.Button(interactive=True),
+                            ),
+                            None,
+                            [text_input, launch_research_btn],
+                        )
+                    except Exception as e:
+                        # Fallback to desktop layout if mobile layout fails
+                        logger.error(f"Mobile layout failed: {e}")
+                        # Re-render desktop layout as fallback
+                        with gr.Blocks(fill_height=True):
+                            gr.Markdown(value=f"<h1 style=\"display:flex; align-items:center; gap:10px; margin:0; text-align:left;\">{_logo_img_html}Hugging Research</h1>")
+                            gr.Markdown("""<img src=\"https://github.githubassets.com/images/modules/logos_page/GitHub-Mark.png\" width=\"20\" height=\"20\" style=\"display: inline-block; vertical-align: middle; margin-right: 8px;\"> <a href=\"https://github.com/mcdaqc/hugging-research\" target=\"_blank\">Github Repository</a>""")
+                            gr.Markdown("⚠️ Mobile layout failed, using desktop layout as fallback.")
+                            
+                            # Simple fallback interface
+                            stored_messages = gr.State([])
+                            file_uploads_log = gr.State([])
+                            chatbot = gr.Chatbot(
+                                label="open-Deep-Research",
+                                type="messages",
+                                avatar_images=(
+                                    None,
+                                    "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/smolagents/mascot_smol.png",
+                                ),
+                            )
+                            with gr.Tabs():
+                                with gr.Tab("Report"):
+                                    fb_report_markdown = gr.Markdown(value="")
+                                    fb_report_dashboard = gr.HTML(value="")
+                            
+                            text_input = gr.Textbox(
+                                lines=1,
+                                label="Your request",
+                                placeholder="Enter your prompt here and press the button",
+                            )
+                            launch_research_btn = gr.Button("Run", variant="primary")
+                            
+                            # Fallback events
+                            text_input.submit(
+                                self.log_user_message,
+                                [text_input, file_uploads_log],
+                                [stored_messages, text_input, launch_research_btn],
+                            ).then(
+                                self.interact_with_agent,
+                                [stored_messages, chatbot],
+                                [chatbot, fb_report_markdown, fb_report_dashboard],
+                            )
+                            launch_research_btn.click(
+                                self.log_user_message,
+                                [text_input, file_uploads_log],
+                                [stored_messages, text_input, launch_research_btn],
+                            ).then(
+                                self.interact_with_agent,
+                                [stored_messages, chatbot],
+                                [chatbot, fb_report_markdown, fb_report_dashboard],
+                            )
+
+        
+        # Configure for Hugging Face Spaces compatibility
+        is_spaces = os.getenv("SPACE_ID") is not None
+        
+        if is_spaces:
+            # Hugging Face Spaces configuration
+            demo.launch(
+                debug=False,
+                server_name="0.0.0.0",
+                server_port=int(os.getenv("PORT", 7860)),
+                share=True,
+                **kwargs
+            )
+        else:
+            # Local development configuration
+            demo.launch(
+                debug=True,
+                server_name="localhost",
+                server_port=7860,
+                share=False,
+                **kwargs
+            )
+
+# Launch the application
+if __name__ == "__main__":
+    try:
+        GradioUI(file_upload_folder="uploads").launch()
+    except KeyboardInterrupt:
+        print("Application stopped by user")
+    except Exception as e:
+        print(f"Error starting application: {e}")
+        raise

assets/images/.gitkeep

@@ -0,0 +1,2 @@
+
+

docs/README.md

@@ -0,0 +1,32 @@
+# Hugging Research — Documentation
+
+Hugging Research is a lightweight research and coding assistant focused on Hugging Face Hub content. It helps you find models, datasets, Spaces, users, collections, and papers, and organizes links into a clean report.
+
+- UI: Gradio Blocks
+- Agent: smolagents `CodeAgent`
+- Tools: `scripts/hf_tools.py` (anonymous, read‑only)
+- Report: server-side generated from the final answer (no model call required)
+
+## Contents
+- [Getting started](./getting-started.md)
+- [Architecture](./architecture.md)
+- [Tools](./tools.md)
+- [Security](./security.md)
+- [Troubleshooting](./troubleshooting.md)
+- [Contributing](./contributing.md)
+- [Roadmap](./roadmap.md)
+
+## What it does
+- Searches Hugging Face Hub (models/datasets/Spaces/papers/users/collections)
+- Pulls tutorials/blog/course content via domain‑restricted search when needed
+- Avoids hallucinated links: only cites URLs from tool outputs
+- Builds an HTML report of links in the Report view automatically
+
+## How it works (quick view)
+- The agent uses `hf_*` tools (return JSON as strings) and `web_search` (returns plain text)
+- The app converts the final answer into a categorized link report
+- No files are written to disk for reports; HTML is rendered in‑app
+
+Start here: [Getting started](./getting-started.md)
+
+

docs/architecture.md

@@ -0,0 +1,28 @@
+# Architecture
+
+## Overview
+Gradio UI with a single Report view. A smolagents `CodeAgent` uses `hf_*` tools to fetch public Hub data, then the app turns the final answer into a categorized HTML link report.
+
+## Components
+- `app.py`
+  - Session and API key setup (for inference model only)
+  - Streams agent messages
+  - Generates report HTML from the final answer (no file writes)
+- `scripts/hf_tools.py`
+  - Anonymous, read‑only wrappers of Hub APIs and domain‑restricted search
+  - Outputs JSON as strings
+- `scripts/report_generator.py`
+  - Parses links in the final answer and renders a self‑contained HTML report
+
+## Flow
+1. User sends a prompt
+2. Agent calls `hf_*` tools and composes an answer with inline links
+3. App converts that answer into an HTML link report and shows it in the Report view
+
+## Privacy
+- Tools never use tokens; gated/private items are marked as not accessible
+- `HF_TOKEN` is only for the inference model
+
+## Extending
+- Add tools in `scripts/hf_tools.py` and register in `create_tools_with_model`
+- Update the system prompt to document tool contracts

docs/contributing.md

@@ -0,0 +1,28 @@
+# Contributing
+
+## Branches and PRs
+- Branch from `main` using `feature/...` or `fix/...` prefixes
+- Keep PRs focused and small; include screenshots for UI changes
+- Link related issues if applicable
+
+## Code style
+- Python 3.10+
+- Descriptive names; functions as verbs, variables as nouns
+- Handle errors explicitly; avoid silent excepts
+- Keep indentation and formatting consistent with the repo
+
+## Commit messages
+- Prefix: `feat:`, `fix:`, `docs:`, `refactor:`, `chore:`
+- Keep them concise and meaningful
+
+## Tests / Manual checks
+- Run the app and verify:
+  - Desktop and mobile UIs load
+  - Single Report view renders the link report from the last answer
+  - HF tools return JSON strings and handle errors
+
+## PR checklist
+- [ ] Code builds and runs locally
+- [ ] No secrets or tokens committed
+- [ ] Updated docs if behavior/UI changed
+- [ ] Included screenshots for UI updates

docs/getting-started.md

@@ -0,0 +1,34 @@
+# Getting started
+
+## Requirements
+- Python 3.10+
+- Internet connection
+
+## Install
+```bash
+git clone https://github.com/mcdaqc/hugging-research
+cd hugging-research
+python -m venv venv
+# Windows
+venv\Scripts\activate
+# macOS/Linux
+# source venv/bin/activate
+pip install -r requirements.txt
+```
+
+## Configure
+Create `.env` (token is only for the inference model; tools are anonymous):
+```ini
+HF_TOKEN=hf_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
+MODEL_ID=Qwen/Qwen3-Coder-480B-A35B-Instruct
+```
+
+## Run
+```bash
+python app.py
+# open http://localhost:7860
+```
+
+## Notes for Spaces
+- Set `HF_TOKEN` as a Space Secret if you want to choose a different inference model via `MODEL_ID`.
+- Tools never use tokens; private/gated items will be marked as not accessible.

docs/roadmap.md

@@ -0,0 +1,7 @@
+# Roadmap
+
+- Report enhancements: optional filters, compact view
+- Optional client-side export (PNG/PDF), no server deps
+- Caching for Hub calls with short TTL
+- Broader tests for tools and UI
+- Split `hf_tools.py` into domain modules

docs/security.md

@@ -0,0 +1,15 @@
+# Security & privacy
+
+## Principles
+- Read‑only: tools never send Authorization headers
+- Respect gated/private resources and label them as not accessible
+- Don’t log secrets; `HF_TOKEN` is only for the inference model
+
+## Details
+- Tools normalize `visibility` and `access` fields
+- The Report view renders HTML in memory; no report files are saved
+
+
+## Scope
+- No write operations to the Hub
+- Only public endpoints and domain‑restricted search are used

docs/tools.md

@@ -0,0 +1,31 @@
+# Tools
+
+All `hf_*` tools are anonymous, read‑only, and return JSON as a string.
+- Listing tools → `{ "results": [...] }`
+- Detail tools → `{ "item": {...} }`
+
+Common metadata: `visibility`, `access`, `type`, `id`, `owner`, `url`, `likes`, `downloads`, `updatedAt`.
+
+## Searches
+- `hf_models_search(query, owner?, task?, tags?, sort?, direction?, limit=10)`
+- `hf_datasets_search(query, owner?, tags?, sort?, direction?, limit=10)`
+- `hf_spaces_search(query, owner?, tags?, sort?, direction?, limit=10)`
+- `hf_site_search(query, limit=10)` — Blog/Learn/Docs discovery (plain links)
+
+## Details
+- `hf_model_info(repo_id)`
+- `hf_dataset_info(repo_id)`
+- `hf_space_info(repo_id)`
+- `hf_user_info(username)`
+- `hf_collections_list(owner)` / `hf_collection_get(owner, slug)`
+- `hf_paper_info(arxiv_id)` / `hf_paper_repos(arxiv_id)` / `hf_daily_papers(date?)`
+- `hf_repo_info(repo_type, repo_id)`
+
+## Parsing rules (important)
+- `web_search` → returns plain text. Do not `json.loads`.
+- `hf_*` tools → return JSON as a string. Always `json.loads(...)` before indexing.
+
+## Examples
+- “Find top Stable Diffusion models.”
+- “Datasets for Spanish sentiment analysis.”
+- “Spaces for document Q&A.”

docs/troubleshooting.md

@@ -0,0 +1,17 @@
+# Troubleshooting
+
+## UI/Queue errors (fn_index KeyError)
+- After code changes, do a hard refresh (Ctrl+F5) or restart the server. Stale frontends can reference removed event handlers.
+
+## Tool output parsing
+- `web_search` returns plain text. Do not `json.loads` or index it.
+- `hf_*` tools return JSON as strings. Always `json.loads` before indexing.
+
+## Rate limits
+- DuckDuckGo/site search can rate limit. Reduce `limit`, vary queries, retry later.
+
+## Windows Unicode
+- If console prints error on Unicode, ensure UTF‑8 code page or encode output explicitly.
+
+## Access errors
+- 401/403 is expected for gated/private. Tools should mark `access=no_access`.

requirements.txt

@@ -0,0 +1,132 @@
+aiofiles==24.1.0
+aiohappyeyeballs==2.6.1
+aiohttp==3.12.15
+aiosignal==1.4.0
+annotated-types==0.7.0
+anthropic==0.54.0
+anyio==4.10.0
+async-timeout==5.0.1
+attrs==25.3.0
+beautifulsoup4==4.13.4
+biopython==1.85
+Brotli==1.1.0
+certifi==2025.8.3
+cffi==1.17.1
+charset-normalizer==3.4.3
+chess==1.11.2
+click==8.2.1
+cobble==0.1.4
+colorama==0.4.6
+cryptography==45.0.6
+datasets==3.6.0
+ddgs==9.5.4
+defusedxml==0.7.1
+dill==0.3.8
+distro==1.9.0
+duckduckgo_search==8.0.4
+et_xmlfile==2.0.0
+exceptiongroup==1.3.0
+fastapi==0.116.1
+ffmpy==0.6.1
+filelock==3.19.1
+frozenlist==1.7.0
+fsspec==2025.3.0
+google_search_results==2.4.2
+gradio==5.42.0
+gradio_client==1.11.1
+groovy==0.1.2
+h11==0.16.0
+httpcore==1.0.9
+httpx==0.28.1
+httpx-sse==0.4.1
+huggingface-hub==0.34.4
+idna==3.10
+Jinja2==3.1.6
+jiter==0.10.0
+joblib==1.5.1
+loguru==0.7.3
+lxml==6.0.0
+mammoth==1.9.1
+markdown-it-py==4.0.0
+markdownify==1.1.0
+MarkupSafe==3.0.2
+mcp==1.9.3
+mdurl==0.1.2
+mpmath==1.3.0
+multidict==6.6.4
+multiprocess==0.70.16
+narwhals==2.1.2
+networkx==3.4.2
+numexpr==2.11.0
+numpy==2.2.6
+openai==1.87.0
+openpyxl==3.1.5
+orjson==3.11.2
+packaging==25.0
+pandas==2.3.0
+pathvalidate==3.3.1
+pdfminer==20191125
+pdfminer.six==20250506
+pillow==11.2.1
+plotly==6.1.2
+primp==0.15.0
+propcache==0.3.2
+PubChemPy==1.0.4
+puremagic==1.29
+pyarrow==21.0.0
+pycparser==2.22
+pycryptodome==3.23.0
+pydantic==2.11.7
+pydantic-settings==2.10.1
+pydantic_core==2.33.2
+pydub==0.25.1
+Pygments==2.19.2
+pypdf==5.6.0
+PyPDF2==3.0.1
+python-dateutil==2.9.0.post0
+python-dotenv==1.1.0
+python-multipart==0.0.20
+python-pptx==1.0.2
+pytz==2025.2
+PyYAML==6.0.2
+regex==2025.7.34
+requests==2.32.4
+rich==14.1.0
+ruff==0.12.9
+safehttpx==0.1.6
+safetensors==0.6.2
+scikit-learn==1.7.0
+scipy==1.15.3
+semantic-version==2.10.0
+serpapi==0.1.5
+shellingham==1.5.4
+simplejson==3.20.1
+six==1.17.0
+smolagents==1.21.1
+sniffio==1.3.1
+soupsieve==2.7
+SpeechRecognition==3.14.3
+sse-starlette==3.0.2
+starlette==0.47.2
+sympy==1.14.0
+threadpoolctl==3.6.0
+tokenizers==0.21.4
+tomlkit==0.13.3
+torch==2.7.1
+torchvision==0.22.1
+tqdm==4.67.1
+transformers==4.52.4
+typer==0.16.1
+typing-inspection==0.4.1
+typing_extensions==4.14.1
+tzdata==2025.2
+urllib3==2.5.0
+uvicorn==0.35.0
+websockets==15.0.1
+win32_setctime==1.2.0
+xlrd==2.0.2
+xlsxwriter==3.2.5
+xxhash==3.5.0
+yahoo-finance==1.4.0
+yarl==1.20.1
+youtube-transcript-api==1.1.0

runtime.txt

@@ -0,0 +1 @@
+python-3.10.0 

scripts/cookies.py

@@ -0,0 +1,715 @@
+from requests.cookies import RequestsCookieJar
+
+
+COOKIES_LIST = [
+    {
+        "domain": ".youtube.com",
+        "expirationDate": 1718884961,
+        "hostOnly": False,
+        "httpOnly": False,
+        "name": "ST-xuwub9",
+        "path": "/",
+        "sameSite": None,
+        "secure": False,
+        "session": False,
+        "storeId": None,
+        "value": "session_logininfo=AFmmF2swRAIgf4gadACOuWOcipI1anW-dakEjtidNLkufnOC8uml7EECIDh2YisqWELDBJPTGUysCucJ3I0wjXxYjVHro1LHrdW0%3AQUQ3MjNmd2Jiajl3OWZYRnpFNnZlWWV5ZGJWZ0hpcmp4LVVPU280bk4zOS03Z0ozZG9fOFhWZ0dXaVo3NG1wTEg1b3hGaG10TFBlaFBnTlJfbER5bEp0aFhoNS1OLVhYNFRZT2F6ajgzOFpDbGhlUjZpMWRETlFFRjFfTTRiM0RnNTROSkdmMTFMVjFic1VuZ2trbGp4aktDa0JJUC1BWDh3",
+    },
+    {
+        "domain": ".youtube.com",
+        "expirationDate": 1753004444.745411,
+        "hostOnly": False,
+        "httpOnly": True,
+        "name": "__Secure-YEC",
+        "path": "/",
+        "sameSite": "lax",
+        "secure": True,
+        "session": False,
+        "storeId": None,
+        "value": "CgtRVnI5LW1zRHlQVSjbtNCzBjIhCgJGUhIbEhcSFRMLFBUWFwwYGRobHB0eHw4PIBAREiAk",
+    },
+    {
+        "domain": ".youtube.com",
+        "expirationDate": 1753434620.050824,
+        "hostOnly": False,
+        "httpOnly": True,
+        "name": "__Secure-3PSID",
+        "path": "/",
+        "sameSite": "no_restriction",
+        "secure": True,
+        "session": False,
+        "storeId": None,
+        "value": "g.a000kwibeLUu8Ea9Y-vLun7u3kU5VNJVuMAZl_jdfJaNm50JyDBB4ezJ_bdWu46a7YwObVn44wACgYKAakSARQSFQHGX2MicJcTzecTKH6bHzqU6TMbTxoVAUF8yKqQYK-MoI6Ql3vI2oYTB3E-0076",
+    },
+    {
+        "domain": ".youtube.com",
+        "expirationDate": 1750420959.974642,
+        "hostOnly": False,
+        "httpOnly": False,
+        "name": "SIDCC",
+        "path": "/",
+        "sameSite": None,
+        "secure": False,
+        "session": False,
+        "storeId": None,
+        "value": "AKEyXzWQZauHKOo8t87zoEcjaVNIYUX54ohoWXT-tX4aAhEuZzIIptxZAcNkHuG2oDXYL6t-lw",
+    },
+    {
+        "domain": ".youtube.com",
+        "expirationDate": 1753434620.050652,
+        "hostOnly": False,
+        "httpOnly": False,
+        "name": "SID",
+        "path": "/",
+        "sameSite": None,
+        "secure": False,
+        "session": False,
+        "storeId": None,
+        "value": "g.a000kwibeLUu8Ea9Y-vLun7u3kU5VNJVuMAZl_jdfJaNm50JyDBB6VHrZcC3gBAsFPbCQ0gF5AACgYKAYkSARQSFQHGX2Mi9kt0gHg5CxCYSkLQGHWaeBoVAUF8yKre_V6r3jZVak6JV4o2Q0FL0076",
+    },
+    {
+        "domain": ".youtube.com",
+        "expirationDate": 1750420958.397534,
+        "hostOnly": False,
+        "httpOnly": True,
+        "name": "__Secure-1PSIDTS",
+        "path": "/",
+        "sameSite": None,
+        "secure": True,
+        "session": False,
+        "storeId": None,
+        "value": "sidts-CjIB3EgAEkYL2L-GfrEzW5Dfy62S9oefGNLgst78S_986htCnGcfkxECch_9oz-qytSsZBAA",
+    },
+    {
+        "domain": ".youtube.com",
+        "expirationDate": 1753433494.44729,
+        "hostOnly": False,
+        "httpOnly": False,
+        "name": "_ga_M0180HEFCY",
+        "path": "/",
+        "sameSite": None,
+        "secure": False,
+        "session": False,
+        "storeId": None,
+        "value": "GS1.1.1718871908.1.0.1718873494.0.0.0",
+    },
+    {
+        "domain": ".youtube.com",
+        "expirationDate": 1753434620.050933,
+        "hostOnly": False,
+        "httpOnly": False,
+        "name": "SAPISID",
+        "path": "/",
+        "sameSite": None,
+        "secure": True,
+        "session": False,
+        "storeId": None,
+        "value": "mfeuiC-HraNJ-A03/ASXvCPNJSw7yTFgd6",
+    },
+    {
+        "domain": ".youtube.com",
+        "expirationDate": 1750420959.974764,
+        "hostOnly": False,
+        "httpOnly": True,
+        "name": "__Secure-1PSIDCC",
+        "path": "/",
+        "sameSite": None,
+        "secure": True,
+        "session": False,
+        "storeId": None,
+        "value": "AKEyXzWHDSoXGCZpZhPxRrnC7B1s8zGIUjeMVyvgtQfsm1fs92lXPtFEI_td9LBUyqVUe0xK",
+    },
+    {
+        "domain": ".youtube.com",
+        "expirationDate": 1753434620.050881,
+        "hostOnly": False,
+        "httpOnly": True,
+        "name": "SSID",
+        "path": "/",
+        "sameSite": None,
+        "secure": True,
+        "session": False,
+        "storeId": None,
+        "value": "AmlwXHnQvOQ10LVd-",
+    },
+    {
+        "domain": ".youtube.com",
+        "expirationDate": 1753434620.050959,
+        "hostOnly": False,
+        "httpOnly": False,
+        "name": "__Secure-1PAPISID",
+        "path": "/",
+        "sameSite": None,
+        "secure": True,
+        "session": False,
+        "storeId": None,
+        "value": "mfeuiC-HraNJ-A03/ASXvCPNJSw7yTFgd6",
+    },
+    {
+        "domain": ".youtube.com",
+        "expirationDate": 1753434620.050795,
+        "hostOnly": False,
+        "httpOnly": True,
+        "name": "__Secure-1PSID",
+        "path": "/",
+        "sameSite": None,
+        "secure": True,
+        "session": False,
+        "storeId": None,
+        "value": "g.a000kwibeLUu8Ea9Y-vLun7u3kU5VNJVuMAZl_jdfJaNm50JyDBBrlk7lRpKQGywAHEon7WGQAACgYKAQsSARQSFQHGX2MirAmnSRdZl6GPG6KLd4hOihoVAUF8yKoV17Tcj1a_OenIOkf2wBjO0076",
+    },
+    {
+        "domain": ".youtube.com",
+        "expirationDate": 1753434620.050993,
+        "hostOnly": False,
+        "httpOnly": False,
+        "name": "__Secure-3PAPISID",
+        "path": "/",
+        "sameSite": "no_restriction",
+        "secure": True,
+        "session": False,
+        "storeId": None,
+        "value": "mfeuiC-HraNJ-A03/ASXvCPNJSw7yTFgd6",
+    },
+    {
+        "domain": ".youtube.com",
+        "expirationDate": 1750420959.974815,
+        "hostOnly": False,
+        "httpOnly": True,
+        "name": "__Secure-3PSIDCC",
+        "path": "/",
+        "sameSite": "no_restriction",
+        "secure": True,
+        "session": False,
+        "storeId": None,
+        "value": "AKEyXzXM5UjKUEXwSHVmRAIo6hGHA4G63adj3EE1VdNriD0f38jZQbsUKiD4LQbA3BValmTFDg",
+    },
+    {
+        "domain": ".youtube.com",
+        "expirationDate": 1750420958.397647,
+        "hostOnly": False,
+        "httpOnly": True,
+        "name": "__Secure-3PSIDTS",
+        "path": "/",
+        "sameSite": "no_restriction",
+        "secure": True,
+        "session": False,
+        "storeId": None,
+        "value": "sidts-CjIB3EgAEkYL2L-GfrEzW5Dfy62S9oefGNLgst78S_986htCnGcfkxECch_9oz-qytSsZBAA",
+    },
+    {
+        "domain": ".youtube.com",
+        "expirationDate": 1753434620.050908,
+        "hostOnly": False,
+        "httpOnly": False,
+        "name": "APISID",
+        "path": "/",
+        "sameSite": None,
+        "secure": False,
+        "session": False,
+        "storeId": None,
+        "value": "IlQWLPjdNqziwCrV/ANG7Z4x5FF-IBxbZk",
+    },
+    {
+        "domain": ".youtube.com",
+        "expirationDate": 1753434620.050855,
+        "hostOnly": False,
+        "httpOnly": True,
+        "name": "HSID",
+        "path": "/",
+        "sameSite": None,
+        "secure": False,
+        "session": False,
+        "storeId": None,
+        "value": "AasA7hmRuTFv7vjoq",
+    },
+    {
+        "domain": ".youtube.com",
+        "expirationDate": 1753435873.577793,
+        "hostOnly": False,
+        "httpOnly": True,
+        "name": "LOGIN_INFO",
+        "path": "/",
+        "sameSite": "no_restriction",
+        "secure": True,
+        "session": False,
+        "storeId": None,
+        "value": "AFmmF2swRAIgf4gadACOuWOcipI1anW-dakEjtidNLkufnOC8uml7EECIDh2YisqWELDBJPTGUysCucJ3I0wjXxYjVHro1LHrdW0:QUQ3MjNmd2Jiajl3OWZYRnpFNnZlWWV5ZGJWZ0hpcmp4LVVPU280bk4zOS03Z0ozZG9fOFhWZ0dXaVo3NG1wTEg1b3hGaG10TFBlaFBnTlJfbER5bEp0aFhoNS1OLVhYNFRZT2F6ajgzOFpDbGhlUjZpMWRETlFFRjFfTTRiM0RnNTROSkdmMTFMVjFic1VuZ2trbGp4aktDa0JJUC1BWDh3",
+    },
+    {
+        "domain": ".youtube.com",
+        "expirationDate": 1753444956.555608,
+        "hostOnly": False,
+        "httpOnly": False,
+        "name": "PREF",
+        "path": "/",
+        "sameSite": None,
+        "secure": True,
+        "session": False,
+        "storeId": None,
+        "value": "f4=4000000&f6=40000000&tz=Europe.Paris&f5=30000&f7=100",
+    },
+]
+
+COOKIES_LIST += [
+    {
+        "domain": ".www.researchgate.net",
+        "hostOnly": False,
+        "httpOnly": True,
+        "name": "isInstIp",
+        "path": "/",
+        "sameSite": None,
+        "secure": True,
+        "session": True,
+        "storeId": None,
+        "value": "False",
+    },
+    {
+        "domain": ".researchgate.net",
+        "expirationDate": 1734423981,
+        "hostOnly": False,
+        "httpOnly": False,
+        "name": "__eoi",
+        "path": "/",
+        "sameSite": None,
+        "secure": False,
+        "session": False,
+        "storeId": None,
+        "value": "ID=c26f752377373146:T=1718871981:RT=1718884914:S=AA-AfjZw-T_OOX2kW2LLaFzXImgc",
+    },
+    {
+        "domain": ".www.researchgate.net",
+        "expirationDate": 1753444909.646103,
+        "hostOnly": False,
+        "httpOnly": True,
+        "name": "ptc",
+        "path": "/",
+        "sameSite": None,
+        "secure": True,
+        "session": False,
+        "storeId": None,
+        "value": "RG1.8947708639250500550.1718872043",
+    },
+    {
+        "domain": ".researchgate.net",
+        "expirationDate": 1750507578,
+        "hostOnly": False,
+        "httpOnly": False,
+        "name": "euconsent-v2-didomi",
+        "path": "/",
+        "sameSite": "lax",
+        "secure": True,
+        "session": False,
+        "storeId": None,
+        "value": "CQAgmoAQAgmoAAHABBENA5EsAP_gAEPgAAYgJ2pB5G5UTWlBIG53YMskIAUFhFBoQEAgAACAAwIBSBIAIIwEAGAAIAgAICACAAIAIBIAIABAGAAAAAAAYIAAIAAIAAAQIAAKIAAAAAAAAgBQAAgIAgggEAAAgEBEABAAgAAAEIIAQNgACgAAACCAAAAAAAABAAAAAAAAQAAAAAAAYCQAAAJIAAAAACAIABAIAAAAAAAAAAAAAAAABBAAIJ2wPIAFAAXABQAFQALgAcAA8ACAAEgALwAZAA0ACIAEcAJgAUgAqgBcADEAGgAPQAfgBEACOAE4AMMAZYA0QBsgDkAHOAO4AfsBBwEIAItARwBHQC6gHUAO2Ae0A_4CHQEXgJ2AUOAo8BT4CpQFqALYAXmAwQBkgDLAGXANjAhCBG8CbAE3gJ1gTtAA.f_wACHwAAAAA",
+    },
+    {
+        "domain": ".researchgate.net",
+        "expirationDate": 1718885236,
+        "hostOnly": False,
+        "httpOnly": False,
+        "name": "_gat",
+        "path": "/",
+        "sameSite": None,
+        "secure": False,
+        "session": False,
+        "storeId": None,
+        "value": "1",
+    },
+    {
+        "domain": "www.researchgate.net",
+        "expirationDate": 1721477183,
+        "hostOnly": True,
+        "httpOnly": False,
+        "name": "_pbjs_userid_consent_data",
+        "path": "/",
+        "sameSite": "lax",
+        "secure": False,
+        "session": False,
+        "storeId": None,
+        "value": "3524755945110770",
+    },
+    {
+        "domain": ".researchgate.net",
+        "expirationDate": 1752567981,
+        "hostOnly": False,
+        "httpOnly": False,
+        "name": "__gads",
+        "path": "/",
+        "sameSite": None,
+        "secure": False,
+        "session": False,
+        "storeId": None,
+        "value": "ID=eca2adb88969c830:T=1718871981:RT=1718884914:S=ALNI_MY2qZchynrhWX6hWMlaI87Pcj9riQ",
+    },
+    {
+        "domain": ".researchgate.net",
+        "expirationDate": 1718886709.646173,
+        "hostOnly": False,
+        "httpOnly": True,
+        "name": "__cf_bm",
+        "path": "/",
+        "sameSite": "no_restriction",
+        "secure": True,
+        "session": False,
+        "storeId": None,
+        "value": "IkQ_J4ciBzKQduRvjqsfSmQu8UygDWbHeROO5JVccfo-1718884909-1.0.1.1-qvNGEdbfI0HfhFP6kwe7R7mkTqODNhFuKhs72lLly6K2BOPMG3kbahpQFGvPK0U8FUfkznkq65gngd1sWj7sDA",
+    },
+    {
+        "domain": ".researchgate.net",
+        "expirationDate": 1752567981,
+        "hostOnly": False,
+        "httpOnly": False,
+        "name": "__gpi",
+        "path": "/",
+        "sameSite": None,
+        "secure": False,
+        "session": False,
+        "storeId": None,
+        "value": "UID=00000e4e9aa2e6f2:T=1718871981:RT=1718884914:S=ALNI_MYFNrgzkKn7K6Bd2y8hC6GJCvDiSg",
+    },
+    {
+        "domain": ".researchgate.net",
+        "hostOnly": False,
+        "httpOnly": True,
+        "name": "_cfuvid",
+        "path": "/",
+        "sameSite": "no_restriction",
+        "secure": True,
+        "session": True,
+        "storeId": None,
+        "value": "_GPmGZkBymiH3UiqTqzakEpi98br3nfFUWC2_u_wqkc-1718884909785-0.0.1.1-604800000",
+    },
+    {
+        "domain": ".researchgate.net",
+        "expirationDate": 1753445177.271667,
+        "hostOnly": False,
+        "httpOnly": False,
+        "name": "_ga",
+        "path": "/",
+        "sameSite": None,
+        "secure": False,
+        "session": False,
+        "storeId": None,
+        "value": "GA1.1.1525244793.1718885177",
+    },
+    {
+        "domain": ".researchgate.net",
+        "expirationDate": 1753445177.271482,
+        "hostOnly": False,
+        "httpOnly": False,
+        "name": "_ga_4P31SJ70EJ",
+        "path": "/",
+        "sameSite": None,
+        "secure": False,
+        "session": False,
+        "storeId": None,
+        "value": "GS1.1.1718885177.1.0.1718885177.0.0.0",
+    },
+    {
+        "domain": ".researchgate.net",
+        "expirationDate": 1718971576,
+        "hostOnly": False,
+        "httpOnly": False,
+        "name": "_gid",
+        "path": "/",
+        "sameSite": None,
+        "secure": False,
+        "session": False,
+        "storeId": None,
+        "value": "GA1.2.854907463.1718885177",
+    },
+    {
+        "domain": ".www.researchgate.net",
+        "expirationDate": 1750407982.506505,
+        "hostOnly": False,
+        "httpOnly": True,
+        "name": "did",
+        "path": "/",
+        "sameSite": None,
+        "secure": True,
+        "session": False,
+        "storeId": None,
+        "value": "1dWLO3C6am8l667Q4VUlBo0O1LI49Qi2Vw21SJEXHavBDYT56DI9007W5rYGVFVH",
+    },
+    {
+        "domain": ".researchgate.net",
+        "expirationDate": 1750507578,
+        "hostOnly": False,
+        "httpOnly": False,
+        "name": "didomi_token",
+        "path": "/",
+        "sameSite": "lax",
+        "secure": True,
+        "session": False,
+        "storeId": None,
+        "value": "eyJ1c2VyX2lkIjoiMTkwMzU4YTUtNWU2My02Y2UzLWJlNzAtZGFjNzVmYjdiY2ExIiwiY3JlYXRlZCI6IjIwMjQtMDYtMjBUMTI6MDY6MTYuODA2WiIsInVwZGF0ZWQiOiIyMDI0LTA2LTIwVDEyOjA2OjE4Ljc4MVoiLCJ2ZW5kb3JzIjp7ImVuYWJsZWQiOlsidHdpdHRlciIsImdvb2dsZSIsImM6bGlua2VkaW4tbWFya2V0aW5nLXNvbHV0aW9ucyIsImM6b3duZXJpcSIsImM6b21uaXR1cmUtYWRvYmUtYW5hbHl0aWNzIiwiYzp0ZWNobm9yYXRpLW1lZGlhIiwiYzppbnRlcmNvbSIsImM6aW50ZW50LWlxIiwiYzppcHJvbSIsImM6bGlua2VkaW4iLCJjOmFtYXpvbmFkdi16Y1hGTEI2WCIsImM6bWVkaWFuZXQtY1V3YUtFNnoiLCJjOmluZGV4ZXhjaC1OWkNRTTY4UCIsImM6emVvdGFwZ21iLWQ3YndtdGp3IiwiYzp0cmlwbGVsaWYtZGRKSDM0clkiLCJjOnJ0YmhvdXNlLWI4Y2RIOHRNIiwiYzptZHByaW1pcy1lYU4yOVdjUCIsImM6bG9vcG1lbGktVGRhWXRCUHEiLCJjOm1hZ25pdGVpbi05d1RZTHFSRCIsImM6Ymlkc3dpdGNoLWQ2N0V3N1c5IiwiYzpvcmFjbGVhZHYtcUhlREptQUwiLCJjOmdvb2dsZWFuYS00VFhuSmlnUiIsImM6bG90YW1lc29sLURIaTdMUmpNIiwiYzpuZXh0bWlsbGUtR0pyZlg4VWMiLCJjOm5yaWNodGVjLXFVVlEyUlFxIiwiYzpicml0ZXBvb2wtQldWeVdHeVUiLCJjOnRhcGFkaW5jLXFxY2tVN1BXIiwiYzppZDV0ZWNobi16Tk1KNGR3ZiIsImM6bWljcm9zb2Z0IiwiYzpwZXJtdXRpdmUtSjdpaHJlTWsiLCJjOm9wZXJhc29mdC1CY1hjRFZKTSIsImM6cG9zdGhvZy1Cakp4RmRGOSJdfSwicHVycG9zZXMiOnsiZW5hYmxlZCI6WyJnZW9sb2NhdGlvbl9kYXRhIiwiZGV2aWNlX2NoYXJhY3RlcmlzdGljcyJdfSwidmVuZG9yc19saSI6eyJlbmFibGVkIjpbImdvb2dsZSIsImM6b3BlcmFzb2Z0LUJjWGNEVkpNIl19LCJ2ZXJzaW9uIjoyLCJhYyI6IkRIU0FvQUZrQWNnQTVnSHFnUUhBeGdCNndEMTRJR0FRTkFqMEJJd0NTY0VyQUtCd1YtZ3MxQmgwREc0R09nQUEuREhTQW9BRmtBY2dBNWdIcWdRSEF4Z0I2d0QxNElHQVFOQWowQkl3Q1NjRXJBS0J3Vi1nczFCaDBERzRHT2dBQSJ9",
+    },
+    {
+        "domain": ".www.researchgate.net",
+        "hostOnly": False,
+        "httpOnly": True,
+        "name": "hasPdpNext",
+        "path": "/",
+        "sameSite": None,
+        "secure": True,
+        "session": True,
+        "storeId": None,
+        "value": "False",
+    },
+    {
+        "domain": ".researchgate.net",
+        "expirationDate": 1750421183,
+        "hostOnly": False,
+        "httpOnly": False,
+        "name": "ph_phc_ma1XTQyee96N1GML6qUTgLQRiDifnRcE9STiHTZ0CfZ_posthog",
+        "path": "/",
+        "sameSite": "lax",
+        "secure": True,
+        "session": False,
+        "storeId": None,
+        "value": "%7B%22distinct_id%22%3A%220190358a-56a1-7313-83b0-d13dddeac787%22%2C%22%24sesid%22%3A%5B1718885183223%2C%220190358a-56a1-7313-83b0-d13b2b87778d%22%2C1718885176993%5D%2C%22%24session_is_sampled%22%3Atrue%7D",
+    },
+    {
+        "domain": ".www.researchgate.net",
+        "hostOnly": False,
+        "httpOnly": True,
+        "name": "sid",
+        "path": "/",
+        "sameSite": None,
+        "secure": True,
+        "session": True,
+        "storeId": None,
+        "value": "qmH5Lc4f0CUJ3zeaxORcV0S8I8V1MuCFZtcIQqPYtv1XPejrbSLAQRbT50PL40TqeKQ1XsQDWt9gtYVzuL80bRmPjw6jn3cQ0ikNqW40maHcQ3JL2Vfa8ZZf0j7p35eJ",
+    },
+]
+
+COOKIES_LIST += [
+    {
+        "domain": "github.com",
+        "hostOnly": True,
+        "httpOnly": True,
+        "name": "_gh_sess",
+        "path": "/",
+        "sameSite": "lax",
+        "secure": True,
+        "session": True,
+        "storeId": None,
+        "value": "P%2Fmof1avuqwHaUQUIJR%2FZYn7jqbT7lgGuTGjp1BGAFIG5UpNDusEE3b8dRjz0eATE5xPdPjLYFqMs%2FI9AOalKX4YuYfSEEnxCMawU01099b4o9Xzzcv%2BmecrmO0Q8q%2Bdq1h8SIv6nvPP7HzlFesl8ysafb9b%2F0q6dTArKdSOurasza8UgLSYD08ofA50Pcm0IG7CTzF8ZCizrGgGTMi%2F%2B7L3E17jav5PM1Sf2vQKg15Gbg1QIOppJJHzlufgQoZigqFv%2BWznaws0Tt7Y2lSFCw%3D%3D--CJRhqMXJnwOaJgk4--DhUErlL4GdROikEjKD4O9g%3D%3D",
+    },
+    {
+        "domain": ".github.com",
+        "expirationDate": 1750408875.763785,
+        "hostOnly": False,
+        "httpOnly": False,
+        "name": "_octo",
+        "path": "/",
+        "sameSite": "lax",
+        "secure": True,
+        "session": False,
+        "storeId": None,
+        "value": "GH1.1.728652011.1718872875",
+    },
+    {
+        "domain": ".github.com",
+        "expirationDate": 1750408875.763926,
+        "hostOnly": False,
+        "httpOnly": True,
+        "name": "logged_in",
+        "path": "/",
+        "sameSite": "lax",
+        "secure": True,
+        "session": False,
+        "storeId": None,
+        "value": "no",
+    },
+    {
+        "domain": ".github.com",
+        "hostOnly": False,
+        "httpOnly": False,
+        "name": "preferred_color_mode",
+        "path": "/",
+        "sameSite": "lax",
+        "secure": True,
+        "session": True,
+        "storeId": None,
+        "value": "dark",
+    },
+    {
+        "domain": ".github.com",
+        "hostOnly": False,
+        "httpOnly": False,
+        "name": "tz",
+        "path": "/",
+        "sameSite": "lax",
+        "secure": True,
+        "session": True,
+        "storeId": None,
+        "value": "Europe%2FParis",
+    },
+]
+
+COOKIES_LIST += [
+    {
+        "domain": ".web.archive.org",
+        "expirationDate": 1718886430,
+        "hostOnly": False,
+        "httpOnly": False,
+        "name": "_gat",
+        "path": "/web/20201123221659/http://orcid.org/",
+        "sameSite": None,
+        "secure": False,
+        "session": False,
+        "storeId": None,
+        "value": "1",
+    },
+    {
+        "domain": ".web.archive.org",
+        "expirationDate": 1718972770,
+        "hostOnly": False,
+        "httpOnly": False,
+        "name": "_gid",
+        "path": "/web/20201123221659/http://orcid.org/",
+        "sameSite": None,
+        "secure": False,
+        "session": False,
+        "storeId": None,
+        "value": "GA1.2.402246368.1606169825",
+    },
+    {
+        "domain": ".web.archive.org",
+        "expirationDate": 1753446370.315621,
+        "hostOnly": False,
+        "httpOnly": False,
+        "name": "_ga",
+        "path": "/web/20201123221659/http://orcid.org/",
+        "sameSite": None,
+        "secure": False,
+        "session": False,
+        "storeId": None,
+        "value": "GA1.2.1301409987.1606169825",
+    },
+    {
+        "domain": ".web.archive.org",
+        "expirationDate": 1750422367,
+        "hostOnly": False,
+        "httpOnly": False,
+        "name": "_hjid",
+        "path": "/web/20201123221659/http://orcid.org/",
+        "sameSite": "lax",
+        "secure": False,
+        "session": False,
+        "storeId": None,
+        "value": "07f80263-a631-4bf4-8ffd-8fc8912085e2",
+    },
+    {
+        "domain": ".web.archive.org",
+        "expirationDate": 1718888167,
+        "hostOnly": False,
+        "httpOnly": False,
+        "name": "_hjFirstSeen",
+        "path": "/web/20201123221659/http://orcid.org/",
+        "sameSite": "lax",
+        "secure": False,
+        "session": False,
+        "storeId": None,
+        "value": "1",
+    },
+]
+COOKIES_LIST += [
+    {
+        "domain": "orcid.org",
+        "hostOnly": True,
+        "httpOnly": False,
+        "name": "AWSELBCORS",
+        "path": "/",
+        "sameSite": "no_restriction",
+        "secure": True,
+        "session": True,
+        "storeId": None,
+        "value": "CBD1D7FF1216388FA48838CBCA4774FD22800B8FB548A40EF92BB0994D5B77A8410307CDEAA69C52236663F2BF89B252C17BC0FCDF790FD59771BDDF6EA8CA4CFD29D8733F",
+    },
+    {
+        "domain": ".orcid.org",
+        "expirationDate": 1753452454.637671,
+        "hostOnly": False,
+        "httpOnly": False,
+        "name": "_ga_9R61FWK9H5",
+        "path": "/",
+        "sameSite": None,
+        "secure": False,
+        "session": False,
+        "storeId": None,
+        "value": "GS1.1.1718892454.1.0.1718892454.0.0.0",
+    },
+    {
+        "domain": ".orcid.org",
+        "expirationDate": 1753452454.63421,
+        "hostOnly": False,
+        "httpOnly": False,
+        "name": "_ga",
+        "path": "/",
+        "sameSite": None,
+        "secure": False,
+        "session": False,
+        "storeId": None,
+        "value": "GA1.1.2021310691.1718892455",
+    },
+    {
+        "domain": "orcid.org",
+        "hostOnly": True,
+        "httpOnly": False,
+        "name": "AWSELB",
+        "path": "/",
+        "sameSite": None,
+        "secure": False,
+        "session": True,
+        "storeId": None,
+        "value": "CBD1D7FF1216388FA48838CBCA4774FD22800B8FB548A40EF92BB0994D5B77A8410307CDEAA69C52236663F2BF89B252C17BC0FCDF790FD59771BDDF6EA8CA4CFD29D8733F",
+    },
+    {
+        "domain": ".orcid.org",
+        "expirationDate": 1750428454,
+        "hostOnly": False,
+        "httpOnly": False,
+        "name": "OptanonAlertBoxClosed",
+        "path": "/",
+        "sameSite": "lax",
+        "secure": False,
+        "session": False,
+        "storeId": None,
+        "value": "2024-06-20T14:07:34.583Z",
+    },
+    {
+        "domain": ".orcid.org",
+        "expirationDate": 1750428454,
+        "hostOnly": False,
+        "httpOnly": False,
+        "name": "OptanonConsent",
+        "path": "/",
+        "sameSite": "lax",
+        "secure": False,
+        "session": False,
+        "storeId": None,
+        "value": "isGpcEnabled=0&datestamp=Thu+Jun+20+2024+16%3A07%3A34+GMT%2B0200+(heure+d%E2%80%99%C3%A9t%C3%A9+d%E2%80%99Europe+centrale)&version=202310.2.0&browserGpcFlag=0&isIABGlobal=False&hosts=&landingPath=NotLandingPage&groups=C0001%3A1%2CC0003%3A1%2CC0002%3A1%2CC0004%3A1",
+    },
+    {
+        "domain": "orcid.org",
+        "hostOnly": True,
+        "httpOnly": False,
+        "name": "XSRF-TOKEN",
+        "path": "/",
+        "sameSite": None,
+        "secure": True,
+        "session": True,
+        "storeId": None,
+        "value": "6957be7a-bcb4-4d59-a522-ea9b6b210ed9",
+    },
+]
+
+# Create a RequestsCookieJar instance
+COOKIES = RequestsCookieJar()
+
+# Add cookies to the jar
+for cookie in COOKIES_LIST:
+    COOKIES.set(cookie["name"], cookie["value"], domain=cookie["domain"], path=cookie["path"])

scripts/gaia_scorer.py

@@ -0,0 +1,124 @@
+import re
+import string
+import warnings
+
+
+def normalize_number_str(number_str: str) -> float:
+    # we replace these common units and commas to allow
+    # conversion to float
+    for char in ["$", "%", ","]:
+        number_str = number_str.replace(char, "")
+    try:
+        return float(number_str)
+    except ValueError:
+        print(f"String {number_str} cannot be normalized to number str.")
+        return float("inf")
+
+
+def split_string(
+    s: str,
+    char_list: list[str] = [",", ";"],
+) -> list[str]:
+    pattern = f"[{''.join(char_list)}]"
+    return re.split(pattern, s)
+
+
+def is_float(element: any) -> bool:
+    try:
+        float(element)
+        return True
+    except ValueError:
+        return False
+
+
+def question_scorer(
+    model_answer: str,
+    ground_truth: str,
+) -> bool:
+    # if gt is a number
+    if is_float(ground_truth):
+        normalized_answer = normalize_number_str(str(model_answer))
+        return normalized_answer == float(ground_truth)
+
+    # if gt is a list
+    elif any(char in ground_truth for char in [",", ";"]):
+        # question with the fish: normalization removes punct
+
+        gt_elems = split_string(ground_truth)
+        ma_elems = split_string(model_answer)
+
+        # check length is the same
+        if len(gt_elems) != len(ma_elems):
+            warnings.warn("Answer lists have different lengths, returning False.", UserWarning)
+            return False
+
+        # compare each element as float or str
+        comparisons = []
+        for ma_elem, gt_elem in zip(ma_elems, gt_elems):
+            if is_float(gt_elem):
+                normalized_ma_elem = normalize_number_str(ma_elem)
+                comparisons.append(normalized_ma_elem == float(gt_elem))
+            else:
+                # we do not remove punct since comparisons can include punct
+                comparisons.append(
+                    normalize_str(ma_elem, remove_punct=False) == normalize_str(gt_elem, remove_punct=False)
+                )
+        return all(comparisons)
+
+    # if gt is a str
+    else:
+        return normalize_str(model_answer) == normalize_str(ground_truth)
+
+
+def check_prediction_contains_answer_letters_in_order(prediction, true_answer):
+    prediction = prediction.lower()
+    true_answer = true_answer.lower()
+    if len(prediction) > len(true_answer) * 3:
+        return False
+    i = 0
+    for letter in true_answer:
+        if letter in prediction[i:]:
+            i += prediction[i:].index(letter)
+        else:
+            return False
+    return True
+
+
+def check_close_call(prediction, true_answer, is_correct):
+    if is_correct:
+        return True
+    else:
+        if is_float(true_answer):
+            return is_correct
+        else:
+            if (
+                check_prediction_contains_answer_letters_in_order(str(prediction), str(true_answer))
+                and len(str(true_answer)) * 0.5 <= len(str(prediction)) <= len(str(true_answer)) * 2
+            ):
+                print(f"Close call: {prediction} vs {true_answer}")
+                return True
+            else:
+                return False
+
+
+def normalize_str(input_str, remove_punct=True) -> str:
+    """
+    Normalize a string by:
+    - Removing all white spaces
+    - Optionally removing punctuation (if remove_punct is True)
+    - Converting to lowercase
+    Parameters:
+    - input_str: str, the string to normalize
+    - remove_punct: bool, whether to remove punctuation (default: True)
+    Returns:
+    - str, the normalized string
+    """
+    # Remove all white spaces. Required e.g for seagull vs. sea gull
+    no_spaces = re.sub(r"\s", "", input_str)
+
+    # Remove punctuation, if specified.
+    if remove_punct:
+        translator = str.maketrans("", "", string.punctuation)
+        return no_spaces.lower().translate(translator)
+    else:
+        return no_spaces.lower()

scripts/hf_tools.py

@@ -0,0 +1,867 @@
+import json
+import time
+from typing import Dict, List, Optional, Tuple
+
+import requests
+from smolagents import Tool
+
+
+# -----------------------------
+# HTTP helpers (anonymous only)
+# -----------------------------
+
+DEFAULT_TIMEOUT = 15
+RETRY_STATUS = {429, 500, 502, 503, 504}
+
+
+def _anonymous_headers() -> Dict[str, str]:
+    return {
+        "Accept": "application/json",
+        "User-Agent": "HuggingResearch-Agent/1.0 (+https://huggingface.co)",
+        # No Authorization header on purpose (public only)
+    }
+
+
+def _http_get_json(url: str, params: Optional[Dict] = None, max_retries: int = 2) -> Tuple[Optional[Dict | List], int, str]:
+    last_err = ""
+    for attempt in range(max_retries + 1):
+        try:
+            resp = requests.get(url, params=params or {}, headers=_anonymous_headers(), timeout=DEFAULT_TIMEOUT)
+            if resp.status_code == 200:
+                try:
+                    return resp.json(), resp.status_code, ""
+                except Exception as je:
+                    return None, resp.status_code, f"invalid_json: {je}"
+            if resp.status_code in {401, 403}:
+                # Private/Gated/Unauthorized
+                return None, resp.status_code, "no_access"
+            if resp.status_code in RETRY_STATUS and attempt < max_retries:
+                time.sleep(1.0 * (attempt + 1))
+                continue
+            return None, resp.status_code, f"http_{resp.status_code}"
+        except Exception as e:
+            last_err = str(e)
+            if attempt < max_retries:
+                time.sleep(0.8 * (attempt + 1))
+                continue
+            return None, 0, f"exception: {last_err}"
+
+
+# -----------------------------
+# Normalization helpers
+# -----------------------------
+
+def _visibility_from_item(item: Dict) -> Tuple[str, str]:
+    if not isinstance(item, dict):
+        return "public", "accessible"
+    if item.get("private") is True:
+        return "private", "no_access"
+    if item.get("gated") is True or item.get("gatedReason") or (isinstance(item.get("cardData"), dict) and item["cardData"].get("gated")):
+        return "gated", "no_access"
+    return "public", "accessible"
+
+
+def _norm_common(item_id: str, item_type: str, owner: str, description: str = "", url_suffix: str = "") -> Dict:
+    url = f"https://huggingface.co/{url_suffix}{item_id}" if url_suffix else f"https://huggingface.co/{item_id}"
+    return {
+        "type": item_type,
+        "id": item_id,
+        "owner": owner,
+        "url": url,
+        "description": description or "",
+    }
+
+
+def _safe_get(item: Dict, key: str, default=None):
+    return item.get(key, default) if isinstance(item, dict) else default
+
+
+# -----------------------------
+# Tools
+# -----------------------------
+
+
+class HFModelsSearchTool(Tool):
+    name = "hf_models_search"
+    description = (
+        "Search public Hugging Face models. Provide a free-text query and optional filters "
+        "(owner, single pipeline_tag, tags CSV, sort/direction, limit). "
+        "Prefer minimal params; add owner/task/tags/sort only when the user implies them. "
+        "Defaults: limit=10, sort omitted, direction omitted. Returns JSON with `results`, `status`, `error`, and `params`."
+    )
+    inputs = {
+        "query": {"type": "string", "description": "Free-text search", "nullable": True},
+        "owner": {"type": "string", "description": "Filter by owner/namespace", "nullable": True},
+        "task": {"type": "string", "description": "Primary pipeline tag, e.g. text-classification", "nullable": True},
+        "tags": {"type": "string", "description": "Comma-separated tags filter", "nullable": True},
+        "sort": {"type": "string", "description": "downloads|likes|modified", "nullable": True},
+        "direction": {"type": "string", "description": "descending|ascending", "nullable": True},
+        "limit": {"type": "number", "description": "Max results", "nullable": True},
+    }
+    output_type = "string"
+
+    def forward(self, query: Optional[str] = None, owner: Optional[str] = None, task: Optional[str] = None, tags: Optional[str] = None, sort: Optional[str] = None, direction: Optional[str] = None, limit: Optional[int] = None) -> str:
+        # Build conservative params
+        params = {}
+        if query:
+            params["search"] = query
+        if owner:
+            params["author"] = owner
+        if task:
+            # pipeline_tag must be a single value; if multiple provided, take the first
+            first_task = task.split(",")[0].strip()
+            if first_task:
+                params["pipeline_tag"] = first_task
+        if tags:
+            # Support comma-separated → repeated tags
+            tag_list = [t.strip() for t in tags.split(",") if t.strip()]
+            if len(tag_list) == 1:
+                params["tags"] = tag_list[0]
+            elif len(tag_list) > 1:
+                params["tags"] = tag_list  # requests will repeat param
+        # Support 'trending' as an alias mapped to downloads+descending for recency/interest
+        if sort in {"downloads", "likes", "modified", "trending"}:
+            params["sort"] = sort
+        if sort == "trending":
+            params["sort"] = "downloads"
+            params["direction"] = "descending"
+        elif direction in {"descending", "ascending"}:
+            params["direction"] = direction
+        # Default limit to 10 if not specified
+        lim = int(limit) if limit else 10
+        params["limit"] = lim
+
+        data, status, err = _http_get_json("https://huggingface.co/api/models", params)
+        # Fallback: retry with minimal params if 400
+        if status == 400:
+            minimal = {"search": query} if query else {}
+            if limit:
+                minimal["limit"] = int(limit)
+            data, status, err = _http_get_json("https://huggingface.co/api/models", minimal)
+        results: List[Dict] = []
+        if isinstance(data, list):
+            for it in data:
+                model_id = _safe_get(it, "id") or _safe_get(it, "modelId") or ""
+                if not model_id:
+                    continue
+                owner_name = model_id.split("/")[0] if "/" in model_id else ""
+                desc = ""
+                # If present, short description may live in cardData/summary when full=true; not guaranteed in list
+                visibility, access = _visibility_from_item(it)
+                norm = _norm_common(model_id, "model", owner_name, desc)
+                norm.update({
+                    "tags": _safe_get(it, "tags", []),
+                    "task": _safe_get(it, "pipeline_tag"),
+                    "likes": _safe_get(it, "likes", 0),
+                    "downloads": _safe_get(it, "downloads", 0),
+                    "updatedAt": _safe_get(it, "lastModified"),
+                    "visibility": visibility,
+                    "access": access,
+                })
+                results.append(norm)
+
+        return json.dumps({
+            "results": results,
+            "status": status,
+            "error": err,
+            "params": params,
+        }, ensure_ascii=False)
+
+
+class HFModelInfoTool(Tool):
+    name = "hf_model_info"
+    description = (
+        "Get detailed public model info by repo id (owner/name). Use this after a model search to fetch richer metadata (cardData, siblings, tags)."
+    )
+    inputs = {
+        "repo_id": {"type": "string", "description": "Model repo id, e.g. bigscience/bloom"}
+    }
+    output_type = "string"
+
+    def forward(self, repo_id: str) -> str:
+        data, status, err = _http_get_json(f"https://huggingface.co/api/models/{repo_id}", {"full": "true"})
+        item: Dict = {}
+        if isinstance(data, dict):
+            model_id = data.get("id") or data.get("modelId") or repo_id
+            owner_name = model_id.split("/")[0] if "/" in model_id else ""
+            visibility, access = _visibility_from_item(data)
+            desc = ""
+            # Some cards put a short summary in cardData/summary
+            if isinstance(data.get("cardData"), dict):
+                desc = data["cardData"].get("summary") or data["cardData"].get("description") or ""
+            item = _norm_common(model_id, "model", owner_name, desc)
+            item.update({
+                "tags": data.get("tags", []),
+                "task": data.get("pipeline_tag"),
+                "likes": data.get("likes", 0),
+                "downloads": data.get("downloads", 0),
+                "updatedAt": data.get("lastModified"),
+                "visibility": visibility,
+                "access": access,
+                "cardData": data.get("cardData"),
+                "siblings": data.get("siblings"),
+            })
+        return json.dumps({"item": item, "status": status, "error": err}, ensure_ascii=False)
+
+
+class HFDatasetsSearchTool(Tool):
+    name = "hf_datasets_search"
+    description = (
+        "Search public datasets with a free-text query and optional filters (owner, tags CSV, sort/direction, limit). "
+        "Prefer minimal params; add filters when implied. Defaults: limit=10. Returns JSON with `results`, `status`, `error`, and `params`."
+    )
+    inputs = {
+        "query": {"type": "string", "description": "Free-text search", "nullable": True},
+        "owner": {"type": "string", "description": "Filter by owner/namespace", "nullable": True},
+        "tags": {"type": "string", "description": "Comma-separated tags filter", "nullable": True},
+        "sort": {"type": "string", "description": "downloads|likes|modified", "nullable": True},
+        "direction": {"type": "string", "description": "descending|ascending", "nullable": True},
+        "limit": {"type": "number", "description": "Max results", "nullable": True},
+    }
+    output_type = "string"
+
+    def forward(self, query: Optional[str] = None, owner: Optional[str] = None, tags: Optional[str] = None, sort: Optional[str] = None, direction: Optional[str] = None, limit: Optional[int] = None) -> str:
+        params = {}
+        if query:
+            params["search"] = query
+        if owner:
+            params["author"] = owner
+        if tags:
+            tag_list = [t.strip() for t in tags.split(",")] if isinstance(tags, str) else []
+            tag_list = [t for t in tag_list if t]
+            if len(tag_list) == 1:
+                params["tags"] = tag_list[0]
+            elif len(tag_list) > 1:
+                params["tags"] = tag_list
+        if sort in {"downloads", "likes", "modified", "trending"}:
+            params["sort"] = sort
+        if sort == "trending":
+            params["sort"] = "downloads"
+            params["direction"] = "descending"
+        elif direction in {"descending", "ascending"}:
+            params["direction"] = direction
+        lim = int(limit) if limit else 10
+        params["limit"] = lim
+
+        data, status, err = _http_get_json("https://huggingface.co/api/datasets", params)
+        if status == 400:
+            minimal = {"search": query} if query else {}
+            if limit:
+                minimal["limit"] = int(limit)
+            data, status, err = _http_get_json("https://huggingface.co/api/datasets", minimal)
+        results: List[Dict] = []
+        if isinstance(data, list):
+            for it in data:
+                ds_id = _safe_get(it, "id") or _safe_get(it, "datasetId") or ""
+                if not ds_id:
+                    continue
+                owner_name = ds_id.split("/")[0] if "/" in ds_id else ""
+                visibility, access = _visibility_from_item(it)
+                norm = _norm_common(ds_id, "dataset", owner_name, "")
+                norm.update({
+                    "tags": _safe_get(it, "tags", []),
+                    "likes": _safe_get(it, "likes", 0),
+                    "downloads": _safe_get(it, "downloads", 0),
+                    "updatedAt": _safe_get(it, "lastModified"),
+                    "visibility": visibility,
+                    "access": access,
+                })
+                results.append(norm)
+        return json.dumps({"results": results, "status": status, "error": err, "params": params}, ensure_ascii=False)
+
+
+class HFDatasetInfoTool(Tool):
+    name = "hf_dataset_info"
+    description = (
+        "Get detailed public dataset info by repo id (owner/name). Use after a dataset search to retrieve cardData and siblings."
+    )
+    inputs = {"repo_id": {"type": "string", "description": "Dataset repo id, e.g. glue"}}
+    output_type = "string"
+
+    def forward(self, repo_id: str) -> str:
+        data, status, err = _http_get_json(f"https://huggingface.co/api/datasets/{repo_id}", {"full": "true"})
+        item: Dict = {}
+        if isinstance(data, dict):
+            ds_id = data.get("id") or data.get("datasetId") or repo_id
+            owner_name = ds_id.split("/")[0] if "/" in ds_id else ""
+            visibility, access = _visibility_from_item(data)
+            desc = ""
+            if isinstance(data.get("cardData"), dict):
+                desc = data["cardData"].get("summary") or data["cardData"].get("description") or ""
+            item = _norm_common(ds_id, "dataset", owner_name, desc)
+            item.update({
+                "tags": data.get("tags", []),
+                "likes": data.get("likes", 0),
+                "downloads": data.get("downloads", 0),
+                "updatedAt": data.get("lastModified"),
+                "visibility": visibility,
+                "access": access,
+                "cardData": data.get("cardData"),
+                "siblings": data.get("siblings"),
+            })
+        return json.dumps({"item": item, "status": status, "error": err}, ensure_ascii=False)
+
+
+class HFSpacesSearchTool(Tool):
+    name = "hf_spaces_search"
+    description = (
+        "Search public Spaces with query and optional filters (owner, tags CSV, sort/direction, limit). "
+        "Good for tutorials/demos related to a topic. Defaults: limit=10. Returns JSON with `results`, `status`, `error`, and `params`."
+    )
+    inputs = {
+        "query": {"type": "string", "description": "Free-text search", "nullable": True},
+        "owner": {"type": "string", "description": "Filter by owner/namespace", "nullable": True},
+        "tags": {"type": "string", "description": "Comma-separated tags filter", "nullable": True},
+        "sort": {"type": "string", "description": "likes|modified", "nullable": True},
+        "direction": {"type": "string", "description": "descending|ascending", "nullable": True},
+        "limit": {"type": "number", "description": "Max results", "nullable": True},
+    }
+    output_type = "string"
+
+    def forward(self, query: Optional[str] = None, owner: Optional[str] = None, tags: Optional[str] = None, sort: Optional[str] = None, direction: Optional[str] = None, limit: Optional[int] = None) -> str:
+        params = {}
+        if query:
+            params["search"] = query
+        if owner:
+            params["author"] = owner
+        if tags:
+            tag_list = [t.strip() for t in tags.split(",")] if isinstance(tags, str) else []
+            tag_list = [t for t in tag_list if t]
+            if len(tag_list) == 1:
+                params["tags"] = tag_list[0]
+            elif len(tag_list) > 1:
+                params["tags"] = tag_list
+        if sort in {"likes", "modified", "trending"}:
+            params["sort"] = sort
+        if sort == "trending":
+            params["sort"] = "likes"
+            params["direction"] = "descending"
+        elif direction in {"descending", "ascending"}:
+            params["direction"] = direction
+        lim = int(limit) if limit else 10
+        params["limit"] = lim
+
+        data, status, err = _http_get_json("https://huggingface.co/api/spaces", params)
+        if status == 400:
+            minimal = {"search": query} if query else {}
+            if limit:
+                minimal["limit"] = int(limit)
+            data, status, err = _http_get_json("https://huggingface.co/api/spaces", minimal)
+        results: List[Dict] = []
+        if isinstance(data, list):
+            for it in data:
+                sp_id = _safe_get(it, "id") or _safe_get(it, "spaceId") or ""
+                if not sp_id:
+                    continue
+                owner_name = sp_id.split("/")[0] if "/" in sp_id else ""
+                visibility, access = _visibility_from_item(it)
+                norm = _norm_common(sp_id, "space", owner_name, "")
+                # Try to extract Space runtime (sdk, app file) when available in list
+                norm.update({
+                    "tags": _safe_get(it, "tags", []),
+                    "likes": _safe_get(it, "likes", 0),
+                    "downloads": _safe_get(it, "downloads", 0),
+                    "updatedAt": _safe_get(it, "lastModified"),
+                    "visibility": visibility,
+                    "access": access,
+                })
+                results.append(norm)
+        return json.dumps({"results": results, "status": status, "error": err, "params": params}, ensure_ascii=False)
+
+
+class HFSpaceInfoTool(Tool):
+    name = "hf_space_info"
+    description = (
+        "Get detailed Space info by repo id (owner/name). Use to inspect tags, likes, and card details after a Space search."
+    )
+    inputs = {"repo_id": {"type": "string", "description": "Space repo id, e.g. user/space-name"}}
+    output_type = "string"
+
+    def forward(self, repo_id: str) -> str:
+        data, status, err = _http_get_json(f"https://huggingface.co/api/spaces/{repo_id}", {"full": "true"})
+        item: Dict = {}
+        if isinstance(data, dict):
+            sp_id = data.get("id") or data.get("spaceId") or repo_id
+            owner_name = sp_id.split("/")[0] if "/" in sp_id else ""
+            visibility, access = _visibility_from_item(data)
+            desc = ""
+            if isinstance(data.get("cardData"), dict):
+                desc = data["cardData"].get("summary") or data["cardData"].get("description") or ""
+            item = _norm_common(sp_id, "space", owner_name, desc)
+            item.update({
+                "tags": data.get("tags", []),
+                "likes": data.get("likes", 0),
+                "downloads": data.get("downloads", 0),
+                "updatedAt": data.get("lastModified"),
+                "visibility": visibility,
+                "access": access,
+                "cardData": data.get("cardData"),
+                "siblings": data.get("siblings"),
+            })
+        return json.dumps({"item": item, "status": status, "error": err}, ensure_ascii=False)
+
+
+class HFUserInfoTool(Tool):
+    name = "hf_user_info"
+    description = (
+        "Fetch public user/org profile by username. Helpful to scope searches by owner or explore maintainers."
+    )
+    inputs = {"username": {"type": "string", "description": "User or organization name"}}
+    output_type = "string"
+
+    def forward(self, username: str) -> str:
+        data, status, err = _http_get_json(f"https://huggingface.co/api/users/{username}")
+        item = data if isinstance(data, dict) else {}
+        visibility = "public"
+        access = "accessible" if status == 200 else "no_access"
+        return json.dumps({"item": item, "status": status, "error": err, "visibility": visibility, "access": access}, ensure_ascii=False)
+
+
+class HFCollectionsListTool(Tool):
+    name = "hf_collections_list"
+    description = (
+        "List public collections, optionally filtered by owner/namespace. Use to surface curated sets of repos. "
+        "Owner may be an object; URL is normalized to https://huggingface.co/collections/{owner_name}/{slug}."
+    )
+    inputs = {"owner": {"type": "string", "description": "Filter by collection owner/namespace", "nullable": True}}
+    output_type = "string"
+
+    def forward(self, owner: Optional[str] = None) -> str:
+        params = {}
+        if owner:
+            params["owner"] = owner
+        data, status, err = _http_get_json("https://huggingface.co/api/collections", params)
+        results = data if isinstance(data, list) else []
+        # Normalize minimally
+        items: List[Dict] = []
+        for it in results:
+            cid = _safe_get(it, "id") or _safe_get(it, "slug") or ""
+            ns_val = _safe_get(it, "owner") or _safe_get(it, "namespace") or ""
+            if isinstance(ns_val, dict):
+                ns = ns_val.get("name") or ns_val.get("fullname") or ""
+            else:
+                ns = ns_val
+            url = ""
+            if ns and cid:
+                # Some APIs return id as "{namespace}/{slug}", so extract slug part only
+                slug = cid.split("/")[-1]
+                url = f"https://huggingface.co/collections/{ns}/{slug}"
+            items.append({
+                "type": "collection",
+                "id": cid,
+                "owner": ns,
+                "title": _safe_get(it, "title", ""),
+                "url": url,
+                "visibility": "public",
+                "access": "accessible",
+            })
+        return json.dumps({"results": items, "status": status, "error": err}, ensure_ascii=False)
+
+
+class HFCollectionGetTool(Tool):
+    name = "hf_collection_get"
+    description = (
+        "Get collection details by namespace and slug id (as in URL). Use after listing to inspect items."
+    )
+    inputs = {
+        "namespace": {"type": "string", "description": "Collection owner/namespace"},
+        "slug_id": {"type": "string", "description": "slug-id part as shown in URL"},
+    }
+    output_type = "string"
+
+    def forward(self, namespace: str, slug_id: str) -> str:
+        data, status, err = _http_get_json(f"https://huggingface.co/api/collections/{namespace}/{slug_id}")
+        item = data if isinstance(data, dict) else {}
+        return json.dumps({"item": item, "status": status, "error": err}, ensure_ascii=False)
+
+
+class HFPaperInfoTool(Tool):
+    name = "hf_paper_info"
+    description = (
+        "Fetch paper metadata by arXiv id (e.g., 1706.03762). Combine with hf_paper_repos to find related repos."
+    )
+    inputs = {"arxiv_id": {"type": "string", "description": "arXiv identifier, e.g. 1706.03762"}}
+    output_type = "string"
+
+    def forward(self, arxiv_id: str) -> str:
+        data, status, err = _http_get_json(f"https://huggingface.co/api/papers/{arxiv_id}")
+        item = data if isinstance(data, dict) else {}
+        return json.dumps({"item": item, "status": status, "error": err}, ensure_ascii=False)
+
+
+class HFPaperReposTool(Tool):
+    name = "hf_paper_repos"
+    description = (
+        "List repos (models/datasets/spaces) referencing an arXiv id. Use alongside hf_paper_info to map research → repos."
+    )
+    inputs = {"arxiv_id": {"type": "string", "description": "arXiv identifier, e.g. 1706.03762"}}
+    output_type = "string"
+
+    def forward(self, arxiv_id: str) -> str:
+        data, status, err = _http_get_json(f"https://huggingface.co/api/arxiv/{arxiv_id}/repos")
+        results = data if isinstance(data, list) else []
+        return json.dumps({"results": results, "status": status, "error": err}, ensure_ascii=False)
+
+
+class HFDailyPapersTool(Tool):
+    name = "hf_daily_papers"
+    description = (
+        "Get the daily curated papers list from Hugging Face. Useful for current research trends."
+    )
+    inputs = {}
+    output_type = "string"
+
+    def forward(self) -> str:  # type: ignore[override]
+        data, status, err = _http_get_json("https://huggingface.co/api/daily_papers")
+        results = data if isinstance(data, list) else []
+        return json.dumps({"results": results, "status": status, "error": err}, ensure_ascii=False)
+
+
+class HFRepoInfoTool(Tool):
+    name = "hf_repo_info"
+    description = (
+        "Generic repo info for model|dataset|space by id. Use if you already know the type and want raw item metadata."
+    )
+    inputs = {
+        "repo_type": {"type": "string", "description": "model|dataset|space"},
+        "repo_id": {"type": "string", "description": "Owner/name or id"},
+    }
+    output_type = "string"
+
+    def forward(self, repo_type: str, repo_id: str) -> str:
+        repo_type = (repo_type or "").strip().lower()
+        if repo_type not in {"model", "dataset", "space"}:
+            return json.dumps({"error": "invalid_repo_type", "status": 400})
+        base = {"model": "models", "dataset": "datasets", "space": "spaces"}[repo_type]
+        data, status, err = _http_get_json(f"https://huggingface.co/api/{base}/{repo_id}", {"full": "true"})
+        item = data if isinstance(data, dict) else {}
+        return json.dumps({"item": item, "status": status, "error": err}, ensure_ascii=False)
+
+
+class HFSiteSearchTool(Tool):
+    name = "hf_site_search"
+    description = (
+        "Search within huggingface.co for blogs, Learn pages, and posts (DuckDuckGo). Prefer this for tutorials and docs not covered by Hub APIs. "
+        "Defaults: limit=10 to reduce rate limiting. Returns JSON with `results`, `status`, and `error`."
+    )
+    inputs = {
+        "query": {"type": "string", "description": "Search query. 'site:huggingface.co' will be added if missing."},
+        "limit": {"type": "number", "description": "Max results (default 20)", "nullable": True},
+    }
+    output_type = "string"
+
+    def forward(self, query: str, limit: Optional[int] = None) -> str:
+        try:
+            from duckduckgo_search import DDGS
+        except Exception:
+            return json.dumps({"results": [], "status": 500, "error": "duckduckgo_search_not_installed"})
+
+        q = f"site:huggingface.co {query}" if "huggingface.co" not in query else query
+        lim = int(limit) if limit else 10
+        results: List[Dict] = []
+        try:
+            with DDGS() as ddgs:
+                for r in ddgs.text(q, safesearch="moderate", timelimit=None, max_results=lim):
+                    if not isinstance(r, dict):
+                        continue
+                    results.append({
+                        "type": "site",
+                        "title": r.get("title"),
+                        "url": r.get("href"),
+                        "snippet": r.get("body"),
+                        "date": r.get("date"),
+                    })
+        except Exception as e:
+            return json.dumps({"results": [], "status": 500, "error": str(e)})
+        return json.dumps({"results": results, "status": 200, "error": ""}, ensure_ascii=False)
+
+
+class HFReportGenerateTool(Tool):
+    name = "hf_report_generate"
+    description = (
+        "Generate a full HTML report from aggregated JSON (string). The app prefers its own dashboard, but this can render custom summaries."
+    )
+    inputs = {
+        "data_json": {"type": "string", "description": "Aggregated search results JSON"},
+        "title": {"type": "string", "description": "Report title", "nullable": True},
+    }
+    output_type = "string"
+
+    def forward(self, data_json: str, title: Optional[str] = None) -> str:
+        try:
+            data = json.loads(data_json) if data_json else {}
+        except Exception as e:
+            data = {"parse_error": str(e)}
+        title = title or "Hugging Face Research Report"
+
+        def card_html(item: Dict) -> str:
+            badge = ""
+            vis = item.get("visibility")
+            access = item.get("access")
+            if vis in {"private", "gated"} or access == "no_access":
+                badge = f"<span class=badge badge-warn>{vis or 'restricted'}</span>"
+            meta = []
+            if item.get("task"):
+                meta.append(f"<span class=meta>Task: {item['task']}</span>")
+            if item.get("tags"):
+                meta.append(f"<span class=meta>Tags: {', '.join(item['tags'][:5])}</span>")
+            if item.get("downloads") is not None:
+                meta.append(f"<span class=stat>⬇️ {item['downloads']}</span>")
+            if item.get("likes") is not None:
+                meta.append(f"<span class=stat>❤️ {item['likes']}</span>")
+            if item.get("updatedAt"):
+                meta.append(f"<span class=meta>Updated: {item['updatedAt']}</span>")
+            desc = (item.get("description") or "").strip()
+            if len(desc) > 220:
+                desc = desc[:217] + "..."
+            return (
+                "<div class=card>"
+                f"<div class=card-title><a href='{item.get('url')}' target=_blank rel=noopener>{item.get('id')}</a> {badge}</div>"
+                f"<div class=card-subtitle>{item.get('type','')} • {item.get('owner','')}</div>"
+                f"<div class=card-desc>{desc}</div>"
+                f"<div class=card-meta>{' | '.join(meta)}</div>"
+                "</div>"
+            )
+
+        def section(title_text: str, items: List[Dict]) -> str:
+            if not items:
+                return ""
+            cards = "\n".join(card_html(it) for it in items)
+            return f"<section><h2>{title_text}</h2><div class=cards>{cards}</div></section>"
+
+        # Accept either a dict with category keys or a flat list
+        models = data.get("models") or data.get("Models") or []
+        datasets = data.get("datasets") or data.get("Datasets") or []
+        spaces = data.get("spaces") or data.get("Spaces") or []
+        papers = data.get("papers") or data.get("Papers") or []
+        daily_papers = data.get("daily_papers") or data.get("DailyPapers") or []
+        users = data.get("users") or data.get("Users") or []
+        collections = data.get("collections") or data.get("Collections") or []
+        site = data.get("site") or data.get("Site") or []
+
+        html = f"""<!DOCTYPE html>
+<html lang=\"en\">
+<head>
+  <meta charset=\"utf-8\" />
+  <meta name=\"viewport\" content=\"width=device-width, initial-scale=1\" />
+  <title>{title}</title>
+  <style>
+    :root {{ --bg:#0b0d12; --fg:#e6e9ef; --muted:#9aa4b2; --card:#121621; --accent:#5ac8fa; --warn:#eab308; }}
+    body {{ background:var(--bg); color:var(--fg); font-family: ui-sans-serif, system-ui, -apple-system, Segoe UI, Roboto, Inter, Arial, sans-serif; margin:0; padding:24px; }}
+    h1 {{ font-size: 24px; margin: 0 0 12px; }}
+    h2 {{ font-size: 18px; margin: 24px 0 8px; color: var(--accent); }}
+    .container {{ max-width: 1120px; margin: 0 auto; }}
+    .subtitle {{ color: var(--muted); margin-bottom: 18px; }}
+    .cards {{ display: grid; grid-template-columns: repeat(auto-fill, minmax(280px,1fr)); gap: 12px; }}
+    .card {{ background: var(--card); border: 1px solid rgba(255,255,255,0.06); border-radius: 10px; padding: 12px; }}
+    .card-title {{ font-weight: 600; margin-bottom: 4px; overflow-wrap:anywhere; }}
+    .card-subtitle {{ color: var(--muted); font-size: 12px; margin-bottom: 8px; }}
+    .card-desc {{ font-size: 13px; line-height: 1.45; min-height: 28px; margin-bottom: 8px; color: #d2d7df; }}
+    .card-meta {{ font-size: 12px; color: var(--muted); display:flex; flex-wrap:wrap; gap:8px; }}
+    .badge {{ background: rgba(234, 179, 8, 0.15); color: #facc15; border:1px solid rgba(250,204,21,0.35); border-radius: 999px; padding: 2px 8px; font-size: 11px; margin-left: 6px; }}
+    .badge-warn {{ background: rgba(234, 179, 8, 0.15); }}
+    a {{ color: #93c5fd; text-decoration: none; }}
+    a:hover {{ text-decoration: underline; }}
+    section {{ margin-bottom: 18px; }}
+  </style>
+  <script>
+    function printToPDF() {{ window.print(); }}
+  </script>
+  <link rel=\"stylesheet\" href=\"https://cdnjs.cloudflare.com/ajax/libs/modern-normalize/2.0.0/modern-normalize.min.css\" />
+  <meta name=\"robots\" content=\"noindex\" />
+  <meta name=\"referrer\" content=\"no-referrer\" />
+  <meta http-equiv=\"Content-Security-Policy\" content=\"default-src 'self' 'unsafe-inline' data: https://cdnjs.cloudflare.com; img-src * data:; style-src 'self' 'unsafe-inline' https://cdnjs.cloudflare.com;\" />
+</head>
+<body>
+  <div class=\"container\">
+    <div style=\"display:flex; align-items:center; justify-content:space-between; gap:12px;\">
+      <div>
+        <h1>{title}</h1>
+        <div class=\"subtitle\">Generated by Hugging Search</div>
+      </div>
+      <button onclick=\"printToPDF()\" style=\"background:#1f2937;color:#e5e7eb;border:1px solid rgba(255,255,255,0.08);border-radius:8px;padding:8px 10px;cursor:pointer;\">Print to PDF</button>
+    </div>
+    {section("Models", models)}
+    {section("Datasets", datasets)}
+    {section("Spaces", spaces)}
+    {section("Papers", papers)}
+    {section("Daily Papers", daily_papers)}
+    {section("Users", users)}
+    {section("Collections", collections)}
+    {section("Site results", site)}
+  </div>
+</body>
+</html>
+"""
+        return html
+
+
+class HFDashboardReportTool(Tool):
+    name = "hf_generate_dashboard_report"
+    description = (
+        "One-click dashboard report from a query. Fetches public models/datasets/spaces/daily_papers and returns a full HTML dashboard."
+    )
+    inputs = {
+        "query": {"type": "string", "description": "User intent / keywords to search across Hub"},
+        "limit": {"type": "number", "description": "Max results per category (default 20)", "nullable": True},
+    }
+    output_type = "string"
+
+    def forward(self, query: str, limit: Optional[int] = None) -> str:
+        lim = int(limit) if limit else 20
+        params_common = {"search": query, "sort": "downloads", "direction": "descending", "limit": lim}
+        # Fetch categories
+        m_data, m_status, _ = _http_get_json("https://huggingface.co/api/models", params_common)
+        d_data, d_status, _ = _http_get_json("https://huggingface.co/api/datasets", params_common)
+        s_data, s_status, _ = _http_get_json("https://huggingface.co/api/spaces", {"search": query, "sort": "likes", "direction": "descending", "limit": lim})
+        dp_data, dp_status, _ = _http_get_json("https://huggingface.co/api/daily_papers")
+
+        models: List[Dict] = []
+        if isinstance(m_data, list):
+            for it in m_data[:lim]:
+                model_id = _safe_get(it, "id") or _safe_get(it, "modelId") or ""
+                if not model_id:
+                    continue
+                owner_name = model_id.split("/")[0] if "/" in model_id else ""
+                visibility, access = _visibility_from_item(it)
+                norm = _norm_common(model_id, "model", owner_name, "")
+                norm.update({
+                    "tags": _safe_get(it, "tags", []),
+                    "task": _safe_get(it, "pipeline_tag"),
+                    "likes": _safe_get(it, "likes", 0),
+                    "downloads": _safe_get(it, "downloads", 0),
+                    "updatedAt": _safe_get(it, "lastModified"),
+                    "visibility": visibility,
+                    "access": access,
+                })
+                models.append(norm)
+
+        datasets: List[Dict] = []
+        if isinstance(d_data, list):
+            for it in d_data[:lim]:
+                ds_id = _safe_get(it, "id") or _safe_get(it, "datasetId") or ""
+                if not ds_id:
+                    continue
+                owner_name = ds_id.split("/")[0] if "/" in ds_id else ""
+                visibility, access = _visibility_from_item(it)
+                norm = _norm_common(ds_id, "dataset", owner_name, "")
+                norm.update({
+                    "tags": _safe_get(it, "tags", []),
+                    "likes": _safe_get(it, "likes", 0),
+                    "downloads": _safe_get(it, "downloads", 0),
+                    "updatedAt": _safe_get(it, "lastModified"),
+                    "visibility": visibility,
+                    "access": access,
+                })
+                datasets.append(norm)
+
+        spaces: List[Dict] = []
+        if isinstance(s_data, list):
+            for it in s_data[:lim]:
+                sp_id = _safe_get(it, "id") or _safe_get(it, "spaceId") or ""
+                if not sp_id:
+                    continue
+                owner_name = sp_id.split("/")[0] if "/" in sp_id else ""
+                visibility, access = _visibility_from_item(it)
+                norm = _norm_common(sp_id, "space", owner_name, "")
+                norm.update({
+                    "tags": _safe_get(it, "tags", []),
+                    "likes": _safe_get(it, "likes", 0),
+                    "downloads": _safe_get(it, "downloads", 0),
+                    "updatedAt": _safe_get(it, "lastModified"),
+                    "visibility": visibility,
+                    "access": access,
+                })
+                spaces.append(norm)
+
+        papers = dp_data if isinstance(dp_data, list) else []
+
+        # Build dashboard HTML
+        def card_html(item: Dict) -> str:
+            badge = ""
+            if item.get("visibility") in {"private", "gated"} or item.get("access") == "no_access":
+                badge = f"<span class=badge badge-warn>{item.get('visibility','restricted')}</span>"
+            meta = []
+            if item.get("task"):
+                meta.append(f"<span class=meta>Task: {item['task']}</span>")
+            if item.get("tags"):
+                meta.append(f"<span class=meta>Tags: {', '.join(item['tags'][:5])}</span>")
+            if item.get("downloads") is not None:
+                meta.append(f"<span class=stat>⬇️ {item['downloads']}</span>")
+            if item.get("likes") is not None:
+                meta.append(f"<span class=stat>❤️ {item['likes']}</span>")
+            if item.get("updatedAt"):
+                meta.append(f"<span class=meta>Updated: {item['updatedAt']}</span>")
+            desc = (item.get("description") or "").strip()
+            if len(desc) > 200:
+                desc = desc[:197] + "..."
+            return (
+                "<div class=card>"
+                f"<div class=card-title><a href='{item.get('url')}' target=_blank rel=noopener>{item.get('id')}</a> {badge}</div>"
+                f"<div class=card-subtitle>{item.get('type','')} • {item.get('owner','')}</div>"
+                f"<div class=card-desc>{desc}</div>"
+                f"<div class=card-meta>{' | '.join(meta)}</div>"
+                "</div>"
+            )
+
+        def section(title_text: str, items: List[Dict]) -> str:
+            if not items:
+                return ""
+            cards = "\n".join(card_html(it) for it in items)
+            return f"<section><h2>{title_text}</h2><div class=cards>{cards}</div></section>"
+
+        html = f"""<!DOCTYPE html>
+<html lang=\"en\">
+<head>
+  <meta charset=\"utf-8\" />
+  <meta name=\"viewport\" content=\"width=device-width, initial-scale=1\" />
+  <title>Hugging Search — Dashboard</title>
+  <style>
+    :root {{ --bg:#0b0d12; --fg:#e6e9ef; --muted:#9aa4b2; --card:#121621; --accent:#5ac8fa; --warn:#eab308; }}
+    body {{ background:var(--bg); color:var(--fg); font-family: ui-sans-serif, system-ui, -apple-system, Segoe UI, Roboto, Inter, Arial, sans-serif; margin:0; padding:24px; }}
+    .container {{ max-width: 1200px; margin: 0 auto; }}
+    .header {{ display:flex; justify-content:space-between; align-items:center; gap:12px; margin-bottom: 16px; }}
+    .title {{ font-size: 22px; margin: 0; }}
+    .subtitle {{ color: var(--muted); }}
+    .stats {{ display:flex; gap:10px; flex-wrap:wrap; margin: 8px 0 18px; }}
+    .stat-chip {{ background: var(--card); border: 1px solid rgba(255,255,255,0.08); border-radius: 999px; padding: 6px 10px; font-size: 12px; color: var(--muted); }}
+    .cards {{ display: grid; grid-template-columns: repeat(auto-fill, minmax(280px,1fr)); gap: 12px; }}
+    .card {{ background: var(--card); border: 1px solid rgba(255,255,255,0.06); border-radius: 10px; padding: 12px; }}
+    .card-title {{ font-weight: 600; margin-bottom: 4px; overflow-wrap:anywhere; }}
+    .card-subtitle {{ color: var(--muted); font-size: 12px; margin-bottom: 8px; }}
+    .card-desc {{ font-size: 13px; line-height: 1.45; min-height: 28px; margin-bottom: 8px; color: #d2d7df; }}
+    .card-meta {{ font-size: 12px; color: var(--muted); display:flex; flex-wrap:wrap; gap:8px; }}
+    .badge {{ background: rgba(234, 179, 8, 0.15); color: #facc15; border:1px solid rgba(250,204,21,0.35); border-radius: 999px; padding: 2px 8px; font-size: 11px; margin-left: 6px; }}
+    h2 {{ font-size: 16px; margin: 18px 0 8px; color: var(--accent); }}
+    .actions {{ display:flex; gap:8px; align-items:center; }}
+    button {{ background:#1f2937;color:#e5e7eb;border:1px solid rgba(255,255,255,0.08);border-radius:8px;padding:8px 10px;cursor:pointer; }}
+  </style>
+  <script>
+    function printToPDF() {{ window.print(); }}
+  </script>
+</head>
+<body>
+  <div class=\"container\">
+    <div class=\"header\">
+      <div>
+        <div class=\"title\">Hugging Search — Dashboard</div>
+        <div class=\"subtitle\">Query: {query}</div>
+      </div>
+      <div class=\"actions\"><button onclick=\"printToPDF()\">Print to PDF</button></div>
+    </div>
+    <div class=\"stats\">
+      <div class=\"stat-chip\">Models: {len(models)}</div>
+      <div class=\"stat-chip\">Datasets: {len(datasets)}</div>
+      <div class=\"stat-chip\">Spaces: {len(spaces)}</div>
+      <div class=\"stat-chip\">Daily papers: {len(papers) if isinstance(papers,list) else 0}</div>
+    </div>
+    {section("Models", models)}
+    {section("Datasets", datasets)}
+    {section("Spaces", spaces)}
+  </div>
+</body>
+</html>
+"""
+        return html
+
+

scripts/mdconvert.py

@@ -0,0 +1,982 @@
+# This is copied from Magentic-one's great repo: https://github.com/microsoft/autogen/blob/v0.4.4/python/packages/autogen-magentic-one/src/autogen_magentic_one/markdown_browser/mdconvert.py
+# Thanks to Microsoft researchers for open-sourcing this!
+# type: ignore
+import base64
+import copy
+import html
+import json
+import mimetypes
+import os
+import re
+import shutil
+import subprocess
+import sys
+import tempfile
+import traceback
+from typing import Any, Dict, List, Optional, Union
+from urllib.parse import parse_qs, quote, unquote, urlparse, urlunparse
+
+import mammoth
+import markdownify
+import pandas as pd
+import pdfminer
+import pdfminer.high_level
+import pptx
+
+# File-format detection
+import puremagic
+import pydub
+import requests
+import speech_recognition as sr
+from bs4 import BeautifulSoup
+from youtube_transcript_api import YouTubeTranscriptApi
+from youtube_transcript_api.formatters import SRTFormatter
+
+
+class _CustomMarkdownify(markdownify.MarkdownConverter):
+    """
+    A custom version of markdownify's MarkdownConverter. Changes include:
+
+    - Altering the default heading style to use '#', '##', etc.
+    - Removing javascript hyperlinks.
+    - Truncating images with large data:uri sources.
+    - Ensuring URIs are properly escaped, and do not conflict with Markdown syntax
+    """
+
+    def __init__(self, **options: Any):
+        options["heading_style"] = options.get("heading_style", markdownify.ATX)
+        # Explicitly cast options to the expected type if necessary
+        super().__init__(**options)
+
+    def convert_hn(self, n: int, el: Any, text: str, convert_as_inline: bool) -> str:
+        """Same as usual, but be sure to start with a new line"""
+        if not convert_as_inline:
+            if not re.search(r"^\n", text):
+                return "\n" + super().convert_hn(n, el, text, convert_as_inline)  # type: ignore
+
+        return super().convert_hn(n, el, text, convert_as_inline)  # type: ignore
+
+    def convert_a(self, el: Any, text: str, convert_as_inline: bool):
+        """Same as usual converter, but removes Javascript links and escapes URIs."""
+        prefix, suffix, text = markdownify.chomp(text)  # type: ignore
+        if not text:
+            return ""
+        href = el.get("href")
+        title = el.get("title")
+
+        # Escape URIs and skip non-http or file schemes
+        if href:
+            try:
+                parsed_url = urlparse(href)  # type: ignore
+                if parsed_url.scheme and parsed_url.scheme.lower() not in ["http", "https", "file"]:  # type: ignore
+                    return "%s%s%s" % (prefix, text, suffix)
+                href = urlunparse(parsed_url._replace(path=quote(unquote(parsed_url.path))))  # type: ignore
+            except ValueError:  # It's not clear if this ever gets thrown
+                return "%s%s%s" % (prefix, text, suffix)
+
+        # For the replacement see #29: text nodes underscores are escaped
+        if (
+            self.options["autolinks"]
+            and text.replace(r"\_", "_") == href
+            and not title
+            and not self.options["default_title"]
+        ):
+            # Shortcut syntax
+            return "<%s>" % href
+        if self.options["default_title"] and not title:
+            title = href
+        title_part = ' "%s"' % title.replace('"', r"\"") if title else ""
+        return "%s[%s](%s%s)%s" % (prefix, text, href, title_part, suffix) if href else text
+
+    def convert_img(self, el: Any, text: str, convert_as_inline: bool) -> str:
+        """Same as usual converter, but removes data URIs"""
+
+        alt = el.attrs.get("alt", None) or ""
+        src = el.attrs.get("src", None) or ""
+        title = el.attrs.get("title", None) or ""
+        title_part = ' "%s"' % title.replace('"', r"\"") if title else ""
+        if convert_as_inline and el.parent.name not in self.options["keep_inline_images_in"]:
+            return alt
+
+        # Remove dataURIs
+        if src.startswith("data:"):
+            src = src.split(",")[0] + "..."
+
+        return "![%s](%s%s)" % (alt, src, title_part)
+
+    def convert_soup(self, soup: Any) -> str:
+        return super().convert_soup(soup)  # type: ignore
+
+
+class DocumentConverterResult:
+    """The result of converting a document to text."""
+
+    def __init__(self, title: Union[str, None] = None, text_content: str = ""):
+        self.title: Union[str, None] = title
+        self.text_content: str = text_content
+
+
+class DocumentConverter:
+    """Abstract superclass of all DocumentConverters."""
+
+    def convert(self, local_path: str, **kwargs: Any) -> Union[None, DocumentConverterResult]:
+        raise NotImplementedError()
+
+
+class PlainTextConverter(DocumentConverter):
+    """Anything with content type text/plain"""
+
+    def convert(self, local_path: str, **kwargs: Any) -> Union[None, DocumentConverterResult]:
+        # Guess the content type from any file extension that might be around
+        content_type, _ = mimetypes.guess_type("__placeholder" + kwargs.get("file_extension", ""))
+
+        # Only accept text files
+        if content_type is None:
+            return None
+        # elif "text/" not in content_type.lower():
+        #     return None
+
+        text_content = ""
+        with open(local_path, "rt", encoding="utf-8") as fh:
+            text_content = fh.read()
+        return DocumentConverterResult(
+            title=None,
+            text_content=text_content,
+        )
+
+
+class JavaScriptConverter(DocumentConverter):
+    """Handle JavaScript files (.js, .mjs) as plain text"""
+
+    def convert(self, local_path: str, **kwargs: Any) -> Union[None, DocumentConverterResult]:
+        # Check if it's a JavaScript file
+        extension = kwargs.get("file_extension", "")
+        if extension.lower() not in [".js", ".mjs"]:
+            return None
+
+        try:
+            # Read the file as text
+            with open(local_path, "rt", encoding="utf-8") as fh:
+                text_content = fh.read()
+            
+            return DocumentConverterResult(
+                title=f"JavaScript file: {os.path.basename(local_path)}",
+                text_content=text_content,
+            )
+        except Exception as e:
+            # If UTF-8 fails, try with different encoding
+            try:
+                with open(local_path, "rt", encoding="latin-1") as fh:
+                    text_content = fh.read()
+                
+                return DocumentConverterResult(
+                    title=f"JavaScript file: {os.path.basename(local_path)}",
+                    text_content=text_content,
+                )
+            except Exception:
+                return None
+
+
+class HtmlConverter(DocumentConverter):
+    """Anything with content type text/html"""
+
+    def convert(self, local_path: str, **kwargs: Any) -> Union[None, DocumentConverterResult]:
+        # Bail if not html
+        extension = kwargs.get("file_extension", "")
+        if extension.lower() not in [".html", ".htm"]:
+            return None
+
+        result = None
+        with open(local_path, "rt", encoding="utf-8") as fh:
+            result = self._convert(fh.read())
+
+        return result
+
+    def _convert(self, html_content: str) -> Union[None, DocumentConverterResult]:
+        """Helper function that converts and HTML string."""
+
+        # Parse the string
+        soup = BeautifulSoup(html_content, "html.parser")
+
+        # Remove javascript and style blocks
+        for script in soup(["script", "style"]):
+            script.extract()
+
+        # Print only the main content
+        body_elm = soup.find("body")
+        webpage_text = ""
+        if body_elm:
+            webpage_text = _CustomMarkdownify().convert_soup(body_elm)
+        else:
+            webpage_text = _CustomMarkdownify().convert_soup(soup)
+
+        assert isinstance(webpage_text, str)
+
+        return DocumentConverterResult(
+            title=None if soup.title is None else soup.title.string, text_content=webpage_text
+        )
+
+
+class WikipediaConverter(DocumentConverter):
+    """Handle Wikipedia pages separately, focusing only on the main document content."""
+
+    def convert(self, local_path: str, **kwargs: Any) -> Union[None, DocumentConverterResult]:
+        # Bail if not Wikipedia
+        extension = kwargs.get("file_extension", "")
+        if extension.lower() not in [".html", ".htm"]:
+            return None
+        url = kwargs.get("url", "")
+        if not re.search(r"^https?:\/\/[a-zA-Z]{2,3}\.wikipedia.org\/", url):
+            return None
+
+        # Parse the file
+        soup = None
+        with open(local_path, "rt", encoding="utf-8") as fh:
+            soup = BeautifulSoup(fh.read(), "html.parser")
+
+        # Remove javascript and style blocks
+        for script in soup(["script", "style"]):
+            script.extract()
+
+        # Print only the main content
+        body_elm = soup.find("div", {"id": "mw-content-text"})
+        title_elm = soup.find("span", {"class": "mw-page-title-main"})
+
+        webpage_text = ""
+        main_title = None if soup.title is None else soup.title.string
+
+        if body_elm:
+            # What's the title
+            if title_elm and len(title_elm) > 0:
+                main_title = title_elm.string  # type: ignore
+                assert isinstance(main_title, str)
+
+            # Convert the page
+            webpage_text = f"# {main_title}\n\n" + _CustomMarkdownify().convert_soup(body_elm)
+        else:
+            webpage_text = _CustomMarkdownify().convert_soup(soup)
+
+        return DocumentConverterResult(
+            title=main_title,
+            text_content=webpage_text,
+        )
+
+
+class YouTubeConverter(DocumentConverter):
+    """Handle YouTube specially, focusing on the video title, description, and transcript."""
+
+    def convert(self, local_path: str, **kwargs: Any) -> Union[None, DocumentConverterResult]:
+        # Bail if not YouTube
+        extension = kwargs.get("file_extension", "")
+        if extension.lower() not in [".html", ".htm"]:
+            return None
+        url = kwargs.get("url", "")
+        if not url.startswith("https://www.youtube.com/watch?"):
+            return None
+
+        # Parse the file
+        soup = None
+        with open(local_path, "rt", encoding="utf-8") as fh:
+            soup = BeautifulSoup(fh.read(), "html.parser")
+
+        # Read the meta tags
+        assert soup.title is not None and soup.title.string is not None
+        metadata: Dict[str, str] = {"title": soup.title.string}
+        for meta in soup(["meta"]):
+            for a in meta.attrs:
+                if a in ["itemprop", "property", "name"]:
+                    metadata[meta[a]] = meta.get("content", "")
+                    break
+
+        # We can also try to read the full description. This is more prone to breaking, since it reaches into the page implementation
+        try:
+            for script in soup(["script"]):
+                content = script.text
+                if "ytInitialData" in content:
+                    lines = re.split(r"\r?\n", content)
+                    obj_start = lines[0].find("{")
+                    obj_end = lines[0].rfind("}")
+                    if obj_start >= 0 and obj_end >= 0:
+                        data = json.loads(lines[0][obj_start : obj_end + 1])
+                        attrdesc = self._findKey(data, "attributedDescriptionBodyText")  # type: ignore
+                        if attrdesc:
+                            metadata["description"] = str(attrdesc["content"])
+                    break
+        except Exception:
+            pass
+
+        # Start preparing the page
+        webpage_text = "# YouTube\n"
+
+        title = self._get(metadata, ["title", "og:title", "name"])  # type: ignore
+        assert isinstance(title, str)
+
+        if title:
+            webpage_text += f"\n## {title}\n"
+
+        stats = ""
+        views = self._get(metadata, ["interactionCount"])  # type: ignore
+        if views:
+            stats += f"- **Views:** {views}\n"
+
+        keywords = self._get(metadata, ["keywords"])  # type: ignore
+        if keywords:
+            stats += f"- **Keywords:** {keywords}\n"
+
+        runtime = self._get(metadata, ["duration"])  # type: ignore
+        if runtime:
+            stats += f"- **Runtime:** {runtime}\n"
+
+        if len(stats) > 0:
+            webpage_text += f"\n### Video Metadata\n{stats}\n"
+
+        description = self._get(metadata, ["description", "og:description"])  # type: ignore
+        if description:
+            webpage_text += f"\n### Description\n{description}\n"
+
+        transcript_text = ""
+        parsed_url = urlparse(url)  # type: ignore
+        params = parse_qs(parsed_url.query)  # type: ignore
+        if "v" in params:
+            assert isinstance(params["v"][0], str)
+            video_id = str(params["v"][0])
+            try:
+                # Must be a single transcript.
+                transcript = YouTubeTranscriptApi.get_transcript(video_id)  # type: ignore
+                # transcript_text = " ".join([part["text"] for part in transcript])  # type: ignore
+                # Alternative formatting:
+                transcript_text = SRTFormatter().format_transcript(transcript)
+            except Exception:
+                pass
+        if transcript_text:
+            webpage_text += f"\n### Transcript\n{transcript_text}\n"
+
+        title = title if title else soup.title.string
+        assert isinstance(title, str)
+
+        return DocumentConverterResult(
+            title=title,
+            text_content=webpage_text,
+        )
+
+    def _get(self, metadata: Dict[str, str], keys: List[str], default: Union[str, None] = None) -> Union[str, None]:
+        for k in keys:
+            if k in metadata:
+                return metadata[k]
+        return default
+
+    def _findKey(self, json: Any, key: str) -> Union[str, None]:  # TODO: Fix json type
+        if isinstance(json, list):
+            for elm in json:
+                ret = self._findKey(elm, key)
+                if ret is not None:
+                    return ret
+        elif isinstance(json, dict):
+            for k in json:
+                if k == key:
+                    return json[k]
+                else:
+                    ret = self._findKey(json[k], key)
+                    if ret is not None:
+                        return ret
+        return None
+
+
+class PdfConverter(DocumentConverter):
+    """
+    Converts PDFs to Markdown. Most style information is ignored, so the results are essentially plain-text.
+    """
+
+    def convert(self, local_path, **kwargs) -> Union[None, DocumentConverterResult]:
+        # Bail if not a PDF
+        extension = kwargs.get("file_extension", "")
+        if extension.lower() != ".pdf":
+            return None
+
+        return DocumentConverterResult(
+            title=None,
+            text_content=pdfminer.high_level.extract_text(local_path),
+        )
+
+
+class DocxConverter(HtmlConverter):
+    """
+    Converts DOCX files to Markdown. Style information (e.g.m headings) and tables are preserved where possible.
+    """
+
+    def convert(self, local_path, **kwargs) -> Union[None, DocumentConverterResult]:
+        # Bail if not a DOCX
+        extension = kwargs.get("file_extension", "")
+        if extension.lower() != ".docx":
+            return None
+
+        result = None
+        with open(local_path, "rb") as docx_file:
+            result = mammoth.convert_to_html(docx_file)
+            html_content = result.value
+            result = self._convert(html_content)
+
+        return result
+
+
+class XlsxConverter(HtmlConverter):
+    """
+    Converts XLSX files to Markdown, with each sheet presented as a separate Markdown table.
+    """
+
+    def convert(self, local_path, **kwargs) -> Union[None, DocumentConverterResult]:
+        # Bail if not a XLSX
+        extension = kwargs.get("file_extension", "")
+        if extension.lower() not in [".xlsx", ".xls"]:
+            return None
+
+        sheets = pd.read_excel(local_path, sheet_name=None)
+        md_content = ""
+        for s in sheets:
+            md_content += f"## {s}\n"
+            html_content = sheets[s].to_html(index=False)
+            md_content += self._convert(html_content).text_content.strip() + "\n\n"
+
+        return DocumentConverterResult(
+            title=None,
+            text_content=md_content.strip(),
+        )
+
+
+class PptxConverter(HtmlConverter):
+    """
+    Converts PPTX files to Markdown. Supports heading, tables and images with alt text.
+    """
+
+    def convert(self, local_path, **kwargs) -> Union[None, DocumentConverterResult]:
+        # Bail if not a PPTX
+        extension = kwargs.get("file_extension", "")
+        if extension.lower() != ".pptx":
+            return None
+
+        md_content = ""
+
+        presentation = pptx.Presentation(local_path)
+        slide_num = 0
+        for slide in presentation.slides:
+            slide_num += 1
+
+            md_content += f"\n\n<!-- Slide number: {slide_num} -->\n"
+
+            title = slide.shapes.title
+            for shape in slide.shapes:
+                # Pictures
+                if self._is_picture(shape):
+                    # https://github.com/scanny/python-pptx/pull/512#issuecomment-1713100069
+                    alt_text = ""
+                    try:
+                        alt_text = shape._element._nvXxPr.cNvPr.attrib.get("descr", "")
+                    except Exception:
+                        pass
+
+                    # A placeholder name
+                    filename = re.sub(r"\W", "", shape.name) + ".jpg"
+                    md_content += "\n![" + (alt_text if alt_text else shape.name) + "](" + filename + ")\n"
+
+                # Tables
+                if self._is_table(shape):
+                    html_table = "<html><body><table>"
+                    first_row = True
+                    for row in shape.table.rows:
+                        html_table += "<tr>"
+                        for cell in row.cells:
+                            if first_row:
+                                html_table += "<th>" + html.escape(cell.text) + "</th>"
+                            else:
+                                html_table += "<td>" + html.escape(cell.text) + "</td>"
+                        html_table += "</tr>"
+                        first_row = False
+                    html_table += "</table></body></html>"
+                    md_content += "\n" + self._convert(html_table).text_content.strip() + "\n"
+
+                # Text areas
+                elif shape.has_text_frame:
+                    if shape == title:
+                        md_content += "# " + shape.text.lstrip() + "\n"
+                    else:
+                        md_content += shape.text + "\n"
+
+            md_content = md_content.strip()
+
+            if slide.has_notes_slide:
+                md_content += "\n\n### Notes:\n"
+                notes_frame = slide.notes_slide.notes_text_frame
+                if notes_frame is not None:
+                    md_content += notes_frame.text
+                md_content = md_content.strip()
+
+        return DocumentConverterResult(
+            title=None,
+            text_content=md_content.strip(),
+        )
+
+    def _is_picture(self, shape):
+        if shape.shape_type == pptx.enum.shapes.MSO_SHAPE_TYPE.PICTURE:
+            return True
+        if shape.shape_type == pptx.enum.shapes.MSO_SHAPE_TYPE.PLACEHOLDER:
+            if hasattr(shape, "image"):
+                return True
+        return False
+
+    def _is_table(self, shape):
+        if shape.shape_type == pptx.enum.shapes.MSO_SHAPE_TYPE.TABLE:
+            return True
+        return False
+
+
+class MediaConverter(DocumentConverter):
+    """
+    Abstract class for multi-modal media (e.g., images and audio)
+    """
+
+    def _get_metadata(self, local_path):
+        exiftool = shutil.which("exiftool")
+        if not exiftool:
+            return None
+        else:
+            try:
+                result = subprocess.run([exiftool, "-json", local_path], capture_output=True, text=True).stdout
+                return json.loads(result)[0]
+            except Exception:
+                return None
+
+
+class WavConverter(MediaConverter):
+    """
+    Converts WAV files to markdown via extraction of metadata (if `exiftool` is installed), and speech transcription (if `speech_recognition` is installed).
+    """
+
+    def convert(self, local_path, **kwargs) -> Union[None, DocumentConverterResult]:
+        # Bail if not a XLSX
+        extension = kwargs.get("file_extension", "")
+        if extension.lower() != ".wav":
+            return None
+
+        md_content = ""
+
+        # Add metadata
+        metadata = self._get_metadata(local_path)
+        if metadata:
+            for f in [
+                "Title",
+                "Artist",
+                "Author",
+                "Band",
+                "Album",
+                "Genre",
+                "Track",
+                "DateTimeOriginal",
+                "CreateDate",
+                "Duration",
+            ]:
+                if f in metadata:
+                    md_content += f"{f}: {metadata[f]}\n"
+
+        # Transcribe
+        try:
+            transcript = self._transcribe_audio(local_path)
+            md_content += "\n\n### Audio Transcript:\n" + ("[No speech detected]" if transcript == "" else transcript)
+        except Exception:
+            md_content += "\n\n### Audio Transcript:\nError. Could not transcribe this audio."
+
+        return DocumentConverterResult(
+            title=None,
+            text_content=md_content.strip(),
+        )
+
+    def _transcribe_audio(self, local_path) -> str:
+        recognizer = sr.Recognizer()
+        with sr.AudioFile(local_path) as source:
+            audio = recognizer.record(source)
+            return recognizer.recognize_google(audio).strip()
+
+
+class Mp3Converter(WavConverter):
+    """
+    Converts MP3 files to markdown via extraction of metadata (if `exiftool` is installed), and speech transcription (if `speech_recognition` AND `pydub` are installed).
+    """
+
+    def convert(self, local_path, **kwargs) -> Union[None, DocumentConverterResult]:
+        # Bail if not a MP3
+        extension = kwargs.get("file_extension", "")
+        if extension.lower() != ".mp3":
+            return None
+
+        md_content = ""
+
+        # Add metadata
+        metadata = self._get_metadata(local_path)
+        if metadata:
+            for f in [
+                "Title",
+                "Artist",
+                "Author",
+                "Band",
+                "Album",
+                "Genre",
+                "Track",
+                "DateTimeOriginal",
+                "CreateDate",
+                "Duration",
+            ]:
+                if f in metadata:
+                    md_content += f"{f}: {metadata[f]}\n"
+
+        # Transcribe
+        handle, temp_path = tempfile.mkstemp(suffix=".wav")
+        os.close(handle)
+        try:
+            sound = pydub.AudioSegment.from_mp3(local_path)
+            sound.export(temp_path, format="wav")
+
+            _args = dict()
+            _args.update(kwargs)
+            _args["file_extension"] = ".wav"
+
+            try:
+                transcript = super()._transcribe_audio(temp_path).strip()
+                md_content += "\n\n### Audio Transcript:\n" + (
+                    "[No speech detected]" if transcript == "" else transcript
+                )
+            except Exception:
+                md_content += "\n\n### Audio Transcript:\nError. Could not transcribe this audio."
+
+        finally:
+            os.unlink(temp_path)
+
+        # Return the result
+        return DocumentConverterResult(
+            title=None,
+            text_content=md_content.strip(),
+        )
+
+
+class ImageConverter(MediaConverter):
+    """
+    Converts images to markdown via extraction of metadata (if `exiftool` is installed), OCR (if `easyocr` is installed), and description via a multimodal LLM (if an mlm_client is configured).
+    """
+
+    def convert(self, local_path, **kwargs) -> Union[None, DocumentConverterResult]:
+        # Bail if not a XLSX
+        extension = kwargs.get("file_extension", "")
+        if extension.lower() not in [".jpg", ".jpeg", ".png"]:
+            return None
+
+        md_content = ""
+
+        # Add metadata
+        metadata = self._get_metadata(local_path)
+        if metadata:
+            for f in [
+                "ImageSize",
+                "Title",
+                "Caption",
+                "Description",
+                "Keywords",
+                "Artist",
+                "Author",
+                "DateTimeOriginal",
+                "CreateDate",
+                "GPSPosition",
+            ]:
+                if f in metadata:
+                    md_content += f"{f}: {metadata[f]}\n"
+
+        # Try describing the image with GPTV
+        mlm_client = kwargs.get("mlm_client")
+        mlm_model = kwargs.get("mlm_model")
+        if mlm_client is not None and mlm_model is not None:
+            md_content += (
+                "\n# Description:\n"
+                + self._get_mlm_description(
+                    local_path, extension, mlm_client, mlm_model, prompt=kwargs.get("mlm_prompt")
+                ).strip()
+                + "\n"
+            )
+
+        return DocumentConverterResult(
+            title=None,
+            text_content=md_content,
+        )
+
+    def _get_mlm_description(self, local_path, extension, client, model, prompt=None):
+        if prompt is None or prompt.strip() == "":
+            prompt = "Write a detailed caption for this image."
+
+        sys.stderr.write(f"MLM Prompt:\n{prompt}\n")
+
+        data_uri = ""
+        with open(local_path, "rb") as image_file:
+            content_type, encoding = mimetypes.guess_type("_dummy" + extension)
+            if content_type is None:
+                content_type = "image/jpeg"
+            image_base64 = base64.b64encode(image_file.read()).decode("utf-8")
+            data_uri = f"data:{content_type};base64,{image_base64}"
+
+        messages = [
+            {
+                "role": "user",
+                "content": [
+                    {"type": "text", "text": prompt},
+                    {
+                        "type": "image_url",
+                        "image_url": {
+                            "url": data_uri,
+                        },
+                    },
+                ],
+            }
+        ]
+
+        response = client.chat.completions.create(model=model, messages=messages)
+        return response.choices[0].message.content
+
+
+class FileConversionException(BaseException):
+    pass
+
+
+class UnsupportedFormatException(BaseException):
+    pass
+
+
+class MarkdownConverter:
+    """(In preview) An extremely simple text-based document reader, suitable for LLM use.
+    This reader will convert common file-types or webpages to Markdown."""
+
+    def __init__(
+        self,
+        requests_session: Optional[requests.Session] = None,
+        mlm_client: Optional[Any] = None,
+        mlm_model: Optional[Any] = None,
+    ):
+        if requests_session is None:
+            self._requests_session = requests.Session()
+        else:
+            self._requests_session = requests_session
+
+        self._mlm_client = mlm_client
+        self._mlm_model = mlm_model
+
+        self._page_converters: List[DocumentConverter] = []
+
+        # Register converters for successful browsing operations
+        # Later registrations are tried first / take higher priority than earlier registrations
+        # To this end, the most specific converters should appear below the most generic converters
+        self.register_page_converter(PlainTextConverter())
+        self.register_page_converter(JavaScriptConverter())
+        self.register_page_converter(HtmlConverter())
+        self.register_page_converter(WikipediaConverter())
+        self.register_page_converter(YouTubeConverter())
+        self.register_page_converter(DocxConverter())
+        self.register_page_converter(XlsxConverter())
+        self.register_page_converter(PptxConverter())
+        self.register_page_converter(WavConverter())
+        self.register_page_converter(Mp3Converter())
+        self.register_page_converter(ImageConverter())
+        self.register_page_converter(PdfConverter())
+
+    def convert(
+        self, source: Union[str, requests.Response], **kwargs: Any
+    ) -> DocumentConverterResult:  # TODO: deal with kwargs
+        """
+        Args:
+            - source: can be a string representing a path or url, or a requests.response object
+            - extension: specifies the file extension to use when interpreting the file. If None, infer from source (path, uri, content-type, etc.)
+        """
+
+        # Local path or url
+        if isinstance(source, str):
+            if source.startswith("http://") or source.startswith("https://") or source.startswith("file://"):
+                return self.convert_url(source, **kwargs)
+            else:
+                return self.convert_local(source, **kwargs)
+        # Request response
+        elif isinstance(source, requests.Response):
+            return self.convert_response(source, **kwargs)
+
+    def convert_local(self, path: str, **kwargs: Any) -> DocumentConverterResult:  # TODO: deal with kwargs
+        # Prepare a list of extensions to try (in order of priority)
+        ext = kwargs.get("file_extension")
+        extensions = [ext] if ext is not None else []
+
+        # Get extension alternatives from the path and puremagic
+        base, ext = os.path.splitext(path)
+        self._append_ext(extensions, ext)
+        self._append_ext(extensions, self._guess_ext_magic(path))
+
+        # Convert
+        return self._convert(path, extensions, **kwargs)
+
+    # TODO what should stream's type be?
+    def convert_stream(self, stream: Any, **kwargs: Any) -> DocumentConverterResult:  # TODO: deal with kwargs
+        # Prepare a list of extensions to try (in order of priority)
+        ext = kwargs.get("file_extension")
+        extensions = [ext] if ext is not None else []
+
+        # Save the file locally to a temporary file. It will be deleted before this method exits
+        handle, temp_path = tempfile.mkstemp()
+        fh = os.fdopen(handle, "wb")
+        result = None
+        try:
+            # Write to the temporary file
+            content = stream.read()
+            if isinstance(content, str):
+                fh.write(content.encode("utf-8"))
+            else:
+                fh.write(content)
+            fh.close()
+
+            # Use puremagic to check for more extension options
+            self._append_ext(extensions, self._guess_ext_magic(temp_path))
+
+            # Convert
+            result = self._convert(temp_path, extensions, **kwargs)
+        # Clean up
+        finally:
+            try:
+                fh.close()
+            except Exception:
+                pass
+            os.unlink(temp_path)
+
+        return result
+
+    def convert_url(self, url: str, **kwargs: Any) -> DocumentConverterResult:  # TODO: fix kwargs type
+        # Send a HTTP request to the URL
+        user_agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36 Edg/119.0.0.0"
+        response = self._requests_session.get(url, stream=True, headers={"User-Agent": user_agent})
+        response.raise_for_status()
+        return self.convert_response(response, **kwargs)
+
+    def convert_response(
+        self, response: requests.Response, **kwargs: Any
+    ) -> DocumentConverterResult:  # TODO fix kwargs type
+        # Prepare a list of extensions to try (in order of priority)
+        ext = kwargs.get("file_extension")
+        extensions = [ext] if ext is not None else []
+
+        # Guess from the mimetype
+        content_type = response.headers.get("content-type", "").split(";")[0]
+        self._append_ext(extensions, mimetypes.guess_extension(content_type))
+
+        # Read the content disposition if there is one
+        content_disposition = response.headers.get("content-disposition", "")
+        m = re.search(r"filename=([^;]+)", content_disposition)
+        if m:
+            base, ext = os.path.splitext(m.group(1).strip("\"'"))
+            self._append_ext(extensions, ext)
+
+        # Read from the extension from the path
+        base, ext = os.path.splitext(urlparse(response.url).path)
+        self._append_ext(extensions, ext)
+
+        # Save the file locally to a temporary file. It will be deleted before this method exits
+        handle, temp_path = tempfile.mkstemp()
+        fh = os.fdopen(handle, "wb")
+        result = None
+        try:
+            # Download the file
+            for chunk in response.iter_content(chunk_size=512):
+                fh.write(chunk)
+            fh.close()
+
+            # Use puremagic to check for more extension options
+            self._append_ext(extensions, self._guess_ext_magic(temp_path))
+
+            # Convert
+            result = self._convert(temp_path, extensions, url=response.url)
+        except Exception as e:
+            print(f"Error in converting: {e}")
+
+        # Clean up
+        finally:
+            try:
+                fh.close()
+            except Exception:
+                pass
+            os.unlink(temp_path)
+
+        return result
+
+    def _convert(self, local_path: str, extensions: List[Union[str, None]], **kwargs) -> DocumentConverterResult:
+        error_trace = ""
+        for ext in extensions + [None]:  # Try last with no extension
+            for converter in self._page_converters:
+                _kwargs = copy.deepcopy(kwargs)
+
+                # Overwrite file_extension appropriately
+                if ext is None:
+                    if "file_extension" in _kwargs:
+                        del _kwargs["file_extension"]
+                else:
+                    _kwargs.update({"file_extension": ext})
+
+                # Copy any additional global options
+                if "mlm_client" not in _kwargs and self._mlm_client is not None:
+                    _kwargs["mlm_client"] = self._mlm_client
+
+                if "mlm_model" not in _kwargs and self._mlm_model is not None:
+                    _kwargs["mlm_model"] = self._mlm_model
+
+                # If we hit an error log it and keep trying
+                try:
+                    res = converter.convert(local_path, **_kwargs)
+                except Exception:
+                    error_trace = ("\n\n" + traceback.format_exc()).strip()
+
+                if res is not None:
+                    # Normalize the content
+                    res.text_content = "\n".join([line.rstrip() for line in re.split(r"\r?\n", res.text_content)])
+                    res.text_content = re.sub(r"\n{3,}", "\n\n", res.text_content)
+
+                    # Todo
+                    return res
+
+        # If we got this far without success, report any exceptions
+        if len(error_trace) > 0:
+            raise FileConversionException(
+                f"Could not convert '{local_path}' to Markdown. File type was recognized as {extensions}. While converting the file, the following error was encountered:\n\n{error_trace}"
+            )
+
+        # Nothing can handle it!
+        raise UnsupportedFormatException(
+            f"Could not convert '{local_path}' to Markdown. The formats {extensions} are not supported."
+        )
+
+    def _append_ext(self, extensions, ext):
+        """Append a unique non-None, non-empty extension to a list of extensions."""
+        if ext is None:
+            return
+        ext = ext.strip()
+        if ext == "":
+            return
+        # if ext not in extensions:
+        if True:
+            extensions.append(ext)
+
+    def _guess_ext_magic(self, path):
+        """Use puremagic (a Python implementation of libmagic) to guess a file's extension based on the first few bytes."""
+        # Use puremagic to guess
+        try:
+            guesses = puremagic.magic_file(path)
+            if len(guesses) > 0:
+                ext = guesses[0].extension.strip()
+                if len(ext) > 0:
+                    return ext
+        except FileNotFoundError:
+            pass
+        except IsADirectoryError:
+            pass
+        except PermissionError:
+            pass
+        return None
+
+    def register_page_converter(self, converter: DocumentConverter) -> None:
+        """Register a page text converter."""
+        self._page_converters.insert(0, converter)

scripts/reformulator.py

@@ -0,0 +1,86 @@
+# Shamelessly stolen from Microsoft Autogen team: thanks to them for this great resource!
+# https://github.com/microsoft/autogen/blob/gaia_multiagent_v01_march_1st/autogen/browser_utils.py
+import copy
+
+from smolagents.models import MessageRole, Model
+
+
+def prepare_response(original_task: str, inner_messages, reformulation_model: Model) -> str:
+    messages = [
+        {
+            "role": MessageRole.SYSTEM,
+            "content": [
+                {
+                    "type": "text",
+                    "text": f"""Earlier you were asked the following:
+
+{original_task}
+
+Your team then worked diligently to address that request. Read below a transcript of that conversation:""",
+                }
+            ],
+        }
+    ]
+
+    # The first message just repeats the question, so remove it
+    # if len(inner_messages) > 1:
+    #    del inner_messages[0]
+
+    # copy them to this context
+    try:
+        for message in inner_messages:
+            if not message.get("content"):
+                continue
+            message = copy.deepcopy(message)
+            message["role"] = MessageRole.USER
+            messages.append(message)
+    except Exception:
+        messages += [{"role": MessageRole.ASSISTANT, "content": str(inner_messages)}]
+
+    # ask for the final answer
+    messages.append(
+        {
+            "role": MessageRole.USER,
+            "content": [
+                {
+                    "type": "text",
+                    "text": f"""
+Read the above conversation and output a FINAL ANSWER to the question. The question is repeated here for convenience:
+
+{original_task}
+
+To output the final answer, use the following template: FINAL ANSWER: [YOUR FINAL ANSWER]
+Your FINAL ANSWER should be a number OR as few words as possible OR a comma separated list of numbers and/or strings.
+ADDITIONALLY, your FINAL ANSWER MUST adhere to any formatting instructions specified in the original question (e.g., alphabetization, sequencing, units, rounding, decimal places, etc.)
+If you are asked for a number, express it numerically (i.e., with digits rather than words), don't use commas, and DO NOT INCLUDE UNITS such as $ or USD or percent signs unless specified otherwise.
+If you are asked for a string, don't use articles or abbreviations (e.g. for cities), unless specified otherwise. Don't output any final sentence punctuation such as '.', '!', or '?'.
+If you are asked for a comma separated list, apply the above rules depending on whether the elements are numbers or strings.
+If you are unable to determine the final answer, output 'FINAL ANSWER: Unable to determine'
+""",
+                }
+            ],
+        }
+    )
+
+    response = reformulation_model(messages).content
+
+    final_answer = response.split("FINAL ANSWER: ")[-1].strip()
+    print("> Reformulated answer: ", final_answer)
+
+    #     if "unable to determine" in final_answer.lower():
+    #         messages.append({"role": MessageRole.ASSISTANT, "content": response })
+    #         messages.append({"role": MessageRole.USER, "content": [{"type": "text", "text": """
+    # I understand that a definitive answer could not be determined. Please make a well-informed EDUCATED GUESS based on the conversation.
+
+    # To output the educated guess, use the following template: EDUCATED GUESS: [YOUR EDUCATED GUESS]
+    # Your EDUCATED GUESS should be a number OR as few words as possible OR a comma separated list of numbers and/or strings. DO NOT OUTPUT 'I don't know', 'Unable to determine', etc.
+    # ADDITIONALLY, your EDUCATED GUESS MUST adhere to any formatting instructions specified in the original question (e.g., alphabetization, sequencing, units, rounding, decimal places, etc.)
+    # If you are asked for a number, express it numerically (i.e., with digits rather than words), don't use commas, and don't include units such as $ or percent signs unless specified otherwise.
+    # If you are asked for a string, don't use articles or abbreviations (e.g. cit for cities), unless specified otherwise. Don't output any final sentence punctuation such as '.', '!', or '?'.
+    # If you are asked for a comma separated list, apply the above rules depending on whether the elements are numbers or strings.
+    # """.strip()}]})
+
+    #         response = model(messages).content
+    #         print("\n>>>Making an educated guess.\n", response)
+    #         final_answer = response.split("EDUCATED GUESS: ")[-1].strip()
+    return final_answer

scripts/report_generator.py

@@ -0,0 +1,153 @@
+from typing import List, Optional, Dict
+from smolagents import Tool
+
+class HFLinkReportTool(Tool):
+    """Generate a single-layout HTML report (cards + counters) from a final textual answer.
+    The tool extracts links from the provided text, categorizes them (HF models/datasets/spaces/papers, blogs, repos, videos, news),
+    and renders a consistent link report. Always returns a full HTML document (starts with <!DOCTYPE html>)."""
+
+    name = "hf_links_to_report"
+    description = (
+        "Create an HTML report from a final answer text. The tool parses links, groups them into categories "
+        "(Hugging Face models/datasets/spaces/papers and external resources like blogs/repos/videos/news), and renders cards. "
+        "Inputs: final_answer (string, required), query (string, optional), title (string, optional). Returns an HTML document."
+    )
+    inputs = {
+        "final_answer": {"type": "string", "description": "Final answer text containing inline links"},
+        "query": {"type": "string", "description": "Original user intent or topic", "nullable": True},
+        "title": {"type": "string", "description": "Dashboard title", "nullable": True},
+    }
+    output_type = "string"
+
+    def forward(self, final_answer: str, query: Optional[str] = None, title: Optional[str] = None) -> str:
+        try:
+            import re
+            import json as _json
+            doc_title = title or "Report"
+            query = (query or "").strip()
+
+            # Extract URLs
+            urls = re.findall(r"https?://[^\s)\]]+", final_answer or "")
+            # Categorize
+            cats = {
+                "models": [], "datasets": [], "spaces": [], "papers": [],
+                "blogs": [], "repos": [], "videos": [], "news": [], "other": []
+            }
+            for u in urls:
+                low = u.lower()
+                if "huggingface.co/" in low:
+                    # Prefer explicit kinds first to avoid misclassifying /datasets/* as generic owner/repo
+                    if "/datasets/" in low:
+                        cats["datasets"].append(u)
+                    elif "/spaces/" in low:
+                        cats["spaces"].append(u)
+                    elif "/papers/" in low:
+                        cats["papers"].append(u)
+                    elif "/models/" in low:
+                        cats["models"].append(u)
+                    else:
+                        # Treat bare owner/repo as models only if it is NOT under known sections
+                        # e.g., huggingface.co/owner/repo → model repo; huggingface.co/blog/... → blog
+                        m = re.search(r"huggingface\.co/([^/]+)/([^/]+)$", low)
+                        if m and m.group(1) not in {"datasets", "spaces", "papers", "blog", "learn", "docs", "organizations", "collections"}:
+                            cats["models"].append(u)
+                        else:
+                            cats["blogs"].append(u)
+                elif "github.com" in low:
+                    cats["repos"].append(u)
+                elif "youtube.com" in low or "youtu.be" in low:
+                    cats["videos"].append(u)
+                elif any(d in low for d in ["arxiv.org", "medium.com", "towardsdatascience.com", "huggingface.co/blog", "huggingface.co/learn"]):
+                    cats["blogs"].append(u)
+                elif any(d in low for d in ["theverge.com", "techcrunch.com", "venturebeat.com", "wired.com", "mit.edu"]):
+                    cats["news"].append(u)
+                else:
+                    cats["other"].append(u)
+
+            def chips_section():
+                chips = [
+                    ("Models", len(cats["models"])),
+                    ("Datasets", len(cats["datasets"])),
+                    ("Spaces", len(cats["spaces"])),
+                    ("Papers", len(cats["papers"])),
+                    ("Blogs/Docs", len(cats["blogs"])),
+                    ("Repos", len(cats["repos"])),
+                    ("Videos", len(cats["videos"])),
+                    ("News", len(cats["news"]))
+                ]
+                return "\n".join([f"<div class=stat-chip>{name}: {count}</div>" for name, count in chips])
+
+            def host_icon(host: str) -> str:
+                return ""
+
+            def card_list(urls: List[str], data_cat: str) -> str:
+                items = []
+                for u in urls:
+                    host = re.sub(r"^https?://", "", u).split("/")[0]
+                    icon = host_icon(host)
+                    favicon = f"https://www.google.com/s2/favicons?sz=32&domain={host}"
+                    items.append(
+                        f"<div class=card data-cat='{data_cat}'>"
+                        f"<div class=card-title>{icon} <img class=\"fav\" src=\"{favicon}\" alt=\"\"/> <a href='{u}' target=_blank rel=noopener>{u}</a></div>"
+                        f"<div class=card-subtitle>{host}</div>"
+                        f"<div class=card-actions><button onclick=\"copyLink('{u}')\">Copy</button></div>"
+                        "</div>"
+                    )
+                return "\n".join(items)
+
+            def section(title_text: str, urls: List[str], key: str) -> str:
+                if not urls:
+                    return ""
+                return f"<section data-key='{key}'><h2>{title_text}</h2><div class=cards>{card_list(urls, key)}</div></section>"
+
+            html = f"""<!DOCTYPE html>
+<html lang=\"en\">
+<head>
+  <meta charset=\"utf-8\" />
+  <meta name=\"viewport\" content=\"width=device-width, initial-scale=1\" />
+  <title>{doc_title}</title>
+  <style>
+    :root {{ --bg:#0b0d12; --fg:#e6e9ef; --muted:#9aa4b2; --card:#121621; --accent:#5ac8fa; }}
+    body {{ background:var(--bg); color:var(--fg); font-family: ui-sans-serif, system-ui, -apple-system, Segoe UI, Roboto, Inter, Arial, sans-serif; margin:0; padding:24px; }}
+    .container {{ max-width: 1200px; margin: 0 auto; }}
+    .header {{ display:flex; justify-content:space-between; align-items:center; gap:12px; margin-bottom: 12px; }}
+    .title {{ font-size: 22px; margin: 0; }}
+    .subtitle {{ color: var(--muted); }}
+    .stats {{ display:flex; gap:10px; flex-wrap:wrap; margin: 8px 0 18px; }}
+    .stat-chip {{ background: var(--card); border: 1px solid rgba(255,255,255,0.08); border-radius: 999px; padding: 6px 10px; font-size: 12px; color: var(--muted); }}
+    h2 {{ font-size: 16px; margin: 18px 0 8px; color: var(--accent); }}
+    .cards {{ display: grid; grid-template-columns: repeat(auto-fill, minmax(280px,1fr)); gap: 12px; }}
+    .card {{ background: var(--card); border: 1px solid rgba(255,255,255,0.06); border-radius: 10px; padding: 12px; }}
+    .card-title {{ font-weight: 600; margin-bottom: 4px; overflow-wrap:anywhere; }}
+    .card-subtitle {{ color: var(--muted); font-size: 12px; }}
+    .answer {{ line-height:1.55; color:#d2d7df; }}
+    .card-actions button {{ background:#1f2937;color:#e5e7eb;border:1px solid rgba(255,255,255,0.08);border-radius:6px;padding:4px 8px;cursor:pointer;font-size:12px; }}
+    .fav {{ width:14px; height:14px; vertical-align:middle; margin-right:6px; border-radius:4px; }}
+    .warn {{ margin-left:6px; cursor: help; }}
+  </style>
+  <script src=\"https://cdn.jsdelivr.net/npm/marked/marked.min.js\"></script>
+  <script src=\"https://cdn.jsdelivr.net/npm/[email protected]/dist/purify.min.js\"></script>
+</head>
+<body>
+  <div class=\"container\">{('<div class=\\"header\\"><div><div class=\\"title\\">' + title + '</div></div></div>') if title else ''}
+    <h2>You may be interested <span class=\"warn\" title=\"Links may be AI‑generated and might not resolve.\">⚠️</span></h2>
+    <div class=\"stats\">{chips_section()}</div>
+    {section('Models', cats['models'], 'models')}
+    {section('Datasets', cats['datasets'], 'datasets')}
+    {section('Spaces', cats['spaces'], 'spaces')}
+    {section('Papers', cats['papers'], 'papers')}
+    {section('Blogs / Docs', cats['blogs'], 'blogs')}
+    {section('Repositories', cats['repos'], 'repos')}
+    {section('Videos', cats['videos'], 'videos')}
+    {section('News', cats['news'], 'news')}
+    {section('Other', cats['other'], 'other')}
+  </div>
+  <script>
+    function copyLink(url){{ try{{navigator.clipboard && navigator.clipboard.writeText(url);}}catch(e){{}} }}
+  </script>
+</body>
+</html>
+"""
+            return html
+        except Exception as e:
+            return f"<!DOCTYPE html><html><body><pre>Error generating report: {str(e)}</pre></body></html>"

scripts/run_agents.py

@@ -0,0 +1,87 @@
+import json
+import os
+import shutil
+import textwrap
+from pathlib import Path
+
+# import tqdm.asyncio
+from smolagents.utils import AgentError
+
+
+def serialize_agent_error(obj):
+    if isinstance(obj, AgentError):
+        return {"error_type": obj.__class__.__name__, "message": obj.message}
+    else:
+        return str(obj)
+
+
+def get_image_description(file_name: str, question: str, visual_inspection_tool) -> str:
+    prompt = f"""Write a caption of 5 sentences for this image. Pay special attention to any details that might be useful for someone answering the following question:
+{question}. But do not try to answer the question directly!
+Do not add any information that is not present in the image."""
+    return visual_inspection_tool(image_path=file_name, question=prompt)
+
+
+def get_document_description(file_path: str, question: str, document_inspection_tool) -> str:
+    prompt = f"""Write a caption of 5 sentences for this document. Pay special attention to any details that might be useful for someone answering the following question:
+{question}. But do not try to answer the question directly!
+Do not add any information that is not present in the document."""
+    return document_inspection_tool.forward_initial_exam_mode(file_path=file_path, question=prompt)
+
+
+def get_single_file_description(file_path: str, question: str, visual_inspection_tool, document_inspection_tool):
+    file_extension = file_path.split(".")[-1]
+    if file_extension in ["png", "jpg", "jpeg"]:
+        file_description = f" - Attached image: {file_path}"
+        file_description += (
+            f"\n     -> Image description: {get_image_description(file_path, question, visual_inspection_tool)}"
+        )
+        return file_description
+    elif file_extension in ["pdf", "xls", "xlsx", "docx", "doc", "xml"]:
+        file_description = f" - Attached document: {file_path}"
+        image_path = file_path.split(".")[0] + ".png"
+        if os.path.exists(image_path):
+            description = get_image_description(image_path, question, visual_inspection_tool)
+        else:
+            description = get_document_description(file_path, question, document_inspection_tool)
+        file_description += f"\n     -> File description: {description}"
+        return file_description
+    elif file_extension in ["mp3", "m4a", "wav"]:
+        return f" - Attached audio: {file_path}"
+    else:
+        return f" - Attached file: {file_path}"
+
+
+def get_zip_description(file_path: str, question: str, visual_inspection_tool, document_inspection_tool):
+    folder_path = file_path.replace(".zip", "")
+    os.makedirs(folder_path, exist_ok=True)
+    shutil.unpack_archive(file_path, folder_path)
+
+    prompt_use_files = ""
+    for root, dirs, files in os.walk(folder_path):
+        for file in files:
+            file_path = os.path.join(root, file)
+            prompt_use_files += "\n" + textwrap.indent(
+                get_single_file_description(file_path, question, visual_inspection_tool, document_inspection_tool),
+                prefix="    ",
+            )
+    return prompt_use_files
+
+
+def get_tasks_to_run(data, total: int, base_filename: Path, tasks_ids: list[int]):
+    f = base_filename.parent / f"{base_filename.stem}_answers.jsonl"
+    done = set()
+    if f.exists():
+        with open(f, encoding="utf-8") as fh:
+            done = {json.loads(line)["task_id"] for line in fh if line.strip()}
+
+    tasks = []
+    for i in range(total):
+        task_id = int(data[i]["task_id"])
+        if task_id not in done:
+            if tasks_ids is not None:
+                if task_id in tasks_ids:
+                    tasks.append(data[i])
+            else:
+                tasks.append(data[i])
+    return tasks

scripts/text_inspector_tool.py

@@ -0,0 +1,88 @@
+from typing import Optional
+import os
+
+from smolagents import Tool
+from smolagents.models import MessageRole, Model
+
+from .mdconvert import MarkdownConverter
+
+
+class TextInspectorTool(Tool):
+    name = "inspect_file_as_text"
+    description = """
+You cannot load files yourself: instead call this tool to read a file as markdown text and ask questions about it.
+This tool handles the following file extensions: [".html", ".htm", ".xlsx", ".pptx", ".wav", ".mp3", ".flac", ".pdf", ".docx", ".mjs", ".js"], and all other types of text files. IT DOES NOT HANDLE IMAGES."""
+
+    inputs = {
+        "file_path": {
+            "description": "The path to the file you want to read as text. Must be a '.something' file, like '.pdf'. If it is an image, use the visualizer tool instead! DO NOT use this tool for an HTML webpage: use the web_search tool instead!",
+            "type": "string",
+        },
+        "question": {
+            "description": "[Optional]: Your question, as a natural language sentence. Provide as much context as possible. Do not pass this parameter if you just want to directly return the content of the file.",
+            "type": "string",
+            "nullable": True,
+        },
+    }
+    output_type = "string"
+    md_converter = MarkdownConverter()
+
+    def __init__(self, model: Model, text_limit: int):
+        super().__init__()
+        self.model = model
+        self.text_limit = text_limit
+
+    def forward_initial_exam_mode(self, file_path, question):
+        try:
+            # Only allow reading files from uploads directory
+            uploads_dir = os.path.abspath(os.path.join(os.getcwd(), "uploads"))
+            candidate_path = os.path.abspath(file_path)
+            if not candidate_path.startswith(uploads_dir + os.sep):
+                # Fallback to uploads/<basename>
+                candidate_path = os.path.join(uploads_dir, os.path.basename(file_path))
+
+            result = self.md_converter.convert(candidate_path)
+
+            if file_path[-4:] in [".png", ".jpg"]:
+                raise Exception("Cannot use inspect_file_as_text tool with images: use visualizer instead!")
+
+            if ".zip" in file_path:
+                return result.text_content
+
+            if not question:
+                return result.text_content
+
+            if len(result.text_content) < 4000:
+                return "Document content: " + result.text_content
+
+            # For larger files, just return the content without model processing to avoid freezing
+            return f"Document title: {result.title}\n\nDocument content:\n{result.text_content[:self.text_limit]}"
+            
+        except Exception as e:
+            return f"Error reading file '{file_path}': {str(e)}. Access is restricted to files uploaded via the interface."
+
+    def forward(self, file_path, question: Optional[str] = None) -> str:
+        try:
+            # Only allow reading files from uploads directory
+            uploads_dir = os.path.abspath(os.path.join(os.getcwd(), "uploads"))
+            candidate_path = os.path.abspath(file_path)
+            if not candidate_path.startswith(uploads_dir + os.sep):
+                # Fallback to uploads/<basename>
+                candidate_path = os.path.join(uploads_dir, os.path.basename(file_path))
+
+            result = self.md_converter.convert(candidate_path)
+
+            if file_path[-4:] in [".png", ".jpg"]:
+                raise Exception("Cannot use inspect_file_as_text tool with images: use visualizer instead!")
+
+            if ".zip" in file_path:
+                return result.text_content
+
+            if not question:
+                return result.text_content
+
+            # For questions, return the content with a note about the question
+            return f"Question: {question}\n\nDocument title: {result.title}\n\nDocument content:\n{result.text_content[:self.text_limit]}"
+            
+        except Exception as e:
+            return f"Error reading file '{file_path}': {str(e)}. Access is restricted to files uploaded via the interface."

scripts/text_web_browser.py

@@ -0,0 +1,564 @@
+# Shamelessly stolen from Microsoft Autogen team: thanks to them for this great resource!
+# https://github.com/microsoft/autogen/blob/gaia_multiagent_v01_march_1st/autogen/browser_utils.py
+import mimetypes
+import os
+import pathlib
+import re
+import time
+import uuid
+from typing import Any, Dict, List, Optional, Tuple, Union
+from urllib.parse import unquote, urljoin, urlparse
+
+import pathvalidate
+import requests
+from serpapi import GoogleSearch
+# from serpapi.google_search import GoogleSearch
+
+from smolagents import Tool
+
+from .cookies import COOKIES
+from .mdconvert import FileConversionException, MarkdownConverter, UnsupportedFormatException
+
+
+class SimpleTextBrowser:
+    """(In preview) An extremely simple text-based web browser comparable to Lynx. Suitable for Agentic use."""
+
+    def __init__(
+        self,
+        start_page: Optional[str] = None,
+        viewport_size: Optional[int] = 1024 * 8,
+        downloads_folder: Optional[Union[str, None]] = None,
+        serpapi_key: Optional[Union[str, None]] = None,
+        request_kwargs: Optional[Union[Dict[str, Any], None]] = None,
+    ):
+        self.start_page: str = start_page if start_page else "about:blank"
+        self.viewport_size = viewport_size  # Applies only to the standard uri types
+        self.downloads_folder = downloads_folder
+        self.history: List[Tuple[str, float]] = list()
+        self.page_title: Optional[str] = None
+        self.viewport_current_page = 0
+        self.viewport_pages: List[Tuple[int, int]] = list()
+        self.set_address(self.start_page)
+        self.serpapi_key = serpapi_key
+        self.request_kwargs = request_kwargs
+        self.request_kwargs["cookies"] = COOKIES
+        self._mdconvert = MarkdownConverter()
+        self._page_content: str = ""
+
+        self._find_on_page_query: Union[str, None] = None
+        self._find_on_page_last_result: Union[int, None] = None  # Location of the last result
+
+    @property
+    def address(self) -> str:
+        """Return the address of the current page."""
+        return self.history[-1][0]
+
+    def set_address(self, uri_or_path: str, filter_year: Optional[int] = None) -> None:
+        # TODO: Handle anchors
+        self.history.append((uri_or_path, time.time()))
+
+        # Handle special URIs
+        if uri_or_path == "about:blank":
+            self._set_page_content("")
+        elif uri_or_path.startswith("google:"):
+            self._serpapi_search(uri_or_path[len("google:") :].strip(), filter_year=filter_year)
+        else:
+            if (
+                not uri_or_path.startswith("http:")
+                and not uri_or_path.startswith("https:")
+                and not uri_or_path.startswith("file:")
+            ):
+                if len(self.history) > 1:
+                    prior_address = self.history[-2][0]
+                    uri_or_path = urljoin(prior_address, uri_or_path)
+                    # Update the address with the fully-qualified path
+                    self.history[-1] = (uri_or_path, self.history[-1][1])
+            self._fetch_page(uri_or_path)
+
+        self.viewport_current_page = 0
+        self.find_on_page_query = None
+        self.find_on_page_viewport = None
+
+    @property
+    def viewport(self) -> str:
+        """Return the content of the current viewport."""
+        bounds = self.viewport_pages[self.viewport_current_page]
+        return self.page_content[bounds[0] : bounds[1]]
+
+    @property
+    def page_content(self) -> str:
+        """Return the full contents of the current page."""
+        return self._page_content
+
+    def _set_page_content(self, content: str) -> None:
+        """Sets the text content of the current page."""
+        self._page_content = content
+        self._split_pages()
+        if self.viewport_current_page >= len(self.viewport_pages):
+            self.viewport_current_page = len(self.viewport_pages) - 1
+
+    def page_down(self) -> None:
+        self.viewport_current_page = min(self.viewport_current_page + 1, len(self.viewport_pages) - 1)
+
+    def page_up(self) -> None:
+        self.viewport_current_page = max(self.viewport_current_page - 1, 0)
+
+    def find_on_page(self, query: str) -> Union[str, None]:
+        """Searches for the query from the current viewport forward, looping back to the start if necessary."""
+
+        # Did we get here via a previous find_on_page search with the same query?
+        # If so, map to find_next
+        if query == self._find_on_page_query and self.viewport_current_page == self._find_on_page_last_result:
+            return self.find_next()
+
+        # Ok it's a new search start from the current viewport
+        self._find_on_page_query = query
+        viewport_match = self._find_next_viewport(query, self.viewport_current_page)
+        if viewport_match is None:
+            self._find_on_page_last_result = None
+            return None
+        else:
+            self.viewport_current_page = viewport_match
+            self._find_on_page_last_result = viewport_match
+            return self.viewport
+
+    def find_next(self) -> Union[str, None]:
+        """Scroll to the next viewport that matches the query"""
+
+        if self._find_on_page_query is None:
+            return None
+
+        starting_viewport = self._find_on_page_last_result
+        if starting_viewport is None:
+            starting_viewport = 0
+        else:
+            starting_viewport += 1
+            if starting_viewport >= len(self.viewport_pages):
+                starting_viewport = 0
+
+        viewport_match = self._find_next_viewport(self._find_on_page_query, starting_viewport)
+        if viewport_match is None:
+            self._find_on_page_last_result = None
+            return None
+        else:
+            self.viewport_current_page = viewport_match
+            self._find_on_page_last_result = viewport_match
+            return self.viewport
+
+    def _find_next_viewport(self, query: str, starting_viewport: int) -> Union[int, None]:
+        """Search for matches between the starting viewport looping when reaching the end."""
+
+        if query is None:
+            return None
+
+        # Normalize the query, and convert to a regular expression
+        nquery = re.sub(r"\*", "__STAR__", query)
+        nquery = " " + (" ".join(re.split(r"\W+", nquery))).strip() + " "
+        nquery = nquery.replace(" __STAR__ ", "__STAR__ ")  # Merge isolated stars with prior word
+        nquery = nquery.replace("__STAR__", ".*").lower()
+
+        if nquery.strip() == "":
+            return None
+
+        idxs = list()
+        idxs.extend(range(starting_viewport, len(self.viewport_pages)))
+        idxs.extend(range(0, starting_viewport))
+
+        for i in idxs:
+            bounds = self.viewport_pages[i]
+            content = self.page_content[bounds[0] : bounds[1]]
+
+            # TODO: Remove markdown links and images
+            ncontent = " " + (" ".join(re.split(r"\W+", content))).strip().lower() + " "
+            if re.search(nquery, ncontent):
+                return i
+
+        return None
+
+    def visit_page(self, path_or_uri: str, filter_year: Optional[int] = None) -> str:
+        """Update the address, visit the page, and return the content of the viewport."""
+        self.set_address(path_or_uri, filter_year=filter_year)
+        return self.viewport
+
+    def _split_pages(self) -> None:
+        # Do not split search results
+        if self.address.startswith("google:"):
+            self.viewport_pages = [(0, len(self._page_content))]
+            return
+
+        # Handle empty pages
+        if len(self._page_content) == 0:
+            self.viewport_pages = [(0, 0)]
+            return
+
+        # Break the viewport into pages
+        self.viewport_pages = []
+        start_idx = 0
+        while start_idx < len(self._page_content):
+            end_idx = min(start_idx + self.viewport_size, len(self._page_content))  # type: ignore[operator]
+            # Adjust to end on a space
+            while end_idx < len(self._page_content) and self._page_content[end_idx - 1] not in [" ", "\t", "\r", "\n"]:
+                end_idx += 1
+            self.viewport_pages.append((start_idx, end_idx))
+            start_idx = end_idx
+
+    def _serpapi_search(self, query: str, filter_year: Optional[int] = None) -> None:
+        if self.serpapi_key is None:
+            raise ValueError("Missing SerpAPI key.")
+
+        params = {
+            "engine": "google",
+            "q": query,
+            "api_key": self.serpapi_key,
+        }
+        if filter_year is not None:
+            params["tbs"] = f"cdr:1,cd_min:01/01/{filter_year},cd_max:12/31/{filter_year}"
+
+        search = GoogleSearch(params)
+        results = search.get_dict()
+        self.page_title = f"{query} - Search"
+        if "organic_results" not in results.keys():
+            raise Exception(f"No results found for query: '{query}'. Use a less specific query.")
+        if len(results["organic_results"]) == 0:
+            year_filter_message = f" with filter year={filter_year}" if filter_year is not None else ""
+            self._set_page_content(
+                f"No results found for '{query}'{year_filter_message}. Try with a more general query, or remove the year filter."
+            )
+            return
+
+        def _prev_visit(url):
+            for i in range(len(self.history) - 1, -1, -1):
+                if self.history[i][0] == url:
+                    return f"You previously visited this page {round(time.time() - self.history[i][1])} seconds ago.\n"
+            return ""
+
+        web_snippets: List[str] = list()
+        idx = 0
+        if "organic_results" in results:
+            for page in results["organic_results"]:
+                idx += 1
+                date_published = ""
+                if "date" in page:
+                    date_published = "\nDate published: " + page["date"]
+
+                source = ""
+                if "source" in page:
+                    source = "\nSource: " + page["source"]
+
+                snippet = ""
+                if "snippet" in page:
+                    snippet = "\n" + page["snippet"]
+
+                redacted_version = f"{idx}. [{page['title']}]({page['link']}){date_published}{source}\n{_prev_visit(page['link'])}{snippet}"
+
+                redacted_version = redacted_version.replace("Your browser can't play this video.", "")
+                web_snippets.append(redacted_version)
+
+        content = (
+            f"A Google search for '{query}' found {len(web_snippets)} results:\n\n## Web Results\n"
+            + "\n\n".join(web_snippets)
+        )
+
+        self._set_page_content(content)
+
+    def _fetch_page(self, url: str) -> None:
+        download_path = ""
+        try:
+            if url.startswith("file://"):
+                download_path = os.path.normcase(os.path.normpath(unquote(url[7:])))
+                res = self._mdconvert.convert_local(download_path)
+                self.page_title = res.title
+                self._set_page_content(res.text_content)
+            else:
+                # Prepare the request parameters
+                request_kwargs = self.request_kwargs.copy() if self.request_kwargs is not None else {}
+                request_kwargs["stream"] = True
+
+                # Send a HTTP request to the URL
+                response = requests.get(url, **request_kwargs)
+                response.raise_for_status()
+
+                # If the HTTP request was successful
+                content_type = response.headers.get("content-type", "")
+
+                # Text or HTML
+                if "text/" in content_type.lower():
+                    res = self._mdconvert.convert_response(response)
+                    self.page_title = res.title
+                    self._set_page_content(res.text_content)
+                # A download
+                else:
+                    # Try producing a safe filename
+                    fname = None
+                    download_path = None
+                    try:
+                        fname = pathvalidate.sanitize_filename(os.path.basename(urlparse(url).path)).strip()
+                        download_path = os.path.abspath(os.path.join(self.downloads_folder, fname))
+
+                        suffix = 0
+                        while os.path.exists(download_path) and suffix < 1000:
+                            suffix += 1
+                            base, ext = os.path.splitext(fname)
+                            new_fname = f"{base}__{suffix}{ext}"
+                            download_path = os.path.abspath(os.path.join(self.downloads_folder, new_fname))
+
+                    except NameError:
+                        pass
+
+                    # No suitable name, so make one
+                    if fname is None:
+                        extension = mimetypes.guess_extension(content_type)
+                        if extension is None:
+                            extension = ".download"
+                        fname = str(uuid.uuid4()) + extension
+                        download_path = os.path.abspath(os.path.join(self.downloads_folder, fname))
+
+                    # Open a file for writing
+                    with open(download_path, "wb") as fh:
+                        for chunk in response.iter_content(chunk_size=512):
+                            fh.write(chunk)
+
+                    # Render it
+                    local_uri = pathlib.Path(download_path).as_uri()
+                    self.set_address(local_uri)
+
+        except UnsupportedFormatException as e:
+            print(e)
+            self.page_title = ("Download complete.",)
+            self._set_page_content(f"# Download complete\n\nSaved file to '{download_path}'")
+        except FileConversionException as e:
+            print(e)
+            self.page_title = ("Download complete.",)
+            self._set_page_content(f"# Download complete\n\nSaved file to '{download_path}'")
+        except FileNotFoundError:
+            self.page_title = "Error 404"
+            self._set_page_content(f"## Error 404\n\nFile not found: {download_path}")
+        except requests.exceptions.RequestException as request_exception:
+            try:
+                self.page_title = f"Error {response.status_code}"
+
+                # If the error was rendered in HTML we might as well render it
+                content_type = response.headers.get("content-type", "")
+                if content_type is not None and "text/html" in content_type.lower():
+                    res = self._mdconvert.convert(response)
+                    self.page_title = f"Error {response.status_code}"
+                    self._set_page_content(f"## Error {response.status_code}\n\n{res.text_content}")
+                else:
+                    text = ""
+                    for chunk in response.iter_content(chunk_size=512, decode_unicode=True):
+                        text += chunk
+                    self.page_title = f"Error {response.status_code}"
+                    self._set_page_content(f"## Error {response.status_code}\n\n{text}")
+            except NameError:
+                self.page_title = "Error"
+                self._set_page_content(f"## Error\n\n{str(request_exception)}")
+
+    def _state(self) -> Tuple[str, str]:
+        header = f"Address: {self.address}\n"
+        if self.page_title is not None:
+            header += f"Title: {self.page_title}\n"
+
+        current_page = self.viewport_current_page
+        total_pages = len(self.viewport_pages)
+
+        address = self.address
+        for i in range(len(self.history) - 2, -1, -1):  # Start from the second last
+            if self.history[i][0] == address:
+                header += f"You previously visited this page {round(time.time() - self.history[i][1])} seconds ago.\n"
+                break
+
+        header += f"Viewport position: Showing page {current_page + 1} of {total_pages}.\n"
+        return (header, self.viewport)
+
+
+class SearchInformationTool(Tool):
+    name = "web_search"
+    description = "Perform a web search query (think a google search) and returns the search results."
+    inputs = {"query": {"type": "string", "description": "The web search query to perform."}}
+    inputs["filter_year"] = {
+        "type": "string",
+        "description": "[Optional parameter]: filter the search results to only include pages from a specific year. For example, '2020' will only include pages from 2020. Make sure to use this parameter if you're trying to search for articles from a specific date!",
+        "nullable": True,
+    }
+    output_type = "string"
+
+    def __init__(self, browser):
+        super().__init__()
+        self.browser = browser
+
+    def forward(self, query: str, filter_year: Optional[int] = None) -> str:
+        self.browser.visit_page(f"google: {query}", filter_year=filter_year)
+        header, content = self.browser._state()
+        return header.strip() + "\n=======================\n" + content
+
+
+class VisitTool(Tool):
+    name = "visit_page"
+    description = "Visit a webpage at a given URL and return its text. Given a url to a YouTube video, this returns the transcript."
+    inputs = {"url": {"type": "string", "description": "The relative or absolute url of the webapge to visit."}}
+    output_type = "string"
+
+    def __init__(self, browser):
+        super().__init__()
+        self.browser = browser
+
+    def forward(self, url: str) -> str:
+        self.browser.visit_page(url)
+        header, content = self.browser._state()
+        return header.strip() + "\n=======================\n" + content
+
+
+class DownloadTool(Tool):
+    name = "download_file"
+    description = """
+Download a file at a given URL. The file should be of this format: [".xlsx", ".pptx", ".wav", ".mp3", ".png", ".docx"]
+After using this tool, for further inspection of this page you should return the download path to your manager via final_answer, and they will be able to inspect it.
+DO NOT use this tool for .pdf or .txt or .htm files: for these types of files use visit_page with the file url instead."""
+    inputs = {"url": {"type": "string", "description": "The relative or absolute url of the file to be downloaded."}}
+    output_type = "string"
+
+    def __init__(self, browser):
+        super().__init__()
+        self.browser = browser
+
+    def forward(self, url: str) -> str:
+        if "arxiv" in url:
+            url = url.replace("abs", "pdf")
+        response = requests.get(url)
+        content_type = response.headers.get("content-type", "")
+        extension = mimetypes.guess_extension(content_type)
+        if extension and isinstance(extension, str):
+            new_path = f"./downloads/file{extension}"
+        else:
+            new_path = "./downloads/file.object"
+
+        with open(new_path, "wb") as f:
+            f.write(response.content)
+
+        if "pdf" in extension or "txt" in extension or "htm" in extension:
+            raise Exception("Do not use this tool for pdf or txt or html files: use visit_page instead.")
+
+        return f"File was downloaded and saved under path {new_path}."
+
+
+class ArchiveSearchTool(Tool):
+    name = "find_archived_url"
+    description = "Given a url, searches the Wayback Machine and returns the archived version of the url that's closest in time to the desired date."
+    inputs = {
+        "url": {"type": "string", "description": "The url you need the archive for."},
+        "date": {
+            "type": "string",
+            "description": "The date that you want to find the archive for. Give this date in the format 'YYYYMMDD', for instance '27 June 2008' is written as '20080627'.",
+        },
+    }
+    output_type = "string"
+
+    def __init__(self, browser):
+        super().__init__()
+        self.browser = browser
+
+    def forward(self, url, date) -> str:
+        no_timestamp_url = f"https://archive.org/wayback/available?url={url}"
+        archive_url = no_timestamp_url + f"&timestamp={date}"
+        response = requests.get(archive_url).json()
+        response_notimestamp = requests.get(no_timestamp_url).json()
+        if "archived_snapshots" in response and "closest" in response["archived_snapshots"]:
+            closest = response["archived_snapshots"]["closest"]
+            print("Archive found!", closest)
+
+        elif "archived_snapshots" in response_notimestamp and "closest" in response_notimestamp["archived_snapshots"]:
+            closest = response_notimestamp["archived_snapshots"]["closest"]
+            print("Archive found!", closest)
+        else:
+            raise Exception(f"Your {url=} was not archived on Wayback Machine, try a different url.")
+        target_url = closest["url"]
+        self.browser.visit_page(target_url)
+        header, content = self.browser._state()
+        return (
+            f"Web archive for url {url}, snapshot taken at date {closest['timestamp'][:8]}:\n"
+            + header.strip()
+            + "\n=======================\n"
+            + content
+        )
+
+
+class PageUpTool(Tool):
+    name = "page_up"
+    description = "Scroll the viewport UP one page-length in the current webpage and return the new viewport content."
+    inputs = {}
+    output_type = "string"
+
+    def __init__(self, browser):
+        super().__init__()
+        self.browser = browser
+
+    def forward(self) -> str:
+        self.browser.page_up()
+        header, content = self.browser._state()
+        return header.strip() + "\n=======================\n" + content
+
+
+class PageDownTool(Tool):
+    name = "page_down"
+    description = (
+        "Scroll the viewport DOWN one page-length in the current webpage and return the new viewport content."
+    )
+    inputs = {}
+    output_type = "string"
+
+    def __init__(self, browser):
+        super().__init__()
+        self.browser = browser
+
+    def forward(self) -> str:
+        self.browser.page_down()
+        header, content = self.browser._state()
+        return header.strip() + "\n=======================\n" + content
+
+
+class FinderTool(Tool):
+    name = "find_on_page_ctrl_f"
+    description = "Scroll the viewport to the first occurrence of the search string. This is equivalent to Ctrl+F."
+    inputs = {
+        "search_string": {
+            "type": "string",
+            "description": "The string to search for on the page. This search string supports wildcards like '*'",
+        }
+    }
+    output_type = "string"
+
+    def __init__(self, browser):
+        super().__init__()
+        self.browser = browser
+
+    def forward(self, search_string: str) -> str:
+        find_result = self.browser.find_on_page(search_string)
+        header, content = self.browser._state()
+
+        if find_result is None:
+            return (
+                header.strip()
+                + f"\n=======================\nThe search string '{search_string}' was not found on this page."
+            )
+        else:
+            return header.strip() + "\n=======================\n" + content
+
+
+class FindNextTool(Tool):
+    name = "find_next"
+    description = "Scroll the viewport to next occurrence of the search string. This is equivalent to finding the next match in a Ctrl+F search."
+    inputs = {}
+    output_type = "string"
+
+    def __init__(self, browser):
+        super().__init__()
+        self.browser = browser
+
+    def forward(self) -> str:
+        find_result = self.browser.find_next()
+        header, content = self.browser._state()
+
+        if find_result is None:
+            return header.strip() + "\n=======================\nThe search string was not found on this page."
+        else:
+            return header.strip() + "\n=======================\n" + content

scripts/visual_qa.py

@@ -0,0 +1,120 @@
+import base64
+import json
+import mimetypes
+import os
+import uuid
+from io import BytesIO
+from typing import Optional
+
+import requests
+from dotenv import load_dotenv
+from PIL import Image
+
+from smolagents import Tool, tool
+
+
+load_dotenv(override=True)
+
+
+def encode_image(image_path):
+    if image_path.startswith("http"):
+        user_agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36 Edg/119.0.0.0"
+        request_kwargs = {
+            "headers": {"User-Agent": user_agent},
+            "stream": True,
+        }
+
+        # Send a HTTP request to the URL
+        response = requests.get(image_path, **request_kwargs)
+        response.raise_for_status()
+        content_type = response.headers.get("content-type", "")
+
+        extension = mimetypes.guess_extension(content_type)
+        if extension is None:
+            extension = ".download"
+
+        fname = str(uuid.uuid4()) + extension
+        download_path = os.path.abspath(os.path.join("downloads", fname))
+
+        with open(download_path, "wb") as fh:
+            for chunk in response.iter_content(chunk_size=512):
+                fh.write(chunk)
+
+        image_path = download_path
+
+    with open(image_path, "rb") as image_file:
+        return base64.b64encode(image_file.read()).decode("utf-8")
+
+
+def resize_image(image_path):
+    img = Image.open(image_path)
+    width, height = img.size
+    img = img.resize((int(width / 2), int(height / 2)))
+    new_image_path = f"resized_{image_path}"
+    img.save(new_image_path)
+    return new_image_path
+
+
+@tool
+def visualizer(image_path: str, question: Optional[str] = None) -> str:
+    """A tool that can answer questions about attached images.
+
+    Args:
+        image_path: The path to the image on which to answer the question. This should be a local path to downloaded image.
+        question: The question to answer.
+    """
+    if not isinstance(image_path, str):
+        raise Exception("You should provide at least `image_path` string argument to this tool!")
+
+    add_note = False
+    if not question:
+        add_note = True
+        question = "Please write a detailed caption for this image."
+
+    mime_type, _ = mimetypes.guess_type(image_path)
+    base64_image = encode_image(image_path)
+
+    # Configuración para Ollama
+    model_id = os.getenv("MODEL_ID", "qwen2.5-coder:3b")
+    api_base = os.getenv("OPENAI_API_BASE", "http://localhost:11434/v1")
+    api_key = os.getenv("OPENAI_API_KEY", "ollama")
+
+    headers = {
+        "Content-Type": "application/json",
+        "Authorization": f"Bearer {api_key}"
+    }
+
+    payload = {
+        "model": model_id,
+        "messages": [
+            {
+                "role": "user",
+                "content": [
+                    {"type": "text", "text": question},
+                    {"type": "image_url", "image_url": {"url": f"data:{mime_type};base64,{base64_image}"}},
+                ],
+            }
+        ],
+        "max_tokens": 1000,
+    }
+
+    try:
+        response = requests.post(f"{api_base}/chat/completions", headers=headers, json=payload)
+        response.raise_for_status()
+        output = response.json()["choices"][0]["message"]["content"]
+    except Exception as e:
+        print(f"Error processing image: {str(e)}")
+        if "Payload Too Large" in str(e):
+            new_image_path = resize_image(image_path)
+            base64_image = encode_image(new_image_path)
+            payload["messages"][0]["content"][1]["image_url"]["url"] = f"data:{mime_type};base64,{base64_image}"
+            response = requests.post(f"{api_base}/chat/completions", headers=headers, json=payload)
+            response.raise_for_status()
+            output = response.json()["choices"][0]["message"]["content"]
+        else:
+            raise Exception(f"Error processing image: {str(e)}")
+
+    if add_note:
+        output = f"You did not provide a particular question, so here is a detailed caption for the image: {output}"
+
+    return output

set-env.bat

@@ -0,0 +1,11 @@
+REM siliconflow rate limited, wont work for deep-research
+REM set OPENAI_API_BASE=https://api.siliconflow.cn/v1
+REM set OPENAI_API_KEY=%SILICONFLOW_API_KEY%
+REM set MODEL_ID=openai/deepseek-ai/DeepSeek-V3
+
+set OPENAI_API_BASE=https://litellm.dattw.eu.org/v1
+set OPENAI_API_KEY=%LITELLM_API_KEY%
+set MODEL_ID=Qwen/Qwen3-Coder-480B-A35B-Instruct
+
+REM set SERPAPI_API_KEY=b84...
+REM set HF_TOKEN=hf_yC...

tests/hf_tools_tests_output_20250822_034011.txt

@@ -0,0 +1,2425 @@
+Hugging Face Tools Test Run — 20250822_034011
+================================================================================
+
+=== Running test_hf_models_search.py ===
+Command: C:\Users\daqc\Documents\GitHub\open-deep-research-vulnerability-intelligence\venv\Scripts\python.exe C:\Users\daqc\Documents\GitHub\open-deep-research-vulnerability-intelligence\tests\test_hf_models_search.py
+--- INPUT (snippet) ---
+tool.forward(
+        query="stable diffusion",
+        task="text-to-image",
+        sort="downloads",
+        direction="descending",
+        limit=5,
+    )
+    try:
+        data = json.loads(result_json_str)
+        print(json.dumps(data, indent=2, ensure_ascii=False))
+    except Exception:
+        print(result_json_str)
+
+
+if __name__ == "__main__":
+    main()
+
+
+--- STDOUT ---
+{
+  "results": [
+    {
+      "type": "model",
+      "id": "stabilityai/stable-diffusion-xl-base-1.0",
+      "owner": "stabilityai",
+      "url": "https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0",
+      "description": "",
+      "tags": [
+        "diffusers",
+        "onnx",
+        "safetensors",
+        "text-to-image",
+        "stable-diffusion",
+        "arxiv:2307.01952",
+        "arxiv:2211.01324",
+        "arxiv:2108.01073",
+        "arxiv:2112.10752",
+        "license:openrail++",
+        "autotrain_compatible",
+        "endpoints_compatible",
+        "diffusers:StableDiffusionXLPipeline",
+        "region:us"
+      ],
+      "task": "text-to-image",
+      "likes": 6862,
+      "downloads": 2281071,
+      "updatedAt": null,
+      "visibility": "public",
+      "access": "accessible"
+    },
+    {
+      "type": "model",
+      "id": "stable-diffusion-v1-5/stable-diffusion-v1-5",
+      "owner": "stable-diffusion-v1-5",
+      "url": "https://huggingface.co/stable-diffusion-v1-5/stable-diffusion-v1-5",
+      "description": "",
+      "tags": [
+        "diffusers",
+        "safetensors",
+        "stable-diffusion",
+        "stable-diffusion-diffusers",
+        "text-to-image",
+        "arxiv:2207.12598",
+        "arxiv:2112.10752",
+        "arxiv:2103.00020",
+        "arxiv:2205.11487",
+        "arxiv:1910.09700",
+        "license:creativeml-openrail-m",
+        "autotrain_compatible",
+        "endpoints_compatible",
+        "diffusers:StableDiffusionPipeline",
+        "region:us"
+      ],
+      "task": "text-to-image",
+      "likes": 755,
+      "downloads": 2789469,
+      "updatedAt": null,
+      "visibility": "public",
+      "access": "accessible"
+    },
+    {
+      "type": "model",
+      "id": "stabilityai/stable-diffusion-xl-refiner-1.0",
+      "owner": "stabilityai",
+      "url": "https://huggingface.co/stabilityai/stable-diffusion-xl-refiner-1.0",
+      "description": "",
+      "tags": [
+        "diffusers",
+        "safetensors",
+        "stable-diffusion",
+        "image-to-image",
+        "arxiv:2307.01952",
+        "arxiv:2211.01324",
+        "arxiv:2108.01073",
+        "arxiv:2112.10752",
+        "license:openrail++",
+        "diffusers:StableDiffusionXLImg2ImgPipeline",
+        "region:us"
+      ],
+      "task": "image-to-image",
+      "likes": 1953,
+      "downloads": 544617,
+      "updatedAt": null,
+      "visibility": "public",
+      "access": "accessible"
+    },
+    {
+      "type": "model",
+      "id": "stabilityai/stable-diffusion-3.5-large",
+      "owner": "stabilityai",
+      "url": "https://huggingface.co/stabilityai/stable-diffusion-3.5-large",
+      "description": "",
+      "tags": [
+        "diffusers",
+        "safetensors",
+        "text-to-image",
+        "stable-diffusion",
+        "en",
+        "arxiv:2403.03206",
+        "license:other",
+        "diffusers:StableDiffusion3Pipeline",
+        "region:us"
+      ],
+      "task": "text-to-image",
+      "likes": 3073,
+      "downloads": 82027,
+      "updatedAt": null,
+      "visibility": "public",
+      "access": "accessible"
+    },
+    {
+      "type": "model",
+      "id": "stabilityai/stable-video-diffusion-img2vid",
+      "owner": "stabilityai",
+      "url": "https://huggingface.co/stabilityai/stable-video-diffusion-img2vid",
+      "description": "",
+      "tags": [
+        "diffusers",
+        "safetensors",
+        "image-to-video",
+        "license:other",
+        "diffusers:StableVideoDiffusionPipeline",
+        "region:us"
+      ],
+      "task": "image-to-video",
+      "likes": 957,
+      "downloads": 46464,
+      "updatedAt": null,
+      "visibility": "public",
+      "access": "accessible"
+    }
+  ],
+  "status": 200,
+  "error": "",
+  "params": {
+    "search": "stable diffusion",
+    "pipeline_tag": "text-to-image",
+    "sort": "downloads",
+    "direction": "descending",
+    "limit": 5
+  }
+}
+--- STDERR ---
+
+[OK] test_hf_models_search.py
+--------------------------------------------------------------------------------
+
+=== Running test_hf_model_info.py ===
+Command: C:\Users\daqc\Documents\GitHub\open-deep-research-vulnerability-intelligence\venv\Scripts\python.exe C:\Users\daqc\Documents\GitHub\open-deep-research-vulnerability-intelligence\tests\test_hf_model_info.py
+--- INPUT (snippet) ---
+tool.forward(repo_id=repo_id)
+    try:
+        data = json.loads(result_json_str)
+        print(json.dumps(data, indent=2, ensure_ascii=False))
+    except Exception:
+        print(result_json_str)
+
+
+if __name__ == "__main__":
+    main()
+
+
+--- STDOUT ---
+{
+  "item": {
+    "type": "model",
+    "id": "sentence-transformers/all-MiniLM-L6-v2",
+    "owner": "sentence-transformers",
+    "url": "https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2",
+    "description": "",
+    "tags": [
+      "sentence-transformers",
+      "pytorch",
+      "tf",
+      "rust",
+      "onnx",
+      "safetensors",
+      "openvino",
+      "bert",
+      "feature-extraction",
+      "sentence-similarity",
+      "transformers",
+      "en",
+      "dataset:s2orc",
+      "dataset:flax-sentence-embeddings/stackexchange_xml",
+      "dataset:ms_marco",
+      "dataset:gooaq",
+      "dataset:yahoo_answers_topics",
+      "dataset:code_search_net",
+      "dataset:search_qa",
+      "dataset:eli5",
+      "dataset:snli",
+      "dataset:multi_nli",
+      "dataset:wikihow",
+      "dataset:natural_questions",
+      "dataset:trivia_qa",
+      "dataset:embedding-data/sentence-compression",
+      "dataset:embedding-data/flickr30k-captions",
+      "dataset:embedding-data/altlex",
+      "dataset:embedding-data/simple-wiki",
+      "dataset:embedding-data/QQP",
+      "dataset:embedding-data/SPECTER",
+      "dataset:embedding-data/PAQ_pairs",
+      "dataset:embedding-data/WikiAnswers",
+      "arxiv:1904.06472",
+      "arxiv:2102.07033",
+      "arxiv:2104.08727",
+      "arxiv:1704.05179",
+      "arxiv:1810.09305",
+      "license:apache-2.0",
+      "autotrain_compatible",
+      "text-embeddings-inference",
+      "endpoints_compatible",
+      "region:us"
+    ],
+    "task": "sentence-similarity",
+    "likes": 3788,
+    "downloads": 91944061,
+    "updatedAt": "2025-03-06T13:37:44.000Z",
+    "visibility": "public",
+    "access": "accessible",
+    "cardData": {
+      "language": "en",
+      "license": "apache-2.0",
+      "library_name": "sentence-transformers",
+      "tags": [
+        "sentence-transformers",
+        "feature-extraction",
+        "sentence-similarity",
+        "transformers"
+      ],
+      "datasets": [
+        "s2orc",
+        "flax-sentence-embeddings/stackexchange_xml",
+        "ms_marco",
+        "gooaq",
+        "yahoo_answers_topics",
+        "code_search_net",
+        "search_qa",
+        "eli5",
+        "snli",
+        "multi_nli",
+        "wikihow",
+        "natural_questions",
+        "trivia_qa",
+        "embedding-data/sentence-compression",
+        "embedding-data/flickr30k-captions",
+        "embedding-data/altlex",
+        "embedding-data/simple-wiki",
+        "embedding-data/QQP",
+        "embedding-data/SPECTER",
+        "embedding-data/PAQ_pairs",
+        "embedding-data/WikiAnswers"
+      ],
+      "pipeline_tag": "sentence-similarity"
+    },
+    "siblings": [
+      {
+        "rfilename": ".gitattributes"
+      },
+      {
+        "rfilename": "1_Pooling/config.json"
+      },
+      {
+        "rfilename": "README.md"
+      },
+      {
+        "rfilename": "config.json"
+      },
+      {
+        "rfilename": "config_sentence_transformers.json"
+      },
+      {
+        "rfilename": "data_config.json"
+      },
+      {
+        "rfilename": "model.safetensors"
+      },
+      {
+        "rfilename": "modules.json"
+      },
+      {
+        "rfilename": "onnx/model.onnx"
+      },
+      {
+        "rfilename": "onnx/model_O1.onnx"
+      },
+      {
+        "rfilename": "onnx/model_O2.onnx"
+      },
+      {
+        "rfilename": "onnx/model_O3.onnx"
+      },
+      {
+        "rfilename": "onnx/model_O4.onnx"
+      },
+      {
+        "rfilename": "onnx/model_qint8_arm64.onnx"
+      },
+      {
+        "rfilename": "onnx/model_qint8_avx512.onnx"
+      },
+      {
+        "rfilename": "onnx/model_qint8_avx512_vnni.onnx"
+      },
+      {
+        "rfilename": "onnx/model_quint8_avx2.onnx"
+      },
+      {
+        "rfilename": "openvino/openvino_model.bin"
+      },
+      {
+        "rfilename": "openvino/openvino_model.xml"
+      },
+      {
+        "rfilename": "openvino/openvino_model_qint8_quantized.bin"
+      },
+      {
+        "rfilename": "openvino/openvino_model_qint8_quantized.xml"
+      },
+      {
+        "rfilename": "pytorch_model.bin"
+      },
+      {
+        "rfilename": "rust_model.ot"
+      },
+      {
+        "rfilename": "sentence_bert_config.json"
+      },
+      {
+        "rfilename": "special_tokens_map.json"
+      },
+      {
+        "rfilename": "tf_model.h5"
+      },
+      {
+        "rfilename": "tokenizer.json"
+      },
+      {
+        "rfilename": "tokenizer_config.json"
+      },
+      {
+        "rfilename": "train_script.py"
+      },
+      {
+        "rfilename": "vocab.txt"
+      }
+    ]
+  },
+  "status": 200,
+  "error": ""
+}
+--- STDERR ---
+
+[OK] test_hf_model_info.py
+--------------------------------------------------------------------------------
+
+=== Running test_hf_datasets_search.py ===
+Command: C:\Users\daqc\Documents\GitHub\open-deep-research-vulnerability-intelligence\venv\Scripts\python.exe C:\Users\daqc\Documents\GitHub\open-deep-research-vulnerability-intelligence\tests\test_hf_datasets_search.py
+--- INPUT (snippet) ---
+tool.forward(
+        query="sentiment analysis",
+        tags="language:es",
+        sort="downloads",
+        direction="descending",
+        limit=5,
+    )
+    try:
+        data = json.loads(result_json_str)
+        print(json.dumps(data, indent=2, ensure_ascii=False))
+    except Exception:
+        print(result_json_str)
+
+
+if __name__ == "__main__":
+    main()
+
+
+--- STDOUT ---
+{
+  "results": [
+    {
+      "type": "dataset",
+      "id": "Renukswamy/Patent_sentiment_analysis",
+      "owner": "Renukswamy",
+      "url": "https://huggingface.co/Renukswamy/Patent_sentiment_analysis",
+      "description": "",
+      "tags": [
+        "size_categories:n<1K",
+        "format:csv",
+        "modality:tabular",
+        "modality:text",
+        "library:datasets",
+        "library:pandas",
+        "library:mlcroissant",
+        "library:polars",
+        "region:us"
+      ],
+      "likes": 2,
+      "downloads": 12,
+      "updatedAt": "2021-11-26T09:18:15.000Z",
+      "visibility": "public",
+      "access": "accessible"
+    },
+    {
+      "type": "dataset",
+      "id": "maydogan/Turkish_SentimentAnalysis_TRSAv1",
+      "owner": "maydogan",
+      "url": "https://huggingface.co/maydogan/Turkish_SentimentAnalysis_TRSAv1",
+      "description": "",
+      "tags": [
+        "task_categories:text-classification",
+        "language:tr",
+        "size_categories:100K<n<1M",
+        "format:csv",
+        "modality:text",
+        "library:datasets",
+        "library:pandas",
+        "library:mlcroissant",
+        "library:polars",
+        "region:us"
+      ],
+      "likes": 6,
+      "downloads": 161,
+      "updatedAt": "2024-10-07T14:16:56.000Z",
+      "visibility": "public",
+      "access": "accessible"
+    },
+    {
+      "type": "dataset",
+      "id": "winvoker/turkish-sentiment-analysis-dataset",
+      "owner": "winvoker",
+      "url": "https://huggingface.co/winvoker/turkish-sentiment-analysis-dataset",
+      "description": "",
+      "tags": [
+        "task_categories:text-classification",
+        "task_ids:sentiment-classification",
+        "annotations_creators:crowdsourced",
+        "annotations_creators:expert-generated",
+        "language_creators:crowdsourced",
+        "multilinguality:monolingual",
+        "language:tr",
+        "license:cc-by-sa-4.0",
+        "size_categories:100K<n<1M",
+        "format:csv",
+        "modality:text",
+        "library:datasets",
+        "library:pandas",
+        "library:mlcroissant",
+        "library:polars",
+        "region:us"
+      ],
+      "likes": 47,
+      "downloads": 459,
+      "updatedAt": "2023-07-19T13:15:13.000Z",
+      "visibility": "public",
+      "access": "accessible"
+    },
+    {
+      "type": "dataset",
+      "id": "ramnika003/autotrain-data-sentiment_analysis_project",
+      "owner": "ramnika003",
+      "url": "https://huggingface.co/ramnika003/autotrain-data-sentiment_analysis_project",
+      "description": "",
+      "tags": [
+        "task_categories:text-classification",
+        "region:us"
+      ],
+      "likes": 0,
+      "downloads": 160,
+      "updatedAt": "2022-04-05T09:16:59.000Z",
+      "visibility": "public",
+      "access": "accessible"
+    },
+    {
+      "type": "dataset",
+      "id": "elmurod1202/uzbek-sentiment-analysis",
+      "owner": "elmurod1202",
+      "url": "https://huggingface.co/elmurod1202/uzbek-sentiment-analysis",
+      "description": "",
+      "tags": [
+        "size_categories:10K<n<100K",
+        "format:text",
+        "modality:image",
+        "modality:text",
+        "library:datasets",
+        "library:mlcroissant",
+        "region:us"
+      ],
+      "likes": 3,
+      "downloads": 236,
+      "updatedAt": "2022-05-11T13:43:59.000Z",
+      "visibility": "public",
+      "access": "accessible"
+    }
+  ],
+  "status": 200,
+  "error": "",
+  "params": {
+    "search": "sentiment analysis",
+    "tags": "language:es",
+    "sort": "downloads",
+    "direction": "descending",
+    "limit": 5
+  }
+}
+--- STDERR ---
+
+[OK] test_hf_datasets_search.py
+--------------------------------------------------------------------------------
+
+=== Running test_hf_dataset_info.py ===
+Command: C:\Users\daqc\Documents\GitHub\open-deep-research-vulnerability-intelligence\venv\Scripts\python.exe C:\Users\daqc\Documents\GitHub\open-deep-research-vulnerability-intelligence\tests\test_hf_dataset_info.py
+--- INPUT (snippet) ---
+tool.forward(repo_id=repo_id)
+    try:
+        data = json.loads(result_json_str)
+        print(json.dumps(data, indent=2, ensure_ascii=False))
+    except Exception:
+        print(result_json_str)
+
+
+if __name__ == "__main__":
+    main()
+
+
+--- STDOUT ---
+{
+  "item": {
+    "type": "dataset",
+    "id": "nyu-mll/glue",
+    "owner": "nyu-mll",
+    "url": "https://huggingface.co/nyu-mll/glue",
+    "description": "",
+    "tags": [
+      "task_categories:text-classification",
+      "task_ids:acceptability-classification",
+      "task_ids:natural-language-inference",
+      "task_ids:semantic-similarity-scoring",
+      "task_ids:sentiment-classification",
+      "task_ids:text-scoring",
+      "annotations_creators:other",
+      "language_creators:other",
+      "multilinguality:monolingual",
+      "source_datasets:original",
+      "language:en",
+      "license:other",
+      "size_categories:1M<n<10M",
+      "format:parquet",
+      "modality:tabular",
+      "modality:text",
+      "library:datasets",
+      "library:pandas",
+      "library:mlcroissant",
+      "library:polars",
+      "arxiv:1804.07461",
+      "region:us",
+      "qa-nli",
+      "coreference-nli",
+      "paraphrase-identification"
+    ],
+    "likes": 435,
+    "downloads": 313025,
+    "updatedAt": "2024-01-30T07:41:18.000Z",
+    "visibility": "public",
+    "access": "accessible",
+    "cardData": {
+      "annotations_creators": [
+        "other"
+      ],
+      "language_creators": [
+        "other"
+      ],
+      "language": [
+        "en"
+      ],
+      "license": [
+        "other"
+      ],
+      "multilinguality": [
+        "monolingual"
+      ],
+      "size_categories": [
+        "10K<n<100K"
+      ],
+      "source_datasets": [
+        "original"
+      ],
+      "task_categories": [
+        "text-classification"
+      ],
+      "task_ids": [
+        "acceptability-classification",
+        "natural-language-inference",
+        "semantic-similarity-scoring",
+        "sentiment-classification",
+        "text-scoring"
+      ],
+      "paperswithcode_id": "glue",
+      "pretty_name": "GLUE (General Language Understanding Evaluation benchmark)",
+      "config_names": [
+        "ax",
+        "cola",
+        "mnli",
+        "mnli_matched",
+        "mnli_mismatched",
+        "mrpc",
+        "qnli",
+        "qqp",
+        "rte",
+        "sst2",
+        "stsb",
+        "wnli"
+      ],
+      "tags": [
+        "qa-nli",
+        "coreference-nli",
+        "paraphrase-identification"
+      ],
+      "dataset_info": [
+        {
+          "config_name": "ax",
+          "features": [
+            {
+              "name": "premise",
+              "dtype": "string"
+            },
+            {
+              "name": "hypothesis",
+              "dtype": "string"
+            },
+            {
+              "name": "label",
+              "dtype": {
+                "class_label": {
+                  "names": {
+                    "0": "entailment",
+                    "1": "neutral",
+                    "2": "contradiction"
+                  }
+                }
+              }
+            },
+            {
+              "name": "idx",
+              "dtype": "int32"
+            }
+          ],
+          "splits": [
+            {
+              "name": "test",
+              "num_bytes": 237694,
+              "num_examples": 1104
+            }
+          ],
+          "download_size": 80767,
+          "dataset_size": 237694
+        },
+        {
+          "config_name": "cola",
+          "features": [
+            {
+              "name": "sentence",
+              "dtype": "string"
+            },
+            {
+              "name": "label",
+              "dtype": {
+                "class_label": {
+                  "names": {
+                    "0": "unacceptable",
+                    "1": "acceptable"
+                  }
+                }
+              }
+            },
+            {
+              "name": "idx",
+              "dtype": "int32"
+            }
+          ],
+          "splits": [
+            {
+              "name": "train",
+              "num_bytes": 484869,
+              "num_examples": 8551
+            },
+            {
+              "name": "validation",
+              "num_bytes": 60322,
+              "num_examples": 1043
+            },
+            {
+              "name": "test",
+              "num_bytes": 60513,
+              "num_examples": 1063
+            }
+          ],
+          "download_size": 326394,
+          "dataset_size": 605704
+        },
+        {
+          "config_name": "mnli",
+          "features": [
+            {
+              "name": "premise",
+              "dtype": "string"
+            },
+            {
+              "name": "hypothesis",
+              "dtype": "string"
+            },
+            {
+              "name": "label",
+              "dtype": {
+                "class_label": {
+                  "names": {
+                    "0": "entailment",
+                    "1": "neutral",
+                    "2": "contradiction"
+                  }
+                }
+              }
+            },
+            {
+              "name": "idx",
+              "dtype": "int32"
+            }
+          ],
+          "splits": [
+            {
+              "name": "train",
+              "num_bytes": 74619646,
+              "num_examples": 392702
+            },
+            {
+              "name": "validation_matched",
+              "num_bytes": 1833783,
+              "num_examples": 9815
+            },
+            {
+              "name": "validation_mismatched",
+              "num_bytes": 1949231,
+              "num_examples": 9832
+            },
+            {
+              "name": "test_matched",
+              "num_bytes": 1848654,
+              "num_examples": 9796
+            },
+            {
+              "name": "test_mismatched",
+              "num_bytes": 1950703,
+              "num_examples": 9847
+            }
+          ],
+          "download_size": 57168425,
+          "dataset_size": 82202017
+        },
+        {
+          "config_name": "mnli_matched",
+          "features": [
+            {
+              "name": "premise",
+              "dtype": "string"
+            },
+            {
+              "name": "hypothesis",
+              "dtype": "string"
+            },
+            {
+              "name": "label",
+              "dtype": {
+                "class_label": {
+                  "names": {
+                    "0": "entailment",
+                    "1": "neutral",
+                    "2": "contradiction"
+                  }
+                }
+              }
+            },
+            {
+              "name": "idx",
+              "dtype": "int32"
+            }
+          ],
+          "splits": [
+            {
+              "name": "validation",
+              "num_bytes": 1833783,
+              "num_examples": 9815
+            },
+            {
+              "name": "test",
+              "num_bytes": 1848654,
+              "num_examples": 9796
+            }
+          ],
+          "download_size": 2435055,
+          "dataset_size": 3682437
+        },
+        {
+          "config_name": "mnli_mismatched",
+          "features": [
+            {
+              "name": "premise",
+              "dtype": "string"
+            },
+            {
+              "name": "hypothesis",
+              "dtype": "string"
+            },
+            {
+              "name": "label",
+              "dtype": {
+                "class_label": {
+                  "names": {
+                    "0": "entailment",
+                    "1": "neutral",
+                    "2": "contradiction"
+                  }
+                }
+              }
+            },
+            {
+              "name": "idx",
+              "dtype": "int32"
+            }
+          ],
+          "splits": [
+            {
+              "name": "validation",
+              "num_bytes": 1949231,
+              "num_examples": 9832
+            },
+            {
+              "name": "test",
+              "num_bytes": 1950703,
+              "num_examples": 9847
+            }
+          ],
+          "download_size": 2509009,
+          "dataset_size": 3899934
+        },
+        {
+          "config_name": "mrpc",
+          "features": [
+            {
+              "name": "sentence1",
+              "dtype": "string"
+            },
+            {
+              "name": "sentence2",
+              "dtype": "string"
+            },
+            {
+              "name": "label",
+              "dtype": {
+                "class_label": {
+                  "names": {
+                    "0": "not_equivalent",
+                    "1": "equivalent"
+                  }
+                }
+              }
+            },
+            {
+              "name": "idx",
+              "dtype": "int32"
+            }
+          ],
+          "splits": [
+            {
+              "name": "train",
+              "num_bytes": 943843,
+              "num_examples": 3668
+            },
+            {
+              "name": "validation",
+              "num_bytes": 105879,
+              "num_examples": 408
+            },
+            {
+              "name": "test",
+              "num_bytes": 442410,
+              "num_examples": 1725
+            }
+          ],
+          "download_size": 1033400,
+          "dataset_size": 1492132
+        },
+        {
+          "config_name": "qnli",
+          "features": [
+            {
+              "name": "question",
+              "dtype": "string"
+            },
+            {
+              "name": "sentence",
+              "dtype": "string"
+            },
+            {
+              "name": "label",
+              "dtype": {
+                "class_label": {
+                  "names": {
+                    "0": "entailment",
+                    "1": "not_entailment"
+                  }
+                }
+              }
+            },
+            {
+              "name": "idx",
+              "dtype": "int32"
+            }
+          ],
+          "splits": [
+            {
+              "name": "train",
+              "num_bytes": 25612443,
+              "num_examples": 104743
+            },
+            {
+              "name": "validation",
+              "num_bytes": 1368304,
+              "num_examples": 5463
+            },
+            {
+              "name": "test",
+              "num_bytes": 1373093,
+              "num_examples": 5463
+            }
+          ],
+          "download_size": 19278324,
+          "dataset_size": 28353840
+        },
+        {
+          "config_name": "qqp",
+          "features": [
+            {
+              "name": "question1",
+              "dtype": "string"
+            },
+            {
+              "name": "question2",
+              "dtype": "string"
+            },
+            {
+              "name": "label",
+              "dtype": {
+                "class_label": {
+                  "names": {
+                    "0": "not_duplicate",
+                    "1": "duplicate"
+                  }
+                }
+              }
+            },
+            {
+              "name": "idx",
+              "dtype": "int32"
+            }
+          ],
+          "splits": [
+            {
+              "name": "train",
+              "num_bytes": 50900820,
+              "num_examples": 363846
+            },
+            {
+              "name": "validation",
+              "num_bytes": 5653754,
+              "num_examples": 40430
+            },
+            {
+              "name": "test",
+              "num_bytes": 55171111,
+              "num_examples": 390965
+            }
+          ],
+          "download_size": 73982265,
+          "dataset_size": 111725685
+        },
+        {
+          "config_name": "rte",
+          "features": [
+            {
+              "name": "sentence1",
+              "dtype": "string"
+            },
+            {
+              "name": "sentence2",
+              "dtype": "string"
+            },
+            {
+              "name": "label",
+              "dtype": {
+                "class_label": {
+                  "names": {
+                    "0": "entailment",
+                    "1": "not_entailment"
+                  }
+                }
+              }
+            },
+            {
+              "name": "idx",
+              "dtype": "int32"
+            }
+          ],
+          "splits": [
+            {
+              "name": "train",
+              "num_bytes": 847320,
+              "num_examples": 2490
+            },
+            {
+              "name": "validation",
+              "num_bytes": 90728,
+              "num_examples": 277
+            },
+            {
+              "name": "test",
+              "num_bytes": 974053,
+              "num_examples": 3000
+            }
+          ],
+          "download_size": 1274409,
+          "dataset_size": 1912101
+        },
+        {
+          "config_name": "sst2",
+          "features": [
+            {
+              "name": "sentence",
+              "dtype": "string"
+            },
+            {
+              "name": "label",
+              "dtype": {
+                "class_label": {
+                  "names": {
+                    "0": "negative",
+                    "1": "positive"
+                  }
+                }
+              }
+            },
+            {
+              "name": "idx",
+              "dtype": "int32"
+            }
+          ],
+          "splits": [
+            {
+              "name": "train",
+              "num_bytes": 4681603,
+              "num_examples": 67349
+            },
+            {
+              "name": "validation",
+              "num_bytes": 106252,
+              "num_examples": 872
+            },
+            {
+              "name": "test",
+              "num_bytes": 216640,
+              "num_examples": 1821
+            }
+          ],
+          "download_size": 3331080,
+          "dataset_size": 5004495
+        },
+        {
+          "config_name": "stsb",
+          "features": [
+            {
+              "name": "sentence1",
+              "dtype": "string"
+            },
+            {
+              "name": "sentence2",
+              "dtype": "string"
+            },
+            {
+              "name": "label",
+              "dtype": "float32"
+            },
+            {
+              "name": "idx",
+              "dtype": "int32"
+            }
+          ],
+          "splits": [
+            {
+              "name": "train",
+              "num_bytes": 754791,
+              "num_examples": 5749
+            },
+            {
+              "name": "validation",
+              "num_bytes": 216064,
+              "num_examples": 1500
+            },
+            {
+              "name": "test",
+              "num_bytes": 169974,
+              "num_examples": 1379
+            }
+          ],
+          "download_size": 766983,
+          "dataset_size": 1140829
+        },
+        {
+          "config_name": "wnli",
+          "features": [
+            {
+              "name": "sentence1",
+              "dtype": "string"
+            },
+            {
+              "name": "sentence2",
+              "dtype": "string"
+            },
+            {
+              "name": "label",
+              "dtype": {
+                "class_label": {
+                  "names": {
+                    "0": "not_entailment",
+                    "1": "entailment"
+                  }
+                }
+              }
+            },
+            {
+              "name": "idx",
+              "dtype": "int32"
+            }
+          ],
+          "splits": [
+            {
+              "name": "train",
+              "num_bytes": 107109,
+              "num_examples": 635
+            },
+            {
+              "name": "validation",
+              "num_bytes": 12162,
+              "num_examples": 71
+            },
+            {
+              "name": "test",
+              "num_bytes": 37889,
+              "num_examples": 146
+            }
+          ],
+          "download_size": 63522,
+          "dataset_size": 157160
+        }
+      ],
+      "configs": [
+        {
+          "config_name": "ax",
+          "data_files": [
+            {
+              "split": "test",
+              "path": "ax/test-*"
+            }
+          ]
+        },
+        {
+          "config_name": "cola",
+          "data_files": [
+            {
+              "split": "train",
+              "path": "cola/train-*"
+            },
+            {
+              "split": "validation",
+              "path": "cola/validation-*"
+            },
+            {
+              "split": "test",
+              "path": "cola/test-*"
+            }
+          ]
+        },
+        {
+          "config_name": "mnli",
+          "data_files": [
+            {
+              "split": "train",
+              "path": "mnli/train-*"
+            },
+            {
+              "split": "validation_matched",
+              "path": "mnli/validation_matched-*"
+            },
+            {
+              "split": "validation_mismatched",
+              "path": "mnli/validation_mismatched-*"
+            },
+            {
+              "split": "test_matched",
+              "path": "mnli/test_matched-*"
+            },
+            {
+              "split": "test_mismatched",
+              "path": "mnli/test_mismatched-*"
+            }
+          ]
+        },
+        {
+          "config_name": "mnli_matched",
+          "data_files": [
+            {
+              "split": "validation",
+              "path": "mnli_matched/validation-*"
+            },
+            {
+              "split": "test",
+              "path": "mnli_matched/test-*"
+            }
+          ]
+        },
+        {
+          "config_name": "mnli_mismatched",
+          "data_files": [
+            {
+              "split": "validation",
+              "path": "mnli_mismatched/validation-*"
+            },
+            {
+              "split": "test",
+              "path": "mnli_mismatched/test-*"
+            }
+          ]
+        },
+        {
+          "config_name": "mrpc",
+          "data_files": [
+            {
+              "split": "train",
+              "path": "mrpc/train-*"
+            },
+            {
+              "split": "validation",
+              "path": "mrpc/validation-*"
+            },
+            {
+              "split": "test",
+              "path": "mrpc/test-*"
+            }
+          ]
+        },
+        {
+          "config_name": "qnli",
+          "data_files": [
+            {
+              "split": "train",
+              "path": "qnli/train-*"
+            },
+            {
+              "split": "validation",
+              "path": "qnli/validation-*"
+            },
+            {
+              "split": "test",
+              "path": "qnli/test-*"
+            }
+          ]
+        },
+        {
+          "config_name": "qqp",
+          "data_files": [
+            {
+              "split": "train",
+              "path": "qqp/train-*"
+            },
+            {
+              "split": "validation",
+              "path": "qqp/validation-*"
+            },
+            {
+              "split": "test",
+              "path": "qqp/test-*"
+            }
+          ]
+        },
+        {
+          "config_name": "rte",
+          "data_files": [
+            {
+              "split": "train",
+              "path": "rte/train-*"
+            },
+            {
+              "split": "validation",
+              "path": "rte/validation-*"
+            },
+            {
+              "split": "test",
+              "path": "rte/test-*"
+            }
+          ]
+        },
+        {
+          "config_name": "sst2",
+          "data_files": [
+            {
+              "split": "train",
+              "path": "sst2/train-*"
+            },
+            {
+              "split": "validation",
+              "path": "sst2/validation-*"
+            },
+            {
+              "split": "test",
+              "path": "sst2/test-*"
+            }
+          ]
+        },
+        {
+          "config_name": "stsb",
+          "data_files": [
+            {
+              "split": "train",
+              "path": "stsb/train-*"
+            },
+            {
+              "split": "validation",
+              "path": "stsb/validation-*"
+            },
+            {
+              "split": "test",
+              "path": "stsb/test-*"
+            }
+          ]
+        },
+        {
+          "config_name": "wnli",
+          "data_files": [
+            {
+              "split": "train",
+              "path": "wnli/train-*"
+            },
+            {
+              "split": "validation",
+              "path": "wnli/validation-*"
+            },
+            {
+              "split": "test",
+              "path": "wnli/test-*"
+            }
+          ]
+        }
+      ],
+      "train-eval-index": [
+        {
+          "config": "cola",
+          "task": "text-classification",
+          "task_id": "binary_classification",
+          "splits": {
+            "train_split": "train",
+            "eval_split": "validation"
+          },
+          "col_mapping": {
+            "sentence": "text",
+            "label": "target"
+          }
+        },
+        {
+          "config": "sst2",
+          "task": "text-classification",
+          "task_id": "binary_classification",
+          "splits": {
+            "train_split": "train",
+            "eval_split": "validation"
+          },
+          "col_mapping": {
+            "sentence": "text",
+            "label": "target"
+          }
+        },
+        {
+          "config": "mrpc",
+          "task": "text-classification",
+          "task_id": "natural_language_inference",
+          "splits": {
+            "train_split": "train",
+            "eval_split": "validation"
+          },
+          "col_mapping": {
+            "sentence1": "text1",
+            "sentence2": "text2",
+            "label": "target"
+          }
+        },
+        {
+          "config": "qqp",
+          "task": "text-classification",
+          "task_id": "natural_language_inference",
+          "splits": {
+            "train_split": "train",
+            "eval_split": "validation"
+          },
+          "col_mapping": {
+            "question1": "text1",
+            "question2": "text2",
+            "label": "target"
+          }
+        },
+        {
+          "config": "stsb",
+          "task": "text-classification",
+          "task_id": "natural_language_inference",
+          "splits": {
+            "train_split": "train",
+            "eval_split": "validation"
+          },
+          "col_mapping": {
+            "sentence1": "text1",
+            "sentence2": "text2",
+            "label": "target"
+          }
+        },
+        {
+          "config": "mnli",
+          "task": "text-classification",
+          "task_id": "natural_language_inference",
+          "splits": {
+            "train_split": "train",
+            "eval_split": "validation_matched"
+          },
+          "col_mapping": {
+            "premise": "text1",
+            "hypothesis": "text2",
+            "label": "target"
+          }
+        },
+        {
+          "config": "mnli_mismatched",
+          "task": "text-classification",
+          "task_id": "natural_language_inference",
+          "splits": {
+            "train_split": "train",
+            "eval_split": "validation"
+          },
+          "col_mapping": {
+            "premise": "text1",
+            "hypothesis": "text2",
+            "label": "target"
+          }
+        },
+        {
+          "config": "mnli_matched",
+          "task": "text-classification",
+          "task_id": "natural_language_inference",
+          "splits": {
+            "train_split": "train",
+            "eval_split": "validation"
+          },
+          "col_mapping": {
+            "premise": "text1",
+            "hypothesis": "text2",
+            "label": "target"
+          }
+        },
+        {
+          "config": "qnli",
+          "task": "text-classification",
+          "task_id": "natural_language_inference",
+          "splits": {
+            "train_split": "train",
+            "eval_split": "validation"
+          },
+          "col_mapping": {
+            "question": "text1",
+            "sentence": "text2",
+            "label": "target"
+          }
+        },
+        {
+          "config": "rte",
+          "task": "text-classification",
+          "task_id": "natural_language_inference",
+          "splits": {
+            "train_split": "train",
+            "eval_split": "validation"
+          },
+          "col_mapping": {
+            "sentence1": "text1",
+            "sentence2": "text2",
+            "label": "target"
+          }
+        },
+        {
+          "config": "wnli",
+          "task": "text-classification",
+          "task_id": "natural_language_inference",
+          "splits": {
+            "train_split": "train",
+            "eval_split": "validation"
+          },
+          "col_mapping": {
+            "sentence1": "text1",
+            "sentence2": "text2",
+            "label": "target"
+          }
+        }
+      ]
+    },
+    "siblings": [
+      {
+        "rfilename": ".gitattributes"
+      },
+      {
+        "rfilename": "README.md"
+      },
+      {
+        "rfilename": "ax/test-00000-of-00001.parquet"
+      },
+      {
+        "rfilename": "cola/test-00000-of-00001.parquet"
+      },
+      {
+        "rfilename": "cola/train-00000-of-00001.parquet"
+      },
+      {
+        "rfilename": "cola/validation-00000-of-00001.parquet"
+      },
+      {
+        "rfilename": "mnli/test_matched-00000-of-00001.parquet"
+      },
+      {
+        "rfilename": "mnli/test_mismatched-00000-of-00001.parquet"
+      },
+      {
+        "rfilename": "mnli/train-00000-of-00001.parquet"
+      },
+      {
+        "rfilename": "mnli/validation_matched-00000-of-00001.parquet"
+      },
+      {
+        "rfilename": "mnli/validation_mismatched-00000-of-00001.parquet"
+      },
+      {
+        "rfilename": "mnli_matched/test-00000-of-00001.parquet"
+      },
+      {
+        "rfilename": "mnli_matched/validation-00000-of-00001.parquet"
+      },
+      {
+        "rfilename": "mnli_mismatched/test-00000-of-00001.parquet"
+      },
+      {
+        "rfilename": "mnli_mismatched/validation-00000-of-00001.parquet"
+      },
+      {
+        "rfilename": "mrpc/test-00000-of-00001.parquet"
+      },
+      {
+        "rfilename": "mrpc/train-00000-of-00001.parquet"
+      },
+      {
+        "rfilename": "mrpc/validation-00000-of-00001.parquet"
+      },
+      {
+        "rfilename": "qnli/test-00000-of-00001.parquet"
+      },
+      {
+        "rfilename": "qnli/train-00000-of-00001.parquet"
+      },
+      {
+        "rfilename": "qnli/validation-00000-of-00001.parquet"
+      },
+      {
+        "rfilename": "qqp/test-00000-of-00001.parquet"
+      },
+      {
+        "rfilename": "qqp/train-00000-of-00001.parquet"
+      },
+      {
+        "rfilename": "qqp/validation-00000-of-00001.parquet"
+      },
+      {
+        "rfilename": "rte/test-00000-of-00001.parquet"
+      },
+      {
+        "rfilename": "rte/train-00000-of-00001.parquet"
+      },
+      {
+        "rfilename": "rte/validation-00000-of-00001.parquet"
+      },
+      {
+        "rfilename": "sst2/test-00000-of-00001.parquet"
+      },
+      {
+        "rfilename": "sst2/train-00000-of-00001.parquet"
+      },
+      {
+        "rfilename": "sst2/validation-00000-of-00001.parquet"
+      },
+      {
+        "rfilename": "stsb/test-00000-of-00001.parquet"
+      },
+      {
+        "rfilename": "stsb/train-00000-of-00001.parquet"
+      },
+      {
+        "rfilename": "stsb/validation-00000-of-00001.parquet"
+      },
+      {
+        "rfilename": "wnli/test-00000-of-00001.parquet"
+      },
+      {
+        "rfilename": "wnli/train-00000-of-00001.parquet"
+      },
+      {
+        "rfilename": "wnli/validation-00000-of-00001.parquet"
+      }
+    ]
+  },
+  "status": 200,
+  "error": ""
+}
+--- STDERR ---
+
+[OK] test_hf_dataset_info.py
+--------------------------------------------------------------------------------
+
+=== Running test_hf_spaces_search.py ===
+Command: C:\Users\daqc\Documents\GitHub\open-deep-research-vulnerability-intelligence\venv\Scripts\python.exe C:\Users\daqc\Documents\GitHub\open-deep-research-vulnerability-intelligence\tests\test_hf_spaces_search.py
+--- INPUT (snippet) ---
+tool.forward(
+        query="document Q&A",
+        sort="likes",
+        direction="descending",
+        limit=5,
+    )
+    try:
+        data = json.loads(result_json_str)
+        print(json.dumps(data, indent=2, ensure_ascii=False))
+    except Exception:
+        print(result_json_str)
+
+
+if __name__ == "__main__":
+    main()
+
+
+--- STDOUT ---
+{
+  "results": [],
+  "status": 200,
+  "error": "",
+  "params": {
+    "search": "document Q&A",
+    "sort": "likes",
+    "direction": "descending",
+    "limit": 5
+  }
+}
+--- STDERR ---
+
+[OK] test_hf_spaces_search.py
+--------------------------------------------------------------------------------
+
+=== Running test_hf_space_info.py ===
+Command: C:\Users\daqc\Documents\GitHub\open-deep-research-vulnerability-intelligence\venv\Scripts\python.exe C:\Users\daqc\Documents\GitHub\open-deep-research-vulnerability-intelligence\tests\test_hf_space_info.py
+--- INPUT (snippet) ---
+tool.forward(repo_id=repo_id)
+    try:
+        data = json.loads(result_json_str)
+        print(json.dumps(data, indent=2, ensure_ascii=False))
+    except Exception:
+        # Fallback: encode to UTF-8 to avoid Windows cp1252 issues
+        try:
+            print(result_json_str.encode("utf-8", errors="replace").decode("utf-8", errors="replace"))
+        except Exception:
+            print("<unprintable> due to encoding error")
+
+
+if __name__ == "__main__":
+    main()
+
+
+--- STDOUT ---
+<unprintable> due to encoding error
+--- STDERR ---
+
+[OK] test_hf_space_info.py
+--------------------------------------------------------------------------------
+
+=== Running test_hf_user_info.py ===
+Command: C:\Users\daqc\Documents\GitHub\open-deep-research-vulnerability-intelligence\venv\Scripts\python.exe C:\Users\daqc\Documents\GitHub\open-deep-research-vulnerability-intelligence\tests\test_hf_user_info.py
+--- INPUT (snippet) ---
+tool.forward(username=username)
+    try:
+        data = json.loads(result_json_str)
+        print(json.dumps(data, indent=2, ensure_ascii=False))
+    except Exception:
+        print(result_json_str)
+
+
+if __name__ == "__main__":
+    main()
+
+
+--- STDOUT ---
+{
+  "item": {},
+  "status": 404,
+  "error": "http_404",
+  "visibility": "public",
+  "access": "no_access"
+}
+--- STDERR ---
+
+[OK] test_hf_user_info.py
+--------------------------------------------------------------------------------
+
+=== Running test_hf_collections_list.py ===
+Command: C:\Users\daqc\Documents\GitHub\open-deep-research-vulnerability-intelligence\venv\Scripts\python.exe C:\Users\daqc\Documents\GitHub\open-deep-research-vulnerability-intelligence\tests\test_hf_collections_list.py
+--- INPUT (snippet) ---
+tool.forward(owner=None)  # or set an owner like "huggingface"
+    try:
+        data = json.loads(result_json_str)
+        print(json.dumps(data, indent=2, ensure_ascii=False))
+    except Exception:
+        print(result_json_str)
+
+
+if __name__ == "__main__":
+    main()
+
+
+--- STDOUT ---
+{
+  "results": [
+    {
+      "type": "collection",
+      "id": "deepseek-ai/deepseek-v31-68a491bed32bd77e7fca048f",
+      "owner": "deepseek-ai",
+      "title": "DeepSeek-V3.1",
+      "url": "https://huggingface.co/collections/deepseek-ai/deepseek-ai/deepseek-v31-68a491bed32bd77e7fca048f",
+      "visibility": "public",
+      "access": "accessible"
+    },
+    {
+      "type": "collection",
+      "id": "facebook/dinov3-68924841bd6b561778e31009",
+      "owner": "facebook",
+      "title": "DINOv3",
+      "url": "https://huggingface.co/collections/facebook/facebook/dinov3-68924841bd6b561778e31009",
+      "visibility": "public",
+      "access": "accessible"
+    },
+    {
+      "type": "collection",
+      "id": "AIDC-AI/ovis25-689ec1474633b2aab8809335",
+      "owner": "AIDC-AI",
+      "title": "Ovis2.5",
+      "url": "https://huggingface.co/collections/AIDC-AI/AIDC-AI/ovis25-689ec1474633b2aab8809335",
+      "visibility": "public",
+      "access": "accessible"
+    },
+    {
+      "type": "collection",
+      "id": "nvidia/nvidia-nemotron-689f6d6e6ead8e77dd641615",
+      "owner": "nvidia",
+      "title": "NVIDIA Nemotron",
+      "url": "https://huggingface.co/collections/nvidia/nvidia/nvidia-nemotron-689f6d6e6ead8e77dd641615",
+      "visibility": "public",
+      "access": "accessible"
+    },
+    {
+      "type": "collection",
+      "id": "ByteDance-Seed/seed-oss-68a609f4201e788db05b5dcd",
+      "owner": "ByteDance-Seed",
+      "title": "Seed-OSS",
+      "url": "https://huggingface.co/collections/ByteDance-Seed/ByteDance-Seed/seed-oss-68a609f4201e788db05b5dcd",
+      "visibility": "public",
+      "access": "accessible"
+    },
+    {
+      "type": "collection",
+      "id": "google/gemma-3-release-67c6c6f89c4f76621268bb6d",
+      "owner": "google",
+      "title": "Gemma 3 Release",
+      "url": "https://huggingface.co/collections/google/google/gemma-3-release-67c6c6f89c4f76621268bb6d",
+      "visibility": "public",
+      "access": "accessible"
+    },
+    {
+      "type": "collection",
+      "id": "openai/gpt-oss-68911959590a1634ba11c7a4",
+      "owner": "openai",
+      "title": "gpt-oss",
+      "url": "https://huggingface.co/collections/openai/openai/gpt-oss-68911959590a1634ba11c7a4",
+      "visibility": "public",
+      "access": "accessible"
+    },
+    {
+      "type": "collection",
+      "id": "Qwen/qwen3-67dd247413f0e2e4f653967f",
+      "owner": "Qwen",
+      "title": "Qwen3",
+      "url": "https://huggingface.co/collections/Qwen/Qwen/qwen3-67dd247413f0e2e4f653967f",
+      "visibility": "public",
+      "access": "accessible"
+    },
+    {
+      "type": "collection",
+      "id": "nvidia/nemotron-pre-training-dataset-689d9de36f84279d83786b35",
+      "owner": "nvidia",
+      "title": "Nemotron-Pre-Training-Dataset",
+      "url": "https://huggingface.co/collections/nvidia/nvidia/nemotron-pre-training-dataset-689d9de36f84279d83786b35",
+      "visibility": "public",
+      "access": "accessible"
+    },
+    {
+      "type": "collection",
+      "id": "inclusionAI/ui-venus-689f2fb01a4234cbce91c56a",
+      "owner": "inclusionAI",
+      "title": "UI-Venus",
+      "url": "https://huggingface.co/collections/inclusionAI/inclusionAI/ui-venus-689f2fb01a4234cbce91c56a",
+      "visibility": "public",
+      "access": "accessible"
+    }
+  ],
+  "status": 200,
+  "error": ""
+}
+--- STDERR ---
+
+[OK] test_hf_collections_list.py
+--------------------------------------------------------------------------------
+
+=== Running test_hf_collection_get.py ===
+Command: C:\Users\daqc\Documents\GitHub\open-deep-research-vulnerability-intelligence\venv\Scripts\python.exe C:\Users\daqc\Documents\GitHub\open-deep-research-vulnerability-intelligence\tests\test_hf_collection_get.py
+--- INPUT (snippet) ---
+tool.forward(namespace=namespace, slug_id=slug_id)
+    try:
+        data = json.loads(result_json_str)
+        print(json.dumps(data, indent=2, ensure_ascii=False))
+    except Exception:
+        print(result_json_str)
+
+
+if __name__ == "__main__":
+    main()
+
+
+--- STDOUT ---
+{
+  "item": {},
+  "status": 404,
+  "error": "http_404"
+}
+--- STDERR ---
+
+[OK] test_hf_collection_get.py
+--------------------------------------------------------------------------------
+
+=== Running test_hf_paper_info.py ===
+Command: C:\Users\daqc\Documents\GitHub\open-deep-research-vulnerability-intelligence\venv\Scripts\python.exe C:\Users\daqc\Documents\GitHub\open-deep-research-vulnerability-intelligence\tests\test_hf_paper_info.py
+--- INPUT (snippet) ---
+tool.forward(arxiv_id=arxiv_id)
+    try:
+        data = json.loads(result_json_str)
+        print(json.dumps(data, indent=2, ensure_ascii=False))
+    except Exception:
+        print(result_json_str)
+
+
+if __name__ == "__main__":
+    main()
+
+
+--- STDOUT ---
+{
+  "item": {
+    "id": "1706.03762",
+    "authors": [
+      {
+        "_id": "6411c77d6b75ddced38902b6",
+        "user": {
+          "_id": "60fe1b231a3e6f5129776bf9",
+          "avatarUrl": "/avatars/c80edad5267c6ed9ecde9b056993d5c3.svg",
+          "isPro": false,
+          "fullname": "Ashish Vaswani",
+          "user": "ashishvaswanigoogle",
+          "type": "user"
+        },
+        "name": "Ashish Vaswani",
+        "status": "extracted_pending",
+        "statusLastChangedAt": "2023-03-15T09:40:25.803Z",
+        "hidden": false
+      },
+      {
+        "_id": "6411c77d6b75ddced38902b7",
+        "name": "Noam Shazeer",
+        "hidden": false
+      },
+      {
+        "_id": "6411c77d6b75ddced38902b8",
+        "user": {
+          "_id": "60fee8e1465daccb9f332e2f",
+          "avatarUrl": "/avatars/ea92bf15e181b4f13a70f18ee3ba7a51.svg",
+          "isPro": false,
+          "fullname": "Niki Parmar",
+          "user": "nikip",
+          "type": "user"
+        },
+        "name": "Niki Parmar",
+        "status": "extracted_pending",
+        "statusLastChangedAt": "2023-03-15T09:40:25.803Z",
+        "hidden": false
+      },
+      {
+        "_id": "6411c77d6b75ddced38902b9",
+        "name": "Jakob Uszkoreit",
+        "hidden": false
+      },
+      {
+        "_id": "6411c77d6b75ddced38902ba",
+        "name": "Llion Jones",
+        "hidden": false
+      },
+      {
+        "_id": "6411c77d6b75ddced38902bb",
+        "name": "Aidan N. Gomez",
+        "hidden": false
+      },
+      {
+        "_id": "6411c77d6b75ddced38902bc",
+        "name": "Lukasz Kaiser",
+        "hidden": false
+      },
+      {
+        "_id": "6411c77d6b75ddced38902bd",
+        "name": "Illia Polosukhin",
+        "hidden": false
+      }
+    ],
+    "publishedAt": "2017-06-12T17:57:34.000Z",
+    "title": "Attention Is All You Need",
+    "summary": "The dominant sequence transduction models are based on complex recurrent or\nconvolutional neural networks in an encoder-decoder configuration. The best\nperforming models also connect the encoder and decoder through an attention\nmechanism. We propose a new simple network architecture, the Transformer, based\nsolely on attention mechanisms, dispensing with recurrence and convolutions\nentirely. Experiments on two machine translation tasks show these models to be\nsuperior in quality while being more parallelizable and requiring significantly\nless time to train. Our model achieves 28.4 BLEU on the WMT 2014\nEnglish-to-German translation task, improving over the existing best results,\nincluding ensembles by over 2 BLEU. On the WMT 2014 English-to-French\ntranslation task, our model establishes a new single-model state-of-the-art\nBLEU score of 41.8 after training for 3.5 days on eight GPUs, a small fraction\nof the training costs of the best models from the literature. We show that the\nTransformer generalizes well to other tasks by applying it successfully to\nEnglish constituency parsing both with large and limited training data.",
+    "upvotes": 79,
+    "discussionId": "641192343ea54b1aa7e2f084",
+    "ai_summary": "The Transformer architecture, based purely on attention mechanisms, achieves superior performance on machine translation and parsing tasks with improved parallelizability and reduced training time.",
+    "ai_keywords": [
+      "recurrent neural networks",
+      "convolutional neural networks",
+      "encoder-decoder configuration",
+      "attention mechanism",
+      "Transformer",
+      "BLEU score",
+      "WMT 2014 English-to-German translation",
+      "WMT 2014 English-to-French translation",
+      "English constituency parsing"
+    ]
+  },
+  "status": 200,
+  "error": ""
+}
+--- STDERR ---
+
+[OK] test_hf_paper_info.py
+--------------------------------------------------------------------------------
+
+=== Running test_hf_paper_repos.py ===
+Command: C:\Users\daqc\Documents\GitHub\open-deep-research-vulnerability-intelligence\venv\Scripts\python.exe C:\Users\daqc\Documents\GitHub\open-deep-research-vulnerability-intelligence\tests\test_hf_paper_repos.py
+--- INPUT (snippet) ---
+tool.forward(arxiv_id=arxiv_id)
+    try:
+        data = json.loads(result_json_str)
+        print(json.dumps(data, indent=2, ensure_ascii=False))
+    except Exception:
+        print(result_json_str)
+
+
+if __name__ == "__main__":
+    main()
+
+
+--- STDOUT ---
+{
+  "results": [],
+  "status": 200,
+  "error": ""
+}
+--- STDERR ---
+
+[OK] test_hf_paper_repos.py
+--------------------------------------------------------------------------------
+
+=== Running test_hf_daily_papers.py ===
+Command: C:\Users\daqc\Documents\GitHub\open-deep-research-vulnerability-intelligence\venv\Scripts\python.exe C:\Users\daqc\Documents\GitHub\open-deep-research-vulnerability-intelligence\tests\test_hf_daily_papers.py
+--- INPUT (snippet) ---
+tool.forward()
+    try:
+        data = json.loads(result_json_str)
+        print(json.dumps(data, indent=2, ensure_ascii=False))
+    except Exception:
+        try:
+            print(result_json_str.encode("utf-8", errors="replace").decode("utf-8", errors="replace"))
+        except Exception:
+            print("<unprintable> due to encoding error")
+
+
+if __name__ == "__main__":
+    main()
+
+
+--- STDOUT ---
+<unprintable> due to encoding error
+--- STDERR ---
+
+[OK] test_hf_daily_papers.py
+--------------------------------------------------------------------------------
+
+=== Running test_hf_repo_info.py ===
+Command: C:\Users\daqc\Documents\GitHub\open-deep-research-vulnerability-intelligence\venv\Scripts\python.exe C:\Users\daqc\Documents\GitHub\open-deep-research-vulnerability-intelligence\tests\test_hf_repo_info.py
+--- INPUT (snippet) ---
+tool.forward(repo_type="model", repo_id="bert-base-uncased")
+    try:
+        data = json.loads(result_json_str)
+        print(json.dumps(data, indent=2, ensure_ascii=False))
+    except Exception:
+        print(result_json_str)
+
+
+if __name__ == "__main__":
+    main()
+
+
+--- STDOUT ---
+{
+  "item": {
+    "_id": "621ffdc036468d709f174338",
+    "id": "google-bert/bert-base-uncased",
+    "private": false,
+    "pipeline_tag": "fill-mask",
+    "library_name": "transformers",
+    "tags": [
+      "transformers",
+      "pytorch",
+      "tf",
+      "jax",
+      "rust",
+      "coreml",
+      "onnx",
+      "safetensors",
+      "bert",
+      "fill-mask",
+      "exbert",
+      "en",
+      "dataset:bookcorpus",
+      "dataset:wikipedia",
+      "arxiv:1810.04805",
+      "license:apache-2.0",
+      "autotrain_compatible",
+      "endpoints_compatible",
+      "region:us"
+    ],
+    "downloads": 52064840,
+    "likes": 2385,
+    "modelId": "google-bert/bert-base-uncased",
+    "author": "google-bert",
+    "sha": "86b5e0934494bd15c9632b12f734a8a67f723594",
+    "lastModified": "2024-02-19T11:06:12.000Z",
+    "gated": false,
+    "disabled": false,
+    "mask_token": "[MASK]",
+    "widgetData": [
+      {
+        "text": "Paris is the [MASK] of France."
+      },
+      {
+        "text": "The goal of life is [MASK]."
+      }
+    ],
+    "model-index": null,
+    "config": {
+      "architectures": [
+        "BertForMaskedLM"
+      ],
+      "model_type": "bert",
+      "tokenizer_config": {}
+    },
+    "cardData": {
+      "language": "en",
+      "tags": [
+        "exbert"
+      ],
+      "license": "apache-2.0",
+      "datasets": [
+        "bookcorpus",
+        "wikipedia"
+      ]
+    },
+    "transformersInfo": {
+      "auto_model": "AutoModelForMaskedLM",
+      "pipeline_tag": "fill-mask",
+      "processor": "AutoTokenizer"
+    },
+    "siblings": [
+      {
+        "rfilename": ".gitattributes"
+      },
+      {
+        "rfilename": "LICENSE"
+      },
+      {
+        "rfilename": "README.md"
+      },
+      {
+        "rfilename": "config.json"
+      },
+      {
+        "rfilename": "coreml/fill-mask/float32_model.mlpackage/Data/com.apple.CoreML/model.mlmodel"
+      },
+      {
+        "rfilename": "coreml/fill-mask/float32_model.mlpackage/Data/com.apple.CoreML/weights/weight.bin"
+      },
+      {
+        "rfilename": "coreml/fill-mask/float32_model.mlpackage/Manifest.json"
+      },
+      {
+        "rfilename": "flax_model.msgpack"
+      },
+      {
+        "rfilename": "model.onnx"
+      },
+      {
+        "rfilename": "model.safetensors"
+      },
+      {
+        "rfilename": "pytorch_model.bin"
+      },
+      {
+        "rfilename": "rust_model.ot"
+      },
+      {
+        "rfilename": "tf_model.h5"
+      },
+      {
+        "rfilename": "tokenizer.json"
+      },
+      {
+        "rfilename": "tokenizer_config.json"
+      },
+      {
+        "rfilename": "vocab.txt"
+      }
+    ],
+    "spaces": [
+      "mteb/leaderboard",
+      "microsoft/HuggingGPT",
+      "Vision-CAIR/minigpt4",
+      "lnyan/stablediffusion-infinity",
+      "multimodalart/latentdiffusion",
+      "mrfakename/MeloTTS",
+      "Salesforce/BLIP",
+      "shi-labs/Versatile-Diffusion",
+      "yizhangliu/Grounded-Segment-Anything",
+      "stepfun-ai/Step1X-Edit",
+      "H-Liu1997/TANGO",
+      "xinyu1205/recognize-anything",
+      "cvlab/zero123-live",
+      "hilamanor/audioEditing",
+      "alexnasa/Chain-of-Zoom",
+      "AIGC-Audio/AudioGPT",
+      "Audio-AGI/AudioSep",
+      "m-ric/chunk_visualizer",
+      "jadechoghari/OpenMusic",
+      "DAMO-NLP-SG/Video-LLaMA",
+      "gligen/demo",
+      "declare-lab/mustango",
+      "Yiwen-ntu/MeshAnything",
+      "exbert-project/exbert",
+      "shgao/EditAnything",
+      "LiruiZhao/Diffree",
+      "Vision-CAIR/MiniGPT-v2",
+      "Yuliang/ECON",
+      "nikigoli/countgd",
+      "THUdyh/Oryx",
+      "IDEA-Research/Grounded-SAM",
+      "Awiny/Image2Paragraph",
+      "ShilongLiu/Grounding_DINO_demo",
+      "OpenSound/CapSpeech-TTS",
+      "merve/Grounding_DINO_demo",
+      "yangheng/Super-Resolution-Anime-Diffusion",
+      "liuyuan-pal/SyncDreamer",
+      "XiangJinYu/SPO",
+      "sam-hq-team/sam-hq",
+      "haotiz/glip-zeroshot-demo",
+      "TencentARC/BrushEdit",
+      "Nick088/Audio-SR",
+      "nateraw/lavila",
+      "abyildirim/inst-inpaint",
+      "Yiwen-ntu/MeshAnythingV2",
+      "Pinwheel/GLIP-BLIP-Object-Detection-VQA",
+      "Junfeng5/GLEE_demo",
+      "shi-labs/Matting-Anything",
+      "fffiloni/Video-Matting-Anything",
+      "burtenshaw/autotrain-mcp",
+      "Vision-CAIR/MiniGPT4-video",
+      "linfanluntan/Grounded-SAM",
+      "magicr/BuboGPT",
+      "WensongSong/Insert-Anything",
+      "nvidia/audio-flamingo-2",
+      "multimodalart/MoDA-fast-talking-head",
+      "clip-italian/clip-italian-demo",
+      "OpenGVLab/InternGPT",
+      "mteb/leaderboard_legacy",
+      "hongfz16/3DTopia",
+      "yenniejun/tokenizers-languages",
+      "mmlab-ntu/relate-anything-model",
+      "amphion/PicoAudio",
+      "byeongjun-park/HarmonyView",
+      "keras-io/bert-semantic-similarity",
+      "MirageML/sjc",
+      "fffiloni/vta-ldm",
+      "NAACL2022/CLIP-Caption-Reward",
+      "society-ethics/model-card-regulatory-check",
+      "fffiloni/miniGPT4-Video-Zero",
+      "AIGC-Audio/AudioLCM",
+      "Gladiator/Text-Summarizer",
+      "SVGRender/DiffSketcher",
+      "ethanchern/Anole",
+      "LittleFrog/IntrinsicAnything",
+      "milyiyo/reimagine-it",
+      "ysharma/text-to-image-to-video",
+      "OpenGVLab/VideoChatGPT",
+      "acmc/whatsapp-chats-finetuning-formatter",
+      "ZebangCheng/Emotion-LLaMA",
+      "zakaria-narjis/photo-enhancer",
+      "kaushalya/medclip-roco",
+      "AIGC-Audio/Make_An_Audio",
+      "avid-ml/bias-detection",
+      "sonalkum/GAMA",
+      "topdu/OpenOCR-Demo",
+      "RitaParadaRamos/SmallCapDemo",
+      "llizhx/TinyGPT-V",
+      "codelion/Grounding_DINO_demo",
+      "flosstradamus/FluxMusicGUI",
+      "bartar/tokenizers",
+      "Tinkering/Pytorch-day-prez",
+      "sasha/BiasDetection",
+      "Pusheen/LoCo",
+      "Jingkang/EgoGPT-7B",
+      "flax-community/koclip",
+      "TencentARC/VLog",
+      "ynhe/AskAnything",
+      "Volkopat/SegmentAnythingxGroundingDINO",
+      "phyloforfun/VoucherVision"
+    ],
+    "createdAt": "2022-03-02T23:29:04.000Z",
+    "safetensors": {
+      "parameters": {
+        "F32": 110106428
+      },
+      "total": 110106428
+    },
+    "inference": "warm",
+    "usedStorage": 13397387509
+  },
+  "status": 200,
+  "error": ""
+}
+--- STDERR ---
+
+[OK] test_hf_repo_info.py
+--------------------------------------------------------------------------------
+
+=== Running test_hf_site_search.py ===
+Command: C:\Users\daqc\Documents\GitHub\open-deep-research-vulnerability-intelligence\venv\Scripts\python.exe C:\Users\daqc\Documents\GitHub\open-deep-research-vulnerability-intelligence\tests\test_hf_site_search.py
+--- INPUT (snippet) ---
+tool.forward(query="fine-tuning tutorial", limit=5)
+    try:
+        data = json.loads(result_json_str)
+        print(json.dumps(data, indent=2, ensure_ascii=False))
+    except Exception:
+        print(result_json_str)
+
+
+if __name__ == "__main__":
+    main()
+
+
+--- STDOUT ---
+{
+  "results": [
+    {
+      "type": "site",
+      "title": "Fine-tuning - Hugging Face",
+      "url": "https://huggingface.co/docs/transformers/training",
+      "snippet": "Fine-tuning adapts a pretrained model to a specific task with a smaller specialized dataset. This approach requires far less data and compute compared to training a model from scratch, which makes it a more accessible option for many users. Transformers provides the Trainer API, which offers a comprehensive set of training features, for fine-tuning any of the models on the Hub.",
+      "date": null
+    },
+    {
+      "type": "site",
+      "title": "Fine-tune a pretrained model - Hugging Face",
+      "url": "https://huggingface.co/docs/transformers/v4.18.0/en/training",
+      "snippet": "Before you can fine-tune a pretrained model, download a dataset and prepare it for training. The previous tutorial showed you how to process data for training, and now you get an opportunity to put those skills to the test!",
+      "date": null
+    },
+    {
+      "type": "site",
+      "title": "Let's Fine-Tune Your Model for Function-Calling - Hugging Face",
+      "url": "https://huggingface.co/learn/agents-course/bonus-unit1/fine-tuning",
+      "snippet": "In this tutorial, we will build a function-calling model based on google/gemma-2-2b-it. We choose the fine-tuned model google/gemma-2-2b-it instead of the base model google/gemma-2-2b because the fine-tuned model has been improved for our use-case.",
+      "date": null
+    },
+    {
+      "type": "site",
+      "title": "Fine-Tuning a Vision Language Model (Qwen2-VL-7B) with the Hugging Face ...",
+      "url": "https://huggingface.co/learn/cookbook/fine_tuning_vlm_trl",
+      "snippet": "Phil Schmid's tutorial: an excellent deep dive into fine-tuning multimodal LLMs with TRL. Merve Noyan's smol-vision repository: a collection of engaging notebooks on cutting-edge vision and multimodal AI topics.",
+      "date": null
+    },
+    {
+      "type": "site",
+      "title": "Fine-tuning a pretrained model - Hugging Face",
+      "url": "https://huggingface.co/docs/transformers/v4.15.0/training",
+      "snippet": "In this tutorial, we will show you how to fine-tune a pretrained model from the Transformers library. In TensorFlow, models can be directly trained using Keras and the fit method.",
+      "date": null
+    }
+  ],
+  "status": 200,
+  "error": ""
+}
+--- STDERR ---
+
+[OK] test_hf_site_search.py
+--------------------------------------------------------------------------------
+
+=== Running test_hf_report_generate.py ===
+Command: C:\Users\daqc\Documents\GitHub\open-deep-research-vulnerability-intelligence\venv\Scripts\python.exe C:\Users\daqc\Documents\GitHub\open-deep-research-vulnerability-intelligence\tests\test_hf_report_generate.py
+--- INPUT (snippet) ---
+tool.forward(data_json=json.dumps(data), title="Test Report")
+    print(html[:500])  # print first 500 chars
+
+
+if __name__ == "__main__":
+    main()
+
+
+--- STDOUT ---
+<!DOCTYPE html>
+<html lang="en">
+<head>
+  <meta charset="utf-8" />
+  <meta name="viewport" content="width=device-width, initial-scale=1" />
+  <title>Test Report</title>
+  <style>
+    :root { --bg:#0b0d12; --fg:#e6e9ef; --muted:#9aa4b2; --card:#121621; --accent:#5ac8fa; --warn:#eab308; }
+    body { background:var(--bg); color:var(--fg); font-family: ui-sans-serif, system-ui, -apple-system, Segoe UI, Roboto, Inter, Arial, sans-serif; margin:0; padding:24px; }
+    h1 { font-size: 24px; margin: 0 0 
+--- STDERR ---
+
+[OK] test_hf_report_generate.py
+--------------------------------------------------------------------------------
+
+=== Running test_hf_generate_dashboard_report.py ===
+Command: C:\Users\daqc\Documents\GitHub\open-deep-research-vulnerability-intelligence\venv\Scripts\python.exe C:\Users\daqc\Documents\GitHub\open-deep-research-vulnerability-intelligence\tests\test_hf_generate_dashboard_report.py
+--- INPUT (snippet) ---
+tool.forward(query="semantic search", limit=5)
+    print(html[:500])  # print first 500 chars
+
+
+if __name__ == "__main__":
+    main()
+
+
+--- STDOUT ---
+<!DOCTYPE html>
+<html lang="en">
+<head>
+  <meta charset="utf-8" />
+  <meta name="viewport" content="width=device-width, initial-scale=1" />
+  <title>Hugging Search � Dashboard</title>
+  <style>
+    :root { --bg:#0b0d12; --fg:#e6e9ef; --muted:#9aa4b2; --card:#121621; --accent:#5ac8fa; --warn:#eab308; }
+    body { background:var(--bg); color:var(--fg); font-family: ui-sans-serif, system-ui, -apple-system, Segoe UI, Roboto, Inter, Arial, sans-serif; margin:0; padding:24px; }
+    .container { max-wi
+--- STDERR ---
+
+[OK] test_hf_generate_dashboard_report.py
+--------------------------------------------------------------------------------
+

tests/run_all_hf_tools_tests.py

@@ -0,0 +1,99 @@
+import subprocess
+import sys
+import os
+from datetime import datetime
+
+TESTS = [
+    "test_hf_models_search.py",
+    "test_hf_model_info.py",
+    "test_hf_datasets_search.py",
+    "test_hf_dataset_info.py",
+    "test_hf_spaces_search.py",
+    "test_hf_space_info.py",
+    "test_hf_user_info.py",
+    "test_hf_collections_list.py",
+    "test_hf_collection_get.py",
+    "test_hf_paper_info.py",
+    "test_hf_paper_repos.py",
+    "test_hf_daily_papers.py",
+    "test_hf_repo_info.py",
+    "test_hf_site_search.py",
+    "test_hf_report_generate.py",
+    "test_hf_generate_dashboard_report.py",
+]
+
+
+def main():
+    base = os.path.dirname(__file__)
+    timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+    out_path = os.path.join(base, f"hf_tools_tests_output_{timestamp}.txt")
+
+    ok = True
+    with open(out_path, "w", encoding="utf-8", errors="replace") as f:
+        f.write(f"Hugging Face Tools Test Run — {timestamp}\n")
+        f.write("=" * 80 + "\n\n")
+        for t in TESTS:
+            path = os.path.join(base, t)
+            header = f"=== Running {t} ===\n"
+            print("\n" + header, end="")
+            f.write(header)
+            # Write simple INPUT info: command and forward() snippet if found
+            f.write(f"Command: {sys.executable} {path}\n")
+            try:
+                with open(path, "r", encoding="utf-8", errors="replace") as tf:
+                    src = tf.read()
+                snippet = ""
+                key = "tool.forward("
+                idx = src.find(key)
+                if idx != -1:
+                    end = idx
+                    # capture up to 500 chars from forward( to show parameters
+                    snippet = src[idx: idx + 500]
+                else:
+                    # fallback: show main() body first 400 chars
+                    m_idx = src.find("def main():")
+                    snippet = src[m_idx: m_idx + 400] if m_idx != -1 else src[:400]
+                f.write("--- INPUT (snippet) ---\n")
+                f.write(snippet)
+                if not snippet.endswith("\n"):
+                    f.write("\n")
+            except Exception as e:
+                f.write(f"--- INPUT (unavailable): {e}\n")
+            try:
+                result = subprocess.run(
+                    [sys.executable, path],
+                    stdout=subprocess.PIPE,
+                    stderr=subprocess.PIPE,
+                    text=True,
+                    encoding="utf-8",
+                    errors="replace",
+                )
+                f.write("--- STDOUT ---\n")
+                f.write(result.stdout or "")
+                if not result.stdout.endswith("\n"):
+                    f.write("\n")
+                f.write("--- STDERR ---\n")
+                f.write(result.stderr or "")
+                if not result.stderr.endswith("\n"):
+                    f.write("\n")
+                status_line = f"[OK] {t}\n" if result.returncode == 0 else f"[FAIL] {t}\n"
+                f.write(status_line)
+                f.write("-" * 80 + "\n\n")
+                if result.returncode != 0:
+                    ok = False
+                print(status_line.strip())
+            except Exception as e:
+                ok = False
+                err_line = f"[ERROR] {t}: {e}\n"
+                f.write(err_line)
+                f.write("-" * 80 + "\n\n")
+                print(err_line.strip())
+
+    print(f"\nResults saved to: {out_path}")
+    sys.exit(0 if ok else 1)
+
+
+if __name__ == "__main__":
+    main()
+
+

tests/test_hf_collection_get.py

@@ -0,0 +1,26 @@
+import json
+import os
+import sys
+
+sys.path.append(os.path.dirname(os.path.dirname(__file__)))
+
+from scripts.hf_tools import HFCollectionGetTool
+
+
+def main():
+    tool = HFCollectionGetTool()
+    # Example namespace/slug - replace with a public collection if needed
+    namespace = "huggingface"
+    slug_id = "trending-models"  # may vary; adjust to an existing collection
+    result_json_str = tool.forward(namespace=namespace, slug_id=slug_id)
+    try:
+        data = json.loads(result_json_str)
+        print(json.dumps(data, indent=2, ensure_ascii=False))
+    except Exception:
+        print(result_json_str)
+
+
+if __name__ == "__main__":
+    main()
+
+

tests/test_hf_collections_list.py

@@ -0,0 +1,23 @@
+import json
+import os
+import sys
+
+sys.path.append(os.path.dirname(os.path.dirname(__file__)))
+
+from scripts.hf_tools import HFCollectionsListTool
+
+
+def main():
+    tool = HFCollectionsListTool()
+    result_json_str = tool.forward(owner=None)  # or set an owner like "huggingface"
+    try:
+        data = json.loads(result_json_str)
+        print(json.dumps(data, indent=2, ensure_ascii=False))
+    except Exception:
+        print(result_json_str)
+
+
+if __name__ == "__main__":
+    main()
+
+

tests/test_hf_daily_papers.py

@@ -0,0 +1,26 @@
+import json
+import os
+import sys
+
+sys.path.append(os.path.dirname(os.path.dirname(__file__)))
+
+from scripts.hf_tools import HFDailyPapersTool
+
+
+def main():
+    tool = HFDailyPapersTool()
+    result_json_str = tool.forward()
+    try:
+        data = json.loads(result_json_str)
+        print(json.dumps(data, indent=2, ensure_ascii=False))
+    except Exception:
+        try:
+            print(result_json_str.encode("utf-8", errors="replace").decode("utf-8", errors="replace"))
+        except Exception:
+            print("<unprintable> due to encoding error")
+
+
+if __name__ == "__main__":
+    main()
+
+

tests/test_hf_dataset_info.py

@@ -0,0 +1,24 @@
+import json
+import os
+import sys
+
+sys.path.append(os.path.dirname(os.path.dirname(__file__)))
+
+from scripts.hf_tools import HFDatasetInfoTool
+
+
+def main():
+    tool = HFDatasetInfoTool()
+    repo_id = "glue"
+    result_json_str = tool.forward(repo_id=repo_id)
+    try:
+        data = json.loads(result_json_str)
+        print(json.dumps(data, indent=2, ensure_ascii=False))
+    except Exception:
+        print(result_json_str)
+
+
+if __name__ == "__main__":
+    main()
+
+

tests/test_hf_datasets_search.py

@@ -0,0 +1,29 @@
+import json
+import os
+import sys
+
+sys.path.append(os.path.dirname(os.path.dirname(__file__)))
+
+from scripts.hf_tools import HFDatasetsSearchTool
+
+
+def main():
+    tool = HFDatasetsSearchTool()
+    result_json_str = tool.forward(
+        query="sentiment analysis",
+        tags="language:es",
+        sort="downloads",
+        direction="descending",
+        limit=5,
+    )
+    try:
+        data = json.loads(result_json_str)
+        print(json.dumps(data, indent=2, ensure_ascii=False))
+    except Exception:
+        print(result_json_str)
+
+
+if __name__ == "__main__":
+    main()
+
+

tests/test_hf_generate_dashboard_report.py

@@ -0,0 +1,18 @@
+import os
+import sys
+
+sys.path.append(os.path.dirname(os.path.dirname(__file__)))
+
+from scripts.hf_tools import HFDashboardReportTool
+
+
+def main():
+    tool = HFDashboardReportTool()
+    html = tool.forward(query="semantic search", limit=5)
+    print(html[:500])  # print first 500 chars
+
+
+if __name__ == "__main__":
+    main()
+
+