update website

configs/paper_agent.py

@@ -0,0 +1,4 @@
+workdir = "workdir"
+tag = "paper_agent"
+exp_path = f"{workdir}/{tag}"
+log_path = "agent.log"

src/config/__init__.py

@@ -0,0 +1,3 @@
+from .config import config
+
+__all__ = ['config']

src/config/config.py

@@ -0,0 +1,86 @@
+import os
+from mmengine import Config as MMConfig
+from argparse import Namespace
+
+from dotenv import load_dotenv
+load_dotenv(verbose=True)
+
+from finworld.utils import assemble_project_path, get_tag_name, Singleton, set_seed
+
+def check_level(level: str) -> bool:
+    """
+    Check if the level is valid.
+    """
+    valid_levels = ['1day', '1min', '5min', '15min', '30min', '1hour', '4hour']
+    if level not in valid_levels:
+        return False
+    return True
+
+def process_general(config: MMConfig) -> MMConfig:
+
+    config.exp_path = assemble_project_path(os.path.join(config.workdir, config.tag))
+    os.makedirs(config.exp_path, exist_ok=True)
+
+    config.log_path = os.path.join(config.exp_path, getattr(config, 'log_path', 'finworld.log'))
+
+    if "checkpoint_path" in config:
+        config.checkpoint_path = os.path.join(config.exp_path, getattr(config, 'checkpoint_path', 'checkpoint'))
+        os.makedirs(config.checkpoint_path, exist_ok=True)
+
+    if "plot_path" in config:
+        config.plot_path = os.path.join(config.exp_path, getattr(config, 'plot_path', 'plot'))
+        os.makedirs(config.plot_path, exist_ok=True)
+
+    if "tracker" in config:
+        for key, value in config.tracker.items():
+            config.tracker[key]['logging_dir'] = os.path.join(config.exp_path, value['logging_dir'])
+
+    if "seed" in config:
+        set_seed(config.seed)
+
+    return config
+
+
+class Config(MMConfig, metaclass=Singleton):
+    def __init__(self):
+        super(Config, self).__init__()
+
+    def init_config(self, config_path: str, args: Namespace) -> None:
+        # Initialize the general configuration
+        mmconfig = MMConfig.fromfile(filename=assemble_project_path(config_path))
+        if 'cfg_options' not in args or args.cfg_options is None:
+            cfg_options = dict()
+        else:
+            cfg_options = args.cfg_options
+        for item in args.__dict__:
+            if item not in ['config', 'cfg_options'] and args.__dict__[item] is not None:
+                cfg_options[item] = args.__dict__[item]
+        mmconfig.merge_from_dict(cfg_options)
+
+        tag = get_tag_name(
+            tag=getattr(mmconfig, 'tag', None),
+            assets_name=getattr(mmconfig, 'assets_name', None),
+            source=getattr(mmconfig, 'source', None),
+            data_type= getattr(mmconfig, 'data_type', None),
+            level= getattr(mmconfig, 'level', None),
+        )
+        mmconfig.tag = tag
+
+        # Process general configuration
+        mmconfig = process_general(mmconfig)
+
+        # Initialize the price downloader configuration
+        if 'downloader' in mmconfig:
+            if "assets_path" in mmconfig.downloader:
+                mmconfig.downloader.assets_path = assemble_project_path(mmconfig.downloader.assets_path)
+                assert check_level(mmconfig.downloader.level), f"Invalid level: {mmconfig.downloader.level}. Valid levels are: ['1day', '1min', '5min', '15min', '30min', '1hour', '4hour']"
+
+        if 'processor' in mmconfig:
+            if "assets_path" in mmconfig.processor:
+                mmconfig.processor.assets_path = assemble_project_path(mmconfig.processor.assets_path)
+            mmconfig.processor.repo_id = f"{os.getenv('HF_REPO_NAME')}/{mmconfig.processor.repo_id}"
+            mmconfig.processor.repo_type = mmconfig.processor.repo_type if 'repo_type' in mmconfig.processor else 'dataset'
+
+        self.__dict__.update(mmconfig.__dict__)
+
+config = Config()

src/database/__init__.py

@@ -0,0 +1,5 @@
+# Database management for paper caching
+
+from .db import PapersDatabase, db
+
+__all__ = ['PapersDatabase', 'db']

src/database/db.py

@@ -0,0 +1,143 @@
+import os
+import json
+import sqlite3
+from datetime import date, datetime, timedelta
+from typing import Any, Dict, List, Optional
+from contextlib import contextmanager
+
+
+class PapersDatabase():
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self.db_path = None
+    
+    def init_db(self, config):
+        """Initialize the database with required tables"""
+        
+        self.db_path = config.db_path
+        
+        with self.get_connection() as conn:
+            cursor = conn.cursor()
+            
+            # Create papers cache table
+            cursor.execute('''
+                CREATE TABLE IF NOT EXISTS papers_cache (
+                    date_str TEXT PRIMARY KEY,
+                    html_content TEXT NOT NULL,
+                    parsed_cards TEXT NOT NULL,
+                    created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+                    updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+                )
+            ''')
+            
+            # Create latest_date table to track the most recent available date
+            cursor.execute('''
+                CREATE TABLE IF NOT EXISTS latest_date (
+                    id INTEGER PRIMARY KEY CHECK (id = 1),
+                    date_str TEXT NOT NULL,
+                    updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+                )
+            ''')
+            
+            # Insert default latest_date record if it doesn't exist
+            cursor.execute('''
+                INSERT OR IGNORE INTO latest_date (id, date_str) 
+                VALUES (1, ?)
+            ''', (date.today().isoformat(),))
+            
+            conn.commit()
+    
+    @contextmanager
+    def get_connection(self):
+        """Context manager for database connections"""
+        conn = sqlite3.connect(self.db_path)
+        conn.row_factory = sqlite3.Row  # Enable dict-like access
+        try:
+            yield conn
+        finally:
+            conn.close()
+    
+    def get_cached_papers(self, date_str: str) -> Optional[Dict[str, Any]]:
+        """Get cached papers for a specific date"""
+        with self.get_connection(self.db_path) as conn:
+            cursor = conn.cursor()
+            cursor.execute('''
+                SELECT parsed_cards, created_at 
+                FROM papers_cache 
+                WHERE date_str = ?
+            ''', (date_str,))
+            
+            row = cursor.fetchone()
+            if row:
+                return {
+                    'cards': json.loads(row['parsed_cards']),
+                    'cached_at': row['created_at']
+                }
+            return None
+    
+    def cache_papers(self, date_str: str, html_content: str, parsed_cards: List[Dict[str, Any]]):
+        """Cache papers for a specific date"""
+        with self.get_connection() as conn:
+            cursor = conn.cursor()
+            cursor.execute('''
+                INSERT OR REPLACE INTO papers_cache 
+                (date_str, html_content, parsed_cards, updated_at) 
+                VALUES (?, ?, ?, CURRENT_TIMESTAMP)
+            ''', (date_str, html_content, json.dumps(parsed_cards)))
+            conn.commit()
+    
+    def get_latest_cached_date(self) -> Optional[str]:
+        """Get the latest cached date"""
+        with self.get_connection() as conn:
+            cursor = conn.cursor()
+            cursor.execute('SELECT date_str FROM latest_date WHERE id = 1')
+            row = cursor.fetchone()
+            return row['date_str'] if row else None
+    
+    def update_latest_date(self, date_str: str):
+        """Update the latest available date"""
+        with self.get_connection() as conn:
+            cursor = conn.cursor()
+            cursor.execute('''
+                UPDATE latest_date 
+                SET date_str = ?, updated_at = CURRENT_TIMESTAMP 
+                WHERE id = 1
+            ''', (date_str,))
+            conn.commit()
+    
+    def is_cache_fresh(self, date_str: str, max_age_hours: int = 24) -> bool:
+        """Check if cache is fresh (within max_age_hours)"""
+        with self.get_connection() as conn:
+            cursor = conn.cursor()
+            cursor.execute('''
+                SELECT updated_at 
+                FROM papers_cache 
+                WHERE date_str = ?
+            ''', (date_str,))
+            
+            row = cursor.fetchone()
+            if not row:
+                return False
+            
+            cached_time = datetime.fromisoformat(row['updated_at'].replace('Z', '+00:00'))
+            age = datetime.now(cached_time.tzinfo) - cached_time
+            return age.total_seconds() < max_age_hours * 3600
+    
+    def cleanup_old_cache(self, days_to_keep: int = 7):
+        """Clean up old cache entries"""
+        cutoff_date = (datetime.now() - timedelta(days=days_to_keep)).isoformat()
+        with self.get_connection() as conn:
+            cursor = conn.cursor()
+            cursor.execute('''
+                DELETE FROM papers_cache 
+                WHERE updated_at < ?
+            ''', (cutoff_date,))
+            conn.commit()
+
+    def __str__(self):
+        return f"PapersDatabase(db_path={self.db_path})"
+
+    def __repr__(self):
+        return self.__str__()
+
+db = PapersDatabase()

src/logger/__init__.py

@@ -0,0 +1,10 @@
+from .logger import logger, LogLevel, AgentLogger, YELLOW_HEX
+from .monitor import Monitor, Timing, TokenUsage
+
+__all__ = ["logger",
+           "LogLevel",
+           "AgentLogger",
+           "Monitor",
+           "YELLOW_HEX",
+           "Timing",
+           "TokenUsage"]

src/logger/logger.py

@@ -0,0 +1,229 @@
+import logging
+import json
+from enum import IntEnum
+
+from rich import box
+from rich.console import Console, Group
+from rich.panel import Panel
+from rich.rule import Rule
+from rich.syntax import Syntax
+from rich.table import Table
+from rich.tree import Tree
+
+from src.utils import (
+    escape_code_brackets,
+    Singleton
+)
+
+YELLOW_HEX = "#d4b702"
+
+class LogLevel(IntEnum):
+    OFF = -1  # No output
+    ERROR = 0  # Only errors
+    INFO = 1  # Normal output (default)
+    DEBUG = 2  # Detailed output
+
+class AgentLogger(logging.Logger, metaclass=Singleton):
+    def __init__(self, name="logger", level=logging.INFO):
+        # Initialize the parent class
+        super().__init__(name, level)
+
+        # Define a formatter for log messages
+        self.formatter = logging.Formatter(
+            fmt="\033[92m%(asctime)s - %(name)s:%(levelname)s\033[0m: %(filename)s:%(lineno)s - %(message)s",
+            datefmt="%H:%M:%S",
+        )
+
+    def init_logger(self, log_path: str, level=logging.INFO):
+        """
+        Initialize the logger with a file path and optional main process check.
+
+        Args:
+            log_path (str): The log file path.
+            level (int, optional): The logging level. Defaults to logging.INFO.
+            accelerator (Accelerator, optional): Accelerator instance to determine the main process.
+        """
+
+        # Add a console handler for logging to the console
+        console_handler = logging.StreamHandler()
+        console_handler.setLevel(level)
+        console_handler.setFormatter(self.formatter)
+        self.addHandler(console_handler)
+
+        # Add a file handler for logging to the file
+        file_handler = logging.FileHandler(
+            log_path, mode="a"
+        )  # 'a' mode appends to the file
+        file_handler.setLevel(level)
+        file_handler.setFormatter(self.formatter)
+        self.addHandler(file_handler)
+
+        self.console = Console(width=100)
+        self.file_console = Console(file=open(log_path, "a"), width=100)
+
+        # Prevent duplicate logs from propagating to the root logger
+        self.propagate = False
+
+    def log(self, *args, level: int | str | LogLevel = LogLevel.INFO, **kwargs) -> None:
+        """Logs a message to the console.
+
+        Args:
+            level (LogLevel, optional): Defaults to LogLevel.INFO.
+        """
+        if isinstance(level, str):
+            level = LogLevel[level.upper()]
+        if level <= self.level:
+            self.info(*args, **kwargs)
+
+    def info(self, msg, *args, **kwargs):
+        """
+        Overridden info method with stacklevel adjustment for correct log location.
+        """
+        if isinstance(msg, (Rule, Panel, Group, Tree, Table, Syntax)):
+            self.console.print(msg)
+            self.file_console.print(msg)
+        else:
+            kwargs.setdefault(
+                "stacklevel", 2
+            )  # Adjust stack level to show the actual caller
+            if "style" in kwargs:
+                kwargs.pop("style")
+            if "level" in kwargs:
+                kwargs.pop("level")
+            super().info(msg, *args, **kwargs)
+
+    def warning(self, msg, *args, **kwargs):
+        kwargs.setdefault("stacklevel", 2)
+        super().warning(msg, *args, **kwargs)
+
+    def error(self, msg, *args, **kwargs):
+        kwargs.setdefault("stacklevel", 2)
+        super().error(msg, *args, **kwargs)
+
+    def critical(self, msg, *args, **kwargs):
+        kwargs.setdefault("stacklevel", 2)
+        super().critical(msg, *args, **kwargs)
+
+    def debug(self, msg, *args, **kwargs):
+        kwargs.setdefault("stacklevel", 2)
+        super().debug(msg, *args, **kwargs)
+
+    def log_error(self, error_message: str) -> None:
+        self.info(escape_code_brackets(error_message), style="bold red", level=LogLevel.ERROR)
+
+    def log_markdown(self, content: str, title: str | None = None, level=LogLevel.INFO, style=YELLOW_HEX) -> None:
+        markdown_content = Syntax(
+            content,
+            lexer="markdown",
+            theme="github-dark",
+            word_wrap=True,
+        )
+        if title:
+            self.info(
+                Group(
+                    Rule(
+                        "[bold italic]" + title,
+                        align="left",
+                        style=style,
+                    ),
+                    markdown_content,
+                ),
+                level=level,
+            )
+        else:
+            self.info(markdown_content, level=level)
+
+    def log_code(self, title: str, content: str, level: int = LogLevel.INFO) -> None:
+        self.info(
+            Panel(
+                Syntax(
+                    content,
+                    lexer="python",
+                    theme="monokai",
+                    word_wrap=True,
+                ),
+                title="[bold]" + title,
+                title_align="left",
+                box=box.HORIZONTALS,
+            ),
+            level=level,
+        )
+
+    def log_rule(self, title: str, level: int = LogLevel.INFO) -> None:
+        self.info(
+            Rule(
+                "[bold]" + title,
+                characters="━",
+                style=YELLOW_HEX,
+            ),
+            level=LogLevel.INFO,
+        )
+
+    def log_task(self, content: str, subtitle: str, title: str | None = None, level: LogLevel = LogLevel.INFO) -> None:
+        self.info(
+            Panel(
+                f"\n[bold]{escape_code_brackets(content)}\n",
+                title="[bold]New run" + (f" - {title}" if title else ""),
+                subtitle=subtitle,
+                border_style=YELLOW_HEX,
+                subtitle_align="left",
+            ),
+            level=level,
+        )
+
+    def log_messages(self, messages: list[dict], level: LogLevel = LogLevel.DEBUG) -> None:
+        messages_as_string = "\n".join([json.dumps(dict(message), indent=4, ensure_ascii=False) for message in messages])
+        self.info(
+            Syntax(
+                messages_as_string,
+                lexer="markdown",
+                theme="github-dark",
+                word_wrap=True,
+            ),
+            level=level,
+        )
+
+    def visualize_agent_tree(self, agent):
+        def create_tools_section(tools_dict):
+            table = Table(show_header=True, header_style="bold")
+            table.add_column("Name", style="#1E90FF")
+            table.add_column("Description")
+            table.add_column("Arguments")
+
+            for name, tool in tools_dict.items():
+                args = [
+                    f"{arg_name} (`{info.get('type', 'Any')}`{', optional' if info.get('optional') else ''}): {info.get('description', '')}"
+                    for arg_name, info in getattr(tool, "inputs", {}).items()
+                ]
+                table.add_row(name, getattr(tool, "description", str(tool)), "\n".join(args))
+
+            return Group("🛠️ [italic #1E90FF]Tools:[/italic #1E90FF]", table)
+
+        def get_agent_headline(agent, name: str | None = None):
+            name_headline = f"{name} | " if name else ""
+            return f"[bold {YELLOW_HEX}]{name_headline}{agent.__class__.__name__} | {agent.model.model_id}"
+
+        def build_agent_tree(parent_tree, agent_obj):
+            """Recursively builds the agent tree."""
+            parent_tree.add(create_tools_section(agent_obj.tools))
+
+            if agent_obj.managed_agents:
+                agents_branch = parent_tree.add("🤖 [italic #1E90FF]Managed agents:")
+                for name, managed_agent in agent_obj.managed_agents.items():
+                    agent_tree = agents_branch.add(get_agent_headline(managed_agent, name))
+                    if managed_agent.__class__.__name__ == "CodeAgent":
+                        agent_tree.add(
+                            f"✅ [italic #1E90FF]Authorized imports:[/italic #1E90FF] {managed_agent.additional_authorized_imports}"
+                        )
+                    agent_tree.add(f"📝 [italic #1E90FF]Description:[/italic #1E90FF] {managed_agent.description}")
+                    build_agent_tree(agent_tree, managed_agent)
+
+        main_tree = Tree(get_agent_headline(agent))
+        if agent.__class__.__name__ == "CodeAgent":
+            main_tree.add(
+                f"✅ [italic #1E90FF]Authorized imports:[/italic #1E90FF] {agent.additional_authorized_imports}"
+            )
+        build_agent_tree(main_tree, agent)
+        self.console.print(main_tree)
+
+logger = AgentLogger()

src/utils/__init__.py

@@ -0,0 +1,8 @@
+from .path_utils import get_project_root, assemble_project_path
+from .singleton import Singleton
+
+__all__ = [
+    "get_project_root",
+    "assemble_project_path",
+    "Singleton"
+]

src/utils/path_utils.py

@@ -0,0 +1,12 @@
+from pathlib import Path
+import os
+
+def get_project_root():
+    root = str(Path(__file__).resolve().parents[2])
+    return root
+
+def assemble_project_path(path):
+    """Assemble a path relative to the project root directory"""
+    if not os.path.isabs(path):
+        path = os.path.join(get_project_root(), path)
+    return path

src/utils/singleton.py

@@ -0,0 +1,25 @@
+"""A singleton metaclass for ensuring only one instance of a class."""
+
+import abc
+
+
+class Singleton(abc.ABCMeta, type):
+    """
+    Singleton metaclass for ensuring only one instance of a class.
+    """
+
+    _instances = {}
+
+    def __call__(cls, *args, **kwargs):
+        """Call method for the singleton metaclass."""
+        if cls not in cls._instances:
+            cls._instances[cls] = super(Singleton, cls).__call__(*args, **kwargs)
+        return cls._instances[cls]
+
+
+class AbstractSingleton(abc.ABC, metaclass=Singleton):
+    """
+    Abstract singleton class for ensuring only one instance of a class.
+    """
+
+    pass