main commit

.gitignore

@@ -0,0 +1,2 @@
+__pycache__/
+.DS_Store

.pre-commit-config.yaml

@@ -0,0 +1,53 @@
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+default_language_version:
+  python: python3
+
+ci:
+  autofix_prs: true
+  autoupdate_commit_msg: '[pre-commit.ci] pre-commit suggestions'
+  autoupdate_schedule: quarterly
+
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.3.0
+    hooks:
+      - id: check-yaml
+      - id: check-case-conflict
+      - id: detect-private-key
+      - id: check-added-large-files
+        args: ['--maxkb=1000']
+      - id: requirements-txt-fixer
+      - id: end-of-file-fixer
+      - id: trailing-whitespace
+
+  - repo: https://github.com/PyCQA/isort
+    rev: 5.12.0
+    hooks:
+      - id: isort
+        name: Format imports
+
+  - repo: https://github.com/psf/black
+    rev: 22.12.0
+    hooks:
+      - id: black
+        name: Format code
+        additional_dependencies: ['click==8.0.2']
+
+  - repo: https://github.com/charliermarsh/ruff-pre-commit
+    # Ruff version.
+    rev: 'v0.0.267'
+    hooks:
+      - id: ruff

Makefile

@@ -0,0 +1,13 @@
+.PHONY: style format
+
+
+style:
+	python -m black --line-length 119 src app.py
+	python -m isort src app.py
+	ruff check --fix src app.py
+
+
+quality:
+	python -m black --check --line-length 119 src app.py
+	python -m isort --check-only src app.py
+	ruff check src app.py

README.md

@@ -1,5 +1,5 @@
 ---
-title: Sb3 Backend
+title: SB3 Backend
 emoji: 🏃
 colorFrom: green
 colorTo: red

app.py

@@ -0,0 +1,90 @@
+import logging
+from functools import partial
+from io import StringIO
+
+import gradio as gr
+from apscheduler.schedulers.background import BackgroundScheduler
+from bs4 import BeautifulSoup
+from rich.console import Console
+from rich.syntax import Syntax
+
+from src.backend import backend_routine
+from src.logging import configure_root_logger, log_file, setup_logger
+
+logging.getLogger("httpx").setLevel(logging.WARNING)
+logging.getLogger("numexpr").setLevel(logging.WARNING)
+logging.getLogger("absl").setLevel(logging.WARNING)
+
+configure_root_logger()
+
+logging.basicConfig(level=logging.INFO)
+logger = setup_logger(__name__)
+
+
+def log_file_to_html_string(reverse=True):
+    with open(log_file, "rt") as f:
+        lines = f.readlines()
+        lines = lines[-300:]
+
+    if reverse:
+        lines = reversed(lines)
+
+    output = "".join(lines)
+    syntax = Syntax(output, "python", theme="monokai", word_wrap=True)
+
+    console = Console(record=True, width=150, style="#272822", file=StringIO())
+    console.print(syntax)
+    html_content = console.export_html(inline_styles=True)
+
+    # Parse the HTML content using BeautifulSoup
+    soup = BeautifulSoup(html_content, "lxml")
+
+    # Modify the <pre> tag and add custom styles
+    pre_tag = soup.pre
+    pre_tag["class"] = "scrollable"
+    del pre_tag["style"]
+
+    # Add your custom styles and the .scrollable CSS to the <style> tag
+    style_tag = soup.style
+    style_tag.append(
+        """
+pre, code {
+    background-color: #272822;
+}
+    .scrollable {
+        font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace;
+        height: 500px;
+        overflow: auto;
+    }
+    """
+    )
+
+    return soup.prettify()
+
+
+REPO_ID = "open-rl-leaderboard/leaderboard"
+RESULTS_REPO = "open-rl-leaderboard/results"
+
+
+links_md = f"""
+# Important links
+| Description     | Link |
+|-----------------|------|
+| Leaderboard     | [{REPO_ID}](https://huggingface.co/spaces/{REPO_ID}) |
+| Results Repo    | [{RESULTS_REPO}](https://huggingface.co/datasets/{RESULTS_REPO}) |
+"""
+
+
+with gr.Blocks() as demo:
+    gr.Markdown(links_md)
+    gr.HTML(partial(log_file_to_html_string), every=1)
+    with gr.Row():
+        gr.DownloadButton("Download Log File", value=log_file)
+
+
+scheduler = BackgroundScheduler()
+scheduler.add_job(func=backend_routine, trigger="interval", seconds=5 * 60, max_instances=1)
+scheduler.start()
+
+if __name__ == "__main__":
+    demo.queue().launch()

packages.txt

@@ -0,0 +1,3 @@
+swig
+libosmesa6-dev
+patchelf

pyproject.toml

@@ -0,0 +1,15 @@
+[tool.ruff]
+line-length = 119
+
+[tool.ruff.lint]
+# Enable pycodestyle (`E`) and Pyflakes (`F`) codes by default.
+select = ["E", "F"]
+ignore = ["E501"] # line too long (black is taking care of this)
+fixable = ["A", "B", "C", "D", "E", "F", "G", "I", "N", "Q", "S", "T", "W", "ANN", "ARG", "BLE", "COM", "DJ", "DTZ", "EM", "ERA", "EXE", "FBT", "ICN", "INP", "ISC", "NPY", "PD", "PGH", "PIE", "PL", "PT", "PTH", "PYI", "RET", "RSE", "RUF", "SIM", "SLF", "TCH", "TID", "TRY", "UP", "YTT"]
+
+[tool.isort]
+profile = "black"
+line_length = 119
+
+[tool.black]
+line-length = 119

requirements.txt

@@ -0,0 +1,27 @@
+APScheduler==3.10.1
+black==23.11.0
+click==8.1.3
+datasets==2.14.5
+gradio==4.20.0
+gradio_client
+gymnasium[all,accept-rom-license]==0.29.1
+huggingface-hub>=0.18.0
+matplotlib==3.7.1
+free-mujoco-py
+mujoco<=2.3.7
+numpy==1.24.2
+pandas==2.0.0
+python-dateutil==2.8.2
+requests==2.28.2
+rliable==1.0.8
+rl-zoo3==2.3.0
+sb3-contrib==2.3.0
+torch==2.2.2
+tqdm==4.65.0
+
+
+
+# Log Visualizer
+BeautifulSoup4==4.12.2
+lxml==4.9.3
+rich==13.3.4

src/backend.py

@@ -0,0 +1,594 @@
+import fnmatch
+import importlib
+import json
+import os
+import re
+import shutil
+import sys
+import tempfile
+import time
+import zipfile
+from pathlib import Path
+from typing import Optional
+
+import numpy as np
+import rl_zoo3.import_envs  # noqa: F401 pylint: disable=unused-import
+import torch as th
+import yaml
+from huggingface_hub import CommitOperationAdd, HfApi
+from huggingface_hub.utils import EntryNotFoundError
+from huggingface_sb3 import EnvironmentName, ModelName, ModelRepoId, load_from_hub
+from requests.exceptions import HTTPError
+from rl_zoo3 import ALGOS, create_test_env, get_latest_run_id, get_saved_hyperparams
+from rl_zoo3.exp_manager import ExperimentManager
+from rl_zoo3.utils import get_model_path
+from stable_baselines3.common.utils import set_random_seed
+
+from src.logging import setup_logger
+
+ALL_ENV_IDS = [
+    "AdventureNoFrameskip-v4",
+    "AirRaidNoFrameskip-v4",
+    "AlienNoFrameskip-v4",
+    "AmidarNoFrameskip-v4",
+    "AssaultNoFrameskip-v4",
+    "AsterixNoFrameskip-v4",
+    "AsteroidsNoFrameskip-v4",
+    "AtlantisNoFrameskip-v4",
+    "BankHeistNoFrameskip-v4",
+    "BattleZoneNoFrameskip-v4",
+    "BeamRiderNoFrameskip-v4",
+    "BerzerkNoFrameskip-v4",
+    "BowlingNoFrameskip-v4",
+    "BoxingNoFrameskip-v4",
+    "BreakoutNoFrameskip-v4",
+    "CarnivalNoFrameskip-v4",
+    "CentipedeNoFrameskip-v4",
+    "ChopperCommandNoFrameskip-v4",
+    "CrazyClimberNoFrameskip-v4",
+    "DefenderNoFrameskip-v4",
+    "DemonAttackNoFrameskip-v4",
+    "DoubleDunkNoFrameskip-v4",
+    "ElevatorActionNoFrameskip-v4",
+    "EnduroNoFrameskip-v4",
+    "FishingDerbyNoFrameskip-v4",
+    "FreewayNoFrameskip-v4",
+    "FrostbiteNoFrameskip-v4",
+    "GopherNoFrameskip-v4",
+    "GravitarNoFrameskip-v4",
+    "HeroNoFrameskip-v4",
+    "IceHockeyNoFrameskip-v4",
+    "JamesbondNoFrameskip-v4",
+    "JourneyEscapeNoFrameskip-v4",
+    "KangarooNoFrameskip-v4",
+    "KrullNoFrameskip-v4",
+    "KungFuMasterNoFrameskip-v4",
+    "MontezumaRevengeNoFrameskip-v4",
+    "MsPacmanNoFrameskip-v4",
+    "NameThisGameNoFrameskip-v4",
+    "PhoenixNoFrameskip-v4",
+    "PitfallNoFrameskip-v4",
+    "PongNoFrameskip-v4",
+    "PooyanNoFrameskip-v4",
+    "PrivateEyeNoFrameskip-v4",
+    "QbertNoFrameskip-v4",
+    "RiverraidNoFrameskip-v4",
+    "RoadRunnerNoFrameskip-v4",
+    "RobotankNoFrameskip-v4",
+    "SeaquestNoFrameskip-v4",
+    "SkiingNoFrameskip-v4",
+    "SolarisNoFrameskip-v4",
+    "SpaceInvadersNoFrameskip-v4",
+    "StarGunnerNoFrameskip-v4",
+    "TennisNoFrameskip-v4",
+    "TimePilotNoFrameskip-v4",
+    "TutankhamNoFrameskip-v4",
+    "UpNDownNoFrameskip-v4",
+    "VentureNoFrameskip-v4",
+    "VideoPinballNoFrameskip-v4",
+    "WizardOfWorNoFrameskip-v4",
+    "YarsRevengeNoFrameskip-v4",
+    "ZaxxonNoFrameskip-v4",
+    # Box2D
+    "BipedalWalker-v3",
+    "BipedalWalkerHardcore-v3",
+    "CarRacing-v2",
+    "LunarLander-v2",
+    "LunarLanderContinuous-v2",
+    # Toy text
+    "Blackjack-v1",
+    "CliffWalking-v0",
+    "FrozenLake-v1",
+    "FrozenLake8x8-v1",
+    # Classic control
+    "Acrobot-v1",
+    "CartPole-v1",
+    "MountainCar-v0",
+    "MountainCarContinuous-v0",
+    "Pendulum-v1",
+    # MuJoCo
+    "Ant-v4",
+    "HalfCheetah-v4",
+    "Hopper-v4",
+    "Humanoid-v4",
+    "HumanoidStandup-v4",
+    "InvertedDoublePendulum-v4",
+    "InvertedPendulum-v4",
+    "Pusher-v4",
+    "Reacher-v4",
+    "Swimmer-v4",
+    "Walker2d-v4",
+]
+
+
+def download_from_hub(
+    algo: str,
+    env_name: EnvironmentName,
+    exp_id: int,
+    folder: str,
+    organization: str,
+    repo_name: Optional[str] = None,
+    force: bool = False,
+) -> None:
+    """
+    Try to load a model from the Huggingface hub
+    and save it following the RL Zoo structure.
+    Default repo name is {organization}/{algo}-{env_id}
+    where repo_name = {algo}-{env_id}
+
+    :param algo: Algorithm
+    :param env_name: Environment name
+    :param exp_id: Experiment id
+    :param folder: Log folder
+    :param organization: Huggingface organization
+    :param repo_name: Overwrite default repository name
+    :param force: Allow overwritting the folder
+        if it already exists.
+    """
+
+    model_name = ModelName(algo, env_name)
+
+    if repo_name is None:
+        repo_name = model_name  # Note: model name is {algo}-{env_name}
+
+    # Note: repo id is {organization}/{repo_name}
+    repo_id = ModelRepoId(organization, repo_name)
+    logger.info(f"Downloading from https://huggingface.co/{repo_id}")
+
+    checkpoint = load_from_hub(repo_id, model_name.filename)
+    try:
+        config_path = load_from_hub(repo_id, "config.yml")
+    except EntryNotFoundError:  # hotfix for old models
+        config_path = load_from_hub(repo_id, "config.json")
+        with open(config_path, "r") as f:
+            config = json.load(f)
+        config_path = config_path.replace(".json", ".yml")
+        with open(config_path, "w") as f:
+            yaml.dump(config, f)
+
+    # If VecNormalize, download
+    try:
+        vec_normalize_stats = load_from_hub(repo_id, "vec_normalize.pkl")
+    except HTTPError:
+        logger.info("No normalization file")
+        vec_normalize_stats = None
+
+    try:
+        saved_args = load_from_hub(repo_id, "args.yml")
+    except EntryNotFoundError:
+        logger.info("No args file")
+        saved_args = None
+
+    try:
+        env_kwargs = load_from_hub(repo_id, "env_kwargs.yml")
+    except EntryNotFoundError:
+        logger.info("No env_kwargs file")
+        env_kwargs = None
+
+    try:
+        train_eval_metrics = load_from_hub(repo_id, "train_eval_metrics.zip")
+    except EntryNotFoundError:
+        logger.info("No train_eval_metrics file")
+        train_eval_metrics = None
+
+    if exp_id == 0:
+        exp_id = get_latest_run_id(os.path.join(folder, algo), env_name) + 1
+    # Sanity checks
+    if exp_id > 0:
+        log_path = os.path.join(folder, algo, f"{env_name}_{exp_id}")
+    else:
+        log_path = os.path.join(folder, algo)
+
+    # Check that the folder does not exist
+    log_folder = Path(log_path)
+    if log_folder.is_dir():
+        if force:
+            logger.info(f"The folder {log_path} already exists, overwritting")
+            # Delete the current one to avoid errors
+            shutil.rmtree(log_path)
+        else:
+            raise ValueError(
+                f"The folder {log_path} already exists, use --force to overwrite it, "
+                "or choose '--exp-id 0' to create a new folder"
+            )
+
+    logger.info(f"Saving to {log_path}")
+    # Create folder structure
+    os.makedirs(log_path, exist_ok=True)
+    config_folder = os.path.join(log_path, env_name)
+    os.makedirs(config_folder, exist_ok=True)
+
+    # Copy config files and saved stats
+    shutil.copy(checkpoint, os.path.join(log_path, f"{env_name}.zip"))
+    if saved_args is not None:
+        shutil.copy(saved_args, os.path.join(config_folder, "args.yml"))
+    shutil.copy(config_path, os.path.join(config_folder, "config.yml"))
+    if env_kwargs is not None:
+        shutil.copy(env_kwargs, os.path.join(config_folder, "env_kwargs.yml"))
+    if vec_normalize_stats is not None:
+        shutil.copy(vec_normalize_stats, os.path.join(config_folder, "vecnormalize.pkl"))
+
+    # Extract monitor file and evaluation file
+    if train_eval_metrics is not None:
+        with zipfile.ZipFile(train_eval_metrics, "r") as zip_ref:
+            zip_ref.extractall(log_path)
+
+
+def pattern_match(patterns, source_list):
+    if isinstance(patterns, str):
+        patterns = [patterns]
+
+    env_ids = set()
+    for pattern in patterns:
+        for matching in fnmatch.filter(source_list, pattern):
+            env_ids.add(matching)
+    return sorted(list(env_ids))
+
+
+def evaluate(
+    user_id,
+    repo_name,
+    env="CartPole-v1",
+    folder="rl-trained-agents",
+    algo="ppo",
+    # n_timesteps=1000,
+    n_episodes=50,
+    num_threads=-1,
+    n_envs=1,
+    exp_id=0,
+    verbose=1,
+    no_render=False,
+    deterministic=False,
+    device="auto",
+    load_best=False,
+    load_checkpoint=None,
+    load_last_checkpoint=False,
+    stochastic=False,
+    norm_reward=False,
+    seed=0,
+    reward_log="",
+    gym_packages=[],
+    env_kwargs=None,
+    custom_objects=False,
+    progress=False,
+):
+    """
+    Enjoy trained agent
+
+    :param env: (str) Environment ID
+    :param folder: (str) Log folder
+    :param algo: (str) RL Algorithm
+    :param n_timesteps: (int) Number of timesteps
+    :param num_threads: (int) Number of threads for PyTorch
+    :param n_envs: (int) Number of environments
+    :param exp_id: (int) Experiment ID (default: 0: latest, -1: no exp folder)
+    :param verbose: (int) Verbose mode (0: no output, 1: INFO)
+    :param no_render: (bool) Do not render the environment (useful for tests)
+    :param deterministic: (bool) Use deterministic actions
+    :param device: (str) PyTorch device to be use (ex: cpu, cuda...)
+    :param load_best: (bool) Load best model instead of last model if available
+    :param load_checkpoint: (int) Load checkpoint instead of last model if available
+    :param load_last_checkpoint: (bool) Load last checkpoint instead of last model if available
+    :param stochastic: (bool) Use stochastic actions
+    :param norm_reward: (bool) Normalize reward if applicable (trained with VecNormalize)
+    :param seed: (int) Random generator seed
+    :param reward_log: (str) Where to log reward
+    :param gym_packages: (List[str]) Additional external Gym environment package modules to import
+    :param env_kwargs: (Dict[str, Any]) Optional keyword argument to pass to the env constructor
+    :param custom_objects: (bool) Use custom objects to solve loading issues
+    :param progress: (bool) if toggled, display a progress bar using tqdm and rich
+    """
+
+    # Going through custom gym packages to let them register in the global registory
+    for env_module in gym_packages:
+        importlib.import_module(env_module)
+
+    env_name = EnvironmentName(env)
+
+    # try:
+    #     _, model_path, log_path = get_model_path(
+    #         exp_id,
+    #         folder,
+    #         algo,
+    #         env_name,
+    #         load_best,
+    #         load_checkpoint,
+    #         load_last_checkpoint,
+    #     )
+    # except (AssertionError, ValueError) as e:
+    #     # Special case for rl-trained agents
+    #     # auto-download from the hub
+    #     if "rl-trained-agents" not in folder:
+    #         raise e
+    #     else:
+    #         logger.info("Pretrained model not found, trying to download it from sb3 Huggingface hub: https://huggingface.co/sb3")
+    # Auto-download
+    download_from_hub(
+        algo=algo,
+        env_name=env_name,
+        exp_id=exp_id,
+        folder=folder,
+        # organization="sb3",
+        organization=user_id,
+        # repo_name=None,
+        repo_name=repo_name,
+        force=False,
+    )
+    # Try again
+    _, model_path, log_path = get_model_path(
+        exp_id,
+        folder,
+        algo,
+        env_name,
+        load_best,
+        load_checkpoint,
+        load_last_checkpoint,
+    )
+
+    logger.info(f"Loading {model_path}")
+
+    # Off-policy algorithm only support one env for now
+    off_policy_algos = ["qrdqn", "dqn", "ddpg", "sac", "her", "td3", "tqc"]
+
+    set_random_seed(seed)
+
+    if num_threads > 0:
+        if verbose > 1:
+            logger.info(f"Setting torch.num_threads to {num_threads}")
+        th.set_num_threads(num_threads)
+
+    is_atari = ExperimentManager.is_atari(env_name.gym_id)
+    is_minigrid = ExperimentManager.is_minigrid(env_name.gym_id)
+
+    stats_path = os.path.join(log_path, env_name)
+    hyperparams, maybe_stats_path = get_saved_hyperparams(stats_path, norm_reward=norm_reward, test_mode=True)
+
+    # load env_kwargs if existing
+    env_kwargs = {}
+    args_path = os.path.join(log_path, env_name, "args.yml")
+    if os.path.isfile(args_path):
+        with open(args_path) as f:
+            loaded_args = yaml.load(f, Loader=yaml.UnsafeLoader)
+            if loaded_args["env_kwargs"] is not None:
+                env_kwargs = loaded_args["env_kwargs"]
+    # overwrite with command line arguments
+    if env_kwargs is not None:
+        env_kwargs.update(env_kwargs)
+
+    log_dir = reward_log if reward_log != "" else None
+
+    env = create_test_env(
+        env_name.gym_id,
+        n_envs=n_envs,
+        stats_path=maybe_stats_path,
+        seed=seed,
+        log_dir=log_dir,
+        should_render=not no_render,
+        hyperparams=hyperparams,
+        env_kwargs=env_kwargs,
+    )
+
+    kwargs = dict(seed=seed)
+    if algo in off_policy_algos:
+        # Dummy buffer size as we don't need memory to enjoy the trained agent
+        kwargs.update(dict(buffer_size=1))
+        # Hack due to breaking change in v1.6
+        # handle_timeout_termination cannot be at the same time
+        # with optimize_memory_usage
+        if "optimize_memory_usage" in hyperparams:
+            kwargs.update(optimize_memory_usage=False)
+
+    # Check if we are running python 3.8+
+    # we need to patch saved model under python 3.6/3.7 to load them
+    newer_python_version = sys.version_info.major == 3 and sys.version_info.minor >= 8
+
+    custom_objects = {}
+    if newer_python_version or custom_objects:
+        custom_objects = {
+            "learning_rate": 0.0,
+            "lr_schedule": lambda _: 0.0,
+            "clip_range": lambda _: 0.0,
+        }
+
+    if "HerReplayBuffer" in hyperparams.get("replay_buffer_class", ""):
+        kwargs["env"] = env
+
+    model = ALGOS[algo].load(model_path, custom_objects=custom_objects, device=device, **kwargs)
+    obs = env.reset()
+
+    # Deterministic by default except for atari games
+    stochastic = stochastic or (is_atari or is_minigrid) and not deterministic
+    deterministic = not stochastic
+
+    episode_reward = 0.0
+    episode_rewards, episode_lengths = [], []
+    ep_len = 0
+    # For HER, monitor success rate
+    successes = []
+    lstm_states = None
+    episode_start = np.ones((env.num_envs,), dtype=bool)
+
+    # generator = range(n_timesteps)
+    # if progress:
+    #     if tqdm is None:
+    #         raise ImportError("Please install tqdm and rich to use the progress bar")
+    #     generator = tqdm(generator)
+
+    try:
+        # for _ in generator:
+        while len(episode_rewards) < n_episodes:
+            action, lstm_states = model.predict(
+                obs,  # type: ignore[arg-type]
+                state=lstm_states,
+                episode_start=episode_start,
+                deterministic=deterministic,
+            )
+            obs, reward, done, infos = env.step(action)
+
+            episode_start = done
+
+            if not no_render:
+                env.render("human")
+
+            episode_reward += reward[0]
+            ep_len += 1
+
+            if n_envs == 1:
+                # For atari the return reward is not the atari score
+                # so we have to get it from the infos dict
+                if is_atari and infos is not None and verbose >= 1:
+                    episode_infos = infos[0].get("episode")
+                    if episode_infos is not None:
+                        logger.info(f"Atari Episode Score: {episode_infos['r']:.2f}")
+                        logger.info("Atari Episode Length", episode_infos["l"])
+
+                if done and not is_atari and verbose > 0:
+                    # NOTE: for env using VecNormalize, the mean reward
+                    # is a normalized reward when `--norm_reward` flag is passed
+                    logger.info(f"Episode Reward: {episode_reward:.2f}")
+                    logger.info("Episode Length", ep_len)
+                    episode_rewards.append(episode_reward)
+                    episode_lengths.append(ep_len)
+                    episode_reward = 0.0
+                    ep_len = 0
+
+                # Reset also when the goal is achieved when using HER
+                if done and infos[0].get("is_success") is not None:
+                    if verbose > 1:
+                        logger.info("Success?", infos[0].get("is_success", False))
+
+                    if infos[0].get("is_success") is not None:
+                        successes.append(infos[0].get("is_success", False))
+                        episode_reward, ep_len = 0.0, 0
+
+    except KeyboardInterrupt:
+        pass
+
+    if verbose > 0 and len(successes) > 0:
+        logger.info(f"Success rate: {100 * np.mean(successes):.2f}%")
+
+    if verbose > 0 and len(episode_rewards) > 0:
+        logger.info(f"{len(episode_rewards)} Episodes")
+        logger.info(f"Mean reward: {np.mean(episode_rewards):.2f} +/- {np.std(episode_rewards):.2f}")
+
+    if verbose > 0 and len(episode_lengths) > 0:
+        logger.info(f"Mean episode length: {np.mean(episode_lengths):.2f} +/- {np.std(episode_lengths):.2f}")
+
+    env.close()
+
+    return episode_rewards
+
+
+logger = setup_logger(__name__)
+
+API = HfApi(token=os.environ.get("TOKEN"))
+RESULTS_REPO = "open-rl-leaderboard/results"
+
+
+def _backend_routine():
+    # List only the text classification models
+    rl_models = list(API.list_models(filter=["reinforcement-learning", "stable-baselines3"]))
+    logger.info(f"Found {len(rl_models)} RL models")
+    compatible_models = []
+    for model in rl_models:
+        compatible_models.append((model.modelId, model.sha))
+
+    logger.info(f"Found {len(compatible_models)} compatible models")
+
+    # Get the results
+    pattern = re.compile(r"^[^/]*/[^/]*/[^/]*results_[a-f0-9]+\.json$")
+    filenames = API.list_repo_files(RESULTS_REPO, repo_type="dataset")
+    filenames = [filename for filename in filenames if pattern.match(filename)]
+
+    evaluated_models = set()
+    for filename in filenames:
+        path = API.hf_hub_download(repo_id=RESULTS_REPO, filename=filename, repo_type="dataset")
+        with open(path) as fp:
+            report = json.load(fp)
+        evaluated_models.add((report["config"]["model_id"], report["config"]["model_sha"]))
+
+    # Find the models that are not associated with any results
+    pending_models = list(set(compatible_models) - evaluated_models)
+    logger.info(f"Found {len(pending_models)} pending models")
+
+    if len(pending_models) == 0:
+        return None
+
+    # Run an evaluation on the models
+    with tempfile.TemporaryDirectory() as tmp_dir:
+        for model_id, sha in pending_models:
+            time.sleep(60)
+            commits = []
+            model_info = API.model_info(model_id, revision=sha)
+
+            # Extract the environment IDs from the tags (usually only one)
+            env_ids = pattern_match(model_info.tags, ALL_ENV_IDS)
+            if len(env_ids) == 0:
+                logger.error(f"No environment found for {model_id}")
+                continue
+            else:
+                env = env_ids[0]
+            user_id, repo_name = model_id.split("/")
+            algo = model_info.model_index[0]["name"].lower()
+
+            logger.info(f"Running evaluation on {model_id}")
+            report = {"config": {"model_id": model_id, "model_sha": sha}}
+            try:
+                episodic_returns = evaluate(
+                    user_id, repo_name, env, "rl-trained-agents", algo, no_render=True, verbose=1
+                )
+                evaluations = {env: {"episodic_returns": episodic_returns}}
+            except Exception as e:
+                logger.error(f"Error evaluating {model_id}: {e}")
+                evaluations = None
+
+            if evaluations is not None:
+                report["results"] = evaluations
+                report["status"] = "DONE"
+            else:
+                report["status"] = "FAILED"
+
+            # Update the results
+            dumped = json.dumps(report, indent=2)
+            path_in_repo = f"{model_id}/results_{sha}.json"
+            local_path = os.path.join(tmp_dir, path_in_repo)
+            os.makedirs(os.path.dirname(local_path), exist_ok=True)
+            with open(local_path, "w") as f:
+                f.write(dumped)
+
+            commits.append(CommitOperationAdd(path_in_repo=path_in_repo, path_or_fileobj=local_path))
+
+            API.create_commit(
+                repo_id=RESULTS_REPO, commit_message="Add evaluation results", operations=commits, repo_type="dataset"
+            )
+
+
+def backend_routine():
+    try:
+        _backend_routine()
+    except Exception as e:
+        logger.error(f"{e.__class__.__name__}: {str(e)}")
+
+
+if __name__ == "__main__":
+    backend_routine()

src/logging.py

@@ -0,0 +1,37 @@
+from pathlib import Path
+
+proj_dir = Path(__file__).parents[1]
+
+log_file = proj_dir / "output.log"
+
+
+import logging
+
+
+def setup_logger(name: str):
+    logger = logging.getLogger(name)
+    logger.setLevel(logging.INFO)
+
+    formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
+
+    # Create a file handler to write logs to a file
+    file_handler = logging.FileHandler(log_file)
+    file_handler.setLevel(logging.INFO)
+    file_handler.setFormatter(formatter)
+    logger.addHandler(file_handler)
+
+    return logger
+
+
+def configure_root_logger():
+    # Configure the root logger
+    logging.basicConfig(level=logging.INFO)
+    root_logger = logging.getLogger()
+
+    formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
+
+    file_handler = logging.FileHandler(log_file)
+    file_handler.setLevel(logging.INFO)
+    file_handler.setFormatter(formatter)
+
+    root_logger.addHandler(file_handler)