dummy _compute() output

ref-metric.py

@@ -99,126 +99,153 @@ class UserFriendlyMetrics(evaluate.Metric):
     ):
         """Returns the scores"""
         # TODO: Compute the different scores of the module
-        return calculate_from_payload(
-            payload, max_iou, filters, recognition_thresholds, debug
-        )
+        return dummy_values()
         # return calculate(predictions, references, max_iou)
 
-    def wandb(
-        self,
-        results,
-        wandb_section: str = None,
-        wandb_project="user_friendly_metrics",
-        log_plots: bool = True,
-        debug: bool = False,
-    ):
-        """
-        Logs metrics to Weights and Biases (wandb) for tracking and visualization, including categorized bar charts for global metrics.
-
-        Args:
-            results (dict): Results dictionary with 'global' and 'per_sequence' keys.
-            wandb_section (str, optional): W&B section for metric grouping. Defaults to None.
-            wandb_project (str, optional): The name of the wandb project. Defaults to 'user_friendly_metrics'.
-            log_plots (bool, optional): Generates categorized bar charts for global metrics. Defaults to True.
-            debug (bool, optional): Logs detailed summaries and histories to the terminal console. Defaults to False.
-        """
-
-        current_datetime = datetime.datetime.now()
-        formatted_datetime = current_datetime.strftime("%Y-%m-%d_%H-%M-%S")
-        wandb.login(key=os.getenv("WANDB_API_KEY"))
-
-        run = wandb.init(
-            project=wandb_project,
-            name=f"evaluation-{formatted_datetime}",
-            reinit=True,
-            settings=wandb.Settings(silent=not debug),
-        )
-
-        categories = {
-            "user_friendly_metrics": {
-                "mostly_tracked_score_0.3",
-                "mostly_tracked_score_0.5",
-                "mostly_tracked_score_0.8",
-            },
-            "evaluation_metrics_dev": {
-                "f1",
-                "recall",
-                "precision",
-            },
-            "user_friendly_metrics_dev": {
-                "mostly_tracked_count_0.3",
-                "mostly_tracked_count_0.5",
-                "mostly_tracked_count_0.8",
-                "unique_obj_count",
-            },
-            "predictions_summary": {
-                "fp",
-                "tp",
-                "fn",
+    def dummy_values():
+        return {
+            "model_1": {
+                "overall": {
+                    "all": {
+                        "tp": 50,
+                        "fp": 20,
+                        "fn": 10,
+                        "precision": 0.71,
+                        "recall": 0.83,
+                        "f1": 0.76
+                    },
+                    "small": {
+                        "tp": 15,
+                        "fp": 5,
+                        "fn": 2,
+                        "precision": 0.75,
+                        "recall": 0.88,
+                        "f1": 0.81
+                    },
+                    "medium": {
+                        "tp": 25,
+                        "fp": 10,
+                        "fn": 5,
+                        "precision": 0.71,
+                        "recall": 0.83,
+                        "f1": 0.76
+                    },
+                    "large": {
+                        "tp": 10,
+                        "fp": 5,
+                        "fn": 3,
+                        "precision": 0.67,
+                        "recall": 0.77,
+                        "f1": 0.71
+                    }
+                },
+                "per_sequence": {
+                    "sequence_1": {
+                        "all": {
+                            "tp": 30,
+                            "fp": 15,
+                            "fn": 7,
+                            "precision": 0.67,
+                            "recall": 0.81,
+                            "f1": 0.73
+                        },
+                        "small": {
+                            "tp": 10,
+                            "fp": 3,
+                            "fn": 1,
+                            "precision": 0.77,
+                            "recall": 0.91,
+                            "f1": 0.83
+                        },
+                        "medium": {
+                            "tp": 15,
+                            "fp": 7,
+                            "fn": 2,
+                            "precision": 0.68,
+                            "recall": 0.88,
+                            "f1": 0.77
+                        },
+                        "large": {
+                            "tp": 5,
+                            "fp": 2,
+                            "fn": 1,
+                            "precision": 0.71,
+                            "recall": 0.83,
+                            "f1": 0.76
+                        }
+                    }
+                }
             },
-        }
-
-        chart_data = {key: [] for key in categories.keys()}
-
-        # Log global metrics
-        if "global" in results:
-            for global_key, global_metrics in results["global"].items():
-                for metric, value in global_metrics["all"].items():
-                    log_key = (
-                        f"{wandb_section}/global/{global_key}/{metric}"
-                        if wandb_section
-                        else f"global/{global_key}/{metric}"
-                    )
-                    run.log({log_key: value})
-
-                    if debug:
-                        print(f" {log_key} = {value}")
-
-                    for category, metrics in categories.items():
-                        if metric in metrics:
-                            chart_data[category].append([metric, value])
-            print("----------------------------------------------------")
-
-        if log_plots:
-            for category, data in chart_data.items():
-                if data:
-                    table_data = [[label, value] for label, value in data]
-                    table = wandb.Table(data=table_data, columns=["metrics", "value"])
-                    run.log(
-                        {
-                            f"{category}_bar_chart": wandb.plot.bar(
-                                table,
-                                "metrics",
-                                "value",
-                                title=f"{category.replace('_', ' ').title()}",
-                            )
+            "model_2": {
+                "overall": {
+                    "all": {
+                        "tp": 60,
+                        "fp": 25,
+                        "fn": 15,
+                        "precision": 0.71,
+                        "recall": 0.80,
+                        "f1": 0.75
+                    },
+                    "small": {
+                        "tp": 20,
+                        "fp": 6,
+                        "fn": 3,
+                        "precision": 0.77,
+                        "recall": 0.87,
+                        "f1": 0.82
+                    },
+                    "medium": {
+                        "tp": 30,
+                        "fp": 12,
+                        "fn": 5,
+                        "precision": 0.71,
+                        "recall": 0.86,
+                        "f1": 0.78
+                    },
+                    "large": {
+                        "tp": 10,
+                        "fp": 7,
+                        "fn": 5,
+                        "precision": 0.59,
+                        "recall": 0.67,
+                        "f1": 0.63
+                    }
+                },
+                "per_sequence": {
+                    "sequence_1": {
+                        "all": {
+                            "tp": 40,
+                            "fp": 18,
+                            "fn": 8,
+                            "precision": 0.69,
+                            "recall": 0.83,
+                            "f1": 0.75
+                        },
+                        "small": {
+                            "tp": 12,
+                            "fp": 4,
+                            "fn": 2,
+                            "precision": 0.75,
+                            "recall": 0.86,
+                            "f1": 0.80
+                        },
+                        "medium": {
+                            "tp": 20,
+                            "fp": 8,
+                            "fn": 3,
+                            "precision": 0.71,
+                            "recall": 0.87,
+                            "f1": 0.78
+                        },
+                        "large": {
+                            "tp": 8,
+                            "fp": 6,
+                            "fn": 3,
+                            "precision": 0.57,
+                            "recall": 0.73,
+                            "f1": 0.64
                         }
-                    )
-
-        if "per_sequence" in results:
-            sorted_sequences = sorted(
-                results["per_sequence"].items(),
-                key=lambda x: next(iter(x[1].values()), {}).get("all", {}).get("f1", 0),
-                reverse=True,  # Set to True for descending order
-            )
-
-            for sequence_name, sequence_data in sorted_sequences:
-                for seq_key, seq_metrics in sequence_data.items():
-                    for metric, value in seq_metrics["all"].items():
-                        log_key = (
-                            f"{wandb_section}/per_sequence/{sequence_name}/{seq_key}/{metric}"
-                            if wandb_section
-                            else f"per_sequence/{sequence_name}/{seq_key}/{metric}"
-                        )
-                        run.log({log_key: value})
-                        if debug:
-                            print(f" {log_key} = {value}")
-                print("----------------------------------------------------")
-
-        if debug:
-            print("\nDebug Mode: Logging Summary and History")
-            print(f"Results Summary:\n{results}")
-            print(f"WandB Settings:\n{run.settings}")
-            print("All metrics have been logged.")
-
-        run.finish()
+                    }
+                }
+            }
+        }