Upload metrics.py with huggingface_hub

metrics.py

@@ -16,7 +16,7 @@ from scipy.stats import bootstrap
 from scipy.stats._warnings_errors import DegenerateDataWarning
 
 from .artifact import Artifact
-from .dataclass import AbstractField, InternalField, OptionalField
+from .dataclass import AbstractField, InternalField, NonPositionalField, OptionalField
 from .logging_utils import get_logger
 from .metric_utils import InstanceInput, MetricRequest, MetricResponse
 from .operator import (
@@ -648,6 +648,9 @@ class InstanceMetric(SingleStreamOperator, MetricWithConfidenceInterval):
 
     reduction_map: Dict[str, List[str]] = AbstractField()
 
+    reference_field: str = NonPositionalField(default="references")
+    prediction_field: str = NonPositionalField(default="prediction")
+
     def _validate_group_mean_reduction(self, instances: List[dict]):
         """Ensure that group_mean reduction_map is properly formatted.
 
@@ -827,10 +830,21 @@ class InstanceMetric(SingleStreamOperator, MetricWithConfidenceInterval):
         instances = []
 
         for instance in stream:
-            refs, pred = instance["references"], instance["prediction"]
+            task_data = instance["task_data"] if "task_data" in instance else {}
+
+            if self.reference_field == "references":
+                refs = instance["references"]
+            else:
+                refs = task_data[self.reference_field]
+                if not isinstance(refs, list):
+                    refs = [refs]
+            if self.prediction_field == "prediction":
+                pred = instance["prediction"]
+            else:
+                pred = task_data[self.prediction_field]
+
             self._validate_prediction(pred)
             self._validate_reference(refs)
-            task_data = instance["task_data"] if "task_data" in instance else {}
 
             instance_score = self.compute(
                 references=refs, prediction=pred, task_data=task_data
@@ -1033,7 +1047,6 @@ class MetricPipeline(MultiStreamOperator, Metric):
                 [f"score/instance/{self.main_score}", "score/instance/score"],
                 [f"score/global/{self.main_score}", "score/global/score"],
             ],
-            use_query=True,
         )
 
     def process(self, multi_stream: MultiStream) -> MultiStream:
@@ -1447,13 +1460,15 @@ class Rouge(HuggingfaceMetric):
 
 
 # Computes char edit distance, ignoring whitespace
-class CharEditDistanceAccuracy(InstanceMetric):
-    reduction_map = {"mean": ["char_edit_dist_accuracy"]}
-    main_score = "char_edit_dist_accuracy"
-    ci_scores = ["char_edit_dist_accuracy"]
+class CharEditDistance(InstanceMetric):
+    main_score = "char_edit_distance"
+    reduction_map = {"mean": [main_score]}
+    ci_scores = [main_score]
     prediction_type = "str"
     single_reference_per_prediction = True
 
+    accuracy_metric = False
+
     _requirements_list: List[str] = ["editdistance"]
 
     def prepare(self):
@@ -1467,9 +1482,21 @@ class CharEditDistanceAccuracy(InstanceMetric):
         formatted_reference = "".join(references[0].split())
         max_length = max(len(formatted_reference), len(formatted_prediction))
         if max_length == 0:
-            return {"char_edit_dist_accuracy": 0.0}
+            return {self.main_score: 0.0}
         edit_dist = self.eval(formatted_reference, formatted_prediction)
-        return {"char_edit_dist_accuracy": (1 - edit_dist / max_length)}
+        if self.accuracy_metric:
+            score = 1 - edit_dist / max_length
+        else:
+            score = edit_dist
+        return {self.main_score: score}
+
+
+class CharEditDistanceAccuracy(CharEditDistance):
+    main_score = "char_edit_dist_accuracy"
+    reduction_map = {"mean": [main_score]}
+    ci_scores = [main_score]
+
+    accuracy_metric = True
 
 
 class Wer(HuggingfaceMetric):
@@ -1853,6 +1880,8 @@ class BertScore(HuggingfaceBulkMetric):
     ci_scores = ["f1", "precision", "recall"]
     model_name: str
 
+    prediction_type = "str"
+
     _requirements_list: List[str] = ["bert_score"]
 
     def prepare(self):
@@ -1949,6 +1978,38 @@ class Reward(BulkInstanceMetric):
         return self.pipe(inputs, batch_size=self.batch_size)
 
 
+class Detector(BulkInstanceMetric):
+    reduction_map = {"mean": ["score"]}
+    main_score = "score"
+    batch_size: int = 32
+
+    prediction_type = "str"
+
+    model_name: str
+
+    _requirements_list: List[str] = ["transformers", "torch"]
+
+    def prepare(self):
+        super().prepare()
+        import torch
+        from transformers import pipeline
+
+        device = "cuda:0" if torch.cuda.is_available() else "cpu"
+        self.pipe = pipeline(
+            "text-classification", model=self.model_name, device=device
+        )
+
+    def compute(
+        self,
+        references: List[List[Any]],
+        predictions: List[Any],
+        task_data: List[Dict],
+    ) -> List[Dict[str, Any]]:
+        # compute the metric
+        # add function_to_apply="none" to disable sigmoid
+        return self.pipe(predictions, batch_size=self.batch_size)
+
+
 class LlamaIndexCorrectness(InstanceMetric):
     """LlamaIndex based metric class for evaluating correctness."""
 
@@ -3320,6 +3381,99 @@ class BinaryMaxAccuracy(GlobalMetric):
         return {self.main_score: best_acc, "best_thr_max_acc": best_thr}
 
 
+######################
+# RerankRecallMetric #
+
+
+def pytrec_eval_at_k(results, qrels, at_k, metric_name):
+    import pandas as pd
+    import pytrec_eval
+
+    metric = {}
+
+    for k in at_k:
+        metric[f"{metric_name}@{k}"] = 0.0
+
+    metric_string = f"{metric_name}." + ",".join([str(k) for k in at_k])
+    # print('metric_string = ', metric_string)
+    evaluator = pytrec_eval.RelevanceEvaluator(
+        qrels, {"ndcg", metric_string}
+    )  # {map_string, ndcg_string, recall_string, precision_string})
+    scores = evaluator.evaluate(results)
+    scores = pd.DataFrame(scores).transpose()
+
+    keys = []
+    column_map = {}
+    for k in at_k:
+        keys.append(f"{metric_name}_{k}")
+        column_map[f"{metric_name}_{k}"] = k
+    scores[keys].rename(columns=column_map)
+
+    return scores
+
+
+class RerankRecall(GlobalMetric):
+    """RerankRecall: measures the quality of reranking with respect to ground truth ranking scores.
+
+    This metric measures ranking performance across a dataset.  The
+    references for a query will have a score of 1 for the gold passage
+    and 0 for all other passages.  The model returns scores in [0,1]
+    for each passage,query pair.  This metric measures recall at k by
+    testing that the predicted score for the gold passage,query pair
+    is at least the k'th highest for all passages for that query.  A
+    query receives 1 if so, and 0 if not.  The 1's and 0's are
+    averaged across the dataset.
+
+    query_id_field selects the field containing the query id for an instance.
+    passage_id_field selects the field containing the passage id for an instance.
+    at_k selects the value of k used to compute recall.
+
+    """
+
+    main_score = "recall_at_5"
+    query_id_field: str = "query_id"
+    passage_id_field: str = "passage_id"
+    at_k: List[int] = [1, 2, 5]
+
+    # This doesn't seem to make sense
+    n_resamples = None
+
+    _requirements_list: List[str] = ["pandas", "pytrec_eval"]
+
+    def compute(
+        self,
+        references: List[List[str]],
+        predictions: List[str],
+        task_data: List[Dict],
+    ):
+        # Collect relevance score and ref per query/passage pair
+        results = {}
+        qrels = {}
+        for ref, pred, data in zip(references, predictions, task_data):
+            qid = data[self.query_id_field]
+            pid = data[self.passage_id_field]
+            if qid not in results:
+                results[qid] = {}
+                qrels[qid] = {}
+            # Convert string-wrapped float to regular float
+            try:
+                results[qid][pid] = float(pred)
+            except ValueError:
+                # Card testing feeds nonnumeric values in, so catch that.
+                results[qid][pid] = np.nan
+
+            # There's always a single reference per pid/qid pair
+            qrels[qid][pid] = int(ref[0])
+
+        # Compute recall @ 5
+        scores = pytrec_eval_at_k(results, qrels, self.at_k, "recall")
+        # print(scores.describe())
+        # pytrec returns numpy float32
+        return {
+            f"recall_at_{i}": float(scores[f"recall_{i}"].mean()) for i in self.at_k
+        }
+
+
 KO_ERROR_MESSAGE = """
 
 Additional dependencies required. To install them, run: