Spaces:
Running
on
Zero
Running
on
Zero
File size: 6,376 Bytes
9b33fca |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 |
"""Binary occupancy evaluator."""
from __future__ import annotations
import numpy as np
from vis4d.common.array import array_to_numpy
from vis4d.common.typing import (
ArrayLike,
MetricLogs,
NDArrayBool,
NDArrayNumber,
)
from vis4d.eval.base import Evaluator
def threshold_and_flatten(
prediction: NDArrayNumber, target: NDArrayNumber, threshold_value: float
) -> tuple[NDArrayBool, NDArrayBool]:
"""Thresholds the predictions based on the provided treshold value.
Applies the following actions:
prediction -> prediction >= threshold_value
pred, gt = pred.ravel().bool(), gt.ravel().bool()
Args:
prediction: Prediction array with continuous values
target: Grondgtruth values {0,1}
threshold_value: Value to use to convert the continuous prediction
into binary.
Returns:
tuple of two boolean arrays, prediction and target
"""
prediction_bin: NDArrayBool = prediction >= threshold_value
return prediction_bin.ravel().astype(bool), target.ravel().astype(bool)
class BinaryEvaluator(Evaluator):
"""Creates a new Evaluater that evaluates binary predictions."""
METRIC_BINARY = "BinaryCls"
KEY_IOU = "IoU"
KEY_ACCURACY = "Accuracy"
KEY_F1 = "F1"
KEY_PRECISION = "Precision"
KEY_RECALL = "Recall"
def __init__(
self,
threshold: float = 0.5,
) -> None:
"""Creates a new binary evaluator.
Args:
threshold (float): Threshold for prediction to convert
to binary. All prediction that are higher than
this value will be assigned the 'True' label
"""
super().__init__()
self.threshold = threshold
self.reset()
self.true_positives: list[float] = []
self.false_positives: list[float] = []
self.true_negatives: list[float] = []
self.false_negatives: list[float] = []
self.n_samples: list[float] = []
self.has_samples = False
def _calc_confusion_matrix(
self, prediction: NDArrayBool, target: NDArrayBool
) -> None:
"""Calculates the confusion matrix and stores them as attributes.
Args:
prediction: the prediction (binary) (N, Pts)
target: the groundtruth (binary) (N, Pts)
"""
tp = int(np.sum(np.logical_and(prediction == 1, target == 1)))
fp = int(np.sum(np.logical_and(prediction == 1, target == 0)))
tn = int(np.sum(np.logical_and(prediction == 0, target == 0)))
fn = int(np.sum(np.logical_and(prediction == 0, target == 1)))
self.true_positives.append(tp)
self.false_positives.append(fp)
self.true_negatives.append(tn)
self.false_negatives.append(fn)
self.n_samples.append(tp + fp + tn + fn)
@property
def metrics(self) -> list[str]:
"""Supported metrics."""
return [self.METRIC_BINARY]
def reset(self) -> None:
"""Reset the saved predictions to start new round of evaluation."""
self.true_positives = []
self.false_positives = []
self.true_negatives = []
self.false_negatives = []
self.n_samples = []
def process_batch(
self,
prediction: ArrayLike,
groundtruth: ArrayLike,
) -> None:
"""Processes a new (batch) of predictions.
Calculates the metrics and caches them internally.
Args:
prediction: the prediction(continuous values or bin) (Batch x Pts)
groundtruth: the groundtruth (binary) (Batch x Pts)
"""
pred, gt = threshold_and_flatten(
array_to_numpy(prediction, n_dims=None, dtype=np.float32),
array_to_numpy(groundtruth, n_dims=None, dtype=np.bool_),
self.threshold,
)
# Confusion Matrix
self._calc_confusion_matrix(pred, gt)
self.has_samples = True
def evaluate(self, metric: str) -> tuple[MetricLogs, str]:
"""Evaluate predictions.
Returns a dict containing the raw data and a
short description string containing a readable result.
Args:
metric (str): Metric to use. See @property metric
Returns:
metric_data, description
tuple containing the metric data (dict with metric name and value)
as well as a short string with shortened information.
Raises:
RuntimeError: if no data has been registered to be evaluated.
ValueError: if metric is not supported.
"""
if not self.has_samples:
raise RuntimeError(
"""No data registered to calculate metric.
Register data using .process() first!"""
)
metric_data: MetricLogs = {}
short_description = ""
if metric == self.METRIC_BINARY:
# IoU
iou = sum(self.true_positives) / (
sum(self.n_samples) - sum(self.true_negatives) + 1e-6
)
metric_data[self.KEY_IOU] = iou
short_description += f"IoU: {iou:.3f}\n"
# Accuracy
acc = (sum(self.true_positives) + sum(self.true_negatives)) / sum(
self.n_samples
)
metric_data[self.KEY_ACCURACY] = acc
short_description += f"Accuracy: {acc:.3f}\n"
# Precision
tp_fp = sum(self.true_positives) + sum(self.false_positives)
precision = sum(self.true_positives) / tp_fp if tp_fp != 0 else 1
metric_data[self.KEY_PRECISION] = precision
short_description += f"Precision: {precision:.3f}\n"
# Recall
tp_fn = sum(self.true_positives) + sum(self.false_negatives)
recall = sum(self.true_positives) / tp_fn if tp_fn != 0 else 1
metric_data[self.KEY_RECALL] = recall
short_description += f"Recall: {acc:.3f}\n"
# F1
f1 = 2 * precision * recall / (precision + recall + 1e-8)
metric_data[self.KEY_F1] = f1
short_description += f"F1: {f1:.3f}\n"
else:
raise ValueError(
f"Unsupported metric: {metric}"
) # pragma: no cover
return metric_data, short_description
|