metrics.py

import numpy as np
import rapidfuzz
import sklearn

from data import ValuesType, BenetechOutput, Annotation


def normalized_rmse(expected: list[float], predicted: list[float]) -> float:
    return (1 - sklearn.metrics.r2_score(expected, predicted)) ** 0.5


def normalized_levenshtein_distance(expected: list[str], predicted: list[str]) -> float:
    total_distance = 0
    for e, p in zip(expected, predicted):
        total_distance += rapidfuzz.distance.Levenshtein.distance(e, p)
    total_length = np.sum([len(e) for e in expected])
    return total_distance / total_length


def sigmoid(x):
    return 1 / (1 + np.exp(-x))


def positive_loss_to_score(x):
    return 2 * sigmoid(-x)


def score_axis_values(values_type, expected, predicted):
    if values_type == ValuesType.numerical:
        loss = normalized_rmse(expected, predicted)
    else:
        loss = normalized_levenshtein_distance(expected, predicted)
    return positive_loss_to_score(loss)


def benetech_score(expected: BenetechOutput, predicted: BenetechOutput) -> float:
    if expected.get_main_characteristics() != predicted.get_main_characteristics():
        return 0
    x_score = score_axis_values(
        expected.x_values_type, expected.x_data, predicted.x_data
    )
    y_score = score_axis_values(
        expected.y_values_type, expected.y_data, predicted.y_data
    )
    return (x_score + y_score) / 2


def benetech_score_string_prediction(expected_data_index: int, predicted_string: str):
    if not BenetechOutput.does_string_match_expected_pattern(predicted_string):
        return 0
    expected_annotation = Annotation.from_image_index(expected_data_index)
    expected_output = BenetechOutput.from_annotation(expected_annotation)
    predicted_output = BenetechOutput.from_string(predicted_string)
    return benetech_score(expected_output, predicted_output)