<filename>eval.py
import numpy as np
import pandas as pd
from math import sqrt, acos, pi, sin, cos
from scipy.spatial.transform import Rotation as R
from sklearn.metrics import average_precision_score
from tqdm import tqdm
from joblib import Parallel, delayed
import argparse


def expand_df(df, PredictionStringCols):
    df = (df.copy()).dropna()
    print(df)
    df['NumCars'] = [int((x.count(' ') + 1) / 7)
                     for x in df['PredictionString']]

    image_id_expanded = [item for item, count in zip(
        df['ImageId'], df['NumCars']) for i in range(count)]
    prediction_strings_expanded = df['PredictionString'].str.split(
        ' ', expand=True).values.reshape(-1, 7).astype(float)
    prediction_strings_expanded = prediction_strings_expanded[~np.isnan(
        prediction_strings_expanded).all(axis=1)]
    df = pd.DataFrame(
        {
            'ImageId': image_id_expanded,
            PredictionStringCols[0]: prediction_strings_expanded[:, 0],
            PredictionStringCols[1]: prediction_strings_expanded[:, 1],
            PredictionStringCols[2]: prediction_strings_expanded[:, 2],
            PredictionStringCols[3]: prediction_strings_expanded[:, 3],
            PredictionStringCols[4]: prediction_strings_expanded[:, 4],
            PredictionStringCols[5]: prediction_strings_expanded[:, 5],
            PredictionStringCols[6]: prediction_strings_expanded[:, 6]
        })
    return df


def str2coords(s, names):
    coords = []
    for l in np.array(s.split()).reshape([-1, 7]):
        coords.append(dict(zip(names, l.astype('float'))))
    return coords


def TranslationDistance(p, g, abs_dist=False):
    dx = p['x'] - g['x']
    dy = p['y'] - g['y']
    dz = p['z'] - g['z']
    diff0 = (g['x']**2 + g['y']**2 + g['z']**2)**0.5
    diff1 = (dx**2 + dy**2 + dz**2)**0.5
    if abs_dist:
        diff = diff1
    else:
        diff = diff1 / diff0
    return diff


def RotationDistance(p, g):
    true = [g['pitch'], g['yaw'], g['roll']]
    pred = [p['pitch'], p['yaw'], p['roll']]
    q1 = R.from_euler('xyz', true)
    q2 = R.from_euler('xyz', pred)
    diff = R.inv(q2) * q1
    W = np.clip(diff.as_quat()[-1], -1., 1.)

    # in the official metrics code:
    # https://www.kaggle.com/c/pku-autonomous-driving/overview/evaluation
    #   return Object3D.RadianToDegree( Math.Acos(diff.W) )
    # this code treat θ and θ+2π differntly.
    # So this should be fixed as follows.
    W = (acos(W) * 360) / pi
    if W > 180:
        W = 180 - W
    return W


def check_match(val_df, train_df, thre_tr_dist, thre_ro_dist, keep_gt=False):
    train_dict = {imgID: str2coords(s, names=['carid_or_score', 'pitch', 'yaw', 'roll', 'x', 'y', 'z'])
                  for imgID, s in zip(train_df['ImageId'], train_df['PredictionString'])}
    val_dict = {imgID: str2coords(s, names=['pitch', 'yaw', 'roll', 'x', 'y', 'z', 'carid_or_score'])
                  for imgID, s in zip(val_df['ImageId'], val_df['PredictionString'])}
    result_flg = []  # 1 for TP, 0 for FP
    scores = []
    MAX_VAL = 10**10
    for img_id in tqdm(val_dict, total=len(val_dict)):
        for pcar in sorted(val_dict[img_id], key=lambda x: -x['carid_or_score']):
            # find nearest GT
            min_tr_dist = MAX_VAL
            min_idx = -1
            for idx, gcar in enumerate(train_dict[img_id]):
                tr_dist = TranslationDistance(pcar, gcar)
                if tr_dist < min_tr_dist:
                    min_tr_dist = tr_dist
                    min_ro_dist = RotationDistance(pcar, gcar)
                    min_idx = idx

            # set the result
            if min_tr_dist < thre_tr_dist and min_ro_dist < thre_ro_dist:
                if not keep_gt:
                    train_dict[img_id].pop(min_idx)
                result_flg.append(1)
            else:
                result_flg.append(0)
            scores.append(pcar['carid_or_score'])

    return result_flg, scores


def mean_average_precision(infile, nrows=None):
    val_df = pd.read_csv(infile, nrows=nrows)
    val_df = val_df.dropna()
    expanded_val_df = expand_df(
        val_df, ['pitch', 'yaw', 'roll', 'x', 'y', 'z', 'Score'])
    val_df = val_df.fillna('')
    print(val_df.head())

    train_df = pd.read_csv('inputs/train.csv')
    train_df = train_df[train_df.ImageId.isin(val_df.ImageId.unique())]
    # data description page says, The pose information is formatted as
    # model type, yaw, pitch, roll, x, y, z
    # but it doesn't, and it should be
    # model type, pitch, yaw, roll, x, y, z
    expanded_train_df = expand_df(
        train_df, ['model_type', 'pitch', 'yaw', 'roll', 'x', 'y', 'z'])
    n_gt = len(expanded_train_df)

    thres_ro_list = [50, 45, 40, 35, 30, 25, 20, 15, 10, 5]
    thres_tr_list = [0.1, 0.09, 0.08, 0.07, 0.06, 0.05, 0.04, 0.03, 0.02, 0.01]
    ap_list = []
    for thre_ro_dist, thre_tr_dist in zip(thres_ro_list, thres_tr_list):
        abs_dist = False
        result_flg, scores = check_match(
            val_df, train_df, thre_tr_dist, thre_ro_dist)
        n_tp = np.sum(result_flg)
        recall = n_tp / n_gt
        ap = average_precision_score(result_flg, scores) * recall
        ap_list.append(ap)
    return np.mean(ap_list)


def parse_args():
    parser = argparse.ArgumentParser()

    parser.add_argument('--name', default=None)

    args = parser.parse_args()

    return args


def main():
    args = parse_args()

    map = mean_average_precision('outputs/submissions/val/%s.csv' %args.name, nrows=None)
    print('map:', map)


if __name__ == '__main__':
    main()

<filename>eval.py
import numpy as np
import pandas as pd
from math import sqrt, acos, pi, sin, cos
from scipy.spatial.transform import Rotation as R
from sklearn.metrics import average_precision_score
from tqdm import tqdm
from joblib import Parallel, delayed
import argparse


def expand_df(df, PredictionStringCols):
    df = (df.copy()).dropna()
    print(df)
    df['NumCars'] = [int((x.count(' ') + 1) / 7)
                     for x in df['PredictionString']]

    image_id_expanded = [item for item, count in zip(
        df['ImageId'], df['NumCars']) for i in range(count)]
    prediction_strings_expanded = df['PredictionString'].str.split(
        ' ', expand=True).values.reshape(-1, 7).astype(float)
    prediction_strings_expanded = prediction_strings_expanded[~np.isnan(
        prediction_strings_expanded).all(axis=1)]
    df = pd.DataFrame(
        {
            'ImageId': image_id_expanded,
            PredictionStringCols[0]: prediction_strings_expanded[:, 0],
            PredictionStringCols[1]: prediction_strings_expanded[:, 1],
            PredictionStringCols[2]: prediction_strings_expanded[:, 2],
            PredictionStringCols[3]: prediction_strings_expanded[:, 3],
            PredictionStringCols[4]: prediction_strings_expanded[:, 4],
            PredictionStringCols[5]: prediction_strings_expanded[:, 5],
            PredictionStringCols[6]: prediction_strings_expanded[:, 6]
        })
    return df


def str2coords(s, names):
    coords = []
    for l in np.array(s.split()).reshape([-1, 7]):
        coords.append(dict(zip(names, l.astype('float'))))
    return coords


def TranslationDistance(p, g, abs_dist=False):
    dx = p['x'] - g['x']
    dy = p['y'] - g['y']
    dz = p['z'] - g['z']
    diff0 = (g['x']**2 + g['y']**2 + g['z']**2)**0.5
    diff1 = (dx**2 + dy**2 + dz**2)**0.5
    if abs_dist:
        diff = diff1
    else:
        diff = diff1 / diff0
    return diff


def RotationDistance(p, g):
    true = [g['pitch'], g['yaw'], g['roll']]
    pred = [p['pitch'], p['yaw'], p['roll']]
    q1 = R.from_euler('xyz', true)
    q2 = R.from_euler('xyz', pred)
    diff = R.inv(q2) * q1
    W = np.clip(diff.as_quat()[-1], -1., 1.)

    # in the official metrics code:
    # https://www.kaggle.com/c/pku-autonomous-driving/overview/evaluation
    #   return Object3D.RadianToDegree( Math.Acos(diff.W) )
    # this code treat θ and θ+2π differntly.
    # So this should be fixed as follows.
    W = (acos(W) * 360) / pi
    if W > 180:
        W = 180 - W
    return W


def check_match(val_df, train_df, thre_tr_dist, thre_ro_dist, keep_gt=False):
    train_dict = {imgID: str2coords(s, names=['carid_or_score', 'pitch', 'yaw', 'roll', 'x', 'y', 'z'])
                  for imgID, s in zip(train_df['ImageId'], train_df['PredictionString'])}
    val_dict = {imgID: str2coords(s, names=['pitch', 'yaw', 'roll', 'x', 'y', 'z', 'carid_or_score'])
                  for imgID, s in zip(val_df['ImageId'], val_df['PredictionString'])}
    result_flg = []  # 1 for TP, 0 for FP
    scores = []
    MAX_VAL = 10**10
    for img_id in tqdm(val_dict, total=len(val_dict)):
        for pcar in sorted(val_dict[img_id], key=lambda x: -x['carid_or_score']):
            # find nearest GT
            min_tr_dist = MAX_VAL
            min_idx = -1
            for idx, gcar in enumerate(train_dict[img_id]):
                tr_dist = TranslationDistance(pcar, gcar)
                if tr_dist < min_tr_dist:
                    min_tr_dist = tr_dist
                    min_ro_dist = RotationDistance(pcar, gcar)
                    min_idx = idx

            # set the result
            if min_tr_dist < thre_tr_dist and min_ro_dist < thre_ro_dist:
                if not keep_gt:
                    train_dict[img_id].pop(min_idx)
                result_flg.append(1)
            else:
                result_flg.append(0)
            scores.append(pcar['carid_or_score'])

    return result_flg, scores


def mean_average_precision(infile, nrows=None):
    val_df = pd.read_csv(infile, nrows=nrows)
    val_df = val_df.dropna()
    expanded_val_df = expand_df(
        val_df, ['pitch', 'yaw', 'roll', 'x', 'y', 'z', 'Score'])
    val_df = val_df.fillna('')
    print(val_df.head())

    train_df = pd.read_csv('inputs/train.csv')
    train_df = train_df[train_df.ImageId.isin(val_df.ImageId.unique())]
    # data description page says, The pose information is formatted as
    # model type, yaw, pitch, roll, x, y, z
    # but it doesn't, and it should be
    # model type, pitch, yaw, roll, x, y, z
    expanded_train_df = expand_df(
        train_df, ['model_type', 'pitch', 'yaw', 'roll', 'x', 'y', 'z'])
    n_gt = len(expanded_train_df)

    thres_ro_list = [50, 45, 40, 35, 30, 25, 20, 15, 10, 5]
    thres_tr_list = [0.1, 0.09, 0.08, 0.07, 0.06, 0.05, 0.04, 0.03, 0.02, 0.01]
    ap_list = []
    for thre_ro_dist, thre_tr_dist in zip(thres_ro_list, thres_tr_list):
        abs_dist = False
        result_flg, scores = check_match(
            val_df, train_df, thre_tr_dist, thre_ro_dist)
        n_tp = np.sum(result_flg)
        recall = n_tp / n_gt
        ap = average_precision_score(result_flg, scores) * recall
        ap_list.append(ap)
    return np.mean(ap_list)


def parse_args():
    parser = argparse.ArgumentParser()

    parser.add_argument('--name', default=None)

    args = parser.parse_args()

    return args


def main():
    args = parse_args()

    map = mean_average_precision('outputs/submissions/val/%s.csv' %args.name, nrows=None)
    print('map:', map)


if __name__ == '__main__':
    main()

# in the official metrics code: # https://www.kaggle.com/c/pku-autonomous-driving/overview/evaluation #   return Object3D.RadianToDegree( Math.Acos(diff.W) ) # this code treat θ and θ+2π differntly. # So this should be fixed as follows. # 1 for TP, 0 for FP # find nearest GT # set the result # data description page says, The pose information is formatted as # model type, yaw, pitch, roll, x, y, z # but it doesn't, and it should be # model type, pitch, yaw, roll, x, y, z