paper/plot_contour_metrics.py

import argparse
import os

import matplotlib.pyplot as plt

import numpy as np

from lydorn_utils import python_utils
from lydorn_utils import print_utils


def get_args():
    argparser = argparse.ArgumentParser(description=__doc__)
    argparser.add_argument(
        '--dirpath',
        default="/home/lydorn/data/mapping_challenge_dataset/eval_runs",
        type=str,
        help='Path to eval directory')

    args = argparser.parse_args()
    return args


def plot_metric(dirpath, info_list):
    legend = []
    for info in info_list:
        metrics_filepath = os.path.join(dirpath, info["metrics_filepath"])
        metrics = python_utils.load_json(metrics_filepath)
        if metrics:
            max_angle_diffs = np.array(metrics["max_angle_diffs"])
            total = len(max_angle_diffs)
            angle_thresholds = range(0, 91)
            fraction_under_threshold_list = []
            for angle_threshold in angle_thresholds:
                fraction_under_threshold = np.sum(max_angle_diffs < angle_threshold) / total
                fraction_under_threshold_list.append(fraction_under_threshold)
            # Plot
            plt.plot(angle_thresholds, fraction_under_threshold_list)

            # Compute mean
            mean_error = np.mean(max_angle_diffs)

            legend.append(f"{info['name']}: {mean_error:.1f}°")

        else:
            print_utils.print_warning("WARNING: could not open {}".format(info["metrics_filepath"]))

    plt.legend(legend, loc='lower right')
    plt.xlabel("Threshold (degrees)")
    plt.ylabel("Fraction of detections")
    axes = plt.gca()
    axes.set_xlim([0, 90])
    axes.set_ylim([0, 1])
    title = f"Cumulative max tangent angle error per detection"
    plt.title(title)
    plt.savefig(title.lower().replace(" ", "_") + ".pdf")
    plt.show()


def main():
    args = get_args()


    # Mapping challenge:
    info_list = [
        {
            "name": "UResNet101 (no field), simple poly.",
            "metrics_filepath": "mapping_dataset.unet_resnet101_pretrained.field_off.train_val | 2020-05-21 08:33:20/test.metrics.test.annotation.poly.simple.tol_0.125.json"
        },
        {
            "name": "UResNet101 (with field), simple poly.",
            "metrics_filepath": "mapping_dataset.unet_resnet101_pretrained.train_val | 2020-05-21 08:32:48/test.metrics.test.annotation.poly.simple.tol_0.125.json"
        },
        {
            "name": "UResNet101 (with field), our poly.",
            "metrics_filepath": "mapping_dataset.unet_resnet101_pretrained.train_val | 2020-05-21 08:32:48/test.metrics.test.annotation.poly.acm.tol_0.125.json"
        },
        {
            "name": "UResNet101 (no $L_{align90}$), our poly.",
            "metrics_filepath": "mapping_dataset.unet_resnet101_pretrained.align90_off.train_val | 2020-11-02 07:34:43/test.metrics.test.annotation.poly.acm.tol_0.125.json"
        },
        {
            "name": "UResNet101 (no $L_{int edge}$), our poly.",
            "metrics_filepath": "mapping_dataset.unet_resnet101_pretrained.edge_int_off.train_val | 2020-11-02 07:34:54/test.metrics.test.annotation.poly.acm.tol_0.125.json"
        },
        {
            "name": "UResNet101 (no $L_{int align}$ and $L_{edge align}$), our poly.",
            "metrics_filepath": "mapping_dataset.unet_resnet101_pretrained.seg_framefield_off.train_val | 2020-10-29 11:27:52/test.metrics.test.annotation.poly.acm.tol_0.125.json"
        },
        {
            "name": "UResNet101 (no $L_{smooth}$), our poly.",
            "metrics_filepath": "mapping_dataset.unet_resnet101_pretrained.smooth_off.train_val | 2020-10-29 11:18:33/test.metrics.test.annotation.poly.acm.tol_0.125.json"
        },

        {
            "name": "PolyMapper",
            "metrics_filepath": "mapping_dataset.polymapper | 0000-00-00 00:00:00/test.metrics.test.annotation.poly.json"
        },
        {
            "name": "U-Net variant, ASIP poly.",
            "metrics_filepath": "mapping_dataset.asip | 0000-00-00 00:00:00/test.metrics.test.annotation.poly.json"
        },
        {
            "name": "Zorzi et al.",
            "metrics_filepath": "mapping_dataset.zorzi | 0000-00-00 00:00:00/test.metrics.test.annotation.poly.json"
        },
        {
            "name": "U-Net variant, UResNet101 poly",
            "metrics_filepath": "mapping_dataset.open_solution_full | 0000-00-00 00:00:00/test.metrics.test.annotation.seg_cleaned.poly.json"
        }
    ]

    # Inria Polygonized Dataset
    # info_list = [
    #     {
    #         "name": "UResNet101 (no field), simple poly.",
    #         "metrics_filepath": "/home/lydorn/data/AerialImageDataset/raw/test/pred_ours_leaderboard_new_losses.field_off/poly_shapefile.simple.tol_1/aggr_metrics.json"
    #     },
    #     {
    #         "name": "UResNet101 (with field), our poly.",
    #         "metrics_filepath": "/home/lydorn/data/AerialImageDataset/raw/test/pred_ours_leaderboard/poly_shapefile.acm.tol_0.125/aggr_metrics.json"
    #     },
    #     {
    #         "name": "Zorzi et al.",
    #         "metrics_filepath": "/home/lydorn/data/AerialImageDataset/raw/test/pred_zorzi/shapes/aggr_metrics.json"
    #     },
    #     {
    #         "name": "ICTNet, simple poly.",
    #         "metrics_filepath": "/home/lydorn/data/AerialImageDataset/raw/test/pred_ictnet/shp/aggr_metrics.json"
    #     },
    #     {
    #         "name": "Khvedchenya, simple poly.",
    #         "metrics_filepath": "/home/lydorn/data/AerialImageDataset/raw/test/pred_khvedchenya/shp/aggr_metrics.json"
    #     },
    # ]

    # LuxCarta's Bangkok image
    # info_list = [
    #     {
    #         "name": "ACM",
    #         "metrics_filepath": "/home/lydorn/repos/lydorn/frame_field_learning/frame_field_learning/test_images/Bangkok/Bangkok3bands.poly_acm.metrics.json"
    #     },
    #     {
    #         "name": "ASM",
    #         "metrics_filepath": "/home/lydorn/repos/lydorn/frame_field_learning/frame_field_learning/test_images/Bangkok/Bangkok3bands.poly_asm.metrics.json"
    #     },
    #     {
    #         "name": "ASM regularized",
    #         "metrics_filepath": "/home/lydorn/repos/lydorn/frame_field_learning/frame_field_learning/test_images/Bangkok/Bangkok3bands.reg.metrics.json"
    #     },
    #     {
    #         "name": "Company",
    #         "metrics_filepath": "/home/lydorn/repos/lydorn/frame_field_learning/frame_field_learning/test_images/Bangkok/Luxcarta/Building_Thailand_Bangkok_pansharpened25.metrics.json"
    #     },
    # ]

    plot_metric(args.dirpath, info_list)


if __name__ == '__main__':
    main()