Update leaderboard

app.py

@@ -31,13 +31,13 @@ def get_data(verified, dataset, ipc, label_type, metric_weights=None):
     data["score"] = data[METRICS[0].lower()] * 0.0
     for i, metric in enumerate(METRICS):
         data["score"] += data[metric.lower()] * metric_weights[i] * METRICS_SIGN[i]
-    data["score"] = (np.exp(-0.01 * data["score"]) - np.exp(-1.0)) / (np.exp(1.0) - np.exp(-1.0))
+    data["score"] = 100 * (np.exp(-0.01 * data["score"]) - np.exp(-1.0)) / (np.exp(1.0) - np.exp(-1.0))
     data = data.sort_values(by="score", ascending=False)
     data["ranking"] = range(1, len(data) + 1)
 
     for metric in METRICS:
         data[metric.lower()] = data[metric.lower()].apply(lambda x: round(x, 3))
-    data["score"] = data["score"].apply(lambda x: round(x, 3))
+    data["score"] = data["score"].apply(lambda x: round(x, 1))
 
     # formatting
     data["method"] = "[" + data["method"] + "](" + data["method_reference"] + ")"
@@ -45,9 +45,9 @@ def get_data(verified, dataset, ipc, label_type, metric_weights=None):
     data = data.drop(columns=["method_reference", "dataset", "ipc"])
     data = data[['ranking', 'method', 'verified', 'date', 'label_type', 'hlr', 'ior', 'score']]
     if label_type == "Hard Label":
-        data = data.rename(columns={"ranking": "Ranking", "method": "Method", "date": "Date", "label_type": "Label Type", "hlr": "HLR%↓", "ior": "IOR%↑", "score": "DDRS↑", "verified": "Verified"})
+        data = data.rename(columns={"ranking": "Ranking", "method": "Method", "date": "Date", "label_type": "Label Type", "hlr": "HLR%↓", "ior": "IOR%↑", "score": "LRS%↑", "verified": "Verified"})
     else:
-        data = data.rename(columns={"ranking": "Ranking", "method": "Method", "date": "Date", "label_type": "Label Type", "hlr": "HLR%↓", "ior": "IOR%↑", "score": "DDRS↑", "verified": "Verified"})
+        data = data.rename(columns={"ranking": "Ranking", "method": "Method", "date": "Date", "label_type": "Label Type", "hlr": "HLR%↓", "ior": "IOR%↑", "score": "LRS%↑", "verified": "Verified"})
     return data
 
 

constants.py

@@ -79,24 +79,26 @@ $\\text{HLR} = \\text{Acc.} \\text{full-hard} - \\text{Acc.} \\text{syn-hard}$:
 $\\text{IOR} = \\text{Acc.} \\text{syn-any} - \\text{Acc.} \\text{rdm-any}$: The improvement over random selection when using personalized evaluation methods (improvement over random).
 """
 
-DATASET_LIST = ["CIFAR-10", "CIFAR-100", "Tiny-ImageNet"]
+DATASET_LIST = ["CIFAR-10", "CIFAR-100", "Tiny-ImageNet", "ImageNet1K"]
 IPC_LIST = ["IPC-1", "IPC-10", "IPC-50"]
 DATASET_IPC_LIST = {
     "CIFAR-10": ["IPC-1", "IPC-10", "IPC-50"],
     "CIFAR-100": ["IPC-1", "IPC-10", "IPC-50"],
-    "Tiny-ImageNet": ["IPC-1", "IPC-10"],
+    "Tiny-ImageNet": ["IPC-1", "IPC-10", "IPC-50"],
+    "ImageNet1K": ["IPC-1", "IPC-10", "IPC-50"],
 }
 LABEL_TYPE_LIST = ["Hard Label", "Soft Label"]
 
 METRICS = ["HLR", "IOR"]
 METRICS_SIGN = [1.0, -1.0]
-COLUMN_NAMES = ["Ranking", "Method", "Verified", "Date", "Label Type", "HLR%", "IOR%", "DDRS"]
+COLUMN_NAMES = ["Ranking", "Method", "Verified", "Date", "Label Type", "HLR%", "IOR%", "LRS"]
 DATA_TITLE_TYPE = ['number', 'markdown', 'markdown', 'markdown', 'markdown', 'number', 'number', 'number']
 
 DATASET_MAPPING = {
     "CIFAR-10": 0,
     "CIFAR-100": 1,
     "Tiny-ImageNet": 2,
+    "ImageNet1K": 3,
 }
 
 IPC_MAPPING = {

data.csv

@@ -1,71 +1,114 @@
 method,method_reference,verified,date,dataset,ipc,label_type,hlr,ior
-DC,https://arxiv.org/abs/2006.05929,1,2025-02-08,0,0,0,52.66,12.40
-DSA,https://arxiv.org/abs/2102.08259,1,2025-02-08,0,0,0,58.93,13.16
-DM,https://arxiv.org/abs/2110.04181,1,2025-02-08,0,0,0,61.37,8.73
-MTT,https://arxiv.org/abs/2203.11932,1,2025-02-08,0,0,0,42.21,27.62
-DATM,https://arxiv.org/abs/2310.05773,1,2025-02-08,0,0,1,,
-SRe2L,https://arxiv.org/abs/2306.13092,1,2025-02-08,0,0,1,69.86,-0.27
-RDED,https://arxiv.org/abs/2312.03526,1,2025-02-08,0,0,1,60.58,2.38
-D4M,https://arxiv.org/abs/2407.15138,1,2025-02-08,0,0,1,51.06,6.68
+DC,https://arxiv.org/abs/2006.05929,1,2025-05-30,0,0,0,52.7,12.4
+DSA,https://arxiv.org/abs/2102.08259,1,2025-05-30,0,0,0,58.9,13.2
+DM,https://arxiv.org/abs/2110.04181,1,2025-05-30,0,0,0,61.4,8.7
+MTT,https://arxiv.org/abs/2203.11932,1,2025-05-30,0,0,0,42.2,27.6
+DataDAM,https://arxiv.org/abs/2310.00093,1,2025-05-30,0,0,0,49.9,15.6
+DATM,https://arxiv.org/abs/2310.05773,1,2025-05-30,0,0,1,41.9,30.8
+SRe2L,https://arxiv.org/abs/2306.13092,1,2025-05-30,0,0,1,69.9,-0.3
+RDED,https://arxiv.org/abs/2312.03526,1,2025-05-30,0,0,1,60.6,2.4
+D4M,https://arxiv.org/abs/2407.15138,1,2025-05-30,0,0,1,51.1,6.7
 
-DC,https://arxiv.org/abs/2006.05929,1,2025-02-08,0,1,0,36.70,18.54
-DSA,https://arxiv.org/abs/2102.08259,1,2025-02-08,0,1,0,35.14,19.62
-DM,https://arxiv.org/abs/2110.04181,1,2025-02-08,0,1,0,39.37,16.12
-MTT,https://arxiv.org/abs/2203.11932,1,2025-02-08,0,1,0,23.73,30.90
-DATM,https://arxiv.org/abs/2310.05773,1,2025-02-08,0,1,1,26.84,35.05
-SRe2L,https://arxiv.org/abs/2306.13092,1,2025-02-08,0,1,1,67.82,-5.77
-RDED,https://arxiv.org/abs/2312.03526,1,2025-02-08,0,1,1,50.65,1.11
-D4M,https://arxiv.org/abs/2407.15138,1,2025-02-08,0,1,1,39.88,9.12
+DC,https://arxiv.org/abs/2006.05929,1,2025-05-30,0,1,0,36.7,18.5
+DSA,https://arxiv.org/abs/2102.08259,1,2025-05-30,0,1,0,35.1,19.6
+DM,https://arxiv.org/abs/2110.04181,1,2025-05-30,0,1,0,39.3,16.1
+MTT,https://arxiv.org/abs/2203.11932,1,2025-05-30,0,1,0,23.7,30.9
+DataDAM,https://arxiv.org/abs/2310.00093,1,2025-05-30,0,1,0,34.8,19.9
+DATM,https://arxiv.org/abs/2310.05773,1,2025-05-30,0,1,1,26.8,35.1
+SRe2L,https://arxiv.org/abs/2306.13092,1,2025-05-30,0,1,1,67.8,-5.7
+RDED,https://arxiv.org/abs/2312.03526,1,2025-05-30,0,1,1,50.7,1.1
+D4M,https://arxiv.org/abs/2407.15138,1,2025-05-30,0,1,1,39.9,9.1
 
-DC,https://arxiv.org/abs/2006.05929,1,2025-02-08,0,2,0,26.31,12.34
-DSA,https://arxiv.org/abs/2102.08259,1,2025-02-08,0,2,0,27.41,10.96
-DM,https://arxiv.org/abs/2110.04181,1,2025-02-08,0,2,0,25.06,12.66
-MTT,https://arxiv.org/abs/2203.11932,1,2025-02-08,0,2,0,16.52,20.52
-DATM,https://arxiv.org/abs/2310.05773,1,2025-02-08,0,2,1,18.89,23.89
-SRe2L,https://arxiv.org/abs/2306.13092,1,2025-02-08,0,2,1,62.85,-6.50
-RDED,https://arxiv.org/abs/2312.03526,1,2025-02-08,0,2,1,36.01,-1.55
-D4M,https://arxiv.org/abs/2407.15138,1,2025-02-08,0,2,1,26.98,6.57
+DC,https://arxiv.org/abs/2006.05929,1,2025-05-30,0,2,0,26.3,12.3
+DSA,https://arxiv.org/abs/2102.08259,1,2025-05-30,0,2,0,27.4,11.0
+DM,https://arxiv.org/abs/2110.04181,1,2025-05-30,0,2,0,25.1,12.7
+MTT,https://arxiv.org/abs/2203.11932,1,2025-05-30,0,2,0,16.5,20.5
+DataDAM,https://arxiv.org/abs/2310.00093,1,2025-05-30,0,2,0,21.9,15.8
+DATM,https://arxiv.org/abs/2310.05773,1,2025-05-30,0,2,1,18.9,23.9
+SRe2L,https://arxiv.org/abs/2306.13092,1,2025-05-30,0,2,1,62.9,-6.5
+RDED,https://arxiv.org/abs/2312.03526,1,2025-05-30,0,2,1,36.0,-1.6
+D4M,https://arxiv.org/abs/2407.15138,1,2025-05-30,0,2,1,27.0,6.6
 
-DC,https://arxiv.org/abs/2006.05929,1,2025-02-08,1,0,0,39.36,8.36
-DSA,https://arxiv.org/abs/2102.08259,1,2025-02-08,1,0,0,46.04,8.50
-DM,https://arxiv.org/abs/2110.04181,1,2025-02-08,1,0,0,48.04,6.13
-MTT,https://arxiv.org/abs/2203.11932,1,2025-02-08,1,0,0,35.21,16.66
-SRe2L,https://arxiv.org/abs/2306.13092,1,2025-02-08,1,0,1,52.69,-1.93
-RDED,https://arxiv.org/abs/2312.03526,1,2025-02-08,1,0,1,45.55,-0.52
-D4M,https://arxiv.org/abs/2407.15138,1,2025-02-08,1,0,1,30.86,9.99
+DC,https://arxiv.org/abs/2006.05929,1,2025-05-30,1,0,0,39.4,8.4
+DSA,https://arxiv.org/abs/2102.08259,1,2025-05-30,1,0,0,46.0,8.5
+DM,https://arxiv.org/abs/2110.04181,1,2025-05-30,1,0,0,48.0,6.1
+MTT,https://arxiv.org/abs/2203.11932,1,2025-05-30,1,0,0,35.2,16.7
+DataDAM,https://arxiv.org/abs/2310.00093,1,2025-05-30,1,0,0,45.2,9.1
+DATM,https://arxiv.org/abs/2310.05773,1,2025-05-30,1,0,1,24.1,18.5
+SRe2L,https://arxiv.org/abs/2306.13092,1,2025-05-30,1,0,1,52.7,-1.9
+RDED,https://arxiv.org/abs/2312.03526,1,2025-05-30,1,0,1,45.6,-0.5
+D4M,https://arxiv.org/abs/2407.15138,1,2025-05-30,1,0,1,30.9,10.0
 
-DC,https://arxiv.org/abs/2006.05929,1,2025-02-08,1,1,0,25.52,12.66
-DSA,https://arxiv.org/abs/2102.08259,1,2025-02-08,1,1,0,27.28,13.49
-DM,https://arxiv.org/abs/2110.04181,1,2025-02-08,1,1,0,30.09,10.72
-MTT,https://arxiv.org/abs/2203.11932,1,2025-02-08,1,1,0,18.01,20.68
-DATM,https://arxiv.org/abs/2310.05773,1,2025-02-08,1,1,1,26.98,18.43
-SRe2L,https://arxiv.org/abs/2306.13092,1,2025-02-08,1,1,1,50.45,-14.86
-RDED,https://arxiv.org/abs/2312.03526,1,2025-02-08,1,1,1,37.46,-1.23
-D4M,https://arxiv.org/abs/2407.15138,1,2025-02-08,1,1,1,40.06,9.69
+DC,https://arxiv.org/abs/2006.05929,1,2025-05-30,1,1,0,25.5,12.7
+DSA,https://arxiv.org/abs/2102.08259,1,2025-05-30,1,1,0,26.1,13.5
+DM,https://arxiv.org/abs/2110.04181,1,2025-05-30,1,1,0,30.1,10.7
+MTT,https://arxiv.org/abs/2203.11932,1,2025-05-30,1,1,0,18.0,20.7
+DataDAM,https://arxiv.org/abs/2310.00093,1,2025-05-30,1,1,0,25.9,14.8
+DATM,https://arxiv.org/abs/2310.05773,1,2025-05-30,1,1,1,18.9,18.4
+SRe2L,https://arxiv.org/abs/2306.13092,1,2025-05-30,1,1,1,50.5,-14.8
+RDED,https://arxiv.org/abs/2312.03526,1,2025-05-30,1,1,1,37.5,-1.2
+D4M,https://arxiv.org/abs/2407.15138,1,2025-05-30,1,1,1,40.1,9.7
 
-DC,https://arxiv.org/abs/2006.05929,1,2025-02-08,1,2,0,21.77,1.11
-DSA,https://arxiv.org/abs/2102.08259,1,2025-02-08,1,2,0,21.19,2.02
-DM,https://arxiv.org/abs/2110.04181,1,2025-02-08,1,2,0,16.58,7.16
-MTT,https://arxiv.org/abs/2203.11932,1,2025-02-08,1,2,0,12.08,11.56
-DATM,https://arxiv.org/abs/2310.05773,1,2025-02-08,1,2,1,10.30,26.05
-SRe2L,https://arxiv.org/abs/2306.13092,1,2025-02-08,1,2,1,46.16,-13.55
-RDED,https://arxiv.org/abs/2312.03526,1,2025-02-08,1,2,1,27.32,-1.54
-D4M,https://arxiv.org/abs/2407.15138,1,2025-02-08,1,2,1,26.66,13.5
+DC,https://arxiv.org/abs/2006.05929,1,2025-05-30,1,2,0,21.8,1.1
+DSA,https://arxiv.org/abs/2102.08259,1,2025-05-30,1,2,0,21.2,2.0
+DM,https://arxiv.org/abs/2110.04181,1,2025-05-30,1,2,0,16.6,7.2
+MTT,https://arxiv.org/abs/2203.11932,1,2025-05-30,1,2,0,12.1,11.6
+DataDAM,https://arxiv.org/abs/2310.00093,1,2025-05-30,1,2,0,12.4,11.8
+DATM,https://arxiv.org/abs/2310.05773,1,2025-05-30,1,2,1,10.3,26.1
+SRe2L,https://arxiv.org/abs/2306.13092,1,2025-05-30,1,2,1,46.2,-13.6
+RDED,https://arxiv.org/abs/2312.03526,1,2025-05-30,1,2,1,27.3,-1.5
+D4M,https://arxiv.org/abs/2407.15138,1,2025-05-30,1,2,1,26.7,13.5
 
-DC,https://arxiv.org/abs/2006.05929,1,2025-02-08,2,0,0,28.61,2.90
-DSA,https://arxiv.org/abs/2102.08259,1,2025-02-08,2,0,0,36.26,2.67
-DM,https://arxiv.org/abs/2110.04181,1,2025-02-08,2,0,0,36.67,2.33
-MTT,https://arxiv.org/abs/2203.11932,1,2025-02-08,2,0,0,30.71,5.76
-DATM,https://arxiv.org/abs/2310.05773,1,2025-02-08,2,0,1,25.38,8.61
-SRe2L,https://arxiv.org/abs/2306.13092,1,2025-02-08,2,0,1,47.57,-3.99
-RDED,https://arxiv.org/abs/2312.03526,1,2025-02-08,2,0,1,34.67,2.38
-D4M,https://arxiv.org/abs/2407.15138,1,2025-02-08,2,0,1,35.92,-3.01
+DC,https://arxiv.org/abs/2006.05929,1,2025-05-30,2,0,0,28.6,3.9
+DSA,https://arxiv.org/abs/2102.08259,1,2025-05-30,2,0,0,30.3,3.7
+DM,https://arxiv.org/abs/2110.04181,1,2025-05-30,2,0,0,36.7,2.3
+MTT,https://arxiv.org/abs/2203.11932,1,2025-05-30,2,0,0,30.7,5.8
+DATM,https://arxiv.org/abs/2310.05773,1,2025-05-30,2,0,1,25.4,8.6
+EDF,https://arxiv.org/abs/2410.17193,1,2025-05-30,2,0,1,25.8,9.2
+SRe2L,https://arxiv.org/abs/2306.13092,1,2025-05-30,2,0,1,45.6,-1.8
+RDED,https://arxiv.org/abs/2312.03526,1,2025-05-30,2,0,1,34.0,3.9
+D4M,https://arxiv.org/abs/2407.15138,1,2025-05-30,2,0,1,40.6,-3.0
 
-DC,https://arxiv.org/abs/2006.05929,1,2025-02-08,2,1,0,21.53,7.14
-DSA,https://arxiv.org/abs/2102.08259,1,2025-02-08,2,1,0,26.33,6.77
-DM,https://arxiv.org/abs/2110.04181,1,2025-02-08,2,1,0,28.16,4.53
-MTT,https://arxiv.org/abs/2203.11932,1,2025-02-08,2,1,0,15.58,14.64
-DATM,https://arxiv.org/abs/2310.05773,1,2025-02-08,2,1,1,18.27,14.24
-SRe2L,https://arxiv.org/abs/2306.13092,1,2025-02-08,2,1,1,44.94,-12.95
-RDED,https://arxiv.org/abs/2312.03526,1,2025-02-08,2,1,1,25.94,1.81
-D4M,https://arxiv.org/abs/2407.15138,1,2025-02-08,2,1,1,40.60,-7.51
+DC,https://arxiv.org/abs/2006.05929,1,2025-05-30,2,1,0,21.5,7.1
+DSA,https://arxiv.org/abs/2102.08259,1,2025-05-30,2,1,0,20.3,6.8
+DM,https://arxiv.org/abs/2110.04181,1,2025-05-30,2,1,0,26.2,7.5
+MTT,https://arxiv.org/abs/2203.11932,1,2025-05-30,2,1,0,15.6,14.6
+DATM,https://arxiv.org/abs/2310.05773,1,2025-05-30,2,1,1,18.3,14.2
+EDF,https://arxiv.org/abs/2410.17193,1,2025-05-30,2,1,1,18.5,15.4
+SRe2L,https://arxiv.org/abs/2306.13092,1,2025-05-30,2,1,1,43.6,-8.5
+RDED,https://arxiv.org/abs/2312.03526,1,2025-05-30,2,1,1,25.6,1.8
+D4M,https://arxiv.org/abs/2407.15138,1,2025-05-30,2,1,1,35.6,-5.8
+
+DC,https://arxiv.org/abs/2006.05929,1,2025-05-30,2,2,0,21.3,-2.1
+DSA,https://arxiv.org/abs/2102.08259,1,2025-05-30,2,2,0,17.6,7.6
+DM,https://arxiv.org/abs/2110.04181,1,2025-05-30,2,2,0,18.9,5.3
+MTT,https://arxiv.org/abs/2203.11932,1,2025-05-30,2,2,0,15.6,10.2
+DATM,https://arxiv.org/abs/2310.05773,1,2025-05-30,2,2,1,13.5,15.1
+EDF,https://arxiv.org/abs/2410.17193,1,2025-05-30,2,2,1,13.8,15.9
+SRe2L,https://arxiv.org/abs/2306.13092,1,2025-05-30,2,2,1,33.6,-9.6
+RDED,https://arxiv.org/abs/2312.03526,1,2025-05-30,2,2,1,15.2,-0.6  
+D4M,https://arxiv.org/abs/2407.15138,1,2025-05-30,2,2,1,27.7,12.8
+
+SRe2L,https://arxiv.org/abs/2306.13092,1,2025-05-30,3,0,1,56.3,-1.5
+RDED,https://arxiv.org/abs/2312.03526,1,2025-05-30,3,0,1,55.7,1.6
+D4M,https://arxiv.org/abs/2407.15138,1,2025-05-30,3,0,1,55.9,-0.6
+DWA,https://arxiv.org/abs/2409.17612,1,2025-05-30,3,0,1,56.1,-1.2
+CDA,https://arxiv.org/abs/2311.18838,1,2025-05-30,3,0,1,56.2,-2.5
+EDC,https://arxiv.org/abs/2404.13733,1,2025-05-30,3,0,1,55.7,-0.8
+G-VBSM,https://arxiv.org/abs/2311.17950,1,2025-05-30,3,0,1,56.3,-1.2
+
+SRe2L,https://arxiv.org/abs/2306.13092,1,2025-05-30,3,1,1,55.0,-15.6
+RDED,https://arxiv.org/abs/2312.03526,1,2025-05-30,3,1,1,50.2,-0.6
+D4M,https://arxiv.org/abs/2407.15138,1,2025-05-30,3,1,1,53.0,-7.7
+DWA,https://arxiv.org/abs/2409.17612,1,2025-05-30,3,1,1,54.4,-4.1
+CDA,https://arxiv.org/abs/2311.18838,1,2025-05-30,3,1,1,54.9,-8.6
+EDC,https://arxiv.org/abs/2404.13733,1,2025-05-30,3,1,1,52.0,-0.4
+G-VBSM,https://arxiv.org/abs/2311.17950,1,2025-05-30,3,1,1,55.0,-7.3
+
+SRe2L,https://arxiv.org/abs/2306.13092,1,2025-05-30,3,1,1,53.4,-13.2
+RDED,https://arxiv.org/abs/2312.03526,1,2025-05-30,3,1,1,39.8,-3.6
+D4M,https://arxiv.org/abs/2407.15138,1,2025-05-30,3,1,1,43.7,-5.8
+DWA,https://arxiv.org/abs/2409.17612,1,2025-05-30,3,1,1,49.7,-7.8
+CDA,https://arxiv.org/abs/2311.18838,1,2025-05-30,3,1,1,52.0,-6.7
+EDC,https://arxiv.org/abs/2404.13733,1,2025-05-30,3,1,1,41.3,-0.1
+G-VBSM,https://arxiv.org/abs/2311.17950,1,2025-05-30,3,1,1,44.9,-5.9