Leaderboard adjustments for JSALT25 workshop

app.py

@@ -3,7 +3,7 @@ import os
 import gradio as gr
 from gradio_modal import Modal
 
-from content import HEADER_MARKDOWN, LEADERBOARD_TAB_TITLE_MARKDOWN, SUBMISSION_TAB_TITLE_MARKDOWN
+from content import HEADER_MARKDOWN, LEADERBOARD_TAB_TITLE_MARKDOWN, SUBMISSION_TAB_TITLE_MARKDOWN, ADDITIONAL_NOTES_MARKDOWN
 from leaderboard_server import LeaderboardServer
 
 # Initialize server and task list
@@ -20,10 +20,10 @@ def get_datasets_for_task(task):
 
 
 def update_datasets(task):
-    return gr.CheckboxGroup.update(choices=get_datasets_for_task(task), value=[])
+    return gr.CheckboxGroup(choices=get_datasets_for_task(task), value=get_datasets_for_task(task))
 
 
-def submit_model(task, datasets, hyp_file, submitted_by, model_id, token):
+def submit_model(task, datasets, hyp_file, submitted_by, model_id, token, normalize):
     if not hyp_file:
         return gr.update(visible=True, value="⚠️ Please upload a hypothesis file.")
 
@@ -35,8 +35,10 @@ def submit_model(task, datasets, hyp_file, submitted_by, model_id, token):
 
     metadata = {
         "submitted_by": submitted_by.strip(),
-        "model_id": model_id.strip()
+        "model_id": model_id.strip(),
+        "normalize": normalize  # Include normalization info in metadata if needed
     }
+
     leaderboard_df = server.get_leaderboard()
     if len(leaderboard_df) > 0:
         existing = leaderboard_df[
@@ -47,10 +49,13 @@ def submit_model(task, datasets, hyp_file, submitted_by, model_id, token):
             return gr.update(value="❌ A submission with this name and model ID already exists.", visible=True)
 
     try:
-        server.prepare_model_for_submission(hyp_file.name, metadata, task, datasets)
+        server.prepare_model_for_submission(
+            hyp_file.name, metadata, task, datasets, normalize=normalize
+        )
         server.update_leaderboard()
         return gr.update(visible=True, value="✅ Submission successful!")
     except Exception as e:
+        print(e)
         return gr.update(visible=True, value=f"❌ Error: {str(e)}")
 
 def get_leaderboard_df():
@@ -74,7 +79,7 @@ with gr.Blocks() as demo:
 
             with gr.Row():
                 task_dropdown = gr.Dropdown(choices=TASKS, value=TASKS[0], label="Select Task")
-                dataset_checkboxes = gr.CheckboxGroup(choices=get_datasets_for_task(TASKS[0]), label="Select Datasets")
+                dataset_checkboxes = gr.CheckboxGroup(choices=get_datasets_for_task(TASKS[0]), label="Select Datasets", value=get_datasets_for_task(TASKS[0]))
             task_dropdown.change(fn=update_datasets, inputs=task_dropdown, outputs=dataset_checkboxes)
 
             with gr.Row():
@@ -83,20 +88,25 @@ with gr.Blocks() as demo:
                 token_input = gr.Text(label="Submission Token", type="password")
 
             hyp_file_upload = gr.File(label="Upload Hypothesis JSON", file_types=[".json"])
+
+            normalize_checkbox = gr.Checkbox(label="Apply Chime-8 Normalization", value=True)
+
             submit_btn = gr.Button("Submit")
+
             with Modal("Submission Feedback", visible=False) as loading_msg:
                 feedback_text = gr.Text(visible=True, label="⏳ Processing your submission...")
+            gr.Markdown(ADDITIONAL_NOTES_MARKDOWN)
 
             submit_btn.click(
                 lambda: gr.update(visible=True),  # Show loading
                 outputs=loading_msg
             ).then(
                 fn=submit_model,
-                inputs=[task_dropdown, dataset_checkboxes, hyp_file_upload, submitted_by_input, model_id_input,
-                        token_input],
+                inputs=[task_dropdown, dataset_checkboxes, hyp_file_upload,
+                        submitted_by_input, model_id_input, token_input, normalize_checkbox],
                 outputs=[feedback_text],
             ).then(
-                lambda: server.get_leaderboard(),  # Show loading
+                lambda: server.get_leaderboard(),
                 outputs=leaderboard_output
             )
 

content.py

@@ -4,13 +4,13 @@ This file contains the text content for the leaderboard client.
 HEADER_MARKDOWN = """
 # EMMA JSALT25 Benchmark – Multi-Talker ASR Evaluation
 
-Welcome to the official leaderboard for benchmarking **multi-talker ASR systems**, hosted by the **EMMA JSALT25 team**. This platform enables model submissions, comparisons, and evaluation on challenging multi-speaker scenarios.
+Welcome to the official leaderboard for benchmarking **multi-talker ASR systems**, hosted by the **EMMA JSALT25 team**.
 """
 
 LEADERBOARD_TAB_TITLE_MARKDOWN = """
 ## Leaderboard
 
-Below you’ll find the latest results submitted to the benchmark. Models are evaluated using **`meeteval`** with **TCP-WER (collar=5s)**.
+Below you’ll find the latest results submitted to the benchmark. Models are evaluated using **`meeteval`** with **TCP-WER [%] (collar=5s)**.
 """
 
 SUBMISSION_TAB_TITLE_MARKDOWN = """
@@ -29,23 +29,17 @@ To submit your MT-ASR hypothesis to the benchmark, complete the form below:
 After clicking **Submit**, your model will be evaluated and results displayed in the leaderboard.
 """
 
+ADDITIONAL_NOTES_MARKDOWN = """
 
-RANKING_AFTER_SUBMISSION_MARKDOWN = """
-📊 Below is how your model compares after evaluation:
-"""
 
-SUBMISSION_DETAILS_MARKDOWN = """
-⚠️ Are you sure you want to finalize your submission? This action is **irreversible** once submitted.
-"""
+### Reference/Hypothesis File Format
 
-MORE_DETAILS_MARKDOWN = """
-## Model Metadata:
-Detailed information about the selected submission.
-"""
+🛠️ Reference annotations were constructed via the `prepare_gt.sh` script. To add a new dataset, please create a pull request modifying `prepare_gt.sh`.  
+📚 For details about SegLST format, please see the [SegLST documentation in MeetEval](https://github.com/fgnt/meeteval?tab=readme-ov-file#segment-wise-long-form-speech-transcription-annotation-seglst).
 
-MODAL_SUBMIT_MARKDOWN = """
-✅ Confirm Submission
+🔄 By default, **Chime-8 normalization** is applied during evaluation for both references and hypotheses.  
+You can choose to disable this using the checkbox above.
 
-Are you ready to submit your model for evaluation?
 """
 
+

leaderboard_server.py

@@ -9,7 +9,6 @@ from txt_norm import get_text_norm
 # Constants
 REFERENCE_BASE_PATH = os.path.abspath("references")  # e.g. ./references/single_channel/dataset1.json
 TASKS_METADATA_PATH = os.path.abspath("tasks_metadata.json")
-import os
 
 def list_files(startpath):
     for root, dirs, files in os.walk(startpath):
@@ -19,7 +18,6 @@ def list_files(startpath):
         subindent = ' ' * 4 * (level + 1)
         for f in files:
             print('{}{}'.format(subindent, f))
-list_files("/data/submissions")
 
 class LeaderboardServer:
     def __init__(self):
@@ -46,10 +44,10 @@ class LeaderboardServer:
             hyp_path = os.path.join(self.local_leaderboard, f"{model_id}_hyp.json")
             self.submisssion_id_to_file[model_id] = hyp_path
 
-    def prepare_model_for_submission(self, file, metadata, task, datasets):
+    def prepare_model_for_submission(self, file, metadata, task, datasets, normalize=False):
         submitted_by = metadata["submitted_by"]
         model_id = metadata["model_id"]
-
+        original_id = f"{submitted_by}_{model_id}"
         # Run WER eval
         results = {}
         hyp_seglst = meeteval.io.load(file)
@@ -59,8 +57,8 @@ class LeaderboardServer:
             ref_seglst = meeteval.io.load(ref_path)
             sessions = ref_seglst.unique('session_id')
             local_hyps = hyp_seglst.filter(lambda seg: seg['session_id'] in sessions)
-            ref_seglst = ref_seglst.map(lambda seg: {**seg, "words":self.text_normalizer(seg["words"])})
-            local_hyps = local_hyps.map(lambda seg: {**seg, "words":self.text_normalizer(seg["words"])})
+            ref_seglst = ref_seglst.map(lambda seg: {**seg, "words":self.text_normalizer(seg["words"]) if normalize else seg["words"]})
+            local_hyps = local_hyps.map(lambda seg: {**seg, "words":self.text_normalizer(seg["words"]) if normalize else seg["words"]})
             per_session_wers = calc_wer(tcp_hyp_seglst=local_hyps, ref_seglst=ref_seglst, collar=5, metrics_list=["tcp_wer"])
             metrics = aggregate_wer_metrics(per_session_wers, ["tcp_wer"])
             results[dataset] = metrics
@@ -73,7 +71,8 @@ class LeaderboardServer:
         else:
             all_results = {}
 
-        all_results[model_id] = {
+        all_results[original_id] = {
+            "model_id": model_id,
             "submitted_by": submitted_by,
             "results": results
         }
@@ -81,6 +80,10 @@ class LeaderboardServer:
         with open(results_path, "w") as f:
             json.dump(all_results, f, indent=2)
 
+        with open(f"{self.local_leaderboard}/{original_id}_hyp.json", "w") as out_f:
+            with open(file, "r") as in_f:
+                out_f.write(in_f.read())
+
     def update_leaderboard(self):
         self.fetch_existing_models()
 
@@ -93,10 +96,15 @@ class LeaderboardServer:
             results = json.load(f)
 
         rows = []
-        for model_id, content in results.items():
-            row = {"Model ID": model_id, "Submitted by": content["submitted_by"]}
+        for content in results.values():
+            row = {"Model ID": content["model_id"], "Submitted by": content["submitted_by"]}
             for k, v in content["results"].items():
                 row[k] = v.get("tcp_wer", None)
             rows.append(row)
 
-        return pd.DataFrame(rows)
+        df =  pd.DataFrame(rows)
+        df.iloc[:, 2:] *= 100.0 # Convert WER to percentage, first two columns are metadata
+        df = df.fillna("-")
+        df = df.round(2)
+
+        return df

prepare_gt.sh

@@ -0,0 +1,103 @@
+#!/bin/bash
+
+source "$(dirname ${BASH_SOURCE[0]})/../../configs/local_paths.sh"
+export PYTHONPATH="$(dirname ${BASH_SOURCE[0]})/../:$PYTHONPATH"
+
+DATA_DIR=$SRC_ROOT/data
+DATA_SCRIPTS_PATH=$SRC_ROOT/scripts/data
+MANIFESTS_DIR=$DATA_DIR/manifests_new
+
+mkdir -p $DATA_DIR
+mkdir -p $MANIFESTS_DIR
+mkdir -p $DATA_DIR/tmp
+
+
+# LS
+librispeech_dir=$DATA_DIR/librispeech/LibriSpeech
+lhotse download librispeech $DATA_DIR/librispeech
+lhotse prepare librispeech $librispeech_dir $MANIFESTS_DIR
+
+
+git clone https://github.com/JorisCos/LibriMix $DATA_DIR/tmp/LibriMix
+pip install -r $DATA_DIR/tmp/LibriMix/requirements.txt
+
+# Download WHAM
+wham_zip_file=$DATA_DIR/tmp/wham/wham_noise.zip
+wham_folder=$DATA_DIR/tmp/wham/wham_noise
+if [ ! -d "$wham_folder" ]; then
+    mkdir -p $DATA_DIR/tmp/wham
+
+    if [ ! -f "$wham_zip_file" ]; then
+        wget -c --tries=0 --read-timeout=20 https://my-bucket-a8b4b49c25c811ee9a7e8bba05fa24c7.s3.amazonaws.com/wham_noise.zip -P $DATA_DIR/tmp/wham
+    fi
+
+    unzip -qn $DATA_DIR/tmp/wham/wham_noise.zip -d $DATA_DIR/tmp/wham
+    rm -rf $DATA_DIR/tmp/wham/wham_noise.zip
+fi
+
+
+python $DATA_DIR/tmp/LibriMix/scripts/augment_train_noise.py --wham_dir $DATA_DIR/tmp/wham/wham_noise
+
+for n_src in 2 3; do
+  metadata_dir=$DATA_DIR/tmp/LibriMix/metadata/Libri$n_src"Mix"
+  python $DATA_DIR/tmp/LibriMix/scripts/create_librimix_from_metadata.py --librispeech_dir $librispeech_dir \
+    --wham_dir $DATA_DIR/tmp/wham/wham_noise \
+    --metadata_dir $metadata_dir \
+    --librimix_outdir $DATA_DIR/librimix \
+    --n_src $n_src \
+    --freqs 16k \
+    --modes max \
+    --types mix_clean mix_both mix_single
+  for type in "clean" "both"; do
+      python $DATA_SCRIPTS_PATH/lsmix_to_lhotse.py --ls_supset $MANIFESTS_DIR/librispeech_supervisions_test-clean.jsonl.gz \
+          --mixture_wavs_dir $DATA_DIR/librimix/Libri${n_src}Mix/wav16k/max/test/mix_$type \
+          --output_manifest $MANIFESTS_DIR/libri${n_src}mix_mix_${type}_sc_test_cutset.jsonl.gz \
+          --type $type
+      python $DATA_SCRIPTS_PATH/extract_supervisions.py \
+          --cutset_path $MANIFESTS_DIR/libri${n_src}mix_mix_${type}_sc_test_cutset.jsonl.gz \
+          --output_path $MANIFESTS_DIR/libri${n_src}mix_mix_${type}_sc_test_supervisions.jsonl.gz
+  done
+done
+
+
+# AMI
+lhotse download ami --mic sdm $DATA_DIR/ami
+lhotse prepare ami --mic sdm --normalize-text none $DATA_DIR/ami $MANIFESTS_DIR
+python3 $DATA_SCRIPTS_PATH/create_cutset.py --input_recset $MANIFESTS_DIR/ami-sdm_recordings_test.jsonl.gz --input_supset $MANIFESTS_DIR/ami-sdm_supervisions_test.jsonl.gz --output $MANIFESTS_DIR/ami-sdm_cutset_test.jsonl.gz
+
+
+# NOTSOFAR1
+chime-utils dgen notsofar1 $DATA_DIR/nsf $DATA_DIR/notsofar  --part="train,dev,eval"
+chime-utils lhotse-prep notsofar1 -d eval_sc --txt-norm none -m sdm $DATA_DIR/notsofar $MANIFESTS_DIR
+chime-utils lhotse-prep notsofar1 -d eval --txt-norm none -m mdm $DATA_DIR/notsofar $MANIFESTS_DIR
+
+python3 $DATA_SCRIPTS_PATH/create_cutset.py --input_recset $MANIFESTS_DIR/notsofar1-sdm_recordings_eval_sc.jsonl.gz --input_supset $MANIFESTS_DIR/notsofar1-sdm_supervisions_eval_sc.jsonl.gz --output $MANIFESTS_DIR/notsofar1-sdm_cutset_eval_sc.jsonl.gz
+python3 $DATA_SCRIPTS_PATH/create_cutset.py --input_recset $MANIFESTS_DIR/notsofar1-mdm_recordings_eval.jsonl.gz --input_supset $MANIFESTS_DIR/notsofar1-mdm_supervisions_eval.jsonl.gz --output $MANIFESTS_DIR/notsofar1-mdm_cutset_eval.jsonl.gz
+
+
+# Extract supervisions
+SC_SUP_MANIFESTS_DIR=$DATA_DIR/manifests_sups_test_sc
+mkdir -p $SC_SUP_MANIFESTS_DIR
+cp $MANIFESTS_DIR/ami-sdm_supervisions_test.jsonl.gz $SC_SUP_MANIFESTS_DIR/ami-sdm.jsonl.gz
+cp $MANIFESTS_DIR/notsofar1-sdm_supervisions_eval_sc.jsonl.gz $SC_SUP_MANIFESTS_DIR/notsofar1-small-sdm.jsonl.gz
+cp $MANIFESTS_DIR/libri2mix_mix_clean_sc_test_supervisions.jsonl.gz $SC_SUP_MANIFESTS_DIR/libri2mix_clean.jsonl.gz
+cp $MANIFESTS_DIR/libri2mix_mix_both_sc_test_supervisions.jsonl.gz $SC_SUP_MANIFESTS_DIR/libri2mix_both.jsonl.gz
+cp $MANIFESTS_DIR/libri3mix_mix_clean_sc_test_supervisions.jsonl.gz $SC_SUP_MANIFESTS_DIR/libri3mix_clean.jsonl.gz
+cp $MANIFESTS_DIR/libri3mix_mix_both_sc_test_supervisions.jsonl.gz $SC_SUP_MANIFESTS_DIR/libri3mix_both.jsonl.gz
+
+
+SC_SUP_JSON_DIR=$DATA_DIR/refs_test_sc
+mkdir -p $SC_SUP_JSON_DIR
+for input_file in "$SC_SUP_MANIFESTS_DIR"/*.jsonl.gz; do
+    # Extract just the filename (no path)
+    filename=$(basename "$input_file")
+
+    # Replace suffix to form output filename
+    output_filename="${filename/.jsonl.gz/.json}"
+
+    # Full path to output file
+    output_file="$SC_SUP_JSON_DIR/$output_filename"
+
+    # Call the Python script
+    python3 $DATA_SCRIPTS_PATH/supervision_to_hyp_json.py --input "$input_file" --output "$output_file"
+done

tasks_metadata.json

@@ -1,9 +1,9 @@
 {
   "tasks": {
-    "single_channel_gt": {
+    "single_channel_gt_diar": {
       "name": "Single Channel - Ground Truth Diarization"
     },
-    "single_channel_real": {
+    "single_channel_real_diar": {
       "name": "Single Channel - Real Diarization"
     }
   }