Upload averaged RVQ (2025-09-02)

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+twi_semantic_tokens_min.json filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,38 @@
+---
+license: apache-2.0
+language:
+- tw
+- ak
+library_name: pytorch
+tags:
+- speechless
+- rvq
+- whisper
+- twi
+- akan
+- vector-quantization
+- semantic-tokens
+---
+
+# Speechless TWI — Stage 1 (RVQ for Whisper Encoder)
+
+Trained RVQ that discretizes Whisper encoder features into semantic tokens for **Twi/Akan**.
+
+## Files
+- `rvq_final.pt` — state dict
+- `config_stage1.json` — training/config params
+- `rvq_wrapper.py` — tiny module defining `RVQWrapper`
+
+## Usage (example)
+```python
+import torch, json
+from huggingface_hub import hf_hub_download
+from rvq_wrapper import RVQWrapper
+
+cfg = json.load(open(hf_hub_download("ik/speechless-twi-stage1-rvq-whisper-medium", "config_stage1.json"), "r"))
+ckpt = torch.load(hf_hub_download("ik/speechless-twi-stage1-rvq-whisper-medium", "rvq_final.pt"), map_location="cpu")
+
+rvq = RVQWrapper(cfg["rvq_dim"], cfg["rvq_num_quantizers"], cfg["rvq_codebook_size"])
+rvq.load_state_dict(ckpt["rvq"])
+rvq.eval()
+```

config_stage1.json

@@ -0,0 +1,21 @@
+{
+  "cv_version": "fsicoli/twi",
+  "cv_lang": "tw",
+  "split_train": "train",
+  "split_eval": "validation",
+  "sample_rate": 16000,
+  "max_audio_seconds": 30.0,
+  "whisper_ckpt": "femursmith/intermediate-asr-ashanti-twi",
+  "batch_size": 2,
+  "num_workers": 0,
+  "rvq_dim": 1280,
+  "rvq_num_quantizers": 12,
+  "rvq_codebook_size": 2048,
+  "rvq_commitment_weight": 0.5,
+  "lr": 0.0003,
+  "epochs": 3,
+  "warmup_steps": 200,
+  "save_every": 1000,
+  "ckpt_file": "rvq_averaged.pt",
+  "ckpt_note": "averaged from multiple checkpoints"
+}

rvq_averaged.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:88826673b566b95f637e478b66ff6243bce7b9c374b3d00df99e55ac7ad56213
+size 264990123

rvq_best.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8a3d66420f6aa832bb656a820e81b10fa87717aec738202670ac99181886c43b
+size 264993023

rvq_final.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:44789a82b97a1636437c097340141b3a6470352e7d486e04d3b068dce98d9c21
+size 264993082

rvq_step1000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:798b246a2623d74e4054979e4c3f5465e8a5bd6d544fe35c34c721a42628bca8
+size 264993259

rvq_step2000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f89dee7a6f40c0790455252d1d31264769b1debef6b5616d75624e998a8fd6b7
+size 264993259

rvq_step3000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d154b844a11b5fea028aa664a0644f1af8e616cfdc6e34f0d45c05c32d3d2ffe
+size 264993259

rvq_wrapper.py

@@ -0,0 +1,18 @@
+
+import torch, torch.nn as nn
+from vector_quantize_pytorch import ResidualVQ
+
+class RVQWrapper(nn.Module):
+    def __init__(self, dim, num_quantizers, codebook_size):
+        super().__init__()
+        self.ln_in  = nn.LayerNorm(dim)
+        self.proj_in  = nn.Linear(dim, dim)
+        self.rvq      = ResidualVQ(dim=dim, num_quantizers=num_quantizers, codebook_size=codebook_size)
+        self.ln_out = nn.LayerNorm(dim)
+        self.proj_out = nn.Linear(dim, dim)
+
+    def forward(self, x):
+        x = self.proj_in(self.ln_in(x))
+        q, indices, commit_loss = self.rvq(x)
+        y = self.proj_out(self.ln_out(q))
+        return y, indices, commit_loss

twi_semantic_tokens_min.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c094d7e334c0da342bac1470459f7e4fb364ff1a9c97e6795a8b81fcc7194be7
+size 11269340