yasserrmd commited on
Commit
1b9197f
·
verified ·
1 Parent(s): 68ab585

🚀 Initial push: Spark TTS fine-tuned on NonverbalTTS

Browse files
.gitattributes CHANGED
@@ -33,4 +33,15 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
36
- tokenizer.json filter=lfs diff=lfs merge=lfs -text
 
 
 
 
 
 
 
 
 
 
 
 
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
36
+ LLM/tokenizer.json filter=lfs diff=lfs merge=lfs -text
37
+ wav2vec2-large-xlsr-53/pytorch_model.bin filter=lfs diff=lfs merge=lfs -text
38
+ LLM/model.safetensors filter=lfs diff=lfs merge=lfs -text
39
+ BiCodec/model.safetensors filter=lfs diff=lfs merge=lfs -text
40
+ src/figures/infer_control.png filter=lfs diff=lfs merge=lfs -text
41
+ src/figures/infer_voice_cloning.png filter=lfs diff=lfs merge=lfs -text
42
+ src/logo/HKUST.jpg filter=lfs diff=lfs merge=lfs -text
43
+ src/logo/NPU.jpg filter=lfs diff=lfs merge=lfs -text
44
+ src/logo/SJU.jpg filter=lfs diff=lfs merge=lfs -text
45
+ src/logo/SparkTTS.png filter=lfs diff=lfs merge=lfs -text
46
+ src/logo/mobvoi.jpg filter=lfs diff=lfs merge=lfs -text
47
+ src/logo/mobvoi.png filter=lfs diff=lfs merge=lfs -text
BiCodec/config.yaml ADDED
@@ -0,0 +1,60 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ audio_tokenizer:
2
+ mel_params:
3
+ sample_rate: 16000
4
+ n_fft: 1024
5
+ win_length: 640
6
+ hop_length: 320
7
+ mel_fmin: 10
8
+ mel_fmax: null
9
+ num_mels: 128
10
+
11
+ encoder:
12
+ input_channels: 1024
13
+ vocos_dim: 384
14
+ vocos_intermediate_dim: 2048
15
+ vocos_num_layers: 12
16
+ out_channels: 1024
17
+ sample_ratios: [1,1]
18
+
19
+ decoder:
20
+ input_channel: 1024
21
+ channels: 1536
22
+ rates: [8, 5, 4, 2]
23
+ kernel_sizes: [16,11,8,4]
24
+
25
+ quantizer:
26
+ input_dim: 1024
27
+ codebook_size: 8192
28
+ codebook_dim: 8
29
+ commitment: 0.25
30
+ codebook_loss_weight: 2.0
31
+ use_l2_normlize: True
32
+ threshold_ema_dead_code: 0.2
33
+
34
+ speaker_encoder:
35
+ input_dim: 128
36
+ out_dim: 1024
37
+ latent_dim: 128
38
+ token_num: 32
39
+ fsq_levels: [4, 4, 4, 4, 4, 4]
40
+ fsq_num_quantizers: 1
41
+
42
+ prenet:
43
+ input_channels: 1024
44
+ vocos_dim: 384
45
+ vocos_intermediate_dim: 2048
46
+ vocos_num_layers: 12
47
+ out_channels: 1024
48
+ condition_dim: 1024
49
+ sample_ratios: [1,1]
50
+ use_tanh_at_final: False
51
+
52
+ postnet:
53
+ input_channels: 1024
54
+ vocos_dim: 384
55
+ vocos_intermediate_dim: 2048
56
+ vocos_num_layers: 6
57
+ out_channels: 1024
58
+ use_tanh_at_final: False
59
+
60
+
BiCodec/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e9940cd48d4446e4340ced82d234bf5618350dd9f5db900ebe47a4fdb03867ec
3
+ size 625518756
README.md CHANGED
@@ -1,181 +1,208 @@
1
  ---
2
- license: apache-2.0
3
- tags:
4
- - spark-tts
5
- - text-to-speech
6
- - nonverbal
7
- - emotional
8
- - audio
9
- - speech-synthesis
10
- - huggingface
11
  language:
12
- - en
13
- model-index:
14
- - name: SparkNV-Voice
15
- results: []
 
 
 
16
  ---
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
17
 
18
- # 🔊 SparkNV-Voice
19
 
20
- **SparkNV-Voice** is a fine-tuned version of the [Spark-TTS](https://huggingface.co/suno-ai/spark-tts) model trained on the [NonverbalTTS](https://huggingface.co/datasets/deepvk/NonverbalTTS) dataset. It enables expressive speech synthesis with **nonverbal cues** (like laughter, sighs, sneezing, etc.) and rich emotional tone.
 
 
 
 
 
 
 
21
 
22
- Built for applications that require **natural, human-like vocalization**, this model produces speech with **semantic tokens** and **global prosody control** using BiCodec detokenization.
23
 
24
- ---
 
25
 
26
- ## 🧾 Model Details
 
 
 
 
27
 
28
- - **Base**: `suno-ai/spark-tts`
29
- - **Dataset**: [`deepvk/NonverbalTTS`](https://huggingface.co/datasets/deepvk/NonverbalTTS)
30
- - **Architecture**: Causal Language Model + BiCodec for audio token generation
31
- - **Language**: English
32
- - **Voice**: Single-speaker (no multi-speaker conditioning)
33
 
34
- ---
 
 
 
 
 
 
35
 
36
- ## 🚀 Inference Code
37
 
 
38
  ```python
39
- import torch
40
- import re
41
- import numpy as np
42
- from transformers import AutoTokenizer, AutoModelForCausalLM
43
- import soundfile as sf
44
- from IPython.display import Audio, display
45
-
46
- # Load model and tokenizer
47
- model = AutoModelForCausalLM.from_pretrained("yasserrmd/SparkNV-Voice", torch_dtype=torch.float16).to("cuda")
48
- tokenizer = AutoTokenizer.from_pretrained("yasserrmd/SparkNV-Voice")
49
-
50
- # Optional: load BiCodec-based audio_tokenizer
51
- from models.audio_codec.bicodec_tokenizer import BiCodecTokenizer
52
- audio_tokenizer = BiCodecTokenizer.from_pretrained("suno-ai/spark-tts", device="cuda")
53
-
54
- # Enable fast inference
55
- from models.fastmodel_patch import FastModel
56
- FastModel.for_inference(model)
57
-
58
- @torch.inference_mode()
59
- def generate_speech_from_text(
60
- text: str,
61
- temperature: float = 0.8,
62
- top_k: int = 50,
63
- top_p: float = 1.0,
64
- max_new_audio_tokens: int = 2048,
65
- device: torch.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
66
- ) -> np.ndarray:
67
-
68
- torch.compiler.reset()
69
-
70
- prompt = "<|task_tts|><|start_content|>" + text + "<|end_content|><|start_global_token|>"
71
- model_inputs = tokenizer([prompt], return_tensors="pt").to(device)
72
-
73
- print("Generating token sequence...")
74
- generated_ids = model.generate(
75
- **model_inputs,
76
- max_new_tokens=max_new_audio_tokens,
77
- do_sample=True,
78
- temperature=temperature,
79
- top_k=top_k,
80
- top_p=top_p,
81
- eos_token_id=tokenizer.eos_token_id,
82
- pad_token_id=tokenizer.pad_token_id
83
- )
84
- print("Token sequence generated.")
85
-
86
- generated_ids_trimmed = generated_ids[:, model_inputs.input_ids.shape[1]:]
87
- predicts_text = tokenizer.batch_decode(generated_ids_trimmed, skip_special_tokens=False)[0]
88
-
89
- # Extract semantic tokens
90
- semantic_matches = re.findall(r"<\|bicodec_semantic_(\d+)\|>", predicts_text)
91
- if not semantic_matches:
92
- print("⚠️ No semantic tokens found.")
93
- return np.array([], dtype=np.float32)
94
-
95
- pred_semantic_ids = torch.tensor([int(token) for token in semantic_matches]).long().unsqueeze(0)
96
-
97
- # Extract global tokens
98
- global_matches = re.findall(r"<\|bicodec_global_(\d+)\|>", predicts_text)
99
- if not global_matches:
100
- print("⚠️ No global tokens found. Using zeros.")
101
- pred_global_ids = torch.zeros((1, 1), dtype=torch.long)
102
- else:
103
- pred_global_ids = torch.tensor([int(token) for token in global_matches]).long().unsqueeze(0)
104
-
105
- pred_global_ids = pred_global_ids.unsqueeze(0)
106
-
107
- print(f"🎯 Found {pred_semantic_ids.shape[1]} semantic tokens.")
108
- print(f"🎯 Found {pred_global_ids.shape[2]} global tokens.")
109
-
110
- # Detokenize audio
111
- audio_tokenizer.device = device
112
- audio_tokenizer.model.to(device)
113
- wav_np = audio_tokenizer.detokenize(
114
- pred_global_ids.to(device).squeeze(0),
115
- pred_semantic_ids.to(device)
116
- )
117
- print("✅ Detokenization complete.")
118
-
119
- return wav_np
120
-
121
-
122
- # === Usage ===
123
- input_text = "Hey there my name is Yasser, and I'm a 🌬️ speech generation model that can sound like a person."
124
- generated_waveform = generate_speech_from_text(input_text)
125
-
126
- if generated_waveform.size > 0:
127
- sample_rate = audio_tokenizer.config.get("sample_rate", 16000)
128
- sf.write("generated_speech.wav", generated_waveform, sample_rate)
129
- print("💾 Saved: generated_speech.wav")
130
- display(Audio(generated_waveform, rate=sample_rate))
131
- else:
132
- print("⚠️ Audio generation failed.")
133
- ````
134
 
135
- ---
 
136
 
137
- ## 🎛️ Supported Nonverbal Cues
 
 
138
 
139
- The model is fine-tuned on sequences containing:
 
140
 
141
- * `<|laughing|>`
142
- * `<|sighing|>`
143
- * `<|groaning|>`
144
- * `<|grunting|>`
145
- * `<|sniffing|>`
146
- * `<|sneezing|>`
147
- * `<|breathing|>`
148
- * `<|coughing|>`
149
- * `<|snoring|>`
150
- * `<|throat_clearing|>`
151
 
152
- You can combine these with your prompt to guide tone/emotion or rely on semantic token generation.
153
 
154
- ---
 
 
 
 
155
 
156
- ## 🧠 Dataset Highlights: `NonverbalTTS`
157
 
158
- * 17+ hours of annotated emotional & nonverbal English speech
159
- * Automatic + human-validated labels
160
- * Sources: VoxCeleb, Expresso
161
- * Paper: [arXiv:2507.13155](https://arxiv.org/abs/2507.13155)
 
 
 
 
 
162
 
163
- ---
164
 
165
- ## 📜 License
166
 
167
- Apache 2.0 — free for commercial and academic use.
168
 
169
- ---
 
 
170
 
171
- ## 🤝 Credits
172
 
173
- * Base model: [`suno-ai/spark-tts`](https://huggingface.co/suno-ai/spark-tts)
174
- * Dataset: [`deepvk/NonverbalTTS`](https://huggingface.co/datasets/deepvk/NonverbalTTS)
175
- * Author: [`@yasserrmd`](https://huggingface.co/yasserrmd)
176
 
177
- ---
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
178
 
179
- ## 💬 Feedback & Contributions
180
 
181
- Open a discussion or issue on this repo. Contributions are welcome!
 
1
  ---
2
+ license: cc-by-nc-sa-4.0
 
 
 
 
 
 
 
 
3
  language:
4
+ - en
5
+ - zh
6
+ tags:
7
+ - text-to-speech
8
+ library_tag: spark-tts
9
+ base_model:
10
+ - SparkAudio/Spark-TTS-0.5B
11
  ---
12
+ <div>
13
+ <p style="margin-bottom: 0; margin-top: 0;">
14
+ <strong>See <a href="https://huggingface.co/collections/unsloth/text-to-speech-tts-models-68007ab12522e96be1e02155">our collection</a> for all our TTS model uploads.</strong>
15
+ </p>
16
+ <p style="margin-bottom: 0;">
17
+ <em>Learn to fine-tune TTS models - <a href="https://docs.unsloth.ai/basics/text-to-speech-tts-fine-tuning">Read our Guide</a>.</em>
18
+ </p>
19
+ <p style="margin-top: 0;margin-bottom: 0;">
20
+ <em><a href="https://docs.unsloth.ai/basics/unsloth-dynamic-v2.0-gguf">Unsloth Dynamic 2.0</a> achieves superior accuracy & outperforms other leading quants.</em>
21
+ </p>
22
+ <div style="display: flex; gap: 5px; align-items: center; ">
23
+ <a href="https://github.com/unslothai/unsloth/">
24
+ <img src="https://github.com/unslothai/unsloth/raw/main/images/unsloth%20new%20logo.png" width="133">
25
+ </a>
26
+ <a href="https://discord.gg/unsloth">
27
+ <img src="https://github.com/unslothai/unsloth/raw/main/images/Discord%20button.png" width="173">
28
+ </a>
29
+ <a href="https://docs.unsloth.ai/basics/text-to-speech-tts-fine-tuning">
30
+ <img src="https://raw.githubusercontent.com/unslothai/unsloth/refs/heads/main/images/documentation%20green%20button.png" width="143">
31
+ </a>
32
+ </div>
33
+ <h1 style="margin-top: 0rem;">✨ Run & Fine-tune TTS models with Unsloth!</h1>
34
+ </div>
35
+
36
+ - Fine-tune TTS models for free using our Google [Colab notebooks here](https://docs.unsloth.ai/get-started/unsloth-notebooks#text-to-speech-tts-notebooks)!
37
+ - Read our Blog about TTS support: [unsloth.ai/blog/tts](https://docs.unsloth.ai/basics/text-to-speech-tts-fine-tuning)
38
+
39
+ | Unsloth supports | Free Notebooks | Performance | Memory use |
40
+ |-----------------|--------------------------------------------------------------------------------------------------------------------------|-------------|----------|
41
+ | **Spark-TTS** | [▶️ Start on Colab](https://colab.research.google.com/github/unslothai/notebooks/blob/main/nb/Spark_TTS_(0_5B).ipynb) | 1.5x faster | 58% less |
42
+ | **Whisper Large V3** | [▶️ Start on Colab](https://colab.research.google.com/github/unslothai/notebooks/blob/main/nb/Whisper.ipynb) | 1.5x faster | 50% less |
43
+ | **Qwen3 (14B)** | [▶️ Start on Colab](https://docs.unsloth.ai/get-started/unsloth-notebooks) | 2x faster | 70% less |
44
+ | **Llama 3.2 Vision (11B)** | [▶️ Start on Colab](https://colab.research.google.com/github/unslothai/notebooks/blob/main/nb/Llama3.2_(11B)-Vision.ipynb) | 1.8x faster | 50% less |
45
+
46
+ <div align="center">
47
+ <h1>
48
+ Spark-TTS
49
+ </h1>
50
+ <p>
51
+ Official model for <br>
52
+ <b><em>Spark-TTS: An Efficient LLM-Based Text-to-Speech Model with Single-Stream Decoupled Speech Tokens</em></b>
53
+ </p>
54
+ <p>
55
+ <img src="src/logo/SparkTTS.jpg" alt="Spark-TTS Logo" style="width: 200px; height: 200px;">
56
+ </p>
57
+ </div>
58
+
59
+
60
+ ## Spark-TTS 🔥
61
+
62
+ ### 👉🏻 [Spark-TTS Demos](https://sparkaudio.github.io/spark-tts/) 👈🏻
63
+
64
+ ### 👉🏻 [Github Repo](https://github.com/SparkAudio/Spark-TTS) 👈🏻
65
+
66
+ ### 👉🏻 [Paper](https://arxiv.org/pdf/2503.01710) 👈🏻
67
+
68
+ ### Overview
69
+
70
+ Spark-TTS is an advanced text-to-speech system that uses the power of large language models (LLM) for highly accurate and natural-sounding voice synthesis. It is designed to be efficient, flexible, and powerful for both research and production use.
71
+
72
+ ### Key Features
73
+
74
+ - **Simplicity and Efficiency**: Built entirely on Qwen2.5, Spark-TTS eliminates the need for additional generation models like flow matching. Instead of relying on separate models to generate acoustic features, it directly reconstructs audio from the code predicted by the LLM. This approach streamlines the process, improving efficiency and reducing complexity.
75
+ - **High-Quality Voice Cloning**: Supports zero-shot voice cloning, which means it can replicate a speaker's voice even without specific training data for that voice. This is ideal for cross-lingual and code-switching scenarios, allowing for seamless transitions between languages and voices without requiring separate training for each one.
76
+ - **Bilingual Support**: Supports both Chinese and English, and is capable of zero-shot voice cloning for cross-lingual and code-switching scenarios, enabling the model to synthesize speech in multiple languages with high naturalness and accuracy.
77
+ - **Controllable Speech Generation**: Supports creating virtual speakers by adjusting parameters such as gender, pitch, and speaking rate.
78
 
79
+ ---
80
 
81
+ <table align="center">
82
+ <tr>
83
+ <td align="center"><b>Inference Overview of Voice Cloning</b><br><img src="src/figures/infer_voice_cloning.png" width="80%" /></td>
84
+ </tr>
85
+ <tr>
86
+ <td align="center"><b>Inference Overview of Controlled Generation</b><br><img src="src/figures/infer_control.png" width="80%" /></td>
87
+ </tr>
88
+ </table>
89
 
 
90
 
91
+ ## Install
92
+ **Clone and Install**
93
 
94
+ - Clone the repo
95
+ ``` sh
96
+ git clone https://github.com/SparkAudio/Spark-TTS.git
97
+ cd Spark-TTS
98
+ ```
99
 
100
+ - Install Conda: please see https://docs.conda.io/en/latest/miniconda.html
101
+ - Create Conda env:
 
 
 
102
 
103
+ ``` sh
104
+ conda create -n sparktts -y python=3.12
105
+ conda activate sparktts
106
+ pip install -r requirements.txt
107
+ # If you are in mainland China, you can set the mirror as follows:
108
+ pip install -r requirements.txt -i https://mirrors.aliyun.com/pypi/simple/ --trusted-host=mirrors.aliyun.com
109
+ ```
110
 
111
+ **Model Download**
112
 
113
+ Download via python:
114
  ```python
115
+ from huggingface_hub import snapshot_download
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
116
 
117
+ snapshot_download("SparkAudio/Spark-TTS-0.5B", local_dir="pretrained_models/Spark-TTS-0.5B")
118
+ ```
119
 
120
+ Download via git clone:
121
+ ```sh
122
+ mkdir -p pretrained_models
123
 
124
+ # Make sure you have git-lfs installed (https://git-lfs.com)
125
+ git lfs install
126
 
127
+ git clone https://huggingface.co/SparkAudio/Spark-TTS-0.5B pretrained_models/Spark-TTS-0.5B
128
+ ```
 
 
 
 
 
 
 
 
129
 
130
+ **Basic Usage**
131
 
132
+ You can simply run the demo with the following commands:
133
+ ``` sh
134
+ cd example
135
+ bash infer.sh
136
+ ```
137
 
138
+ Alternatively, you can directly execute the following command in the command line to perform inference:
139
 
140
+ ``` sh
141
+ python -m cli.inference \
142
+ --text "text to synthesis." \
143
+ --device 0 \
144
+ --save_dir "path/to/save/audio" \
145
+ --model_dir pretrained_models/Spark-TTS-0.5B \
146
+ --prompt_text "transcript of the prompt audio" \
147
+ --prompt_speech_path "path/to/prompt_audio"
148
+ ```
149
 
150
+ **UI Usage**
151
 
152
+ You can start the UI interface by running `python webui.py`, which allows you to perform Voice Cloning and Voice Creation. Voice Cloning supports uploading reference audio or directly recording the audio.
153
 
 
154
 
155
+ | **Voice Cloning** | **Voice Creation** |
156
+ |:-------------------:|:-------------------:|
157
+ | ![Image 1](src/figures/gradio_TTS.png) | ![Image 2](src/figures/gradio_control.png) |
158
 
 
159
 
160
+ ## To-Do List
 
 
161
 
162
+ - [x] Release the Spark-TTS paper.
163
+ - [ ] Release the training code.
164
+ - [ ] Release the training dataset, VoxBox.
165
+
166
+ ## Citation
167
+
168
+ ```
169
+ @misc{wang2025sparktts,
170
+ title={Spark-TTS: An Efficient LLM-Based Text-to-Speech Model with Single-Stream Decoupled Speech Tokens},
171
+ author={Xinsheng Wang and Mingqi Jiang and Ziyang Ma and Ziyu Zhang and Songxiang Liu and Linqin Li and Zheng Liang and Qixi Zheng and Rui Wang and Xiaoqin Feng and Weizhen Bian and Zhen Ye and Sitong Cheng and Ruibin Yuan and Zhixian Zhao and Xinfa Zhu and Jiahao Pan and Liumeng Xue and Pengcheng Zhu and Yunlin Chen and Zhifei Li and Xie Chen and Lei Xie and Yike Guo and Wei Xue},
172
+ year={2025},
173
+ eprint={2503.01710},
174
+ archivePrefix={arXiv},
175
+ primaryClass={cs.SD},
176
+ url={https://arxiv.org/abs/2503.01710},
177
+ }
178
+ ```
179
+
180
+
181
+ ## ⚠ License Update
182
+
183
+ The model's license has been updated from Apache 2.0 to CC BY-NC-SA due to the licensing terms of some training data.
184
+
185
+ Key Changes:
186
+
187
+ - The model can only be used for non-commercial purposes.
188
+
189
+ - Any modifications or derivatives must also be released under CC BY-NC-SA 4.0.
190
+
191
+ - Proper attribution is required when using or modifying the model.
192
+
193
+ Please ensure compliance with the new license terms.
194
+
195
+
196
+ ## ⚠️ Usage Disclaimer
197
+
198
+ This project provides a zero-shot voice cloning TTS model intended for academic research, educational purposes, and legitimate applications, such as personalized speech synthesis, assistive technologies, and linguistic research.
199
+
200
+ Please note:
201
+
202
+ - Do not use this model for unauthorized voice cloning, impersonation, fraud, scams, deepfakes, or any illegal activities.
203
+
204
+ - Ensure compliance with local laws and regulations when using this model and uphold ethical standards.
205
 
206
+ - The developers assume no liability for any misuse of this model.
207
 
208
+ We advocate for the responsible development and use of AI and encourage the community to uphold safety and ethical principles in AI research and applications. If you have any concerns regarding ethics or misuse, please contact us.
config.yaml ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ highpass_cutoff_freq: 40
2
+ sample_rate: 16000
3
+ segment_duration: 2.4 # (s)
4
+ max_val_duration: 12 # (s)
5
+ latent_hop_length: 320
6
+ ref_segment_duration: 6
7
+ volume_normalize: true
src/figures/gradio_TTS.png ADDED
src/figures/gradio_control.png ADDED
src/figures/infer_control.png ADDED

Git LFS Details

  • SHA256: bfb1617c623cc25f5af6ce1e18331f1e0c61892e21bb90702c9ce2b2b5c0e8e2
  • Pointer size: 131 Bytes
  • Size of remote file: 127 kB
src/figures/infer_voice_cloning.png ADDED

Git LFS Details

  • SHA256: 9cb85e0217bfa18ce35dc5237c85b46fd1eeac7434b3a27c89a9136b3707d920
  • Pointer size: 131 Bytes
  • Size of remote file: 119 kB
src/logo/HKUST.jpg ADDED

Git LFS Details

  • SHA256: 65810d8063ffa90864943ff6a7aec52c0704c9230b71fc46b302732950facace
  • Pointer size: 131 Bytes
  • Size of remote file: 102 kB
src/logo/NPU.jpg ADDED

Git LFS Details

  • SHA256: 7d5af77d53b2ad6bd409c068c47ea30c8a4c23fa53e12efab77354f862625bb2
  • Pointer size: 131 Bytes
  • Size of remote file: 152 kB
src/logo/NTU.jpg ADDED
src/logo/SJU.jpg ADDED

Git LFS Details

  • SHA256: 82d9ab05b6c7ceed18348a4ad43ad147ae5607ba629265482cd049e812e65e1b
  • Pointer size: 131 Bytes
  • Size of remote file: 364 kB
src/logo/SparkAudio.jpg ADDED
src/logo/SparkAudio2.jpg ADDED
src/logo/SparkTTS.jpg ADDED
src/logo/SparkTTS.png ADDED

Git LFS Details

  • SHA256: d257dd65c174aa28e21a341906f18b242e08a859e14cc72d8b86a54ad7e096ec
  • Pointer size: 131 Bytes
  • Size of remote file: 102 kB
src/logo/mobvoi.jpg ADDED

Git LFS Details

  • SHA256: 74ac35394031e708f738fd99508f8f941f15bbcb99ea5abbc98cd8795ed44831
  • Pointer size: 131 Bytes
  • Size of remote file: 431 kB
src/logo/mobvoi.png ADDED

Git LFS Details

  • SHA256: 48af5c5a1a8077e6b8db7bfe63c249ed99bf2e6a085ecbceb728614f6a3c94ae
  • Pointer size: 131 Bytes
  • Size of remote file: 120 kB
wav2vec2-large-xlsr-53/README.md ADDED
@@ -0,0 +1,29 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ language: multilingual
3
+ datasets:
4
+ - common_voice
5
+ tags:
6
+ - speech
7
+ license: apache-2.0
8
+ ---
9
+
10
+ # Wav2Vec2-XLSR-53
11
+
12
+ [Facebook's XLSR-Wav2Vec2](https://ai.facebook.com/blog/wav2vec-20-learning-the-structure-of-speech-from-raw-audio/)
13
+
14
+ The base model pretrained on 16kHz sampled speech audio. When using the model make sure that your speech input is also sampled at 16Khz. Note that this model should be fine-tuned on a downstream task, like Automatic Speech Recognition. Check out [this blog](https://huggingface.co/blog/fine-tune-wav2vec2-english) for more information.
15
+
16
+ [Paper](https://arxiv.org/abs/2006.13979)
17
+
18
+ Authors: Alexis Conneau, Alexei Baevski, Ronan Collobert, Abdelrahman Mohamed, Michael Auli
19
+
20
+ **Abstract**
21
+ This paper presents XLSR which learns cross-lingual speech representations by pretraining a single model from the raw waveform of speech in multiple languages. We build on wav2vec 2.0 which is trained by solving a contrastive task over masked latent speech representations and jointly learns a quantization of the latents shared across languages. The resulting model is fine-tuned on labeled data and experiments show that cross-lingual pretraining significantly outperforms monolingual pretraining. On the CommonVoice benchmark, XLSR shows a relative phoneme error rate reduction of 72% compared to the best known results. On BABEL, our approach improves word error rate by 16% relative compared to a comparable system. Our approach enables a single multilingual speech recognition model which is competitive to strong individual models. Analysis shows that the latent discrete speech representations are shared across languages with increased sharing for related languages. We hope to catalyze research in low-resource speech understanding by releasing XLSR-53, a large model pretrained in 53 languages.
22
+
23
+ The original model can be found under https://github.com/pytorch/fairseq/tree/master/examples/wav2vec#wav2vec-20.
24
+
25
+ # Usage
26
+
27
+ See [this notebook](https://colab.research.google.com/github/patrickvonplaten/notebooks/blob/master/Fine_Tune_XLSR_Wav2Vec2_on_Turkish_ASR_with_%F0%9F%A4%97_Transformers.ipynb) for more information on how to fine-tune the model.
28
+
29
+ ![model image](https://raw.githubusercontent.com/patrickvonplaten/scientific_images/master/xlsr_wav2vec2.png)
wav2vec2-large-xlsr-53/config.json ADDED
@@ -0,0 +1,83 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "activation_dropout": 0.0,
3
+ "apply_spec_augment": true,
4
+ "architectures": [
5
+ "Wav2Vec2ForPreTraining"
6
+ ],
7
+ "attention_dropout": 0.1,
8
+ "bos_token_id": 1,
9
+ "codevector_dim": 768,
10
+ "contrastive_logits_temperature": 0.1,
11
+ "conv_bias": true,
12
+ "conv_dim": [
13
+ 512,
14
+ 512,
15
+ 512,
16
+ 512,
17
+ 512,
18
+ 512,
19
+ 512
20
+ ],
21
+ "conv_kernel": [
22
+ 10,
23
+ 3,
24
+ 3,
25
+ 3,
26
+ 3,
27
+ 2,
28
+ 2
29
+ ],
30
+ "conv_stride": [
31
+ 5,
32
+ 2,
33
+ 2,
34
+ 2,
35
+ 2,
36
+ 2,
37
+ 2
38
+ ],
39
+ "ctc_loss_reduction": "sum",
40
+ "ctc_zero_infinity": false,
41
+ "diversity_loss_weight": 0.1,
42
+ "do_stable_layer_norm": true,
43
+ "eos_token_id": 2,
44
+ "feat_extract_activation": "gelu",
45
+ "feat_extract_dropout": 0.0,
46
+ "feat_extract_norm": "layer",
47
+ "feat_proj_dropout": 0.1,
48
+ "feat_quantizer_dropout": 0.0,
49
+ "final_dropout": 0.0,
50
+ "gradient_checkpointing": false,
51
+ "hidden_act": "gelu",
52
+ "hidden_dropout": 0.1,
53
+ "hidden_size": 1024,
54
+ "initializer_range": 0.02,
55
+ "intermediate_size": 4096,
56
+ "layer_norm_eps": 1e-05,
57
+ "layerdrop": 0.1,
58
+ "mask_channel_length": 10,
59
+ "mask_channel_min_space": 1,
60
+ "mask_channel_other": 0.0,
61
+ "mask_channel_prob": 0.0,
62
+ "mask_channel_selection": "static",
63
+ "mask_feature_length": 10,
64
+ "mask_feature_prob": 0.0,
65
+ "mask_time_length": 10,
66
+ "mask_time_min_space": 1,
67
+ "mask_time_other": 0.0,
68
+ "mask_time_prob": 0.075,
69
+ "mask_time_selection": "static",
70
+ "model_type": "wav2vec2",
71
+ "num_attention_heads": 16,
72
+ "num_codevector_groups": 2,
73
+ "num_codevectors_per_group": 320,
74
+ "num_conv_pos_embedding_groups": 16,
75
+ "num_conv_pos_embeddings": 128,
76
+ "num_feat_extract_layers": 7,
77
+ "num_hidden_layers": 24,
78
+ "num_negatives": 100,
79
+ "pad_token_id": 0,
80
+ "proj_codevector_dim": 768,
81
+ "transformers_version": "4.7.0.dev0",
82
+ "vocab_size": 32
83
+ }
wav2vec2-large-xlsr-53/preprocessor_config.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "do_normalize": true,
3
+ "feature_extractor_type": "Wav2Vec2FeatureExtractor",
4
+ "feature_size": 1,
5
+ "padding_side": "right",
6
+ "padding_value": 0,
7
+ "return_attention_mask": true,
8
+ "sampling_rate": 16000
9
+ }
wav2vec2-large-xlsr-53/pytorch_model.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:314340227371a608f71adcd5f0de5933824fe77e55822aa4b24dba9c1c364dcb
3
+ size 1269737156