update

toolbox/k2_sherpa/examples.py

@@ -19,4 +19,36 @@ examples = [
         "Yes",
         "./data/test_wavs/librispeech/1089-134686-0001.wav",
     ],
+    [
+        "Chinese+English",
+        "csukuangfj/sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20",
+        "greedy_search",
+        4,
+        "Yes",
+        "./data/test_wavs/tal_csasr/0.wav",
+    ],
+    [
+        "Chinese+English+Cantonese",
+        "csukuangfj/sherpa-onnx-paraformer-trilingual-zh-cantonese-en",
+        "greedy_search",
+        4,
+        "Yes",
+        "./data/test_wavs/cantonese/2.wav",
+    ],
+    [
+        "Cantonese",
+        "zrjin/icefall-asr-mdcc-zipformer-2024-03-11",
+        "greedy_search",
+        4,
+        "Yes",
+        "./data/test_wavs/cantonese/1.wav",
+    ],
+    [
+        "Tibetan",
+        "syzym/icefall-asr-xbmu-amdo31-pruned-transducer-stateless7-2022-12-02",
+        "greedy_search",
+        4,
+        "No",
+        "./data/test_wavs/tibetan/a_0_cacm-A70_31117.wav",
+    ],
 ]

toolbox/k2_sherpa/nn_models.py

@@ -65,7 +65,7 @@ model_map = {
             "joiner_model_file_sub_folder": ".",
             "tokens_file": "tokens.txt",
             "tokens_file_sub_folder": ".",
-            "loader": "load_sherpa_offline_recognizer_from_transducer",
+            "loader": "load_sherpa_onnx_offline_recognizer_from_transducer",
         },
         {
             "repo_id": "zrjin/icefall-asr-aishell-zipformer-large-2023-10-24",
@@ -77,7 +77,7 @@ model_map = {
             "joiner_model_file_sub_folder": "exp",
             "tokens_file": "tokens.txt",
             "tokens_file_sub_folder": "data/lang_char",
-            "loader": "load_sherpa_offline_recognizer_from_transducer",
+            "loader": "load_sherpa_onnx_offline_recognizer_from_transducer",
         },
         {
             "repo_id": "zrjin/icefall-asr-aishell-zipformer-small-2023-10-24",
@@ -89,7 +89,7 @@ model_map = {
             "joiner_model_file_sub_folder": "exp",
             "tokens_file": "tokens.txt",
             "tokens_file_sub_folder": "data/lang_char",
-            "loader": "load_sherpa_offline_recognizer_from_transducer",
+            "loader": "load_sherpa_onnx_offline_recognizer_from_transducer",
         },
         {
             "repo_id": "zrjin/icefall-asr-aishell-zipformer-2023-10-24",
@@ -101,7 +101,7 @@ model_map = {
             "joiner_model_file_sub_folder": "exp",
             "tokens_file": "tokens.txt",
             "tokens_file_sub_folder": "data/lang_char",
-            "loader": "load_sherpa_offline_recognizer_from_transducer",
+            "loader": "load_sherpa_onnx_offline_recognizer_from_transducer",
         },
         {
             "repo_id": "desh2608/icefall-asr-alimeeting-pruned-transducer-stateless7",
@@ -158,7 +158,7 @@ model_map = {
             "decoder_model_file_sub_folder": ".",
             "tokens_file": "tiny.en-tokens.txt",
             "tokens_file_sub_folder": ".",
-            "loader": "load_sherpa_offline_recognizer_from_whisper",
+            "loader": "load_sherpa_onnx_offline_recognizer_from_whisper",
         },
         {
             "repo_id": "csukuangfj/sherpa-onnx-whisper-base.en",
@@ -168,7 +168,7 @@ model_map = {
             "decoder_model_file_sub_folder": ".",
             "tokens_file": "base.en-tokens.txt",
             "tokens_file_sub_folder": ".",
-            "loader": "load_sherpa_offline_recognizer_from_whisper",
+            "loader": "load_sherpa_onnx_offline_recognizer_from_whisper",
         },
         {
             "repo_id": "csukuangfj/sherpa-onnx-whisper-small.en",
@@ -178,7 +178,7 @@ model_map = {
             "decoder_model_file_sub_folder": ".",
             "tokens_file": "small.en-tokens.txt",
             "tokens_file_sub_folder": ".",
-            "loader": "load_sherpa_offline_recognizer_from_whisper",
+            "loader": "load_sherpa_onnx_offline_recognizer_from_whisper",
         },
         {
             "repo_id": "csukuangfj/sherpa-onnx-paraformer-en-2024-03-09",
@@ -198,7 +198,7 @@ model_map = {
             "joiner_model_file_sub_folder": "exp",
             "tokens_file": "tokens.txt",
             "tokens_file_sub_folder": "data/lang_bpe_500",
-            "loader": "load_sherpa_offline_recognizer_from_transducer",
+            "loader": "load_sherpa_onnx_offline_recognizer_from_transducer",
         },
         {
             "repo_id": "wgb14/icefall-asr-gigaspeech-pruned-transducer-stateless2",
@@ -332,7 +332,7 @@ model_map = {
             "joiner_model_file_sub_folder": ".",
             "tokens_file": "tokens.txt",
             "tokens_file_sub_folder": ".",
-            "loader": "load_sherpa_online_recognizer_from_transducer",
+            "loader": "load_sherpa_onnx_online_recognizer_from_transducer",
         },
         {
             "repo_id": "csukuangfj/sherpa-onnx-paraformer-zh-2023-03-28",
@@ -359,16 +359,58 @@ model_map = {
             "loader": "load_sherpa_offline_recognizer",
         },
     ],
-    "Chinese+Cantonese+English": [
+    "Chinese+English+Cantonese": [
         {
             "repo_id": "csukuangfj/sherpa-onnx-paraformer-trilingual-zh-cantonese-en",
             "nn_model_file": "model.int8.onnx",
             "nn_model_file_sub_folder": ".",
             "tokens_file": "tokens.txt",
             "tokens_file_sub_folder": ".",
-            "loader": "load_sherpa_offline_recognizer_from_paraformer",
+            "loader": "load_sherpa_onnx_offline_recognizer_from_paraformer",
+        },
+        {
+            "repo_id": "csukuangfj/sherpa-onnx-streaming-paraformer-trilingual-zh-cantonese-en",
+            "encoder_model_file": "encoder.int8.onnx",
+            "encoder_model_file_sub_folder": ".",
+            "decoder_model_file": "decoder.int8.onnx",
+            "decoder_model_file_sub_folder": ".",
+            "tokens_file": "tokens.txt",
+            "tokens_file_sub_folder": ".",
+            "loader": "load_sherpa_onnx_online_recognizer_from_paraformer",
+        },
+    ],
+    "Cantonese": [
+        {
+            "repo_id": "zrjin/icefall-asr-mdcc-zipformer-2024-03-11",
+            "encoder_model_file": "encoder-epoch-45-avg-35.int8.onnx",
+            "encoder_model_file_sub_folder": "exp",
+            "decoder_model_file": "decoder-epoch-45-avg-35.onnx",
+            "decoder_model_file_sub_folder": "exp",
+            "joiner_model_file": "joiner-epoch-45-avg-35.int8.onnx",
+            "joiner_model_file_sub_folder": "exp",
+            "tokens_file": "tokens.txt",
+            "tokens_file_sub_folder": "data/lang_char",
+            "loader": "load_sherpa_onnx_offline_recognizer_from_transducer",
+        },
+    ],
+    "Tibetan": [
+        {
+            "repo_id": "syzym/icefall-asr-xbmu-amdo31-pruned-transducer-stateless7-2022-12-02",
+            "nn_model_file": "cpu_jit.pt",
+            "nn_model_file_sub_folder": "exp",
+            "tokens_file": "tokens.txt",
+            "tokens_file_sub_folder": "data/lang_bpe_500",
+            "loader": "load_sherpa_offline_recognizer",
         },
-    ]
+        {
+            "repo_id": "syzym/icefall-asr-xbmu-amdo31-pruned-transducer-stateless5-2022-11-29",
+            "nn_model_file": "cpu_jit-epoch-28-avg-23-torch-1.10.0.pt",
+            "nn_model_file_sub_folder": "exp",
+            "tokens_file": "tokens.txt",
+            "tokens_file_sub_folder": "data/lang_bpe_500",
+            "loader": "load_sherpa_offline_recognizer",
+        },
+    ],
 }
 
 
@@ -456,13 +498,13 @@ def load_sherpa_offline_recognizer(nn_model_file: str,
     return recognizer
 
 
-def load_sherpa_offline_recognizer_from_paraformer(nn_model_file: str,
-                                                   tokens_file: str,
-                                                   sample_rate: int = 16000,
-                                                   decoding_method: str = "greedy_search",
-                                                   feature_dim: int = 80,
-                                                   num_threads: int = 2,
-                                                   ):
+def load_sherpa_onnx_offline_recognizer_from_paraformer(nn_model_file: str,
+                                                        tokens_file: str,
+                                                        sample_rate: int = 16000,
+                                                        decoding_method: str = "greedy_search",
+                                                        feature_dim: int = 80,
+                                                        num_threads: int = 2,
+                                                        ):
     recognizer = sherpa_onnx.OfflineRecognizer.from_paraformer(
         paraformer=nn_model_file,
         tokens=tokens_file,
@@ -475,16 +517,16 @@ def load_sherpa_offline_recognizer_from_paraformer(nn_model_file: str,
     return recognizer
 
 
-def load_sherpa_offline_recognizer_from_transducer(encoder_model_file: str,
-                                                   decoder_model_file: str,
-                                                   joiner_model_file: str,
-                                                   tokens_file: str,
-                                                   sample_rate: int = 16000,
-                                                   decoding_method: str = "greedy_search",
-                                                   feature_dim: int = 80,
-                                                   num_threads: int = 2,
-                                                   num_active_paths: int = 2,
-                                                   ):
+def load_sherpa_onnx_offline_recognizer_from_transducer(encoder_model_file: str,
+                                                        decoder_model_file: str,
+                                                        joiner_model_file: str,
+                                                        tokens_file: str,
+                                                        sample_rate: int = 16000,
+                                                        decoding_method: str = "greedy_search",
+                                                        feature_dim: int = 80,
+                                                        num_threads: int = 2,
+                                                        num_active_paths: int = 2,
+                                                        ):
     recognizer = sherpa_onnx.OfflineRecognizer.from_transducer(
         encoder=encoder_model_file,
         decoder=decoder_model_file,
@@ -499,11 +541,11 @@ def load_sherpa_offline_recognizer_from_transducer(encoder_model_file: str,
     return recognizer
 
 
-def load_sherpa_offline_recognizer_from_whisper(encoder_model_file: str,
-                                                decoder_model_file: str,
-                                                tokens_file: str,
-                                                num_threads: int = 2,
-                                                ):
+def load_sherpa_onnx_offline_recognizer_from_whisper(encoder_model_file: str,
+                                                     decoder_model_file: str,
+                                                     tokens_file: str,
+                                                     num_threads: int = 2,
+                                                     ):
     recognizer = sherpa_onnx.OfflineRecognizer.from_whisper(
         encoder=encoder_model_file,
         decoder=decoder_model_file,
@@ -513,17 +555,17 @@ def load_sherpa_offline_recognizer_from_whisper(encoder_model_file: str,
     return recognizer
 
 
-def load_sherpa_online_recognizer_from_transducer(encoder_model_file: str,
-                                                  decoder_model_file: str,
-                                                  joiner_model_file: str,
-                                                  tokens_file: str,
-                                                  sample_rate: int = 16000,
-                                                  decoding_method: str = "greedy_search",
-                                                  feature_dim: int = 80,
-                                                  num_threads: int = 2,
-                                                  num_active_paths: int = 2,
-                                                  ):
-    recognizer = sherpa_onnx.OfflineRecognizer.from_transducer(
+def load_sherpa_onnx_online_recognizer_from_transducer(encoder_model_file: str,
+                                                       decoder_model_file: str,
+                                                       joiner_model_file: str,
+                                                       tokens_file: str,
+                                                       sample_rate: int = 16000,
+                                                       decoding_method: str = "greedy_search",
+                                                       feature_dim: int = 80,
+                                                       num_threads: int = 2,
+                                                       num_active_paths: int = 2,
+                                                       ):
+    recognizer = sherpa_onnx.OnlineRecognizer.from_transducer(
         encoder=encoder_model_file,
         decoder=decoder_model_file,
         joiner=joiner_model_file,
@@ -537,6 +579,26 @@ def load_sherpa_online_recognizer_from_transducer(encoder_model_file: str,
     return recognizer
 
 
+def load_sherpa_onnx_online_recognizer_from_paraformer(encoder_model_file: str,
+                                                       decoder_model_file: str,
+                                                       tokens_file: str,
+                                                       sample_rate: int = 16000,
+                                                       decoding_method: str = "greedy_search",
+                                                       feature_dim: int = 80,
+                                                       num_threads: int = 2,
+                                                       ):
+    recognizer = sherpa_onnx.OnlineRecognizer.from_paraformer(
+        encoder=encoder_model_file,
+        decoder=decoder_model_file,
+        tokens=tokens_file,
+        num_threads=num_threads,
+        sample_rate=sample_rate,
+        feature_dim=feature_dim,
+        decoding_method=decoding_method,
+    )
+    return recognizer
+
+
 def load_recognizer(local_model_dir: Path,
                     decoding_method: str = "greedy_search",
                     num_active_paths: int = 4,
@@ -577,22 +639,26 @@ def load_recognizer(local_model_dir: Path,
             num_active_paths=num_active_paths,
             **kwargs_
         )
-    elif loader == "load_sherpa_offline_recognizer_from_paraformer":
-        recognizer = load_sherpa_offline_recognizer_from_paraformer(
+    elif loader == "load_sherpa_onnx_offline_recognizer_from_paraformer":
+        recognizer = load_sherpa_onnx_offline_recognizer_from_paraformer(
             decoding_method=decoding_method,
             **kwargs_
         )
-    elif loader == "load_sherpa_offline_recognizer_from_transducer":
-        recognizer = load_sherpa_offline_recognizer_from_transducer(
+    elif loader == "load_sherpa_onnx_offline_recognizer_from_transducer":
+        recognizer = load_sherpa_onnx_offline_recognizer_from_transducer(
             decoding_method=decoding_method,
             **kwargs_
         )
-    elif loader == "load_sherpa_offline_recognizer_from_whisper":
-        recognizer = load_sherpa_offline_recognizer_from_whisper(
+    elif loader == "load_sherpa_onnx_offline_recognizer_from_whisper":
+        recognizer = load_sherpa_onnx_offline_recognizer_from_whisper(
+            **kwargs_
+        )
+    elif loader == "load_sherpa_onnx_online_recognizer_from_transducer":
+        recognizer = load_sherpa_onnx_online_recognizer_from_transducer(
             **kwargs_
         )
-    elif loader == "load_sherpa_online_recognizer_from_transducer":
-        recognizer = load_sherpa_online_recognizer_from_transducer(
+    elif loader == "load_sherpa_onnx_online_recognizer_from_paraformer":
+        recognizer = load_sherpa_onnx_online_recognizer_from_paraformer(
             **kwargs_
         )
     else: