Update moroccan dialect model card.

README.md

@@ -55,7 +55,7 @@ model = Wav2Vec2ForCTC.from_pretrained("othrif/wav2vec2-large-xlsr-moroccan")
 model.to("cuda")
 
 
-chars_to_ignore_regex = '[\,\?\.\!\-\;\:\"\“\'\�]'
+chars_to_ignore_regex = '[\\,\\?\\.\\!\\-\\;\\:\\"\\“\\'\\�]'
 
 def remove_special_characters(batch):
     batch["text"] = re.sub(chars_to_ignore_regex, "", batch["sentence"]).lower() + " "
@@ -105,67 +105,60 @@ The model can be evaluated as follows on the Arabic test data of Common Voice.
 
 
 ```python
+import re
 import torch
+import librosa
 import torchaudio
 from datasets import load_dataset, load_metric
 from transformers import Wav2Vec2ForCTC, Wav2Vec2Processor
-import re
+import soundfile as sf
 
-test_dataset = load_dataset("ma_speech_corpus", split="test") 
+eval_dataset = load_dataset("ma_speech_corpus", split="test")
 wer = load_metric("wer")
 
-processor = Wav2Vec2Processor.from_pretrained("othrif/wav2vec2-large-xlsr-moroccan") 
+processor = Wav2Vec2Processor.from_pretrained("othrif/wav2vec2-large-xlsr-moroccan")
 model = Wav2Vec2ForCTC.from_pretrained("othrif/wav2vec2-large-xlsr-moroccan")
 model.to("cuda")
 
-chars_to_ignore_regex = '[0\\\\,\\\\?\\\\.\\\\!\\\\-\\\\;\\\\:\\\\"\\\\“\\\\%\\\\‘\\\\”\\\\�\\
-\\\\@\\\\ـ\\\\؟\\\\*\\\\ \\\\#\\\\'\\\\ \\\\…\\\\\\\\u2003]'
-#resampler = torchaudio.transforms.Resample(48_000, 16_000)
+chars_to_ignore_regex = '[\,\?\.\!\-\;\:\"\“\'\�]'
+
+def remove_special_characters(batch):
+    batch["text"] = re.sub(chars_to_ignore_regex, "", batch["sentence"]).lower() + " "
+    return batch
+
+
+eval_dataset = eval_dataset.map(remove_special_characters, remove_columns=["sentence"])
+#eval_dataset = eval_dataset.select(range(100))
 
-# Preprocessing the datasets.
-# We need to read the audio files as arrays
 def speech_file_to_array_fn(batch):
-    batch["text"] = re.sub(chars_to_ignore_regex, '', batch["text"]).lower()
-    batch["text"] = re.sub('[a-zA-z]', '', batch["text"]).lower() + " "
-        batch["text"] = re.sub('[ًٌٍَُِ~]', '', batch["text"]).lower() + " "
-
-        # batch["text"] = re.sub('\\\\\\
-','', batch["text"])
-        batch["text"] = re.sub("[إأٱآا]", "ا", batch["text"])
-        batch["text"] = re.sub("ڸ", "ل", batch["text"])
-        noise = re.compile(""" ّ    | # Tashdid
-                             َ    | # Fatha
-                             ً    | # Tanwin Fath
-                             ُ    | # Damma
-                             ٌ    | # Tanwin Damm
-                             ِ    | # Kasra
-                             ٍ    | # Tanwin Kasr
-                             ْ    | # Sukun
-                             ـ     # Tatwil/Kashida
-                         """, re.VERBOSE)
-        batch["text"] = re.sub(noise, '', batch["text"])
-        batch["text"] = re.sub('ٖ', '', batch["text"]).lower() + " "
+    start, stop = batch['segment'].split('_')
     speech_array, sampling_rate = torchaudio.load(batch["path"])
-    batch["speech"] = resampler(speech_array).squeeze().numpy()
+    speech_array, sampling_rate = sf.read(batch["path"], start=int(float(start) * sampling_rate),
+                                          stop=int(float(stop) * sampling_rate))
+    batch["speech"] = librosa.resample(speech_array, sampling_rate, 16_000)
+    batch["sampling_rate"] = 16_000
+    batch["target_text"] = batch["text"]
     return batch
 
-test_dataset = test_dataset.map(speech_file_to_array_fn)
 
-# Preprocessing the datasets.
-# We need to read the audio files as arrays
+eval_dataset = eval_dataset.map(
+    speech_file_to_array_fn,
+    remove_columns=eval_dataset.column_names
+)
+
 def evaluate(batch):
     inputs = processor(batch["speech"], sampling_rate=16_000, return_tensors="pt", padding=True)
 
     with torch.no_grad():
-    logits = model(inputs.input_values.to("cuda"), attention_mask=inputs.attention_mask.to("cuda")).logits
+        logits = model(inputs.input_values.to("cuda"), attention_mask=inputs.attention_mask.to("cuda")).logits
 
     pred_ids = torch.argmax(logits, dim=-1)
     batch["pred_strings"] = processor.batch_decode(pred_ids)
     return batch
 
-result = test_dataset.map(evaluate, batched=True, batch_size=8)
+result = eval_dataset.map(evaluate, batched=True, batch_size=32)
 
-print("WER: {:2f}".format(100 * wer.compute(predictions=result["pred_strings"], references=result["text"])))
+print("WER: {:2f}".format(100 * wer.compute(predictions=result["pred_strings"], references=result["target_text"])))
 ```
 
 **Test Result**: 66.45