Spaces:
Running
Running
Add english to the model name if English in included
Browse files
app.py
CHANGED
@@ -119,7 +119,7 @@ def prune_model(model_name: str, language: str, hf_username: str, hf_token: str,
|
|
119 |
|
120 |
with st.status("Pruning the model...", expanded=True) as status:
|
121 |
st.write("- *Updating the tokenizer*")
|
122 |
-
outdir = f"{language}-{model_name.split('/')[-1]}"
|
123 |
|
124 |
# Export the tokenizer to a JSON string and access its vocabulary (list of lists: [[token, score], ...])
|
125 |
tokenizer_json = json.loads(tokenizer.backend_tokenizer.to_str())
|
|
|
119 |
|
120 |
with st.status("Pruning the model...", expanded=True) as status:
|
121 |
st.write("- *Updating the tokenizer*")
|
122 |
+
outdir = f"{language}-{'english-' if keep_english else ''}{model_name.split('/')[-1]}"
|
123 |
|
124 |
# Export the tokenizer to a JSON string and access its vocabulary (list of lists: [[token, score], ...])
|
125 |
tokenizer_json = json.loads(tokenizer.backend_tokenizer.to_str())
|