initial commit

flores200_codes.py

@@ -4,9 +4,10 @@ English	eng_Latn
 Twi	twi_Latn
 '''
 
-codes_as_string = codes_as_string.split('\n')
+codes_as_string = codes_as_string.strip().split('\n')
 
 flores_codes = {}
 for code in codes_as_string:
-    lang, lang_code = code.split('\t')
-    flores_codes[lang] = lang_code
+    if code.strip():  # Skip empty lines
+        lang, lang_code = code.split('\t')
+        flores_codes[lang] = lang_code

nllb.py

@@ -2,7 +2,8 @@ import os
 from transformers import AutoTokenizer, AutoModelForSeq2SeqLM, pipeline
 from flores200_codes import flores_codes
 
-hf_token = auth_token = os.environ.get("HF_TOKEN") or True
+# Use HF_TOKEN from environment or fall back to True (for public models)
+hf_token = auth_token = os.environ.get("HF_TOKEN") or os.environ.get("HUGGINGFACE_HUB_TOKEN") or True
 model_dict = {}