Reprise de lang id par DL

main_dl.py

@@ -14,7 +14,7 @@ import string
 import re
 import json
 import csv
-# import tiktoken
+import tiktoken
 from sklearn.preprocessing import LabelEncoder
 from tensorflow import keras
 # import keras
@@ -244,7 +244,7 @@ def load_all_data():
 rnn_en_fr, rnn_fr_en, transformer_en_fr, transformer_fr_en = load_all_data() 
       
 #  ==== Language identifier ====
-'''
+
 def encode_text(textes):
     global tokenizer
 
@@ -273,7 +273,7 @@ def init_dl_identifier():
     merge = Merge(dataPath+"/dl_id_lang_split",  dataPath, "dl_tiktoken_id_language_model.h5").merge(cleanup=False)
     dl_model = keras.models.load_model(dataPath+"/dl_tiktoken_id_language_model.h5")
     return
-
+'''
 def lang_id_dl(sentences):
     global dl_model, label_encoder, lan_to_language
     
@@ -290,8 +290,7 @@ def lang_id_dl(sentences):
 @api.get('/', name="Vérification que l'API fonctionne")
 def check_api():
     load_all_data()
-    # read_list_lan()
-    # init_dl_identifier()
+    init_dl_identifier()
     return {'message': "L'API fonctionne"}
 
 @api.get('/small_vocab/rnn', name="Traduction par RNN")

requirements.txt

@@ -12,7 +12,7 @@ pydantic==2.6.1
 pydantic-core==2.16.2
 python-dateutil==2.8.2
 pytz==2024.1
-requests==2.21.0
+requests==2.26.0
 six==1.16.0
 sniffio==1.3.0
 starlette==0.36.3
@@ -27,4 +27,5 @@ sentencepiece==0.1.99
 filesplit==4.0.1
 pydot==2.0.0
 graphviz==0.20.1
-scikit-learn==1.1.3
+scikit-learn==1.1.3
+tiktoken