Update app.py

app.py

@@ -8,6 +8,8 @@ import os
 from transformers import AutoModelForCausalLM, AutoTokenizer, Trainer, TrainingArguments, DataCollatorForLanguageModeling
 from datasets import load_dataset, Dataset
 from huggingface_hub import HfApi, HfFolder
+import requests
+from io import BytesIO
 
 # Récupérer token depuis les variables d'environnement
 hf_token = os.getenv("MisterAI_bigscience_bloom_560m")
@@ -31,20 +33,37 @@ def generate_response(input_text):
     return response
 
 # Fonction pour le fine-tuning
+#def fine_tune_model(dataset_path, dataset_file, epochs, batch_size, prefix):
+#    # Chargement du dataset
+#    if dataset_path.startswith("https://huggingface.co/datasets/"):
+#        dataset = load_dataset('json', data_files={dataset_file: dataset_path})
+#    else:
+#        dataset = load_dataset('json', data_files={dataset_file: dataset_path})
+#
+#    # Préparation des données
+#    dataset = Dataset.from_dict(dataset[dataset_file])
+#    dataset = dataset.map(lambda x: tokenizer(x['question'] + ' ' + x['chosen'], truncation=True, padding='max_length'), batched=True)
+#    dataset.set_format(type='torch', columns=['input_ids', 'attention_mask', 'labels'])
+#
+#    data_collator = DataCollatorForLanguageModeling(tokenizer=tokenizer, mlm=False)
+
+
+
+
 def fine_tune_model(dataset_path, dataset_file, epochs, batch_size, prefix):
-    # Chargement du dataset
-    if dataset_path.startswith("https://huggingface.co/datasets/"):
-        dataset = load_dataset('json', data_files={dataset_file: dataset_path})
-    else:
-        dataset = load_dataset('json', data_files={dataset_file: dataset_path})
+    # Récupération du fichier à partir de l'URL fournie
+    response = requests.get(dataset_path)
+    dataset = list(response.iter_lines())
 
     # Préparation des données
-    dataset = Dataset.from_dict(dataset[dataset_file])
+    dataset = Dataset.from_dict({'data': dataset})
     dataset = dataset.map(lambda x: tokenizer(x['question'] + ' ' + x['chosen'], truncation=True, padding='max_length'), batched=True)
     dataset.set_format(type='torch', columns=['input_ids', 'attention_mask', 'labels'])
 
     data_collator = DataCollatorForLanguageModeling(tokenizer=tokenizer, mlm=False)
 
+
+
     # Configuration de l'entraînement
     training_args = TrainingArguments(
         output_dir=f"./{prefix}_{model_name.split('/')[-1]}",