Model save

README.md

@@ -0,0 +1,71 @@
+---
+license: mit
+library_name: peft
+tags:
+- trl
+- sft
+- generated_from_trainer
+base_model: HuggingFaceH4/zephyr-7b-beta
+datasets:
+- generator
+model-index:
+- name: zephyr-rag-agent
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# zephyr-rag-agent
+
+This model is a fine-tuned version of [HuggingFaceH4/zephyr-7b-beta](https://huggingface.co/HuggingFaceH4/zephyr-7b-beta) on the generator dataset.
+It achieves the following results on the evaluation set:
+- Loss: 1.1623
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 0.0002
+- train_batch_size: 2
+- eval_batch_size: 2
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 2
+- gradient_accumulation_steps: 16
+- total_train_batch_size: 64
+- total_eval_batch_size: 4
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: cosine
+- lr_scheduler_warmup_ratio: 0.1
+- num_epochs: 3
+
+### Training results
+
+| Training Loss | Epoch | Step | Validation Loss |
+|:-------------:|:-----:|:----:|:---------------:|
+| 1.1559        | 0.99  | 27   | 1.1650          |
+| 1.0887        | 1.98  | 54   | 1.1555          |
+| 1.0566        | 2.97  | 81   | 1.1623          |
+
+
+### Framework versions
+
+- PEFT 0.7.1
+- Transformers 4.36.2
+- Pytorch 2.2.2+cu121
+- Datasets 2.14.6
+- Tokenizers 0.15.2

all_results.json

@@ -0,0 +1,13 @@
+{
+    "epoch": 2.97,
+    "eval_loss": 1.1623399257659912,
+    "eval_runtime": 279.5897,
+    "eval_samples": 716,
+    "eval_samples_per_second": 2.167,
+    "eval_steps_per_second": 0.544,
+    "train_loss": 1.1298090528558802,
+    "train_runtime": 14861.7425,
+    "train_samples": 1922,
+    "train_samples_per_second": 0.352,
+    "train_steps_per_second": 0.005
+}

eval_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 2.97,
+    "eval_loss": 1.1623399257659912,
+    "eval_runtime": 279.5897,
+    "eval_samples": 716,
+    "eval_samples_per_second": 2.167,
+    "eval_steps_per_second": 0.544
+}

train_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 2.97,
+    "train_loss": 1.1298090528558802,
+    "train_runtime": 14861.7425,
+    "train_samples": 1922,
+    "train_samples_per_second": 0.352,
+    "train_steps_per_second": 0.005
+}

trainer_state.json

@@ -0,0 +1,156 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 2.965675057208238,
+  "eval_steps": 500,
+  "global_step": 81,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.04,
+      "learning_rate": 2.2222222222222223e-05,
+      "loss": 1.4708,
+      "step": 1
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.00011111111111111112,
+      "loss": 1.4228,
+      "step": 5
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.0001999048221581858,
+      "loss": 1.2725,
+      "step": 10
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.00019659258262890683,
+      "loss": 1.2311,
+      "step": 15
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.00018870108331782217,
+      "loss": 1.1712,
+      "step": 20
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 0.0001766044443118978,
+      "loss": 1.1559,
+      "step": 25
+    },
+    {
+      "epoch": 0.99,
+      "eval_loss": 1.1649807691574097,
+      "eval_runtime": 283.9384,
+      "eval_samples_per_second": 2.134,
+      "eval_steps_per_second": 0.535,
+      "step": 27
+    },
+    {
+      "epoch": 1.1,
+      "learning_rate": 0.00016087614290087208,
+      "loss": 1.1485,
+      "step": 30
+    },
+    {
+      "epoch": 1.28,
+      "learning_rate": 0.00014226182617406996,
+      "loss": 1.11,
+      "step": 35
+    },
+    {
+      "epoch": 1.46,
+      "learning_rate": 0.00012164396139381029,
+      "loss": 1.1045,
+      "step": 40
+    },
+    {
+      "epoch": 1.65,
+      "learning_rate": 0.0001,
+      "loss": 1.0982,
+      "step": 45
+    },
+    {
+      "epoch": 1.83,
+      "learning_rate": 7.835603860618972e-05,
+      "loss": 1.0887,
+      "step": 50
+    },
+    {
+      "epoch": 1.98,
+      "eval_loss": 1.1555219888687134,
+      "eval_runtime": 283.2377,
+      "eval_samples_per_second": 2.14,
+      "eval_steps_per_second": 0.537,
+      "step": 54
+    },
+    {
+      "epoch": 2.01,
+      "learning_rate": 5.773817382593008e-05,
+      "loss": 1.0711,
+      "step": 55
+    },
+    {
+      "epoch": 2.2,
+      "learning_rate": 3.9123857099127936e-05,
+      "loss": 1.0764,
+      "step": 60
+    },
+    {
+      "epoch": 2.38,
+      "learning_rate": 2.339555568810221e-05,
+      "loss": 1.04,
+      "step": 65
+    },
+    {
+      "epoch": 2.56,
+      "learning_rate": 1.129891668217783e-05,
+      "loss": 1.0344,
+      "step": 70
+    },
+    {
+      "epoch": 2.75,
+      "learning_rate": 3.40741737109318e-06,
+      "loss": 1.0026,
+      "step": 75
+    },
+    {
+      "epoch": 2.93,
+      "learning_rate": 9.517784181422019e-08,
+      "loss": 1.0566,
+      "step": 80
+    },
+    {
+      "epoch": 2.97,
+      "eval_loss": 1.1623399257659912,
+      "eval_runtime": 283.2011,
+      "eval_samples_per_second": 2.14,
+      "eval_steps_per_second": 0.537,
+      "step": 81
+    },
+    {
+      "epoch": 2.97,
+      "step": 81,
+      "total_flos": 173916983132160.0,
+      "train_loss": 1.1298090528558802,
+      "train_runtime": 14861.7425,
+      "train_samples_per_second": 0.352,
+      "train_steps_per_second": 0.005
+    }
+  ],
+  "logging_steps": 5,
+  "max_steps": 81,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 500,
+  "total_flos": 173916983132160.0,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}