Model save

README.md

@@ -2,14 +2,9 @@
 license: apache-2.0
 base_model: alignment-handbook/zephyr-7b-sft-full
 tags:
-- alignment-handbook
-- generated_from_trainer
 - trl
 - dpo
 - generated_from_trainer
-datasets:
-- updated
-- original
 model-index:
 - name: zephyr-7b-dpo-full
   results: []
@@ -20,17 +15,7 @@ should probably proofread and complete it, then remove this comment. -->
 
 # zephyr-7b-dpo-full
 
-This model is a fine-tuned version of [alignment-handbook/zephyr-7b-sft-full](https://huggingface.co/alignment-handbook/zephyr-7b-sft-full) on the updated and the original datasets.
-It achieves the following results on the evaluation set:
-- Loss: nan
-- Rewards/chosen: 0.0
-- Rewards/rejected: 0.0
-- Rewards/accuracies: 0.0
-- Rewards/margins: 0.0
-- Logps/rejected: -262.7752
-- Logps/chosen: -296.9845
-- Logits/rejected: -2.8821
-- Logits/chosen: -2.9654
+This model is a fine-tuned version of [alignment-handbook/zephyr-7b-sft-full](https://huggingface.co/alignment-handbook/zephyr-7b-sft-full) on the None dataset.
 
 ## Model description
 
@@ -50,14 +35,14 @@ More information needed
 
 The following hyperparameters were used during training:
 - learning_rate: 5e-07
-- train_batch_size: 4
-- eval_batch_size: 1
+- train_batch_size: 2
+- eval_batch_size: 8
 - seed: 42
 - distributed_type: multi-GPU
-- num_devices: 8
+- num_devices: 9
 - gradient_accumulation_steps: 2
-- total_train_batch_size: 64
-- total_eval_batch_size: 8
+- total_train_batch_size: 36
+- total_eval_batch_size: 72
 - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
 - lr_scheduler_type: cosine
 - lr_scheduler_warmup_ratio: 0.1
@@ -69,7 +54,7 @@ The following hyperparameters were used during training:
 
 ### Framework versions
 
-- Transformers 4.36.2
-- Pytorch 2.1.2
+- Transformers 4.38.2
+- Pytorch 2.2.1+cu121
 - Datasets 2.14.6
 - Tokenizers 0.15.2

all_results.json

@@ -1,21 +1,8 @@
 {
     "epoch": 1.0,
-    "eval_logits/chosen": -2.965409755706787,
-    "eval_logits/rejected": -2.88214111328125,
-    "eval_logps/chosen": -296.9845275878906,
-    "eval_logps/rejected": -262.7751770019531,
-    "eval_loss": NaN,
-    "eval_rewards/accuracies": 0.0,
-    "eval_rewards/chosen": 0.0,
-    "eval_rewards/margins": 0.0,
-    "eval_rewards/rejected": 0.0,
-    "eval_runtime": 1977.0954,
-    "eval_samples": 2000,
-    "eval_samples_per_second": 1.012,
-    "eval_steps_per_second": 0.126,
-    "train_loss": 0.3465735912322998,
-    "train_runtime": 27.0943,
-    "train_samples": 30,
-    "train_samples_per_second": 1.107,
-    "train_steps_per_second": 0.037
+    "train_loss": 0.686960334777832,
+    "train_runtime": 795.5553,
+    "train_samples": 900,
+    "train_samples_per_second": 1.131,
+    "train_steps_per_second": 0.031
 }

config.json

@@ -20,7 +20,7 @@
   "sliding_window": 4096,
   "tie_word_embeddings": false,
   "torch_dtype": "bfloat16",
-  "transformers_version": "4.36.2",
-  "use_cache": true,
+  "transformers_version": "4.38.2",
+  "use_cache": false,
   "vocab_size": 32000
 }

generation_config.json

@@ -2,5 +2,5 @@
   "_from_model_config": true,
   "bos_token_id": 1,
   "eos_token_id": 2,
-  "transformers_version": "4.36.2"
+  "transformers_version": "4.38.2"
 }

model-00001-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ab3121e8d66828f8888b456267d5ce7e87063138a04ddde287016df065852277
+oid sha256:168aea527589d0aa22f36d9c544f076047b12e4813c1b5dc31c89d4b7de2ff31
 size 4943162336

model-00002-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9fa2d53c86d1b508a7440b1a6aa57652ac00fe7307f67b3b184cfbe2e84d8d55
+oid sha256:f4eaec014f0066bdbf8c588a16ffd89772ad91b0199a26acd3354ea429dba0ac
 size 4999819336

model-00003-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f5a120540ca1ffa8b3b7e66aa3a45760092305697b9e0b23a2399ab56b3f0771
+oid sha256:796fee778caa45be4448a1ab0b7afbe59dc4c37a3f555473404ca53439f1652a
 size 4540516344

runs/Mar04_21-34-50_nebula/events.out.tfevents.1709609732.nebula.325150.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:38e468d786f4eb7f62d4f49510839e4ce3cb5576d2cdd1b64cf3cdcb7b8f62c8
+size 7458

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.3465735912322998,
-    "train_runtime": 27.0943,
-    "train_samples": 30,
-    "train_samples_per_second": 1.107,
-    "train_steps_per_second": 0.037
+    "train_loss": 0.686960334777832,
+    "train_runtime": 795.5553,
+    "train_samples": 900,
+    "train_samples_per_second": 1.131,
+    "train_steps_per_second": 0.031
 }

trainer_state.json

@@ -2,43 +2,80 @@
   "best_metric": null,
   "best_model_checkpoint": null,
   "epoch": 1.0,
-  "eval_steps": 100,
-  "global_step": 1,
+  "eval_steps": 500,
+  "global_step": 25,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 1.0,
-      "learning_rate": 5e-07,
-      "logits/chosen": -2.8170723915100098,
-      "logits/rejected": -2.8022234439849854,
-      "logps/chosen": -362.7491455078125,
-      "logps/rejected": -374.0892639160156,
-      "loss": 0.3466,
+      "epoch": 0.04,
+      "grad_norm": 16.178188065123834,
+      "learning_rate": 1.6666666666666665e-07,
+      "logits/chosen": -2.86794376373291,
+      "logits/rejected": -2.7572906017303467,
+      "logps/chosen": -175.52011108398438,
+      "logps/pi_response": -211.910888671875,
+      "logps/ref_response": -211.910888671875,
+      "logps/rejected": -176.99070739746094,
+      "loss": 0.6931,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
       "rewards/margins": 0.0,
       "rewards/rejected": 0.0,
       "step": 1
     },
+    {
+      "epoch": 0.4,
+      "grad_norm": 16.903841435721354,
+      "learning_rate": 3.851602043638994e-07,
+      "logits/chosen": -2.7485318183898926,
+      "logits/rejected": -2.6173155307769775,
+      "logps/chosen": -275.3558349609375,
+      "logps/pi_response": -155.98851013183594,
+      "logps/ref_response": -156.92300415039062,
+      "logps/rejected": -253.09535217285156,
+      "loss": 0.6914,
+      "rewards/accuracies": 0.4722222089767456,
+      "rewards/chosen": 0.010561762377619743,
+      "rewards/margins": 0.002931868424639106,
+      "rewards/rejected": 0.007629893720149994,
+      "step": 10
+    },
+    {
+      "epoch": 0.8,
+      "grad_norm": 13.969736792308046,
+      "learning_rate": 6.106260641143546e-08,
+      "logits/chosen": -2.744602680206299,
+      "logits/rejected": -2.706644296646118,
+      "logps/chosen": -293.81134033203125,
+      "logps/pi_response": -228.12771606445312,
+      "logps/ref_response": -232.7391815185547,
+      "logps/rejected": -310.80322265625,
+      "loss": 0.6837,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.0664653405547142,
+      "rewards/margins": 0.01980663277208805,
+      "rewards/rejected": 0.0466587133705616,
+      "step": 20
+    },
     {
       "epoch": 1.0,
-      "step": 1,
+      "step": 25,
       "total_flos": 0.0,
-      "train_loss": 0.3465735912322998,
-      "train_runtime": 27.0943,
-      "train_samples_per_second": 1.107,
-      "train_steps_per_second": 0.037
+      "train_loss": 0.686960334777832,
+      "train_runtime": 795.5553,
+      "train_samples_per_second": 1.131,
+      "train_steps_per_second": 0.031
     }
   ],
   "logging_steps": 10,
-  "max_steps": 1,
+  "max_steps": 25,
   "num_input_tokens_seen": 0,
   "num_train_epochs": 1,
-  "save_steps": 100,
+  "save_steps": 50,
   "total_flos": 0.0,
-  "train_batch_size": 4,
+  "train_batch_size": 2,
   "trial_name": null,
   "trial_params": null
 }

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0d7118937d7589746905aa37ff2449f33c09ca03c6f8808837f3488d42fc8f0c
-size 5944
+oid sha256:d7e2f0ad6f5b91a2f546d4b6db7dc443ce15c0a03b6e3b63e7562dcbff75bdd2
+size 6136