Model save

README.md

@@ -2,16 +2,9 @@
 license: apache-2.0
 base_model: alignment-handbook/zephyr-7b-sft-full
 tags:
-- alignment-handbook
 - trl
 - dpo
 - generated_from_trainer
-- trl
-- dpo
-- generated_from_trainer
-datasets:
-- updated
-- original
 model-index:
 - name: zephyr-7b-dpo-full
   results: []
@@ -22,7 +15,7 @@ should probably proofread and complete it, then remove this comment. -->
 
 # zephyr-7b-dpo-full
 
-This model is a fine-tuned version of [alignment-handbook/zephyr-7b-sft-full](https://huggingface.co/alignment-handbook/zephyr-7b-sft-full) on the updated and the original datasets.
+This model is a fine-tuned version of [alignment-handbook/zephyr-7b-sft-full](https://huggingface.co/alignment-handbook/zephyr-7b-sft-full) on the None dataset.
 
 ## Model description
 
@@ -61,7 +54,7 @@ The following hyperparameters were used during training:
 
 ### Framework versions
 
-- Transformers 4.38.2
+- Transformers 4.36.2
 - Pytorch 2.2.1+cu121
 - Datasets 2.14.6
 - Tokenizers 0.15.2

all_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.686960334777832,
-    "train_runtime": 795.5553,
-    "train_samples": 900,
-    "train_samples_per_second": 1.131,
-    "train_steps_per_second": 0.031
+    "train_loss": 0.6818081367583502,
+    "train_runtime": 1329.5916,
+    "train_samples": 1500,
+    "train_samples_per_second": 1.128,
+    "train_steps_per_second": 0.032
 }

config.json

@@ -20,7 +20,7 @@
   "sliding_window": 4096,
   "tie_word_embeddings": false,
   "torch_dtype": "bfloat16",
-  "transformers_version": "4.38.2",
-  "use_cache": true,
+  "transformers_version": "4.36.2",
+  "use_cache": false,
   "vocab_size": 32000
 }

generation_config.json

@@ -2,5 +2,5 @@
   "_from_model_config": true,
   "bos_token_id": 1,
   "eos_token_id": 2,
-  "transformers_version": "4.38.2"
+  "transformers_version": "4.36.2"
 }

model-00001-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:168aea527589d0aa22f36d9c544f076047b12e4813c1b5dc31c89d4b7de2ff31
+oid sha256:3dbc16450b8385d558193d968d1a3489cdfa85b0eb4e37348f31c0c267ff267a
 size 4943162336

model-00002-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f4eaec014f0066bdbf8c588a16ffd89772ad91b0199a26acd3354ea429dba0ac
+oid sha256:71f20b31ad6779e594346c1b98da9a5d7839ad16a131ccf7db462bebe1b273df
 size 4999819336

model-00003-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:796fee778caa45be4448a1ab0b7afbe59dc4c37a3f555473404ca53439f1652a
+oid sha256:753b944c7793a85217d4f0b1e395a4f61df49699a0904522077061a880acd47f
 size 4540516344

runs/Mar08_19-46-41_nebula/events.out.tfevents.1709948908.nebula.42292.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c99a89235d90876e288541ca01ea59ee93bc229c564826976b754b1b95d2d3cd
+size 5666

runs/Mar08_20-02-54_nebula/events.out.tfevents.1709949910.nebula.43699.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:73334323b4a7acb3da8f653e28b3e591f70b1e48834fc1523646b475caf6602d
+size 8650

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.686960334777832,
-    "train_runtime": 795.5553,
-    "train_samples": 900,
-    "train_samples_per_second": 1.131,
-    "train_steps_per_second": 0.031
+    "train_loss": 0.6818081367583502,
+    "train_runtime": 1329.5916,
+    "train_samples": 1500,
+    "train_samples_per_second": 1.128,
+    "train_steps_per_second": 0.032
 }

trainer_state.json

@@ -3,21 +3,20 @@
   "best_model_checkpoint": null,
   "epoch": 1.0,
   "eval_steps": 500,
-  "global_step": 25,
+  "global_step": 42,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 0.04,
-      "grad_norm": 16.178188065123834,
-      "learning_rate": 1.6666666666666665e-07,
-      "logits/chosen": -2.86794376373291,
-      "logits/rejected": -2.7572906017303467,
-      "logps/chosen": -175.52011108398438,
-      "logps/pi_response": -211.910888671875,
-      "logps/ref_response": -211.910888671875,
-      "logps/rejected": -176.99070739746094,
+      "epoch": 0.02,
+      "learning_rate": 1e-07,
+      "logits/chosen": -2.961165428161621,
+      "logits/rejected": -2.9065260887145996,
+      "logps/chosen": -301.09039306640625,
+      "logps/pi_response": -154.45700073242188,
+      "logps/ref_response": -154.45700073242188,
+      "logps/rejected": -312.41326904296875,
       "loss": 0.6931,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
@@ -26,51 +25,81 @@
       "step": 1
     },
     {
-      "epoch": 0.4,
-      "grad_norm": 16.903841435721354,
-      "learning_rate": 3.851602043638994e-07,
-      "logits/chosen": -2.7485318183898926,
-      "logits/rejected": -2.6173155307769775,
-      "logps/chosen": -275.3558349609375,
-      "logps/pi_response": -155.98851013183594,
-      "logps/ref_response": -156.92300415039062,
-      "logps/rejected": -253.09535217285156,
-      "loss": 0.6914,
-      "rewards/accuracies": 0.4722222089767456,
-      "rewards/chosen": 0.010561762377619743,
-      "rewards/margins": 0.002931868424639106,
-      "rewards/rejected": 0.007629893720149994,
+      "epoch": 0.24,
+      "learning_rate": 4.778071225970339e-07,
+      "logits/chosen": -2.816866159439087,
+      "logits/rejected": -2.6957647800445557,
+      "logps/chosen": -267.56488037109375,
+      "logps/pi_response": -182.86256408691406,
+      "logps/ref_response": -182.52423095703125,
+      "logps/rejected": -226.00772094726562,
+      "loss": 0.6925,
+      "rewards/accuracies": 0.4444444477558136,
+      "rewards/chosen": -0.0020490202587097883,
+      "rewards/margins": 0.002598464023321867,
+      "rewards/rejected": -0.004647484514862299,
       "step": 10
     },
     {
-      "epoch": 0.8,
-      "grad_norm": 13.969736792308046,
-      "learning_rate": 6.106260641143546e-08,
-      "logits/chosen": -2.744602680206299,
-      "logits/rejected": -2.706644296646118,
-      "logps/chosen": -293.81134033203125,
-      "logps/pi_response": -228.12771606445312,
-      "logps/ref_response": -232.7391815185547,
-      "logps/rejected": -310.80322265625,
-      "loss": 0.6837,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": 0.0664653405547142,
-      "rewards/margins": 0.01980663277208805,
-      "rewards/rejected": 0.0466587133705616,
+      "epoch": 0.48,
+      "learning_rate": 3.2320569281913754e-07,
+      "logits/chosen": -2.7879672050476074,
+      "logits/rejected": -2.814840078353882,
+      "logps/chosen": -255.08975219726562,
+      "logps/pi_response": -235.3483428955078,
+      "logps/ref_response": -233.54464721679688,
+      "logps/rejected": -263.6041564941406,
+      "loss": 0.6868,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.014762332662940025,
+      "rewards/margins": 0.012134673073887825,
+      "rewards/rejected": -0.026897007599473,
       "step": 20
     },
+    {
+      "epoch": 0.71,
+      "learning_rate": 1.189231791106921e-07,
+      "logits/chosen": -2.702427387237549,
+      "logits/rejected": -2.7164032459259033,
+      "logps/chosen": -194.59146118164062,
+      "logps/pi_response": -153.37057495117188,
+      "logps/ref_response": -152.309814453125,
+      "logps/rejected": -225.66796875,
+      "loss": 0.6759,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.0013326064217835665,
+      "rewards/margins": 0.022419044747948647,
+      "rewards/rejected": -0.02108643762767315,
+      "step": 30
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 3.5960224130728858e-09,
+      "logits/chosen": -2.7800514698028564,
+      "logits/rejected": -2.6079983711242676,
+      "logps/chosen": -291.4418029785156,
+      "logps/pi_response": -189.13177490234375,
+      "logps/ref_response": -187.91622924804688,
+      "logps/rejected": -244.4289093017578,
+      "loss": 0.6732,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.02857491932809353,
+      "rewards/margins": 0.013249958865344524,
+      "rewards/rejected": -0.041824884712696075,
+      "step": 40
+    },
     {
       "epoch": 1.0,
-      "step": 25,
+      "step": 42,
       "total_flos": 0.0,
-      "train_loss": 0.686960334777832,
-      "train_runtime": 795.5553,
-      "train_samples_per_second": 1.131,
-      "train_steps_per_second": 0.031
+      "train_loss": 0.6818081367583502,
+      "train_runtime": 1329.5916,
+      "train_samples_per_second": 1.128,
+      "train_steps_per_second": 0.032
     }
   ],
   "logging_steps": 10,
-  "max_steps": 25,
+  "max_steps": 42,
   "num_input_tokens_seen": 0,
   "num_train_epochs": 1,
   "save_steps": 50,

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d7e2f0ad6f5b91a2f546d4b6db7dc443ce15c0a03b6e3b63e7562dcbff75bdd2
-size 6136
+oid sha256:55111e9360cd23b86bd1915f7803c8b695ba266ed14c148454f14bcfd958a0a6
+size 6008