Model save

README.md

@@ -0,0 +1,60 @@
+---
+license: apache-2.0
+base_model: alignment-handbook/zephyr-7b-sft-full
+tags:
+- trl
+- dpo
+- generated_from_trainer
+model-index:
+- name: dpo-v
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# dpo-v
+
+This model is a fine-tuned version of [alignment-handbook/zephyr-7b-sft-full](https://huggingface.co/alignment-handbook/zephyr-7b-sft-full) on the None dataset.
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 5e-07
+- train_batch_size: 8
+- eval_batch_size: 4
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 4
+- gradient_accumulation_steps: 2
+- total_train_batch_size: 64
+- total_eval_batch_size: 16
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: linear
+- lr_scheduler_warmup_ratio: 0.1
+- num_epochs: 1
+
+### Training results
+
+
+
+### Framework versions
+
+- Transformers 4.39.0.dev0
+- Pytorch 2.3.0+cu121
+- Datasets 2.14.6
+- Tokenizers 0.15.2

all_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 1.0,
+    "train_loss": 0.6206242706955777,
+    "train_runtime": 6118.3001,
+    "train_samples": 50000,
+    "train_samples_per_second": 8.172,
+    "train_steps_per_second": 0.128
+}

generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "transformers_version": "4.39.0.dev0"
+}

model-00001-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5e1afa36cefc97cfd8eeeb4173a3f1ec838f8f988be3b1db8de237e1732093c1
+size 4943162336

model-00002-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:60b161d8b4213a46e441b835a7b307b3c86a113f76fe893190fc060cb017532d
+size 4999819336

model-00003-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:65f0eb0c8fcb4520ad0e274fd0d7700bb999c815e59ebe8f25613ae5d795019c
+size 4540516344

model.safetensors.index.json

@@ -0,0 +1,298 @@
+{
+  "metadata": {
+    "total_size": 14483464192
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00003-of-00003.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.norm.weight": "model-00003-of-00003.safetensors"
+  }
+}

runs/May31_21-32-54_gilbreth-j001.rcac.purdue.edu/events.out.tfevents.1717206161.gilbreth-j001.rcac.purdue.edu.116123.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c80361f37fd005f152f9ac792dcde2bd8d5a90d85041f9f9f463a1f79ad43455
-size 39631
+oid sha256:f6c203cd1f6b397b9743c6ef89a5065714c06be6f92199cc24c08ac69ed3247b
+size 59249

train_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 1.0,
+    "train_loss": 0.6206242706955777,
+    "train_runtime": 6118.3001,
+    "train_samples": 50000,
+    "train_samples_per_second": 8.172,
+    "train_steps_per_second": 0.128
+}

trainer_state.json

@@ -0,0 +1,1215 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.999360204734485,
+  "eval_steps": 100,
+  "global_step": 781,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0,
+      "grad_norm": 73.88290603506073,
+      "learning_rate": 6.329113924050633e-09,
+      "logits/chosen": -2.681492805480957,
+      "logits/rejected": -2.721985340118408,
+      "logps/chosen": -124.80496215820312,
+      "logps/rejected": -118.02874755859375,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 1
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 75.17519334145489,
+      "learning_rate": 6.329113924050633e-08,
+      "logits/chosen": -2.7303428649902344,
+      "logits/rejected": -2.6930150985717773,
+      "logps/chosen": -116.66114807128906,
+      "logps/rejected": -117.21126556396484,
+      "loss": 0.6926,
+      "rewards/accuracies": 0.4861111044883728,
+      "rewards/chosen": 0.0020587260369211435,
+      "rewards/margins": 0.005753005389124155,
+      "rewards/rejected": -0.0036942793522030115,
+      "step": 10
+    },
+    {
+      "epoch": 0.03,
+      "grad_norm": 69.81383180991632,
+      "learning_rate": 1.2658227848101266e-07,
+      "logits/chosen": -2.798924684524536,
+      "logits/rejected": -2.7042040824890137,
+      "logps/chosen": -137.05197143554688,
+      "logps/rejected": -109.82391357421875,
+      "loss": 0.6805,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": 0.05025916546583176,
+      "rewards/margins": 0.040553655475378036,
+      "rewards/rejected": 0.00970550999045372,
+      "step": 20
+    },
+    {
+      "epoch": 0.04,
+      "grad_norm": 69.2956842290247,
+      "learning_rate": 1.89873417721519e-07,
+      "logits/chosen": -2.7918338775634766,
+      "logits/rejected": -2.765650749206543,
+      "logps/chosen": -124.9291000366211,
+      "logps/rejected": -107.6879653930664,
+      "loss": 0.6664,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": 0.1035560593008995,
+      "rewards/margins": 0.06856445968151093,
+      "rewards/rejected": 0.03499160334467888,
+      "step": 30
+    },
+    {
+      "epoch": 0.05,
+      "grad_norm": 72.27827498257822,
+      "learning_rate": 2.5316455696202533e-07,
+      "logits/chosen": -2.7633156776428223,
+      "logits/rejected": -2.7486460208892822,
+      "logps/chosen": -124.75370788574219,
+      "logps/rejected": -116.74095153808594,
+      "loss": 0.6418,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.10472150146961212,
+      "rewards/margins": 0.2144879400730133,
+      "rewards/rejected": -0.31920942664146423,
+      "step": 40
+    },
+    {
+      "epoch": 0.06,
+      "grad_norm": 62.36401533736533,
+      "learning_rate": 3.1645569620253163e-07,
+      "logits/chosen": -2.741940975189209,
+      "logits/rejected": -2.7087416648864746,
+      "logps/chosen": -121.89395904541016,
+      "logps/rejected": -111.40077209472656,
+      "loss": 0.6266,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.4552551805973053,
+      "rewards/margins": 0.20323333144187927,
+      "rewards/rejected": -0.6584885120391846,
+      "step": 50
+    },
+    {
+      "epoch": 0.08,
+      "grad_norm": 59.55527237747593,
+      "learning_rate": 3.79746835443038e-07,
+      "logits/chosen": -2.8621158599853516,
+      "logits/rejected": -2.788179636001587,
+      "logps/chosen": -139.24362182617188,
+      "logps/rejected": -123.8193130493164,
+      "loss": 0.606,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -0.5659990310668945,
+      "rewards/margins": 0.3793627917766571,
+      "rewards/rejected": -0.9453617930412292,
+      "step": 60
+    },
+    {
+      "epoch": 0.09,
+      "grad_norm": 69.987091869762,
+      "learning_rate": 4.4303797468354424e-07,
+      "logits/chosen": -2.8053669929504395,
+      "logits/rejected": -2.7556328773498535,
+      "logps/chosen": -125.49467468261719,
+      "logps/rejected": -119.021728515625,
+      "loss": 0.6002,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -0.6134187579154968,
+      "rewards/margins": 0.4472305178642273,
+      "rewards/rejected": -1.0606492757797241,
+      "step": 70
+    },
+    {
+      "epoch": 0.1,
+      "grad_norm": 66.00612011794634,
+      "learning_rate": 4.992877492877492e-07,
+      "logits/chosen": -2.823204755783081,
+      "logits/rejected": -2.7928357124328613,
+      "logps/chosen": -131.06190490722656,
+      "logps/rejected": -125.10595703125,
+      "loss": 0.5735,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.38157644867897034,
+      "rewards/margins": 0.5670378804206848,
+      "rewards/rejected": -0.948614239692688,
+      "step": 80
+    },
+    {
+      "epoch": 0.12,
+      "grad_norm": 61.14332881533426,
+      "learning_rate": 4.921652421652421e-07,
+      "logits/chosen": -2.671870708465576,
+      "logits/rejected": -2.659498929977417,
+      "logps/chosen": -133.38272094726562,
+      "logps/rejected": -116.47938537597656,
+      "loss": 0.6161,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.5839901566505432,
+      "rewards/margins": 0.48598217964172363,
+      "rewards/rejected": -1.0699723958969116,
+      "step": 90
+    },
+    {
+      "epoch": 0.13,
+      "grad_norm": 58.38104880977938,
+      "learning_rate": 4.850427350427351e-07,
+      "logits/chosen": -2.7319064140319824,
+      "logits/rejected": -2.672062873840332,
+      "logps/chosen": -132.8414764404297,
+      "logps/rejected": -128.17662048339844,
+      "loss": 0.605,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.1853744387626648,
+      "rewards/margins": 0.604332685470581,
+      "rewards/rejected": -0.7897071838378906,
+      "step": 100
+    },
+    {
+      "epoch": 0.14,
+      "grad_norm": 69.26950325034447,
+      "learning_rate": 4.779202279202279e-07,
+      "logits/chosen": -2.7438206672668457,
+      "logits/rejected": -2.71018648147583,
+      "logps/chosen": -125.17506408691406,
+      "logps/rejected": -125.14253234863281,
+      "loss": 0.6448,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.3857839107513428,
+      "rewards/margins": 0.6138359904289246,
+      "rewards/rejected": -0.9996198415756226,
+      "step": 110
+    },
+    {
+      "epoch": 0.15,
+      "grad_norm": 57.955141437140774,
+      "learning_rate": 4.707977207977208e-07,
+      "logits/chosen": -2.6172096729278564,
+      "logits/rejected": -2.543118715286255,
+      "logps/chosen": -132.1810760498047,
+      "logps/rejected": -127.86735534667969,
+      "loss": 0.5746,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.5412696599960327,
+      "rewards/margins": 0.7538528442382812,
+      "rewards/rejected": -1.2951223850250244,
+      "step": 120
+    },
+    {
+      "epoch": 0.17,
+      "grad_norm": 55.240124761941935,
+      "learning_rate": 4.6367521367521367e-07,
+      "logits/chosen": -2.63382887840271,
+      "logits/rejected": -2.5919692516326904,
+      "logps/chosen": -129.01876831054688,
+      "logps/rejected": -120.71885681152344,
+      "loss": 0.6371,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.4330121874809265,
+      "rewards/margins": 0.592444121837616,
+      "rewards/rejected": -1.0254563093185425,
+      "step": 130
+    },
+    {
+      "epoch": 0.18,
+      "grad_norm": 58.555560789763724,
+      "learning_rate": 4.5655270655270654e-07,
+      "logits/chosen": -2.6655454635620117,
+      "logits/rejected": -2.6445982456207275,
+      "logps/chosen": -137.69711303710938,
+      "logps/rejected": -121.81614685058594,
+      "loss": 0.6616,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.29771968722343445,
+      "rewards/margins": 0.5459628105163574,
+      "rewards/rejected": -0.843682587146759,
+      "step": 140
+    },
+    {
+      "epoch": 0.19,
+      "grad_norm": 42.11757209844078,
+      "learning_rate": 4.494301994301994e-07,
+      "logits/chosen": -2.6000969409942627,
+      "logits/rejected": -2.5133461952209473,
+      "logps/chosen": -128.32022094726562,
+      "logps/rejected": -121.820068359375,
+      "loss": 0.5996,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.04844401031732559,
+      "rewards/margins": 0.6256290078163147,
+      "rewards/rejected": -0.6740728616714478,
+      "step": 150
+    },
+    {
+      "epoch": 0.2,
+      "grad_norm": 59.4223024628302,
+      "learning_rate": 4.423076923076923e-07,
+      "logits/chosen": -2.616772174835205,
+      "logits/rejected": -2.562450408935547,
+      "logps/chosen": -136.19554138183594,
+      "logps/rejected": -123.13197326660156,
+      "loss": 0.6287,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.3573322296142578,
+      "rewards/margins": 0.7098425030708313,
+      "rewards/rejected": -1.0671746730804443,
+      "step": 160
+    },
+    {
+      "epoch": 0.22,
+      "grad_norm": 48.19732444247329,
+      "learning_rate": 4.3518518518518514e-07,
+      "logits/chosen": -2.4560000896453857,
+      "logits/rejected": -2.3460609912872314,
+      "logps/chosen": -124.30397033691406,
+      "logps/rejected": -115.36116027832031,
+      "loss": 0.5999,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -0.26509490609169006,
+      "rewards/margins": 0.7949749231338501,
+      "rewards/rejected": -1.0600697994232178,
+      "step": 170
+    },
+    {
+      "epoch": 0.23,
+      "grad_norm": 52.40953829832909,
+      "learning_rate": 4.2806267806267807e-07,
+      "logits/chosen": -2.4244871139526367,
+      "logits/rejected": -2.4024457931518555,
+      "logps/chosen": -124.49531555175781,
+      "logps/rejected": -123.16343688964844,
+      "loss": 0.624,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.6049500703811646,
+      "rewards/margins": 0.594744086265564,
+      "rewards/rejected": -1.1996941566467285,
+      "step": 180
+    },
+    {
+      "epoch": 0.24,
+      "grad_norm": 53.88264831350734,
+      "learning_rate": 4.2094017094017093e-07,
+      "logits/chosen": -2.4857068061828613,
+      "logits/rejected": -2.4509663581848145,
+      "logps/chosen": -135.95523071289062,
+      "logps/rejected": -135.1319122314453,
+      "loss": 0.6651,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.6826471090316772,
+      "rewards/margins": 0.7532482147216797,
+      "rewards/rejected": -1.435895323753357,
+      "step": 190
+    },
+    {
+      "epoch": 0.26,
+      "grad_norm": 47.89753703525624,
+      "learning_rate": 4.138176638176638e-07,
+      "logits/chosen": -2.3674235343933105,
+      "logits/rejected": -2.3760576248168945,
+      "logps/chosen": -117.1615219116211,
+      "logps/rejected": -118.3133316040039,
+      "loss": 0.6295,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.0264689922332764,
+      "rewards/margins": 0.5901049375534058,
+      "rewards/rejected": -1.6165739297866821,
+      "step": 200
+    },
+    {
+      "epoch": 0.27,
+      "grad_norm": 50.34845565038448,
+      "learning_rate": 4.0669515669515667e-07,
+      "logits/chosen": -2.530383348464966,
+      "logits/rejected": -2.430882692337036,
+      "logps/chosen": -137.52378845214844,
+      "logps/rejected": -129.38522338867188,
+      "loss": 0.592,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.568996012210846,
+      "rewards/margins": 0.7430901527404785,
+      "rewards/rejected": -1.3120858669281006,
+      "step": 210
+    },
+    {
+      "epoch": 0.28,
+      "grad_norm": 81.3533014179546,
+      "learning_rate": 3.995726495726496e-07,
+      "logits/chosen": -2.47497820854187,
+      "logits/rejected": -2.4513750076293945,
+      "logps/chosen": -144.29660034179688,
+      "logps/rejected": -127.96578216552734,
+      "loss": 0.6322,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.6084749698638916,
+      "rewards/margins": 0.5566127896308899,
+      "rewards/rejected": -1.1650875806808472,
+      "step": 220
+    },
+    {
+      "epoch": 0.29,
+      "grad_norm": 55.826997268717314,
+      "learning_rate": 3.924501424501424e-07,
+      "logits/chosen": -2.467301607131958,
+      "logits/rejected": -2.399174451828003,
+      "logps/chosen": -132.32093811035156,
+      "logps/rejected": -115.2757339477539,
+      "loss": 0.6108,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.43962016701698303,
+      "rewards/margins": 0.6990880966186523,
+      "rewards/rejected": -1.1387083530426025,
+      "step": 230
+    },
+    {
+      "epoch": 0.31,
+      "grad_norm": 58.06866296891129,
+      "learning_rate": 3.853276353276353e-07,
+      "logits/chosen": -2.5078091621398926,
+      "logits/rejected": -2.3649613857269287,
+      "logps/chosen": -141.54556274414062,
+      "logps/rejected": -124.65989685058594,
+      "loss": 0.6346,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.6636645197868347,
+      "rewards/margins": 0.7334359884262085,
+      "rewards/rejected": -1.3971006870269775,
+      "step": 240
+    },
+    {
+      "epoch": 0.32,
+      "grad_norm": 54.41319496753548,
+      "learning_rate": 3.782051282051282e-07,
+      "logits/chosen": -2.3201379776000977,
+      "logits/rejected": -2.2878081798553467,
+      "logps/chosen": -122.65084075927734,
+      "logps/rejected": -120.03619384765625,
+      "loss": 0.6478,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.006847620010376,
+      "rewards/margins": 0.5119993090629578,
+      "rewards/rejected": -1.518846869468689,
+      "step": 250
+    },
+    {
+      "epoch": 0.33,
+      "grad_norm": 47.03372844469492,
+      "learning_rate": 3.7108262108262107e-07,
+      "logits/chosen": -2.3756680488586426,
+      "logits/rejected": -2.360100746154785,
+      "logps/chosen": -133.3583984375,
+      "logps/rejected": -142.30650329589844,
+      "loss": 0.6107,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.9201425313949585,
+      "rewards/margins": 0.6678245663642883,
+      "rewards/rejected": -1.5879669189453125,
+      "step": 260
+    },
+    {
+      "epoch": 0.35,
+      "grad_norm": 53.73333643313856,
+      "learning_rate": 3.6396011396011393e-07,
+      "logits/chosen": -2.3205008506774902,
+      "logits/rejected": -2.3311374187469482,
+      "logps/chosen": -128.84361267089844,
+      "logps/rejected": -132.9806671142578,
+      "loss": 0.6659,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.25936436653137207,
+      "rewards/margins": 0.8723883628845215,
+      "rewards/rejected": -1.1317527294158936,
+      "step": 270
+    },
+    {
+      "epoch": 0.36,
+      "grad_norm": 57.82638792874447,
+      "learning_rate": 3.5683760683760686e-07,
+      "logits/chosen": -2.4364991188049316,
+      "logits/rejected": -2.3067660331726074,
+      "logps/chosen": -134.5369873046875,
+      "logps/rejected": -126.17820739746094,
+      "loss": 0.6213,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -0.33614638447761536,
+      "rewards/margins": 0.930611789226532,
+      "rewards/rejected": -1.2667582035064697,
+      "step": 280
+    },
+    {
+      "epoch": 0.37,
+      "grad_norm": 51.702039738808836,
+      "learning_rate": 3.497150997150997e-07,
+      "logits/chosen": -2.3851261138916016,
+      "logits/rejected": -2.331394910812378,
+      "logps/chosen": -132.78955078125,
+      "logps/rejected": -122.19071960449219,
+      "loss": 0.6496,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.6341385841369629,
+      "rewards/margins": 0.6531113386154175,
+      "rewards/rejected": -1.2872498035430908,
+      "step": 290
+    },
+    {
+      "epoch": 0.38,
+      "grad_norm": 48.12825157367101,
+      "learning_rate": 3.425925925925926e-07,
+      "logits/chosen": -2.505466938018799,
+      "logits/rejected": -2.362255573272705,
+      "logps/chosen": -140.35035705566406,
+      "logps/rejected": -123.61454010009766,
+      "loss": 0.6493,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.048560380935669,
+      "rewards/margins": 0.4846443235874176,
+      "rewards/rejected": -1.5332047939300537,
+      "step": 300
+    },
+    {
+      "epoch": 0.4,
+      "grad_norm": 55.213032188588954,
+      "learning_rate": 3.354700854700854e-07,
+      "logits/chosen": -2.349586009979248,
+      "logits/rejected": -2.3298165798187256,
+      "logps/chosen": -136.4185791015625,
+      "logps/rejected": -135.18222045898438,
+      "loss": 0.6519,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.2685641050338745,
+      "rewards/margins": 0.744733989238739,
+      "rewards/rejected": -2.0132980346679688,
+      "step": 310
+    },
+    {
+      "epoch": 0.41,
+      "grad_norm": 53.19339082185396,
+      "learning_rate": 3.2834757834757833e-07,
+      "logits/chosen": -2.4016404151916504,
+      "logits/rejected": -2.2783358097076416,
+      "logps/chosen": -141.6110076904297,
+      "logps/rejected": -128.9848175048828,
+      "loss": 0.6288,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.2477556467056274,
+      "rewards/margins": 0.5872876644134521,
+      "rewards/rejected": -1.8350433111190796,
+      "step": 320
+    },
+    {
+      "epoch": 0.42,
+      "grad_norm": 45.4410487899538,
+      "learning_rate": 3.212250712250712e-07,
+      "logits/chosen": -2.4078369140625,
+      "logits/rejected": -2.293262004852295,
+      "logps/chosen": -144.69619750976562,
+      "logps/rejected": -131.64419555664062,
+      "loss": 0.6485,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.055912733078003,
+      "rewards/margins": 0.651071310043335,
+      "rewards/rejected": -1.7069839239120483,
+      "step": 330
+    },
+    {
+      "epoch": 0.44,
+      "grad_norm": 50.49590110170781,
+      "learning_rate": 3.1410256410256407e-07,
+      "logits/chosen": -2.3417906761169434,
+      "logits/rejected": -2.281079053878784,
+      "logps/chosen": -128.5661163330078,
+      "logps/rejected": -126.07405853271484,
+      "loss": 0.6417,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.4270094633102417,
+      "rewards/margins": 0.5071158409118652,
+      "rewards/rejected": -1.934125542640686,
+      "step": 340
+    },
+    {
+      "epoch": 0.45,
+      "grad_norm": 48.85971072038047,
+      "learning_rate": 3.06980056980057e-07,
+      "logits/chosen": -2.3560805320739746,
+      "logits/rejected": -2.291642665863037,
+      "logps/chosen": -133.58302307128906,
+      "logps/rejected": -133.82498168945312,
+      "loss": 0.569,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -0.8126407861709595,
+      "rewards/margins": 1.1261043548583984,
+      "rewards/rejected": -1.938745141029358,
+      "step": 350
+    },
+    {
+      "epoch": 0.46,
+      "grad_norm": 48.41008983073053,
+      "learning_rate": 2.9985754985754986e-07,
+      "logits/chosen": -2.412980318069458,
+      "logits/rejected": -2.3227477073669434,
+      "logps/chosen": -139.5390167236328,
+      "logps/rejected": -131.8283233642578,
+      "loss": 0.6305,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.0040569305419922,
+      "rewards/margins": 0.8860235214233398,
+      "rewards/rejected": -1.890080451965332,
+      "step": 360
+    },
+    {
+      "epoch": 0.47,
+      "grad_norm": 50.57958255232871,
+      "learning_rate": 2.927350427350427e-07,
+      "logits/chosen": -2.4325506687164307,
+      "logits/rejected": -2.334515333175659,
+      "logps/chosen": -134.35159301757812,
+      "logps/rejected": -124.22066497802734,
+      "loss": 0.624,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.1691282987594604,
+      "rewards/margins": 0.8201874494552612,
+      "rewards/rejected": -1.9893157482147217,
+      "step": 370
+    },
+    {
+      "epoch": 0.49,
+      "grad_norm": 52.614236574936854,
+      "learning_rate": 2.8561253561253565e-07,
+      "logits/chosen": -2.383819103240967,
+      "logits/rejected": -2.2645859718322754,
+      "logps/chosen": -139.4959716796875,
+      "logps/rejected": -130.26382446289062,
+      "loss": 0.634,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.1070435047149658,
+      "rewards/margins": 0.6042343974113464,
+      "rewards/rejected": -1.711277723312378,
+      "step": 380
+    },
+    {
+      "epoch": 0.5,
+      "grad_norm": 43.435192010665325,
+      "learning_rate": 2.7849002849002846e-07,
+      "logits/chosen": -2.4008045196533203,
+      "logits/rejected": -2.361043930053711,
+      "logps/chosen": -130.10952758789062,
+      "logps/rejected": -124.78385925292969,
+      "loss": 0.6355,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.36943140625953674,
+      "rewards/margins": 0.9072279930114746,
+      "rewards/rejected": -1.276659369468689,
+      "step": 390
+    },
+    {
+      "epoch": 0.51,
+      "grad_norm": 46.89858433964344,
+      "learning_rate": 2.7136752136752133e-07,
+      "logits/chosen": -2.4596517086029053,
+      "logits/rejected": -2.4161503314971924,
+      "logps/chosen": -142.84959411621094,
+      "logps/rejected": -142.54122924804688,
+      "loss": 0.6148,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.0021470785140991,
+      "rewards/margins": 0.7816516757011414,
+      "rewards/rejected": -1.7837988138198853,
+      "step": 400
+    },
+    {
+      "epoch": 0.52,
+      "grad_norm": 46.643170390788974,
+      "learning_rate": 2.642450142450142e-07,
+      "logits/chosen": -2.325350046157837,
+      "logits/rejected": -2.2754597663879395,
+      "logps/chosen": -139.17689514160156,
+      "logps/rejected": -126.951171875,
+      "loss": 0.6212,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.1546385288238525,
+      "rewards/margins": 0.9383947253227234,
+      "rewards/rejected": -2.0930333137512207,
+      "step": 410
+    },
+    {
+      "epoch": 0.54,
+      "grad_norm": 47.637664508073385,
+      "learning_rate": 2.571225071225071e-07,
+      "logits/chosen": -2.438371181488037,
+      "logits/rejected": -2.386777639389038,
+      "logps/chosen": -140.59593200683594,
+      "logps/rejected": -139.70594787597656,
+      "loss": 0.6396,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.085671305656433,
+      "rewards/margins": 0.7094321846961975,
+      "rewards/rejected": -1.795103669166565,
+      "step": 420
+    },
+    {
+      "epoch": 0.55,
+      "grad_norm": 52.93627701697263,
+      "learning_rate": 2.5e-07,
+      "logits/chosen": -2.458922863006592,
+      "logits/rejected": -2.3807573318481445,
+      "logps/chosen": -123.66154479980469,
+      "logps/rejected": -113.27176666259766,
+      "loss": 0.6373,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.9193522334098816,
+      "rewards/margins": 0.7583762407302856,
+      "rewards/rejected": -1.6777284145355225,
+      "step": 430
+    },
+    {
+      "epoch": 0.56,
+      "grad_norm": 44.15921734659201,
+      "learning_rate": 2.4287749287749286e-07,
+      "logits/chosen": -2.438318967819214,
+      "logits/rejected": -2.3401882648468018,
+      "logps/chosen": -132.58303833007812,
+      "logps/rejected": -130.18482971191406,
+      "loss": 0.5897,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.2183334827423096,
+      "rewards/margins": 0.751697838306427,
+      "rewards/rejected": -1.970031499862671,
+      "step": 440
+    },
+    {
+      "epoch": 0.58,
+      "grad_norm": 50.1560080282361,
+      "learning_rate": 2.3575498575498575e-07,
+      "logits/chosen": -2.441657304763794,
+      "logits/rejected": -2.364025592803955,
+      "logps/chosen": -141.0766143798828,
+      "logps/rejected": -135.3448486328125,
+      "loss": 0.6182,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -1.2046372890472412,
+      "rewards/margins": 0.9371660351753235,
+      "rewards/rejected": -2.14180326461792,
+      "step": 450
+    },
+    {
+      "epoch": 0.59,
+      "grad_norm": 51.86298846305774,
+      "learning_rate": 2.2863247863247862e-07,
+      "logits/chosen": -2.437382936477661,
+      "logits/rejected": -2.405207633972168,
+      "logps/chosen": -145.0140380859375,
+      "logps/rejected": -139.2124786376953,
+      "loss": 0.6286,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.2423968315124512,
+      "rewards/margins": 0.7574957013130188,
+      "rewards/rejected": -1.9998924732208252,
+      "step": 460
+    },
+    {
+      "epoch": 0.6,
+      "grad_norm": 46.2482863900479,
+      "learning_rate": 2.215099715099715e-07,
+      "logits/chosen": -2.3723092079162598,
+      "logits/rejected": -2.3213183879852295,
+      "logps/chosen": -124.37519836425781,
+      "logps/rejected": -130.12850952148438,
+      "loss": 0.5962,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.1122941970825195,
+      "rewards/margins": 0.7544301748275757,
+      "rewards/rejected": -1.8667243719100952,
+      "step": 470
+    },
+    {
+      "epoch": 0.61,
+      "grad_norm": 55.18071165480592,
+      "learning_rate": 2.1438746438746438e-07,
+      "logits/chosen": -2.416285991668701,
+      "logits/rejected": -2.3140416145324707,
+      "logps/chosen": -138.05996704101562,
+      "logps/rejected": -127.0961685180664,
+      "loss": 0.6686,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.1176459789276123,
+      "rewards/margins": 0.6477065682411194,
+      "rewards/rejected": -1.7653526067733765,
+      "step": 480
+    },
+    {
+      "epoch": 0.63,
+      "grad_norm": 46.171723981999456,
+      "learning_rate": 2.0726495726495728e-07,
+      "logits/chosen": -2.399726390838623,
+      "logits/rejected": -2.275796890258789,
+      "logps/chosen": -144.00631713867188,
+      "logps/rejected": -135.67333984375,
+      "loss": 0.6217,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.2068181037902832,
+      "rewards/margins": 0.722368597984314,
+      "rewards/rejected": -1.9291868209838867,
+      "step": 490
+    },
+    {
+      "epoch": 0.64,
+      "grad_norm": 49.48414547787998,
+      "learning_rate": 2.0014245014245012e-07,
+      "logits/chosen": -2.4761524200439453,
+      "logits/rejected": -2.4091053009033203,
+      "logps/chosen": -143.36672973632812,
+      "logps/rejected": -134.77394104003906,
+      "loss": 0.6301,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.3807828426361084,
+      "rewards/margins": 0.9061687588691711,
+      "rewards/rejected": -2.2869515419006348,
+      "step": 500
+    },
+    {
+      "epoch": 0.65,
+      "grad_norm": 49.90109558156681,
+      "learning_rate": 1.9301994301994302e-07,
+      "logits/chosen": -2.395216464996338,
+      "logits/rejected": -2.3614308834075928,
+      "logps/chosen": -133.65074157714844,
+      "logps/rejected": -127.56480407714844,
+      "loss": 0.6394,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.0472420454025269,
+      "rewards/margins": 0.7637320756912231,
+      "rewards/rejected": -1.81097412109375,
+      "step": 510
+    },
+    {
+      "epoch": 0.67,
+      "grad_norm": 53.069722657467004,
+      "learning_rate": 1.8589743589743588e-07,
+      "logits/chosen": -2.450152635574341,
+      "logits/rejected": -2.37846302986145,
+      "logps/chosen": -135.0297088623047,
+      "logps/rejected": -127.63478088378906,
+      "loss": 0.5602,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.1925435066223145,
+      "rewards/margins": 0.9132286310195923,
+      "rewards/rejected": -2.105772018432617,
+      "step": 520
+    },
+    {
+      "epoch": 0.68,
+      "grad_norm": 46.68933325282183,
+      "learning_rate": 1.7877492877492878e-07,
+      "logits/chosen": -2.4857304096221924,
+      "logits/rejected": -2.4155218601226807,
+      "logps/chosen": -140.39657592773438,
+      "logps/rejected": -130.3267059326172,
+      "loss": 0.6292,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.0456947088241577,
+      "rewards/margins": 0.7321243286132812,
+      "rewards/rejected": -1.777819037437439,
+      "step": 530
+    },
+    {
+      "epoch": 0.69,
+      "grad_norm": 44.8061648155278,
+      "learning_rate": 1.7165242165242165e-07,
+      "logits/chosen": -2.510542631149292,
+      "logits/rejected": -2.4133083820343018,
+      "logps/chosen": -135.0148468017578,
+      "logps/rejected": -127.9262466430664,
+      "loss": 0.5935,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.187801718711853,
+      "rewards/margins": 0.6901987791061401,
+      "rewards/rejected": -1.878000259399414,
+      "step": 540
+    },
+    {
+      "epoch": 0.7,
+      "grad_norm": 43.11034699833336,
+      "learning_rate": 1.6452991452991452e-07,
+      "logits/chosen": -2.4958064556121826,
+      "logits/rejected": -2.417062997817993,
+      "logps/chosen": -131.7577362060547,
+      "logps/rejected": -129.5203857421875,
+      "loss": 0.6496,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.8051038980484009,
+      "rewards/margins": 0.6998610496520996,
+      "rewards/rejected": -1.5049649477005005,
+      "step": 550
+    },
+    {
+      "epoch": 0.72,
+      "grad_norm": 60.327683231194165,
+      "learning_rate": 1.574074074074074e-07,
+      "logits/chosen": -2.501108407974243,
+      "logits/rejected": -2.4132328033447266,
+      "logps/chosen": -135.57064819335938,
+      "logps/rejected": -128.8811492919922,
+      "loss": 0.6208,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -0.8196865916252136,
+      "rewards/margins": 0.8410652875900269,
+      "rewards/rejected": -1.6607520580291748,
+      "step": 560
+    },
+    {
+      "epoch": 0.73,
+      "grad_norm": 47.23991316598195,
+      "learning_rate": 1.5028490028490028e-07,
+      "logits/chosen": -2.4793362617492676,
+      "logits/rejected": -2.426276922225952,
+      "logps/chosen": -129.61695861816406,
+      "logps/rejected": -129.7327423095703,
+      "loss": 0.6285,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -0.614374041557312,
+      "rewards/margins": 1.074138879776001,
+      "rewards/rejected": -1.6885130405426025,
+      "step": 570
+    },
+    {
+      "epoch": 0.74,
+      "grad_norm": 49.790355934987595,
+      "learning_rate": 1.4316239316239315e-07,
+      "logits/chosen": -2.454620838165283,
+      "logits/rejected": -2.4066927433013916,
+      "logps/chosen": -142.7227020263672,
+      "logps/rejected": -142.06124877929688,
+      "loss": 0.5731,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -0.620644748210907,
+      "rewards/margins": 1.1552683115005493,
+      "rewards/rejected": -1.7759130001068115,
+      "step": 580
+    },
+    {
+      "epoch": 0.75,
+      "grad_norm": 54.193112825604594,
+      "learning_rate": 1.3603988603988604e-07,
+      "logits/chosen": -2.462184429168701,
+      "logits/rejected": -2.3979437351226807,
+      "logps/chosen": -136.3149871826172,
+      "logps/rejected": -131.81707763671875,
+      "loss": 0.6148,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.1002075672149658,
+      "rewards/margins": 0.6897586584091187,
+      "rewards/rejected": -1.789966344833374,
+      "step": 590
+    },
+    {
+      "epoch": 0.77,
+      "grad_norm": 47.41349722997695,
+      "learning_rate": 1.289173789173789e-07,
+      "logits/chosen": -2.5727453231811523,
+      "logits/rejected": -2.412508010864258,
+      "logps/chosen": -139.8438720703125,
+      "logps/rejected": -125.3525161743164,
+      "loss": 0.6157,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.9148397445678711,
+      "rewards/margins": 0.8886247873306274,
+      "rewards/rejected": -1.8034645318984985,
+      "step": 600
+    },
+    {
+      "epoch": 0.78,
+      "grad_norm": 41.12450859291961,
+      "learning_rate": 1.2179487179487178e-07,
+      "logits/chosen": -2.439976215362549,
+      "logits/rejected": -2.3754196166992188,
+      "logps/chosen": -138.5695037841797,
+      "logps/rejected": -130.4626007080078,
+      "loss": 0.564,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -0.6952188014984131,
+      "rewards/margins": 1.181563138961792,
+      "rewards/rejected": -1.8767818212509155,
+      "step": 610
+    },
+    {
+      "epoch": 0.79,
+      "grad_norm": 41.76957543268299,
+      "learning_rate": 1.1467236467236467e-07,
+      "logits/chosen": -2.5016493797302246,
+      "logits/rejected": -2.373262643814087,
+      "logps/chosen": -137.6613311767578,
+      "logps/rejected": -119.12971496582031,
+      "loss": 0.6353,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.046036720275879,
+      "rewards/margins": 0.856979250907898,
+      "rewards/rejected": -1.9030160903930664,
+      "step": 620
+    },
+    {
+      "epoch": 0.81,
+      "grad_norm": 42.53042372069615,
+      "learning_rate": 1.0754985754985754e-07,
+      "logits/chosen": -2.4848201274871826,
+      "logits/rejected": -2.3861021995544434,
+      "logps/chosen": -130.31802368164062,
+      "logps/rejected": -125.7854232788086,
+      "loss": 0.6179,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.9321087598800659,
+      "rewards/margins": 0.7969751358032227,
+      "rewards/rejected": -1.7290840148925781,
+      "step": 630
+    },
+    {
+      "epoch": 0.82,
+      "grad_norm": 49.31782451064985,
+      "learning_rate": 1.0042735042735042e-07,
+      "logits/chosen": -2.4740242958068848,
+      "logits/rejected": -2.3682055473327637,
+      "logps/chosen": -140.46707153320312,
+      "logps/rejected": -130.07626342773438,
+      "loss": 0.6121,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.9649416208267212,
+      "rewards/margins": 0.8673871755599976,
+      "rewards/rejected": -1.8323287963867188,
+      "step": 640
+    },
+    {
+      "epoch": 0.83,
+      "grad_norm": 45.44250026093777,
+      "learning_rate": 9.33048433048433e-08,
+      "logits/chosen": -2.4244422912597656,
+      "logits/rejected": -2.3046741485595703,
+      "logps/chosen": -137.98712158203125,
+      "logps/rejected": -119.65657043457031,
+      "loss": 0.5566,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -0.9997569918632507,
+      "rewards/margins": 0.918919563293457,
+      "rewards/rejected": -1.9186766147613525,
+      "step": 650
+    },
+    {
+      "epoch": 0.84,
+      "grad_norm": 47.21715125647828,
+      "learning_rate": 8.618233618233619e-08,
+      "logits/chosen": -2.4901123046875,
+      "logits/rejected": -2.4289157390594482,
+      "logps/chosen": -129.42507934570312,
+      "logps/rejected": -139.14041137695312,
+      "loss": 0.6202,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.017300009727478,
+      "rewards/margins": 0.8294160962104797,
+      "rewards/rejected": -1.8467161655426025,
+      "step": 660
+    },
+    {
+      "epoch": 0.86,
+      "grad_norm": 52.73206161351472,
+      "learning_rate": 7.905982905982906e-08,
+      "logits/chosen": -2.478940486907959,
+      "logits/rejected": -2.3513381481170654,
+      "logps/chosen": -133.02786254882812,
+      "logps/rejected": -129.9757080078125,
+      "loss": 0.6032,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.9857560992240906,
+      "rewards/margins": 0.7195991277694702,
+      "rewards/rejected": -1.705354928970337,
+      "step": 670
+    },
+    {
+      "epoch": 0.87,
+      "grad_norm": 38.687343792092506,
+      "learning_rate": 7.193732193732194e-08,
+      "logits/chosen": -2.4151899814605713,
+      "logits/rejected": -2.40562105178833,
+      "logps/chosen": -134.22622680664062,
+      "logps/rejected": -129.08303833007812,
+      "loss": 0.6337,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.2530044317245483,
+      "rewards/margins": 0.6666024923324585,
+      "rewards/rejected": -1.9196069240570068,
+      "step": 680
+    },
+    {
+      "epoch": 0.88,
+      "grad_norm": 42.96291096586097,
+      "learning_rate": 6.481481481481481e-08,
+      "logits/chosen": -2.520855665206909,
+      "logits/rejected": -2.444889545440674,
+      "logps/chosen": -132.0274658203125,
+      "logps/rejected": -143.20416259765625,
+      "loss": 0.5952,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.0581846237182617,
+      "rewards/margins": 0.8191283941268921,
+      "rewards/rejected": -1.8773130178451538,
+      "step": 690
+    },
+    {
+      "epoch": 0.9,
+      "grad_norm": 50.35365087276543,
+      "learning_rate": 5.7692307692307695e-08,
+      "logits/chosen": -2.4175751209259033,
+      "logits/rejected": -2.3466906547546387,
+      "logps/chosen": -146.04676818847656,
+      "logps/rejected": -136.0231475830078,
+      "loss": 0.5951,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.9246633648872375,
+      "rewards/margins": 1.0189440250396729,
+      "rewards/rejected": -1.9436070919036865,
+      "step": 700
+    },
+    {
+      "epoch": 0.91,
+      "grad_norm": 48.474517033669926,
+      "learning_rate": 5.056980056980057e-08,
+      "logits/chosen": -2.3717129230499268,
+      "logits/rejected": -2.328782558441162,
+      "logps/chosen": -127.49263763427734,
+      "logps/rejected": -126.10953521728516,
+      "loss": 0.6341,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.229093313217163,
+      "rewards/margins": 0.6647592782974243,
+      "rewards/rejected": -1.8938528299331665,
+      "step": 710
+    },
+    {
+      "epoch": 0.92,
+      "grad_norm": 52.42628807392078,
+      "learning_rate": 4.3447293447293445e-08,
+      "logits/chosen": -2.489219903945923,
+      "logits/rejected": -2.431892156600952,
+      "logps/chosen": -142.9352264404297,
+      "logps/rejected": -131.1381072998047,
+      "loss": 0.6151,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.0513079166412354,
+      "rewards/margins": 0.6682957410812378,
+      "rewards/rejected": -1.7196037769317627,
+      "step": 720
+    },
+    {
+      "epoch": 0.93,
+      "grad_norm": 43.20371681135118,
+      "learning_rate": 3.632478632478633e-08,
+      "logits/chosen": -2.460447311401367,
+      "logits/rejected": -2.3597540855407715,
+      "logps/chosen": -134.65219116210938,
+      "logps/rejected": -128.0625,
+      "loss": 0.604,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.1143229007720947,
+      "rewards/margins": 0.7751682996749878,
+      "rewards/rejected": -1.889491319656372,
+      "step": 730
+    },
+    {
+      "epoch": 0.95,
+      "grad_norm": 50.1036581755337,
+      "learning_rate": 2.92022792022792e-08,
+      "logits/chosen": -2.5251357555389404,
+      "logits/rejected": -2.4169728755950928,
+      "logps/chosen": -133.37075805664062,
+      "logps/rejected": -118.67008972167969,
+      "loss": 0.6673,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.9731636047363281,
+      "rewards/margins": 0.8216039538383484,
+      "rewards/rejected": -1.7947673797607422,
+      "step": 740
+    },
+    {
+      "epoch": 0.96,
+      "grad_norm": 46.47262858763026,
+      "learning_rate": 2.2079772079772077e-08,
+      "logits/chosen": -2.433579921722412,
+      "logits/rejected": -2.34560489654541,
+      "logps/chosen": -130.1377716064453,
+      "logps/rejected": -122.85064697265625,
+      "loss": 0.59,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -0.8935017585754395,
+      "rewards/margins": 0.899819016456604,
+      "rewards/rejected": -1.793320655822754,
+      "step": 750
+    },
+    {
+      "epoch": 0.97,
+      "grad_norm": 44.611084335744415,
+      "learning_rate": 1.4957264957264956e-08,
+      "logits/chosen": -2.506635904312134,
+      "logits/rejected": -2.386122941970825,
+      "logps/chosen": -133.47280883789062,
+      "logps/rejected": -132.68231201171875,
+      "loss": 0.5849,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.768479585647583,
+      "rewards/margins": 0.8516393899917603,
+      "rewards/rejected": -1.6201190948486328,
+      "step": 760
+    },
+    {
+      "epoch": 0.99,
+      "grad_norm": 46.24182857920438,
+      "learning_rate": 7.834757834757834e-09,
+      "logits/chosen": -2.461017608642578,
+      "logits/rejected": -2.4049289226531982,
+      "logps/chosen": -135.73080444335938,
+      "logps/rejected": -125.0393295288086,
+      "loss": 0.5804,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.8659018278121948,
+      "rewards/margins": 0.9317833781242371,
+      "rewards/rejected": -1.7976852655410767,
+      "step": 770
+    },
+    {
+      "epoch": 1.0,
+      "grad_norm": 48.37476187102993,
+      "learning_rate": 7.122507122507123e-10,
+      "logits/chosen": -2.5103912353515625,
+      "logits/rejected": -2.4379806518554688,
+      "logps/chosen": -128.02320861816406,
+      "logps/rejected": -132.70835876464844,
+      "loss": 0.5762,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.9150331616401672,
+      "rewards/margins": 0.8675880432128906,
+      "rewards/rejected": -1.7826210260391235,
+      "step": 780
+    },
+    {
+      "epoch": 1.0,
+      "step": 781,
+      "total_flos": 0.0,
+      "train_loss": 0.6206242706955777,
+      "train_runtime": 6118.3001,
+      "train_samples_per_second": 8.172,
+      "train_steps_per_second": 0.128
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 781,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 500,
+  "total_flos": 0.0,
+  "train_batch_size": 8,
+  "trial_name": null,
+  "trial_params": null
+}