Training in progress, epoch 3

config.json

@@ -29,8 +29,8 @@
   },
   "rope_theta": 500000.0,
   "tie_word_embeddings": true,
-  "torch_dtype": "bfloat16",
-  "transformers_version": "4.46.1",
+  "torch_dtype": "float16",
+  "transformers_version": "4.46.2",
   "use_cache": false,
   "vocab_size": 128256
 }

model-00001-of-00002.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:01c5b0a8b5e7cd6ce31312ec7f3613565626b594f0ba71a6a409c1260308e426
-size 4965801344
+oid sha256:53e398468bbe19f8601d992c08d1a2ba3517ed7ac0c9ea1250cdf26789193d17
+size 4965801152

model-00002-of-00002.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b4ceb4e31b2ebc30f66aaded057b6cc38b87d17b1eb44c7e1765ccc44bf940a4
-size 1547842640
+oid sha256:0d6c9110ee616e0e7ae4642794095286f8b44b20ed18d0d9e9118081a7ccbf77
+size 1528966512

model.safetensors.index.json

@@ -1,16 +1,58 @@
 {
   "metadata": {
-    "total_size": 6513610752
+    "total_size": 6494730240
   },
   "weight_map": {
-    "cross_attention.Wkv.bias": "model-00002-of-00002.safetensors",
-    "cross_attention.Wkv.weight": "model-00002-of-00002.safetensors",
-    "cross_attention.Wq.bias": "model-00002-of-00002.safetensors",
-    "cross_attention.Wq.weight": "model-00002-of-00002.safetensors",
-    "cross_attention.out_proj.bias": "model-00002-of-00002.safetensors",
-    "cross_attention.out_proj.weight": "model-00002-of-00002.safetensors",
-    "image_proj.bias": "model-00002-of-00002.safetensors",
-    "image_proj.weight": "model-00002-of-00002.safetensors",
+    "cross_attention.heads.0.key.bias": "model-00002-of-00002.safetensors",
+    "cross_attention.heads.0.key.weight": "model-00002-of-00002.safetensors",
+    "cross_attention.heads.0.query.bias": "model-00002-of-00002.safetensors",
+    "cross_attention.heads.0.query.weight": "model-00002-of-00002.safetensors",
+    "cross_attention.heads.0.value.bias": "model-00002-of-00002.safetensors",
+    "cross_attention.heads.0.value.weight": "model-00002-of-00002.safetensors",
+    "cross_attention.heads.1.key.bias": "model-00002-of-00002.safetensors",
+    "cross_attention.heads.1.key.weight": "model-00002-of-00002.safetensors",
+    "cross_attention.heads.1.query.bias": "model-00002-of-00002.safetensors",
+    "cross_attention.heads.1.query.weight": "model-00002-of-00002.safetensors",
+    "cross_attention.heads.1.value.bias": "model-00002-of-00002.safetensors",
+    "cross_attention.heads.1.value.weight": "model-00002-of-00002.safetensors",
+    "cross_attention.heads.2.key.bias": "model-00002-of-00002.safetensors",
+    "cross_attention.heads.2.key.weight": "model-00002-of-00002.safetensors",
+    "cross_attention.heads.2.query.bias": "model-00002-of-00002.safetensors",
+    "cross_attention.heads.2.query.weight": "model-00002-of-00002.safetensors",
+    "cross_attention.heads.2.value.bias": "model-00002-of-00002.safetensors",
+    "cross_attention.heads.2.value.weight": "model-00002-of-00002.safetensors",
+    "cross_attention.heads.3.key.bias": "model-00002-of-00002.safetensors",
+    "cross_attention.heads.3.key.weight": "model-00002-of-00002.safetensors",
+    "cross_attention.heads.3.query.bias": "model-00002-of-00002.safetensors",
+    "cross_attention.heads.3.query.weight": "model-00002-of-00002.safetensors",
+    "cross_attention.heads.3.value.bias": "model-00002-of-00002.safetensors",
+    "cross_attention.heads.3.value.weight": "model-00002-of-00002.safetensors",
+    "cross_attention.heads.4.key.bias": "model-00002-of-00002.safetensors",
+    "cross_attention.heads.4.key.weight": "model-00002-of-00002.safetensors",
+    "cross_attention.heads.4.query.bias": "model-00002-of-00002.safetensors",
+    "cross_attention.heads.4.query.weight": "model-00002-of-00002.safetensors",
+    "cross_attention.heads.4.value.bias": "model-00002-of-00002.safetensors",
+    "cross_attention.heads.4.value.weight": "model-00002-of-00002.safetensors",
+    "cross_attention.heads.5.key.bias": "model-00002-of-00002.safetensors",
+    "cross_attention.heads.5.key.weight": "model-00002-of-00002.safetensors",
+    "cross_attention.heads.5.query.bias": "model-00002-of-00002.safetensors",
+    "cross_attention.heads.5.query.weight": "model-00002-of-00002.safetensors",
+    "cross_attention.heads.5.value.bias": "model-00002-of-00002.safetensors",
+    "cross_attention.heads.5.value.weight": "model-00002-of-00002.safetensors",
+    "cross_attention.heads.6.key.bias": "model-00002-of-00002.safetensors",
+    "cross_attention.heads.6.key.weight": "model-00002-of-00002.safetensors",
+    "cross_attention.heads.6.query.bias": "model-00002-of-00002.safetensors",
+    "cross_attention.heads.6.query.weight": "model-00002-of-00002.safetensors",
+    "cross_attention.heads.6.value.bias": "model-00002-of-00002.safetensors",
+    "cross_attention.heads.6.value.weight": "model-00002-of-00002.safetensors",
+    "cross_attention.heads.7.key.bias": "model-00002-of-00002.safetensors",
+    "cross_attention.heads.7.key.weight": "model-00002-of-00002.safetensors",
+    "cross_attention.heads.7.query.bias": "model-00002-of-00002.safetensors",
+    "cross_attention.heads.7.query.weight": "model-00002-of-00002.safetensors",
+    "cross_attention.heads.7.value.bias": "model-00002-of-00002.safetensors",
+    "cross_attention.heads.7.value.weight": "model-00002-of-00002.safetensors",
+    "cross_attention.output_linear.bias": "model-00002-of-00002.safetensors",
+    "cross_attention.output_linear.weight": "model-00002-of-00002.safetensors",
     "llama_model.model.embed_tokens.weight": "model-00001-of-00002.safetensors",
     "llama_model.model.layers.0.input_layernorm.weight": "model-00001-of-00002.safetensors",
     "llama_model.model.layers.0.mlp.down_proj.weight": "model-00001-of-00002.safetensors",

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4b98c07ee1994bcc03bd5d507efed729bcd395fce7601eb2e411b564467ea482
-size 6456
+oid sha256:5a561bc35ea25ffc08623bb5f07cd8e4757f3c0b68fa2ea16f086ce3920e1c49
+size 6075