Training in progress, step 38

adapter_config.json

@@ -12,14 +12,14 @@
   "layers_pattern": null,
   "layers_to_transform": null,
   "loftq_config": {},
-  "lora_alpha": 32,
+  "lora_alpha": 128,
   "lora_bias": false,
   "lora_dropout": 0.05,
   "megatron_config": null,
   "megatron_core": "megatron.core",
   "modules_to_save": null,
   "peft_type": "LORA",
-  "r": 16,
+  "r": 64,
   "rank_pattern": {},
   "revision": null,
   "target_modules": [

adapter_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:63e521c9270d5756d364dbbfca0c0461a9ed73e2a30e7b6fcdd7f37bdf5c52c3
-size 160069834
+oid sha256:36b50997fcab777e5f9d18f8df491b7357f387f269b60d9e9eb4995373e939db
+size 475252480

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:17cd4f2d619ce4a981f695b78045348edd710801815fcaba2b95f767c5326d26
-size 159967880
+oid sha256:0ac06b5dd2c49d323927cdaf25da64976f4e896f119c9f9876ef58fb42b32e6e
+size 639691872

config.json

@@ -14,7 +14,7 @@
   "initializer_range": 0.02,
   "intermediate_size": 11008,
   "max_length": 4096,
-  "max_position_embeddings": 4096,
+  "max_position_embeddings": 2048,
   "mlp_bias": false,
   "model_type": "llama",
   "num_attention_heads": 32,

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0e32fe29f5088d466addaeddb817db439aaee1294b1533d62ee6d9d5e7f33d12
+oid sha256:f6e0f94144cf6ee4f3d9d40c98b0898e3feda45b0a5cde6ebc010e83076337f2
 size 6392