End of training

README.md

@@ -41,11 +41,11 @@ early_stopping_patience: null
 eval_max_new_tokens: 128
 eval_table_size: null
 evals_per_epoch: 1
-flash_attention: false
+flash_attention: true
 fp16: null
 fsdp: null
 fsdp_config: null
-gradient_accumulation_steps: 8
+gradient_accumulation_steps: 16
 gradient_checkpointing: true
 group_by_length: false
 hub_model_id: error577/202543ff-c8cb-4cd4-a8ca-da56264dae6e
@@ -82,7 +82,7 @@ tf32: false
 tokenizer_type: AutoTokenizer
 train_on_inputs: false
 trust_remote_code: true
-val_set_size: 0.02
+val_set_size: 0.001
 wandb_entity: null
 wandb_mode: online
 wandb_name: a79dc58a-b30f-4b42-a703-e31cc51332ed
@@ -101,7 +101,7 @@ xformers_attention: null
 
 This model is a fine-tuned version of [oopsung/llama2-7b-koNqa-test-v1](https://huggingface.co/oopsung/llama2-7b-koNqa-test-v1) on the None dataset.
 It achieves the following results on the evaluation set:
-- Loss: nan
+- Loss: 2.1406
 
 ## Model description
 
@@ -124,8 +124,8 @@ The following hyperparameters were used during training:
 - train_batch_size: 1
 - eval_batch_size: 1
 - seed: 42
-- gradient_accumulation_steps: 8
-- total_train_batch_size: 8
+- gradient_accumulation_steps: 16
+- total_train_batch_size: 16
 - optimizer: Use OptimizerNames.ADAMW_BNB with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
 - lr_scheduler_type: cosine
 - lr_scheduler_warmup_steps: 10
@@ -135,7 +135,7 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch  | Step | Validation Loss |
 |:-------------:|:------:|:----:|:---------------:|
-| 0.0           | 0.0020 | 100  | nan             |
+| 1.4357        | 0.0038 | 100  | 2.1406          |
 
 
 ### Framework versions

adapter_config.json

@@ -20,13 +20,13 @@
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
+    "o_proj",
+    "gate_proj",
     "v_proj",
     "up_proj",
-    "gate_proj",
+    "down_proj",
     "q_proj",
-    "o_proj",
-    "k_proj",
-    "down_proj"
+    "k_proj"
   ],
   "task_type": "CAUSAL_LM",
   "use_dora": false,

adapter_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:448d79c355f7f154f10031130f3112521c83938fb84dc4535dfc7ca4b6cbd6fe
+oid sha256:1793593aee751493d462b407e8dd4822d17c993111b28bec02f359232cdf6e82
 size 319977674

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9d8dc13d2518bb660921f4bd58e81b17ace1239b9852e7f4e49ddd8315091788
+oid sha256:9f6941aad94e301671ce4ec56f0ccceede6972fcaf8c253d4caa13024003889a
 size 319876032

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7ce62b2704cdc4ed29b6f2c9d03327e9b86048a8213f6988a75495efc7e32ae0
+oid sha256:a28dc31871e5040bd4850398ba09ae43aa80a5ef753a46b4245c30c1e1a1daac
 size 6776