ericrisco/llama2-instruct-tune-500s

README.md

@@ -1,12 +1,12 @@
 ---
+base_model: NousResearch/Llama-2-7b-hf
+datasets:
+- generator
 library_name: peft
 tags:
 - trl
 - sft
 - generated_from_trainer
-datasets:
-- generator
-base_model: NousResearch/Llama-2-7b-hf
 model-index:
 - name: llama2_instruct_generation
   results: []
@@ -19,7 +19,7 @@ should probably proofread and complete it, then remove this comment. -->
 
 This model is a fine-tuned version of [NousResearch/Llama-2-7b-hf](https://huggingface.co/NousResearch/Llama-2-7b-hf) on the generator dataset.
 It achieves the following results on the evaluation set:
-- Loss: 1.6737
+- Loss: 1.6761
 
 ## Model description
 
@@ -44,44 +44,43 @@ The following hyperparameters were used during training:
 - seed: 42
 - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
 - lr_scheduler_type: constant
-- lr_scheduler_warmup_steps: 0.03
 - training_steps: 500
 
 ### Training results
 
-| Training Loss | Epoch | Step | Validation Loss |
-|:-------------:|:-----:|:----:|:---------------:|
-| 1.9238        | 0.0   | 20   | 1.8060          |
-| 1.9181        | 0.01  | 40   | 1.7781          |
-| 1.8185        | 0.01  | 60   | 1.7652          |
-| 1.8336        | 0.01  | 80   | 1.7504          |
-| 1.8205        | 0.01  | 100  | 1.7275          |
-| 1.7673        | 0.02  | 120  | 1.7090          |
-| 1.7934        | 0.02  | 140  | 1.6996          |
-| 1.7844        | 0.02  | 160  | 1.6953          |
-| 1.7847        | 0.02  | 180  | 1.6932          |
-| 1.7772        | 0.03  | 200  | 1.6897          |
-| 1.7883        | 0.03  | 220  | 1.6896          |
-| 1.8346        | 0.03  | 240  | 1.6852          |
-| 1.6646        | 0.04  | 260  | 1.6834          |
-| 1.7433        | 0.04  | 280  | 1.6822          |
-| 1.7762        | 0.04  | 300  | 1.6809          |
-| 1.8508        | 0.04  | 320  | 1.6796          |
-| 1.7275        | 0.05  | 340  | 1.6791          |
-| 1.8002        | 0.05  | 360  | 1.6755          |
-| 1.855         | 0.05  | 380  | 1.6768          |
-| 1.7529        | 0.05  | 400  | 1.6751          |
-| 1.7728        | 0.06  | 420  | 1.6737          |
-| 1.8053        | 0.06  | 440  | 1.6737          |
-| 1.8579        | 0.06  | 460  | 1.6736          |
-| 1.7383        | 0.07  | 480  | 1.6735          |
-| 1.9282        | 0.07  | 500  | 1.6737          |
+| Training Loss | Epoch  | Step | Validation Loss |
+|:-------------:|:------:|:----:|:---------------:|
+| 1.9468        | 0.0027 | 20   | 1.8141          |
+| 1.8737        | 0.0054 | 40   | 1.7848          |
+| 1.8769        | 0.0081 | 60   | 1.7718          |
+| 1.8634        | 0.0108 | 80   | 1.7598          |
+| 1.8584        | 0.0135 | 100  | 1.7469          |
+| 1.8271        | 0.0163 | 120  | 1.7170          |
+| 1.8706        | 0.0190 | 140  | 1.7042          |
+| 1.8306        | 0.0217 | 160  | 1.7005          |
+| 1.7954        | 0.0244 | 180  | 1.6948          |
+| 1.8616        | 0.0271 | 200  | 1.6947          |
+| 1.81          | 0.0298 | 220  | 1.6915          |
+| 1.8003        | 0.0325 | 240  | 1.6900          |
+| 1.9069        | 0.0352 | 260  | 1.6880          |
+| 1.8266        | 0.0379 | 280  | 1.6868          |
+| 1.8615        | 0.0406 | 300  | 1.6849          |
+| 1.7728        | 0.0433 | 320  | 1.6832          |
+| 1.806         | 0.0461 | 340  | 1.6824          |
+| 1.8843        | 0.0488 | 360  | 1.6812          |
+| 1.7655        | 0.0515 | 380  | 1.6803          |
+| 1.812         | 0.0542 | 400  | 1.6795          |
+| 1.8058        | 0.0569 | 420  | 1.6779          |
+| 1.7424        | 0.0596 | 440  | 1.6779          |
+| 1.8976        | 0.0623 | 460  | 1.6782          |
+| 1.8237        | 0.0650 | 480  | 1.6778          |
+| 1.8981        | 0.0677 | 500  | 1.6761          |
 
 
 ### Framework versions
 
-- PEFT 0.7.1
-- Transformers 4.36.2
-- Pytorch 2.1.0+cu121
-- Datasets 2.16.1
-- Tokenizers 0.15.0
+- PEFT 0.13.0
+- Transformers 4.45.0
+- Pytorch 2.4.1+cu121
+- Datasets 3.0.1
+- Tokenizers 0.20.0

adapter_config.json

@@ -6,6 +6,7 @@
   "fan_in_fan_out": false,
   "inference_mode": true,
   "init_lora_weights": true,
+  "layer_replication": null,
   "layers_pattern": null,
   "layers_to_transform": null,
   "loftq_config": {},
@@ -22,5 +23,7 @@
     "q_proj",
     "v_proj"
   ],
-  "task_type": "CAUSAL_LM"
+  "task_type": "CAUSAL_LM",
+  "use_dora": false,
+  "use_rslora": false
 }

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:899c28f6b49a4b08069ad1c2e1c95d0eac312b73b25d6bbb2f9b9bcebb8711a8
+oid sha256:c43dc04532b014d1c7de792f57ae7b7d5a3f8e6bd497a9677993c63d5ee08096
 size 134235048

runs/Sep26_13-01-01_3d63c6bcbcfd/events.out.tfevents.1727355710.3d63c6bcbcfd.3916.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:05dd11a33f815e82e6781d7b1fcc26afd888ad6eeadc61afc9a09d4a2cacea44
+size 23452

tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347
+size 499723

tokenizer_config.json

@@ -1,6 +1,7 @@
 {
   "add_bos_token": true,
   "add_eos_token": false,
+  "add_prefix_space": null,
   "added_tokens_decoder": {
     "0": {
       "content": "<unk>",

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:17e7ba8e5f6bd62a4c225faedf20cbd892301185759ca884b25fa624a3f1aa2c
-size 4728
+oid sha256:0d7d05ef4184ef5384d4cbcc938ea6aefb352a1c32c3276f279e3f32f1fe21f7
+size 5496