Model save

README.md

@@ -1,12 +1,8 @@
 ---
 library_name: transformers
-license: llama2
-base_model: meta-llama/Llama-2-7b-chat-hf
+license: apache-2.0
+base_model: mistralai/Mistral-7B-Instruct-v0.1
 tags:
-- alignment-handbook
-- trl
-- sft
-- generated_from_trainer
 - trl
 - sft
 - generated_from_trainer
@@ -20,9 +16,7 @@ should probably proofread and complete it, then remove this comment. -->
 
 # mistral_o1_005_full
 
-This model is a fine-tuned version of [meta-llama/Llama-2-7b-chat-hf](https://huggingface.co/meta-llama/Llama-2-7b-chat-hf) on the None dataset.
-It achieves the following results on the evaluation set:
-- Loss: 0.8031
+This model is a fine-tuned version of [mistralai/Mistral-7B-Instruct-v0.1](https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.1) on the None dataset.
 
 ## Model description
 

all_results.json

@@ -1,14 +1,9 @@
 {
     "epoch": 0.9565217391304348,
-    "eval_loss": 0.803080677986145,
-    "eval_runtime": 1.3886,
-    "eval_samples": 20,
-    "eval_samples_per_second": 14.403,
-    "eval_steps_per_second": 1.44,
-    "total_flos": 1083185692672.0,
-    "train_loss": 0.9741405790502374,
-    "train_runtime": 62.2828,
+    "total_flos": 1081332727808.0,
+    "train_loss": 0.8119967471469532,
+    "train_runtime": 65.397,
     "train_samples": 368,
-    "train_samples_per_second": 5.909,
-    "train_steps_per_second": 0.177
+    "train_samples_per_second": 5.627,
+    "train_steps_per_second": 0.168
 }

config.json

@@ -1,29 +1,27 @@
 {
-  "_name_or_path": "meta-llama/Llama-2-7b-chat-hf",
+  "_name_or_path": "mistralai/Mistral-7B-Instruct-v0.1",
   "architectures": [
-    "LlamaForCausalLM"
+    "MistralForCausalLM"
   ],
-  "attention_bias": false,
   "attention_dropout": 0.0,
   "bos_token_id": 1,
   "eos_token_id": 2,
+  "head_dim": 128,
   "hidden_act": "silu",
   "hidden_size": 4096,
   "initializer_range": 0.02,
-  "intermediate_size": 11008,
-  "max_position_embeddings": 4096,
-  "mlp_bias": false,
-  "model_type": "llama",
+  "intermediate_size": 14336,
+  "max_position_embeddings": 32768,
+  "model_type": "mistral",
   "num_attention_heads": 32,
   "num_hidden_layers": 32,
-  "num_key_value_heads": 32,
-  "pretraining_tp": 1,
+  "num_key_value_heads": 8,
   "rms_norm_eps": 1e-05,
-  "rope_scaling": null,
   "rope_theta": 10000.0,
+  "sliding_window": 4096,
   "tie_word_embeddings": false,
   "torch_dtype": "bfloat16",
   "transformers_version": "4.44.2",
-  "use_cache": true,
+  "use_cache": false,
   "vocab_size": 32000
 }

generation_config.json

@@ -1,10 +1,6 @@
 {
+  "_from_model_config": true,
   "bos_token_id": 1,
-  "do_sample": true,
   "eos_token_id": 2,
-  "max_length": 4096,
-  "pad_token_id": 0,
-  "temperature": 0.6,
-  "top_p": 0.9,
   "transformers_version": "4.44.2"
 }

model-00001-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e70093c66dc0e8fc3ad279b524c73b67d8a2d1e6229958c69376332271e278c0
-size 4938985352
+oid sha256:c1a91f7e17b20f6f785ac3121e63de20e0312735ad4e7b6edee5b125c02ff5b3
+size 4943162336

model-00002-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b824e7aab109170f398e9bd2ea075df9a5419de7d03a504a600b3e897ee2679c
-size 4947390880
+oid sha256:8fd5c8d88cf7ed1822c75cb15cb4428175f2a7dea4128d41e4ff77d5acf4bd0f
+size 4999819336

model-00003-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:45cac3041405070e7b71ae58d7677d06f46602f6e36f5917354b43de23ac7674
-size 3590488816
+oid sha256:dcbd48a79233d29350e3b29bca8ae69f17ed2ac8fee107b5a408941e50c8ae25
+size 4540516344

model.safetensors.index.json

@@ -1,6 +1,6 @@
 {
   "metadata": {
-    "total_size": 13476831232
+    "total_size": 14483464192
   },
   "weight_map": {
     "lm_head.weight": "model-00003-of-00003.safetensors",
@@ -23,24 +23,24 @@
     "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
     "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
     "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
-    "model.layers.10.input_layernorm.weight": "model-00001-of-00003.safetensors",
-    "model.layers.10.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
     "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
     "model.layers.10.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
-    "model.layers.10.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
     "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
     "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
     "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
     "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
     "model.layers.11.input_layernorm.weight": "model-00002-of-00003.safetensors",
     "model.layers.11.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
-    "model.layers.11.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
     "model.layers.11.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
     "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
-    "model.layers.11.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
-    "model.layers.11.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
-    "model.layers.11.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
-    "model.layers.11.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
     "model.layers.12.input_layernorm.weight": "model-00002-of-00003.safetensors",
     "model.layers.12.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
     "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
@@ -140,24 +140,24 @@
     "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
     "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
     "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
-    "model.layers.22.input_layernorm.weight": "model-00002-of-00003.safetensors",
-    "model.layers.22.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
-    "model.layers.22.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
-    "model.layers.22.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
-    "model.layers.22.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
     "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
     "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
     "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
     "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
     "model.layers.23.input_layernorm.weight": "model-00003-of-00003.safetensors",
     "model.layers.23.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
-    "model.layers.23.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
-    "model.layers.23.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
     "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
-    "model.layers.23.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
-    "model.layers.23.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
-    "model.layers.23.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
-    "model.layers.23.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
     "model.layers.24.input_layernorm.weight": "model-00003-of-00003.safetensors",
     "model.layers.24.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
     "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",

runs/Jan07_20-04-55_dgx-a100-12/events.out.tfevents.1736277019.dgx-a100-12.2303065.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1bcc9d740357b0f3347fa734cd2c77371d7be89b3be41f77c359bc5e4bfc0c22
+size 6155

tokenizer.model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347
-size 499723
+oid sha256:dadfd56d766715c61d2ef780a525ab43b8e6da4de6865bda3d95fdef5e134055
+size 493443

tokenizer_config.json

@@ -28,6 +28,7 @@
       "special": true
     }
   },
+  "additional_special_tokens": [],
   "bos_token": "<s>",
   "chat_template": "{{ bos_token + 'System: ' + (messages[0]['content'] | trim + '\n\n' if messages[0]['role'] == 'system' else '') }}{% set messages = messages[1:] if messages[0]['role'] == 'system' else messages %}{% for message in messages %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') if (message['role'] == 'user') != (loop.index0 % 2 == 0) else '' }}{{ '[INST] ' + message['content'] | trim + ' [/INST]' if message['role'] == 'user' else ' ' + message['content'] | trim + eos_token }}{% endfor %}",
   "clean_up_tokenization_spaces": false,
@@ -35,8 +36,8 @@
   "legacy": false,
   "model_max_length": 2048,
   "pad_token": "</s>",
-  "padding_side": "right",
   "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
   "tokenizer_class": "LlamaTokenizer",
   "unk_token": "<unk>",
   "use_default_system_prompt": false

train_results.json

@@ -1,9 +1,9 @@
 {
     "epoch": 0.9565217391304348,
-    "total_flos": 1083185692672.0,
-    "train_loss": 0.9741405790502374,
-    "train_runtime": 62.2828,
+    "total_flos": 1081332727808.0,
+    "train_loss": 0.8119967471469532,
+    "train_runtime": 65.397,
     "train_samples": 368,
-    "train_samples_per_second": 5.909,
-    "train_steps_per_second": 0.177
+    "train_samples_per_second": 5.627,
+    "train_steps_per_second": 0.168
 }

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b251f1c82b0822b8ab316a9cf5e968795149fba290f76e09fba4db75d272f73c
+oid sha256:78a20881c257c3665ecb1a6b3c1f24256f69f94481025b9f4fca67cd1f94d43d
 size 6968