Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,60 @@
+---
+library_name: transformers
+license: other
+base_model: Qwen/Qwen2.5-Coder-7B-Instruct
+tags:
+- llama-factory
+- freeze
+- generated_from_trainer
+model-index:
+- name: qwen_under8_nlx
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# qwen_under8_nlx
+
+This model is a fine-tuned version of [Qwen/Qwen2.5-Coder-7B-Instruct](https://huggingface.co/Qwen/Qwen2.5-Coder-7B-Instruct) on the codes_nlx_under8 dataset.
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 5e-05
+- train_batch_size: 16
+- eval_batch_size: 8
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 3
+- gradient_accumulation_steps: 8
+- total_train_batch_size: 384
+- total_eval_batch_size: 24
+- optimizer: Use adamw_torch with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
+- lr_scheduler_type: cosine
+- num_epochs: 1.0
+
+### Training results
+
+
+
+### Framework versions
+
+- Transformers 4.48.2
+- Pytorch 2.5.1+cu124
+- Datasets 3.2.0
+- Tokenizers 0.21.0

added_tokens.json

@@ -0,0 +1,24 @@
+{
+  "</tool_call>": 151658,
+  "<tool_call>": 151657,
+  "<|box_end|>": 151649,
+  "<|box_start|>": 151648,
+  "<|endoftext|>": 151643,
+  "<|file_sep|>": 151664,
+  "<|fim_middle|>": 151660,
+  "<|fim_pad|>": 151662,
+  "<|fim_prefix|>": 151659,
+  "<|fim_suffix|>": 151661,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644,
+  "<|image_pad|>": 151655,
+  "<|object_ref_end|>": 151647,
+  "<|object_ref_start|>": 151646,
+  "<|quad_end|>": 151651,
+  "<|quad_start|>": 151650,
+  "<|repo_name|>": 151663,
+  "<|video_pad|>": 151656,
+  "<|vision_end|>": 151653,
+  "<|vision_pad|>": 151654,
+  "<|vision_start|>": 151652
+}

all_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 0.991869918699187,
+    "num_input_tokens_seen": 95944704,
+    "total_flos": 4.0703306623541576e+18,
+    "train_loss": 0.5785222493234228,
+    "train_runtime": 9437.3939,
+    "train_samples_per_second": 2.499,
+    "train_steps_per_second": 0.006
+}

config.json

@@ -0,0 +1,35 @@
+{
+  "_name_or_path": "Qwen/Qwen2.5-Coder-7B-Instruct",
+  "architectures": [
+    "Qwen2ForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 151643,
+  "eos_token_id": 151645,
+  "hidden_act": "silu",
+  "hidden_size": 3584,
+  "initializer_range": 0.02,
+  "intermediate_size": 18944,
+  "max_position_embeddings": 32768,
+  "max_window_layers": 28,
+  "model_type": "qwen2",
+  "num_attention_heads": 28,
+  "num_hidden_layers": 28,
+  "num_key_value_heads": 4,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": {
+    "factor": 1.0,
+    "high_freq_factor": 4.0,
+    "low_freq_factor": 1.0,
+    "original_max_position_embeddings": 32768,
+    "rope_type": "llama3"
+  },
+  "rope_theta": 1000000.0,
+  "sliding_window": null,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.48.2",
+  "use_cache": false,
+  "use_sliding_window": false,
+  "vocab_size": 152064
+}

generation_config.json

@@ -0,0 +1,14 @@
+{
+  "bos_token_id": 151643,
+  "do_sample": true,
+  "eos_token_id": [
+    151645,
+    151643
+  ],
+  "pad_token_id": 151643,
+  "repetition_penalty": 1.1,
+  "temperature": 0.7,
+  "top_k": 20,
+  "top_p": 0.8,
+  "transformers_version": "4.48.2"
+}

llamaboard_config.yaml

@@ -0,0 +1,77 @@
+top.booster: liger_kernel
+top.checkpoint_path: null
+top.finetuning_type: freeze
+top.model_name: Qwen2.5-Coder-7B-Instruct
+top.quantization_bit: none
+top.quantization_method: bitsandbytes
+top.rope_scaling: llama3
+top.template: qwen
+train.additional_target: ''
+train.apollo_rank: 256
+train.apollo_scale: 1
+train.apollo_target: all
+train.apollo_update_interval: 200
+train.badam_mode: layer
+train.badam_switch_interval: 50
+train.badam_switch_mode: ascending
+train.badam_update_ratio: 0.05
+train.batch_size: 16
+train.compute_type: bf16
+train.create_new_adapter: false
+train.cutoff_len: 4096
+train.dataset:
+- codes_nlx_under8
+train.dataset_dir: data
+train.ds_offload: false
+train.ds_stage: none
+train.extra_args: '{}'
+train.freeze_extra_modules: ''
+train.freeze_trainable_layers: 2
+train.freeze_trainable_modules: all
+train.galore_rank: 16
+train.galore_scale: 2
+train.galore_target: all
+train.galore_update_interval: 200
+train.gradient_accumulation_steps: 8
+train.learning_rate: 5e-5
+train.logging_steps: 1
+train.lora_alpha: 16
+train.lora_dropout: 0
+train.lora_rank: 8
+train.lora_target: ''
+train.loraplus_lr_ratio: 0
+train.lr_scheduler_type: cosine
+train.mask_history: false
+train.max_grad_norm: '1.0'
+train.max_samples: '50000000'
+train.neat_packing: true
+train.neftune_alpha: 0
+train.num_train_epochs: '1'
+train.packing: true
+train.ppo_score_norm: false
+train.ppo_whiten_rewards: false
+train.pref_beta: 0.1
+train.pref_ftx: 0
+train.pref_loss: sigmoid
+train.report_to:
+- none
+train.resize_vocab: false
+train.reward_model: null
+train.save_steps: 1000
+train.swanlab_api_key: ''
+train.swanlab_mode: cloud
+train.swanlab_project: llamafactory
+train.swanlab_run_name: ''
+train.swanlab_workspace: ''
+train.train_on_prompt: false
+train.training_stage: Supervised Fine-Tuning
+train.use_apollo: true
+train.use_badam: false
+train.use_dora: false
+train.use_galore: false
+train.use_llama_pro: true
+train.use_pissa: false
+train.use_rslora: false
+train.use_swanlab: false
+train.val_size: 0
+train.warmup_steps: 0

model-00001-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0b6f069918b07c064cbba8ae4f00f529aa9bbf84b7cdfcb7fc2694a40f6aa8ef
+size 4877660776

model-00002-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9bd2dfc1a309d8e54f229c8e53e7ad3716292c710b40069f5ecab0c556d03599
+size 4932749648

model-00003-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7bc1b28714c15822444b8ecbfe3b28bf9e1da312eb4ab48a500c555fbea0f502
+size 4991480296

model-00004-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7e979e0d662c63653908e40664174cb6c89be0ee91500ef3887f1a7cec1b7fcc
+size 1361612328

model.safetensors.index.json

@@ -0,0 +1,346 @@
+{
+  "metadata": {
+    "total_size": 16163464192
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00004-of-00004.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.17.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.27.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.norm.weight": "model-00004-of-00004.safetensors"
+  }
+}

running_log.txt

@@ -0,0 +1,369 @@
+[INFO|2025-07-07 19:00:02] configuration_utils.py:696 >> loading configuration file config.json from cache at /home/kiho/.cache/huggingface/hub/models--Qwen--Qwen2.5-Coder-7B-Instruct/snapshots/c03e6d358207e414f1eca0bb1891e29f1db0e242/config.json
+
+[INFO|2025-07-07 19:00:02] configuration_utils.py:768 >> Model config Qwen2Config {
+  "_name_or_path": "Qwen/Qwen2.5-Coder-7B-Instruct",
+  "architectures": [
+    "Qwen2ForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 151643,
+  "eos_token_id": 151645,
+  "hidden_act": "silu",
+  "hidden_size": 3584,
+  "initializer_range": 0.02,
+  "intermediate_size": 18944,
+  "max_position_embeddings": 32768,
+  "max_window_layers": 28,
+  "model_type": "qwen2",
+  "num_attention_heads": 28,
+  "num_hidden_layers": 28,
+  "num_key_value_heads": 4,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 1000000.0,
+  "sliding_window": null,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.48.2",
+  "use_cache": true,
+  "use_sliding_window": false,
+  "vocab_size": 152064
+}
+
+
+[INFO|2025-07-07 19:00:02] tokenization_utils_base.py:2034 >> loading file vocab.json from cache at /home/kiho/.cache/huggingface/hub/models--Qwen--Qwen2.5-Coder-7B-Instruct/snapshots/c03e6d358207e414f1eca0bb1891e29f1db0e242/vocab.json
+
+[INFO|2025-07-07 19:00:02] tokenization_utils_base.py:2034 >> loading file merges.txt from cache at /home/kiho/.cache/huggingface/hub/models--Qwen--Qwen2.5-Coder-7B-Instruct/snapshots/c03e6d358207e414f1eca0bb1891e29f1db0e242/merges.txt
+
+[INFO|2025-07-07 19:00:02] tokenization_utils_base.py:2034 >> loading file tokenizer.json from cache at /home/kiho/.cache/huggingface/hub/models--Qwen--Qwen2.5-Coder-7B-Instruct/snapshots/c03e6d358207e414f1eca0bb1891e29f1db0e242/tokenizer.json
+
+[INFO|2025-07-07 19:00:02] tokenization_utils_base.py:2034 >> loading file added_tokens.json from cache at None
+
+[INFO|2025-07-07 19:00:02] tokenization_utils_base.py:2034 >> loading file special_tokens_map.json from cache at None
+
+[INFO|2025-07-07 19:00:02] tokenization_utils_base.py:2034 >> loading file tokenizer_config.json from cache at /home/kiho/.cache/huggingface/hub/models--Qwen--Qwen2.5-Coder-7B-Instruct/snapshots/c03e6d358207e414f1eca0bb1891e29f1db0e242/tokenizer_config.json
+
+[INFO|2025-07-07 19:00:02] tokenization_utils_base.py:2034 >> loading file chat_template.jinja from cache at None
+
+[INFO|2025-07-07 19:00:03] tokenization_utils_base.py:2304 >> Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.
+
+[INFO|2025-07-07 19:00:03] configuration_utils.py:696 >> loading configuration file config.json from cache at /home/kiho/.cache/huggingface/hub/models--Qwen--Qwen2.5-Coder-7B-Instruct/snapshots/c03e6d358207e414f1eca0bb1891e29f1db0e242/config.json
+
+[INFO|2025-07-07 19:00:03] configuration_utils.py:768 >> Model config Qwen2Config {
+  "_name_or_path": "Qwen/Qwen2.5-Coder-7B-Instruct",
+  "architectures": [
+    "Qwen2ForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 151643,
+  "eos_token_id": 151645,
+  "hidden_act": "silu",
+  "hidden_size": 3584,
+  "initializer_range": 0.02,
+  "intermediate_size": 18944,
+  "max_position_embeddings": 32768,
+  "max_window_layers": 28,
+  "model_type": "qwen2",
+  "num_attention_heads": 28,
+  "num_hidden_layers": 28,
+  "num_key_value_heads": 4,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 1000000.0,
+  "sliding_window": null,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.48.2",
+  "use_cache": true,
+  "use_sliding_window": false,
+  "vocab_size": 152064
+}
+
+
+[INFO|2025-07-07 19:00:04] tokenization_utils_base.py:2034 >> loading file vocab.json from cache at /home/kiho/.cache/huggingface/hub/models--Qwen--Qwen2.5-Coder-7B-Instruct/snapshots/c03e6d358207e414f1eca0bb1891e29f1db0e242/vocab.json
+
+[INFO|2025-07-07 19:00:04] tokenization_utils_base.py:2034 >> loading file merges.txt from cache at /home/kiho/.cache/huggingface/hub/models--Qwen--Qwen2.5-Coder-7B-Instruct/snapshots/c03e6d358207e414f1eca0bb1891e29f1db0e242/merges.txt
+
+[INFO|2025-07-07 19:00:04] tokenization_utils_base.py:2034 >> loading file tokenizer.json from cache at /home/kiho/.cache/huggingface/hub/models--Qwen--Qwen2.5-Coder-7B-Instruct/snapshots/c03e6d358207e414f1eca0bb1891e29f1db0e242/tokenizer.json
+
+[INFO|2025-07-07 19:00:04] tokenization_utils_base.py:2034 >> loading file added_tokens.json from cache at None
+
+[INFO|2025-07-07 19:00:04] tokenization_utils_base.py:2034 >> loading file special_tokens_map.json from cache at None
+
+[INFO|2025-07-07 19:00:04] tokenization_utils_base.py:2034 >> loading file tokenizer_config.json from cache at /home/kiho/.cache/huggingface/hub/models--Qwen--Qwen2.5-Coder-7B-Instruct/snapshots/c03e6d358207e414f1eca0bb1891e29f1db0e242/tokenizer_config.json
+
+[INFO|2025-07-07 19:00:04] tokenization_utils_base.py:2034 >> loading file chat_template.jinja from cache at None
+
+[INFO|2025-07-07 19:00:04] tokenization_utils_base.py:2304 >> Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.
+
+[INFO|2025-07-07 19:00:04] logging.py:157 >> Add <|im_end|> to stop words.
+
+[INFO|2025-07-07 19:00:04] logging.py:157 >> Loading dataset Codes3_query_filtered_553474_mark_less_than_8.0.json...
+
+[INFO|2025-07-07 19:00:43] configuration_utils.py:696 >> loading configuration file config.json from cache at /home/kiho/.cache/huggingface/hub/models--Qwen--Qwen2.5-Coder-7B-Instruct/snapshots/c03e6d358207e414f1eca0bb1891e29f1db0e242/config.json
+
+[INFO|2025-07-07 19:00:43] configuration_utils.py:768 >> Model config Qwen2Config {
+  "_name_or_path": "Qwen/Qwen2.5-Coder-7B-Instruct",
+  "architectures": [
+    "Qwen2ForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 151643,
+  "eos_token_id": 151645,
+  "hidden_act": "silu",
+  "hidden_size": 3584,
+  "initializer_range": 0.02,
+  "intermediate_size": 18944,
+  "max_position_embeddings": 32768,
+  "max_window_layers": 28,
+  "model_type": "qwen2",
+  "num_attention_heads": 28,
+  "num_hidden_layers": 28,
+  "num_key_value_heads": 4,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 1000000.0,
+  "sliding_window": null,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.48.2",
+  "use_cache": true,
+  "use_sliding_window": false,
+  "vocab_size": 152064
+}
+
+
+[WARNING|2025-07-07 19:00:43] logging.py:162 >> Input length is smaller than max length. Consider increase input length.
+
+[INFO|2025-07-07 19:00:43] logging.py:157 >> Using llama3 scaling strategy and setting scaling factor to 1.0.
+
+[INFO|2025-07-07 19:00:43] logging.py:157 >> Using block diagonal attention for sequence packing without cross-attention.
+
+[INFO|2025-07-07 19:00:44] logging.py:157 >> Liger kernel has been applied to the model.
+
+[INFO|2025-07-07 19:00:44] modeling_utils.py:3904 >> loading weights file model.safetensors from cache at /home/kiho/.cache/huggingface/hub/models--Qwen--Qwen2.5-Coder-7B-Instruct/snapshots/c03e6d358207e414f1eca0bb1891e29f1db0e242/model.safetensors.index.json
+
+[INFO|2025-07-07 19:00:44] modeling_utils.py:1582 >> Instantiating Qwen2ForCausalLM model under default dtype torch.bfloat16.
+
+[INFO|2025-07-07 19:00:44] configuration_utils.py:1140 >> Generate config GenerationConfig {
+  "bos_token_id": 151643,
+  "eos_token_id": 151645
+}
+
+
+[INFO|2025-07-07 19:00:52] modeling_utils.py:4888 >> All model checkpoint weights were used when initializing Qwen2ForCausalLM.
+
+
+[INFO|2025-07-07 19:00:52] modeling_utils.py:4896 >> All the weights of Qwen2ForCausalLM were initialized from the model checkpoint at Qwen/Qwen2.5-Coder-7B-Instruct.
+If your task is similar to the task the model of the checkpoint was trained on, you can already use Qwen2ForCausalLM for predictions without further training.
+
+[INFO|2025-07-07 19:00:52] configuration_utils.py:1095 >> loading configuration file generation_config.json from cache at /home/kiho/.cache/huggingface/hub/models--Qwen--Qwen2.5-Coder-7B-Instruct/snapshots/c03e6d358207e414f1eca0bb1891e29f1db0e242/generation_config.json
+
+[INFO|2025-07-07 19:00:52] configuration_utils.py:1140 >> Generate config GenerationConfig {
+  "bos_token_id": 151643,
+  "do_sample": true,
+  "eos_token_id": [
+    151645,
+    151643
+  ],
+  "pad_token_id": 151643,
+  "repetition_penalty": 1.1,
+  "temperature": 0.7,
+  "top_k": 20,
+  "top_p": 0.8
+}
+
+
+[INFO|2025-07-07 19:00:52] logging.py:157 >> Gradient checkpointing enabled.
+
+[INFO|2025-07-07 19:00:52] logging.py:157 >> Using torch SDPA for faster training and inference.
+
+[INFO|2025-07-07 19:00:52] logging.py:157 >> Upcasting trainable params to float32.
+
+[INFO|2025-07-07 19:00:52] logging.py:157 >> Fine-tuning method: Freeze
+
+[INFO|2025-07-07 19:00:52] logging.py:157 >> Set trainable layers: .13.,.27.
+
+[INFO|2025-07-07 19:00:52] logging.py:157 >> trainable params: 466,115,584 || all params: 7,615,616,512 || trainable%: 6.1205
+
+[INFO|2025-07-07 19:00:52] trainer.py:741 >> Using auto half precision backend
+
+[INFO|2025-07-07 19:00:53] logging.py:157 >> Found linear modules: q_proj,gate_proj,k_proj,down_proj,v_proj,o_proj,up_proj
+
+[INFO|2025-07-07 19:00:53] logging.py:157 >> Using APOLLO optimizer with args: {'rank': 256, 'proj': 'random', 'proj_type': 'std', 'update_proj_gap': 200, 'scale': 1, 'scale_type': 'channel', 'scale_front': False}.
+
+[INFO|2025-07-07 19:00:53] trainer.py:2369 >> ***** Running training *****
+
+[INFO|2025-07-07 19:00:53] trainer.py:2370 >>   Num examples = 23,588
+
+[INFO|2025-07-07 19:00:53] trainer.py:2371 >>   Num Epochs = 1
+
+[INFO|2025-07-07 19:00:53] trainer.py:2372 >>   Instantaneous batch size per device = 16
+
+[INFO|2025-07-07 19:00:53] trainer.py:2375 >>   Total train batch size (w. parallel, distributed & accumulation) = 384
+
+[INFO|2025-07-07 19:00:53] trainer.py:2376 >>   Gradient Accumulation steps = 8
+
+[INFO|2025-07-07 19:00:53] trainer.py:2377 >>   Total optimization steps = 61
+
+[INFO|2025-07-07 19:00:53] trainer.py:2378 >>   Number of trainable parameters = 466,115,584
+
+[INFO|2025-07-07 19:03:35] logging.py:157 >> {'loss': 0.8835, 'learning_rate': 4.9967e-05, 'epoch': 0.02, 'throughput': 9741.84}
+
+[INFO|2025-07-07 19:06:09] logging.py:157 >> {'loss': 0.8172, 'learning_rate': 4.9867e-05, 'epoch': 0.03, 'throughput': 9978.43}
+
+[INFO|2025-07-07 19:08:43] logging.py:157 >> {'loss': 0.7415, 'learning_rate': 4.9702e-05, 'epoch': 0.05, 'throughput': 10069.50}
+
+[INFO|2025-07-07 19:11:16] logging.py:157 >> {'loss': 0.7198, 'learning_rate': 4.9471e-05, 'epoch': 0.07, 'throughput': 10117.11}
+
+[INFO|2025-07-07 19:13:50] logging.py:157 >> {'loss': 0.6985, 'learning_rate': 4.9176e-05, 'epoch': 0.08, 'throughput': 10129.94}
+
+[INFO|2025-07-07 19:16:24] logging.py:157 >> {'loss': 0.6642, 'learning_rate': 4.8816e-05, 'epoch': 0.10, 'throughput': 10146.98}
+
+[INFO|2025-07-07 19:18:58] logging.py:157 >> {'loss': 0.6677, 'learning_rate': 4.8393e-05, 'epoch': 0.11, 'throughput': 10159.48}
+
+[INFO|2025-07-07 19:21:32] logging.py:157 >> {'loss': 0.6451, 'learning_rate': 4.7908e-05, 'epoch': 0.13, 'throughput': 10164.48}
+
+[INFO|2025-07-07 19:24:06] logging.py:157 >> {'loss': 0.6327, 'learning_rate': 4.7362e-05, 'epoch': 0.15, 'throughput': 10172.04}
+
+[INFO|2025-07-07 19:26:41] logging.py:157 >> {'loss': 0.6331, 'learning_rate': 4.6757e-05, 'epoch': 0.16, 'throughput': 10169.75}
+
+[INFO|2025-07-07 19:29:17] logging.py:157 >> {'loss': 0.6219, 'learning_rate': 4.6094e-05, 'epoch': 0.18, 'throughput': 10161.43}
+
+[INFO|2025-07-07 19:31:50] logging.py:157 >> {'loss': 0.6205, 'learning_rate': 4.5376e-05, 'epoch': 0.20, 'throughput': 10168.14}
+
+[INFO|2025-07-07 19:34:24] logging.py:157 >> {'loss': 0.6010, 'learning_rate': 4.4603e-05, 'epoch': 0.21, 'throughput': 10172.37}
+
+[INFO|2025-07-07 19:36:58] logging.py:157 >> {'loss': 0.6077, 'learning_rate': 4.3778e-05, 'epoch': 0.23, 'throughput': 10175.33}
+
+[INFO|2025-07-07 19:39:32] logging.py:157 >> {'loss': 0.5958, 'learning_rate': 4.2904e-05, 'epoch': 0.24, 'throughput': 10179.71}
+
+[INFO|2025-07-07 19:42:05] logging.py:157 >> {'loss': 0.5838, 'learning_rate': 4.1982e-05, 'epoch': 0.26, 'throughput': 10183.00}
+
+[INFO|2025-07-07 19:44:39] logging.py:157 >> {'loss': 0.5629, 'learning_rate': 4.1015e-05, 'epoch': 0.28, 'throughput': 10185.76}
+
+[INFO|2025-07-07 19:47:13] logging.py:157 >> {'loss': 0.5848, 'learning_rate': 4.0005e-05, 'epoch': 0.29, 'throughput': 10188.71}
+
+[INFO|2025-07-07 19:49:47] logging.py:157 >> {'loss': 0.5772, 'learning_rate': 3.8956e-05, 'epoch': 0.31, 'throughput': 10190.23}
+
+[INFO|2025-07-07 19:52:20] logging.py:157 >> {'loss': 0.5719, 'learning_rate': 3.7870e-05, 'epoch': 0.33, 'throughput': 10192.39}
+
+[INFO|2025-07-07 19:54:54] logging.py:157 >> {'loss': 0.5445, 'learning_rate': 3.6749e-05, 'epoch': 0.34, 'throughput': 10195.72}
+
+[INFO|2025-07-07 19:57:27] logging.py:157 >> {'loss': 0.5560, 'learning_rate': 3.5598e-05, 'epoch': 0.36, 'throughput': 10198.63}
+
+[INFO|2025-07-07 20:00:01] logging.py:157 >> {'loss': 0.5736, 'learning_rate': 3.4418e-05, 'epoch': 0.37, 'throughput': 10199.96}
+
+[INFO|2025-07-07 20:02:34] logging.py:157 >> {'loss': 0.5350, 'learning_rate': 3.3214e-05, 'epoch': 0.39, 'throughput': 10203.32}
+
+[INFO|2025-07-07 20:05:07] logging.py:157 >> {'loss': 0.5634, 'learning_rate': 3.1987e-05, 'epoch': 0.41, 'throughput': 10205.81}
+
+[INFO|2025-07-07 20:07:42] logging.py:157 >> {'loss': 0.5648, 'learning_rate': 3.0742e-05, 'epoch': 0.42, 'throughput': 10203.48}
+
+[INFO|2025-07-07 20:10:16] logging.py:157 >> {'loss': 0.5467, 'learning_rate': 2.9482e-05, 'epoch': 0.44, 'throughput': 10202.72}
+
+[INFO|2025-07-07 20:12:50] logging.py:157 >> {'loss': 0.5567, 'learning_rate': 2.8210e-05, 'epoch': 0.46, 'throughput': 10203.80}
+
+[INFO|2025-07-07 20:15:24] logging.py:157 >> {'loss': 0.5847, 'learning_rate': 2.6929e-05, 'epoch': 0.47, 'throughput': 10203.36}
+
+[INFO|2025-07-07 20:17:58] logging.py:157 >> {'loss': 0.5429, 'learning_rate': 2.5644e-05, 'epoch': 0.49, 'throughput': 10204.19}
+
+[INFO|2025-07-07 20:20:32] logging.py:157 >> {'loss': 0.5435, 'learning_rate': 2.4356e-05, 'epoch': 0.50, 'throughput': 10204.68}
+
+[INFO|2025-07-07 20:23:06] logging.py:157 >> {'loss': 0.5482, 'learning_rate': 2.3071e-05, 'epoch': 0.52, 'throughput': 10205.83}
+
+[INFO|2025-07-07 20:25:40] logging.py:157 >> {'loss': 0.5418, 'learning_rate': 2.1790e-05, 'epoch': 0.54, 'throughput': 10206.16}
+
+[INFO|2025-07-07 20:28:14] logging.py:157 >> {'loss': 0.5320, 'learning_rate': 2.0518e-05, 'epoch': 0.55, 'throughput': 10205.44}
+
+[INFO|2025-07-07 20:30:48] logging.py:157 >> {'loss': 0.5360, 'learning_rate': 1.9258e-05, 'epoch': 0.57, 'throughput': 10205.42}
+
+[INFO|2025-07-07 20:33:23] logging.py:157 >> {'loss': 0.5314, 'learning_rate': 1.8013e-05, 'epoch': 0.59, 'throughput': 10204.05}
+
+[INFO|2025-07-07 20:35:56] logging.py:157 >> {'loss': 0.5595, 'learning_rate': 1.6786e-05, 'epoch': 0.60, 'throughput': 10205.77}
+
+[INFO|2025-07-07 20:38:32] logging.py:157 >> {'loss': 0.5418, 'learning_rate': 1.5582e-05, 'epoch': 0.62, 'throughput': 10203.12}
+
+[INFO|2025-07-07 20:41:06] logging.py:157 >> {'loss': 0.5438, 'learning_rate': 1.4402e-05, 'epoch': 0.63, 'throughput': 10203.74}
+
+[INFO|2025-07-07 20:43:39] logging.py:157 >> {'loss': 0.5239, 'learning_rate': 1.3251e-05, 'epoch': 0.65, 'throughput': 10204.70}
+
+[INFO|2025-07-07 20:46:13] logging.py:157 >> {'loss': 0.5459, 'learning_rate': 1.2130e-05, 'epoch': 0.67, 'throughput': 10204.78}
+
+[INFO|2025-07-07 20:48:47] logging.py:157 >> {'loss': 0.5373, 'learning_rate': 1.1044e-05, 'epoch': 0.68, 'throughput': 10204.67}
+
+[INFO|2025-07-07 20:51:22] logging.py:157 >> {'loss': 0.5474, 'learning_rate': 9.9946e-06, 'epoch': 0.70, 'throughput': 10203.24}
+
+[INFO|2025-07-07 20:53:56] logging.py:157 >> {'loss': 0.5236, 'learning_rate': 8.9852e-06, 'epoch': 0.72, 'throughput': 10203.82}
+
+[INFO|2025-07-07 20:56:29] logging.py:157 >> {'loss': 0.5336, 'learning_rate': 8.0182e-06, 'epoch': 0.73, 'throughput': 10205.18}
+
+[INFO|2025-07-07 20:59:05] logging.py:157 >> {'loss': 0.5198, 'learning_rate': 7.0962e-06, 'epoch': 0.75, 'throughput': 10203.27}
+
+[INFO|2025-07-07 21:01:40] logging.py:157 >> {'loss': 0.5419, 'learning_rate': 6.2217e-06, 'epoch': 0.76, 'throughput': 10202.64}
+
+[INFO|2025-07-07 21:04:17] logging.py:157 >> {'loss': 0.5544, 'learning_rate': 5.3970e-06, 'epoch': 0.78, 'throughput': 10198.78}
+
+[INFO|2025-07-07 21:06:50] logging.py:157 >> {'loss': 0.5689, 'learning_rate': 4.6243e-06, 'epoch': 0.80, 'throughput': 10200.12}
+
+[INFO|2025-07-07 21:09:24] logging.py:157 >> {'loss': 0.5170, 'learning_rate': 3.9056e-06, 'epoch': 0.81, 'throughput': 10200.21}
+
+[INFO|2025-07-07 21:11:58] logging.py:157 >> {'loss': 0.5440, 'learning_rate': 3.2429e-06, 'epoch': 0.83, 'throughput': 10200.78}
+
+[INFO|2025-07-07 21:14:34] logging.py:157 >> {'loss': 0.5265, 'learning_rate': 2.6378e-06, 'epoch': 0.85, 'throughput': 10198.05}
+
+[INFO|2025-07-07 21:17:09] logging.py:157 >> {'loss': 0.5301, 'learning_rate': 2.0921e-06, 'epoch': 0.86, 'throughput': 10197.60}
+
+[INFO|2025-07-07 21:19:44] logging.py:157 >> {'loss': 0.5357, 'learning_rate': 1.6071e-06, 'epoch': 0.88, 'throughput': 10196.24}
+
+[INFO|2025-07-07 21:22:18] logging.py:157 >> {'loss': 0.5220, 'learning_rate': 1.1841e-06, 'epoch': 0.89, 'throughput': 10196.15}
+
+[INFO|2025-07-07 21:24:54] logging.py:157 >> {'loss': 0.5282, 'learning_rate': 8.2431e-07, 'epoch': 0.91, 'throughput': 10194.76}
+
+[INFO|2025-07-07 21:27:28] logging.py:157 >> {'loss': 0.5327, 'learning_rate': 5.2861e-07, 'epoch': 0.93, 'throughput': 10195.16}
+
+[INFO|2025-07-07 21:30:02] logging.py:157 >> {'loss': 0.5237, 'learning_rate': 2.9780e-07, 'epoch': 0.94, 'throughput': 10195.52}
+
+[INFO|2025-07-07 21:32:36] logging.py:157 >> {'loss': 0.5499, 'learning_rate': 1.3250e-07, 'epoch': 0.96, 'throughput': 10195.79}
+
+[INFO|2025-07-07 21:35:10] logging.py:157 >> {'loss': 0.5576, 'learning_rate': 3.3148e-08, 'epoch': 0.98, 'throughput': 10195.29}
+
+[INFO|2025-07-07 21:37:45] logging.py:157 >> {'loss': 0.5487, 'learning_rate': 0.0000e+00, 'epoch': 0.99, 'throughput': 10195.34}
+
+[INFO|2025-07-07 21:37:45] trainer.py:3910 >> Saving model checkpoint to saves/Qwen2.5-Coder-7B-Instruct/freeze/qwen_under8_nlx/checkpoint-61
+
+[INFO|2025-07-07 21:37:45] configuration_utils.py:420 >> Configuration saved in saves/Qwen2.5-Coder-7B-Instruct/freeze/qwen_under8_nlx/checkpoint-61/config.json
+
+[INFO|2025-07-07 21:37:45] configuration_utils.py:909 >> Configuration saved in saves/Qwen2.5-Coder-7B-Instruct/freeze/qwen_under8_nlx/checkpoint-61/generation_config.json
+
+[INFO|2025-07-07 21:38:10] modeling_utils.py:2996 >> The model is bigger than the maximum size per checkpoint (5GB) and is going to be split in 4 checkpoint shards. You can find where each parameters has been saved in the index located at saves/Qwen2.5-Coder-7B-Instruct/freeze/qwen_under8_nlx/checkpoint-61/model.safetensors.index.json.
+
+[INFO|2025-07-07 21:38:10] tokenization_utils_base.py:2491 >> tokenizer config file saved in saves/Qwen2.5-Coder-7B-Instruct/freeze/qwen_under8_nlx/checkpoint-61/tokenizer_config.json
+
+[INFO|2025-07-07 21:38:10] tokenization_utils_base.py:2500 >> Special tokens file saved in saves/Qwen2.5-Coder-7B-Instruct/freeze/qwen_under8_nlx/checkpoint-61/special_tokens_map.json
+
+[INFO|2025-07-07 21:38:10] trainer.py:2643 >> 
+
+Training completed. Do not forget to share your model on huggingface.co/models =)
+
+
+
+[INFO|2025-07-07 21:38:10] trainer.py:3910 >> Saving model checkpoint to saves/Qwen2.5-Coder-7B-Instruct/freeze/qwen_under8_nlx
+
+[INFO|2025-07-07 21:38:10] configuration_utils.py:420 >> Configuration saved in saves/Qwen2.5-Coder-7B-Instruct/freeze/qwen_under8_nlx/config.json
+
+[INFO|2025-07-07 21:38:10] configuration_utils.py:909 >> Configuration saved in saves/Qwen2.5-Coder-7B-Instruct/freeze/qwen_under8_nlx/generation_config.json
+
+[INFO|2025-07-07 21:38:35] modeling_utils.py:2996 >> The model is bigger than the maximum size per checkpoint (5GB) and is going to be split in 4 checkpoint shards. You can find where each parameters has been saved in the index located at saves/Qwen2.5-Coder-7B-Instruct/freeze/qwen_under8_nlx/model.safetensors.index.json.
+
+[INFO|2025-07-07 21:38:35] tokenization_utils_base.py:2491 >> tokenizer config file saved in saves/Qwen2.5-Coder-7B-Instruct/freeze/qwen_under8_nlx/tokenizer_config.json
+
+[INFO|2025-07-07 21:38:35] tokenization_utils_base.py:2500 >> Special tokens file saved in saves/Qwen2.5-Coder-7B-Instruct/freeze/qwen_under8_nlx/special_tokens_map.json
+
+[WARNING|2025-07-07 21:38:35] logging.py:162 >> No metric eval_loss to plot.
+
+[WARNING|2025-07-07 21:38:35] logging.py:162 >> No metric eval_accuracy to plot.
+
+[INFO|2025-07-07 21:38:35] modelcard.py:449 >> Dropping the following result as it does not have all the necessary fields:
+{'task': {'name': 'Causal Language Modeling', 'type': 'text-generation'}}
+

special_tokens_map.json

@@ -0,0 +1,31 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9c5ae00e602b8860cbd784ba82a8aa14e8feecec692e7076590d014d7b7fdafa
+size 11421896

tokenizer_config.json

@@ -0,0 +1,209 @@
+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151646": {
+      "content": "<|object_ref_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151647": {
+      "content": "<|object_ref_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151648": {
+      "content": "<|box_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151649": {
+      "content": "<|box_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151650": {
+      "content": "<|quad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151651": {
+      "content": "<|quad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151652": {
+      "content": "<|vision_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151653": {
+      "content": "<|vision_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151654": {
+      "content": "<|vision_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151655": {
+      "content": "<|image_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151656": {
+      "content": "<|video_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151657": {
+      "content": "<tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151658": {
+      "content": "</tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151659": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151660": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151661": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151662": {
+      "content": "<|fim_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151663": {
+      "content": "<|repo_name|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151664": {
+      "content": "<|file_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "bos_token": null,
+  "chat_template": "{%- if tools %}\n    {{- '<|im_start|>system\\n' }}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- messages[0]['content'] }}\n    {%- else %}\n        {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}\n    {%- endif %}\n    {{- \"\\n\\n# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within <tools></tools> XML tags:\\n<tools>\" }}\n    {%- for tool in tools %}\n        {{- \"\\n\" }}\n        {{- tool | tojson }}\n    {%- endfor %}\n    {{- \"\\n</tools>\\n\\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\\n<tool_call>\\n{\\\"name\\\": <function-name>, \\\"arguments\\\": <args-json-object>}\\n</tool_call><|im_end|>\\n\" }}\n{%- else %}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- '<|im_start|>system\\n' + messages[0]['content'] + '<|im_end|>\\n' }}\n    {%- else %}\n        {{- '<|im_start|>system\\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\\n' }}\n    {%- endif %}\n{%- endif %}\n{%- for message in messages %}\n    {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) or (message.role == \"assistant\" and not message.tool_calls) %}\n        {{- '<|im_start|>' + message.role + '\\n' + message.content + '<|im_end|>' + '\\n' }}\n    {%- elif message.role == \"assistant\" %}\n        {{- '<|im_start|>' + message.role }}\n        {%- if message.content %}\n            {{- '\\n' + message.content }}\n        {%- endif %}\n        {%- for tool_call in message.tool_calls %}\n            {%- if tool_call.function is defined %}\n                {%- set tool_call = tool_call.function %}\n            {%- endif %}\n            {{- '\\n<tool_call>\\n{\"name\": \"' }}\n            {{- tool_call.name }}\n            {{- '\", \"arguments\": ' }}\n            {{- tool_call.arguments | tojson }}\n            {{- '}\\n</tool_call>' }}\n        {%- endfor %}\n        {{- '<|im_end|>\\n' }}\n    {%- elif message.role == \"tool\" %}\n        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != \"tool\") %}\n            {{- '<|im_start|>user' }}\n        {%- endif %}\n        {{- '\\n<tool_response>\\n' }}\n        {{- message.content }}\n        {{- '\\n</tool_response>' }}\n        {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n            {{- '<|im_end|>\\n' }}\n        {%- endif %}\n    {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {{- '<|im_start|>assistant\\n' }}\n{%- endif %}\n",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": {},
+  "model_max_length": 4096,
+  "pad_token": "<|endoftext|>",
+  "padding_side": "right",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 0.991869918699187,
+    "num_input_tokens_seen": 95944704,
+    "total_flos": 4.0703306623541576e+18,
+    "train_loss": 0.5785222493234228,
+    "train_runtime": 9437.3939,
+    "train_samples_per_second": 2.499,
+    "train_steps_per_second": 0.006
+}

trainer_log.jsonl

@@ -0,0 +1,62 @@
+{"current_steps": 1, "total_steps": 61, "loss": 0.8835, "lr": 4.9966852247120764e-05, "epoch": 0.016260162601626018, "percentage": 1.64, "elapsed_time": "0:02:41", "remaining_time": "2:41:27", "throughput": 9741.84, "total_tokens": 1572864}
+{"current_steps": 2, "total_steps": 61, "loss": 0.8172, "lr": 4.9867496890364726e-05, "epoch": 0.032520325203252036, "percentage": 3.28, "elapsed_time": "0:05:15", "remaining_time": "2:34:59", "throughput": 9978.43, "total_tokens": 3145728}
+{"current_steps": 3, "total_steps": 61, "loss": 0.7415, "lr": 4.970219740227693e-05, "epoch": 0.04878048780487805, "percentage": 4.92, "elapsed_time": "0:07:48", "remaining_time": "2:30:59", "throughput": 10069.5, "total_tokens": 4718592}
+{"current_steps": 4, "total_steps": 61, "loss": 0.7198, "lr": 4.947139212738395e-05, "epoch": 0.06504065040650407, "percentage": 6.56, "elapsed_time": "0:10:21", "remaining_time": "2:27:41", "throughput": 10117.11, "total_tokens": 6291456}
+{"current_steps": 5, "total_steps": 61, "loss": 0.6985, "lr": 4.9175693119783013e-05, "epoch": 0.08130081300813008, "percentage": 8.2, "elapsed_time": "0:12:56", "remaining_time": "2:24:55", "throughput": 10129.94, "total_tokens": 7864320}
+{"current_steps": 6, "total_steps": 61, "loss": 0.6642, "lr": 4.881588452008456e-05, "epoch": 0.0975609756097561, "percentage": 9.84, "elapsed_time": "0:15:30", "remaining_time": "2:22:05", "throughput": 10146.98, "total_tokens": 9437184}
+{"current_steps": 7, "total_steps": 61, "loss": 0.6677, "lr": 4.839292047601234e-05, "epoch": 0.11382113821138211, "percentage": 11.48, "elapsed_time": "0:18:03", "remaining_time": "2:19:20", "throughput": 10159.48, "total_tokens": 11010048}
+{"current_steps": 8, "total_steps": 61, "loss": 0.6451, "lr": 4.790792261217512e-05, "epoch": 0.13008130081300814, "percentage": 13.11, "elapsed_time": "0:20:37", "remaining_time": "2:16:41", "throughput": 10164.48, "total_tokens": 12582912}
+{"current_steps": 9, "total_steps": 61, "loss": 0.6327, "lr": 4.736217705571989e-05, "epoch": 0.14634146341463414, "percentage": 14.75, "elapsed_time": "0:23:11", "remaining_time": "2:14:00", "throughput": 10172.04, "total_tokens": 14155776}
+{"current_steps": 10, "total_steps": 61, "loss": 0.6331, "lr": 4.6757131025753886e-05, "epoch": 0.16260162601626016, "percentage": 16.39, "elapsed_time": "0:25:46", "remaining_time": "2:11:27", "throughput": 10169.75, "total_tokens": 15728640}
+{"current_steps": 11, "total_steps": 61, "loss": 0.6219, "lr": 4.609438899557964e-05, "epoch": 0.17886178861788618, "percentage": 18.03, "elapsed_time": "0:28:22", "remaining_time": "2:08:59", "throughput": 10161.43, "total_tokens": 17301504}
+{"current_steps": 12, "total_steps": 61, "loss": 0.6205, "lr": 4.5375708437920284e-05, "epoch": 0.1951219512195122, "percentage": 19.67, "elapsed_time": "0:30:56", "remaining_time": "2:06:19", "throughput": 10168.14, "total_tokens": 18874368}
+{"current_steps": 13, "total_steps": 61, "loss": 0.601, "lr": 4.460299516441777e-05, "epoch": 0.21138211382113822, "percentage": 21.31, "elapsed_time": "0:33:30", "remaining_time": "2:03:41", "throughput": 10172.37, "total_tokens": 20447232}
+{"current_steps": 14, "total_steps": 61, "loss": 0.6077, "lr": 4.3778298271762995e-05, "epoch": 0.22764227642276422, "percentage": 22.95, "elapsed_time": "0:36:04", "remaining_time": "2:01:05", "throughput": 10175.33, "total_tokens": 22020096}
+{"current_steps": 15, "total_steps": 61, "loss": 0.5958, "lr": 4.2903804707859835e-05, "epoch": 0.24390243902439024, "percentage": 24.59, "elapsed_time": "0:38:37", "remaining_time": "1:58:27", "throughput": 10179.71, "total_tokens": 23592960}
+{"current_steps": 16, "total_steps": 61, "loss": 0.5838, "lr": 4.198183347243233e-05, "epoch": 0.2601626016260163, "percentage": 26.23, "elapsed_time": "0:41:11", "remaining_time": "1:55:50", "throughput": 10183.0, "total_tokens": 25165824}
+{"current_steps": 17, "total_steps": 61, "loss": 0.5629, "lr": 4.101482946745439e-05, "epoch": 0.2764227642276423, "percentage": 27.87, "elapsed_time": "0:43:45", "remaining_time": "1:53:14", "throughput": 10185.76, "total_tokens": 26738688}
+{"current_steps": 18, "total_steps": 61, "loss": 0.5848, "lr": 4.000535701370921e-05, "epoch": 0.2926829268292683, "percentage": 29.51, "elapsed_time": "0:46:18", "remaining_time": "1:50:38", "throughput": 10188.71, "total_tokens": 28311552}
+{"current_steps": 19, "total_steps": 61, "loss": 0.5772, "lr": 3.895609305067162e-05, "epoch": 0.3089430894308943, "percentage": 31.15, "elapsed_time": "0:48:52", "remaining_time": "1:48:02", "throughput": 10190.23, "total_tokens": 29884416}
+{"current_steps": 20, "total_steps": 61, "loss": 0.5719, "lr": 3.7869820037745776e-05, "epoch": 0.3252032520325203, "percentage": 32.79, "elapsed_time": "0:51:26", "remaining_time": "1:45:27", "throughput": 10192.39, "total_tokens": 31457280}
+{"current_steps": 21, "total_steps": 61, "loss": 0.5445, "lr": 3.6749418575683e-05, "epoch": 0.34146341463414637, "percentage": 34.43, "elapsed_time": "0:53:59", "remaining_time": "1:42:50", "throughput": 10195.72, "total_tokens": 33030144}
+{"current_steps": 22, "total_steps": 61, "loss": 0.556, "lr": 3.5597859767746524e-05, "epoch": 0.35772357723577236, "percentage": 36.07, "elapsed_time": "0:56:32", "remaining_time": "1:40:14", "throughput": 10198.63, "total_tokens": 34603008}
+{"current_steps": 23, "total_steps": 61, "loss": 0.5736, "lr": 3.4418197340879635e-05, "epoch": 0.37398373983739835, "percentage": 37.7, "elapsed_time": "0:59:06", "remaining_time": "1:37:39", "throughput": 10199.96, "total_tokens": 36175872}
+{"current_steps": 24, "total_steps": 61, "loss": 0.535, "lr": 3.321355954777087e-05, "epoch": 0.3902439024390244, "percentage": 39.34, "elapsed_time": "1:01:39", "remaining_time": "1:35:03", "throughput": 10203.32, "total_tokens": 37748736}
+{"current_steps": 25, "total_steps": 61, "loss": 0.5634, "lr": 3.1987140871290236e-05, "epoch": 0.4065040650406504, "percentage": 40.98, "elapsed_time": "1:04:12", "remaining_time": "1:32:28", "throughput": 10205.81, "total_tokens": 39321600}
+{"current_steps": 26, "total_steps": 61, "loss": 0.5648, "lr": 3.07421935532949e-05, "epoch": 0.42276422764227645, "percentage": 42.62, "elapsed_time": "1:06:47", "remaining_time": "1:29:55", "throughput": 10203.48, "total_tokens": 40894464}
+{"current_steps": 27, "total_steps": 61, "loss": 0.5467, "lr": 2.9482018970268393e-05, "epoch": 0.43902439024390244, "percentage": 44.26, "elapsed_time": "1:09:22", "remaining_time": "1:27:21", "throughput": 10202.72, "total_tokens": 42467328}
+{"current_steps": 28, "total_steps": 61, "loss": 0.5567, "lr": 2.8209958878663778e-05, "epoch": 0.45528455284552843, "percentage": 45.9, "elapsed_time": "1:11:56", "remaining_time": "1:24:46", "throughput": 10203.8, "total_tokens": 44040192}
+{"current_steps": 29, "total_steps": 61, "loss": 0.5847, "lr": 2.6929386553166164e-05, "epoch": 0.4715447154471545, "percentage": 47.54, "elapsed_time": "1:14:30", "remaining_time": "1:22:12", "throughput": 10203.36, "total_tokens": 45613056}
+{"current_steps": 30, "total_steps": 61, "loss": 0.5429, "lr": 2.564369784137472e-05, "epoch": 0.4878048780487805, "percentage": 49.18, "elapsed_time": "1:17:04", "remaining_time": "1:19:38", "throughput": 10204.19, "total_tokens": 47185920}
+{"current_steps": 31, "total_steps": 61, "loss": 0.5435, "lr": 2.4356302158625288e-05, "epoch": 0.5040650406504065, "percentage": 50.82, "elapsed_time": "1:19:38", "remaining_time": "1:17:03", "throughput": 10204.68, "total_tokens": 48758784}
+{"current_steps": 32, "total_steps": 61, "loss": 0.5482, "lr": 2.3070613446833842e-05, "epoch": 0.5203252032520326, "percentage": 52.46, "elapsed_time": "1:22:11", "remaining_time": "1:14:29", "throughput": 10205.83, "total_tokens": 50331648}
+{"current_steps": 33, "total_steps": 61, "loss": 0.5418, "lr": 2.1790041121336225e-05, "epoch": 0.5365853658536586, "percentage": 54.1, "elapsed_time": "1:24:45", "remaining_time": "1:11:55", "throughput": 10206.16, "total_tokens": 51904512}
+{"current_steps": 34, "total_steps": 61, "loss": 0.532, "lr": 2.0517981029731616e-05, "epoch": 0.5528455284552846, "percentage": 55.74, "elapsed_time": "1:27:20", "remaining_time": "1:09:21", "throughput": 10205.44, "total_tokens": 53477376}
+{"current_steps": 35, "total_steps": 61, "loss": 0.536, "lr": 1.9257806446705116e-05, "epoch": 0.5691056910569106, "percentage": 57.38, "elapsed_time": "1:29:54", "remaining_time": "1:06:47", "throughput": 10205.42, "total_tokens": 55050240}
+{"current_steps": 36, "total_steps": 61, "loss": 0.5314, "lr": 1.8012859128709766e-05, "epoch": 0.5853658536585366, "percentage": 59.02, "elapsed_time": "1:32:29", "remaining_time": "1:04:13", "throughput": 10204.05, "total_tokens": 56623104}
+{"current_steps": 37, "total_steps": 61, "loss": 0.5595, "lr": 1.6786440452229134e-05, "epoch": 0.6016260162601627, "percentage": 60.66, "elapsed_time": "1:35:02", "remaining_time": "1:01:38", "throughput": 10205.77, "total_tokens": 58195968}
+{"current_steps": 38, "total_steps": 61, "loss": 0.5418, "lr": 1.558180265912037e-05, "epoch": 0.6178861788617886, "percentage": 62.3, "elapsed_time": "1:37:37", "remaining_time": "0:59:05", "throughput": 10203.12, "total_tokens": 59768832}
+{"current_steps": 39, "total_steps": 61, "loss": 0.5438, "lr": 1.4402140232253486e-05, "epoch": 0.6341463414634146, "percentage": 63.93, "elapsed_time": "1:40:11", "remaining_time": "0:56:31", "throughput": 10203.74, "total_tokens": 61341696}
+{"current_steps": 40, "total_steps": 61, "loss": 0.5239, "lr": 1.325058142431701e-05, "epoch": 0.6504065040650406, "percentage": 65.57, "elapsed_time": "1:42:45", "remaining_time": "0:53:56", "throughput": 10204.7, "total_tokens": 62914560}
+{"current_steps": 41, "total_steps": 61, "loss": 0.5459, "lr": 1.213017996225424e-05, "epoch": 0.6666666666666666, "percentage": 67.21, "elapsed_time": "1:45:19", "remaining_time": "0:51:22", "throughput": 10204.78, "total_tokens": 64487424}
+{"current_steps": 42, "total_steps": 61, "loss": 0.5373, "lr": 1.1043906949328387e-05, "epoch": 0.6829268292682927, "percentage": 68.85, "elapsed_time": "1:47:53", "remaining_time": "0:48:48", "throughput": 10204.67, "total_tokens": 66060288}
+{"current_steps": 43, "total_steps": 61, "loss": 0.5474, "lr": 9.994642986290797e-06, "epoch": 0.6991869918699187, "percentage": 70.49, "elapsed_time": "1:50:28", "remaining_time": "0:46:14", "throughput": 10203.24, "total_tokens": 67633152}
+{"current_steps": 44, "total_steps": 61, "loss": 0.5236, "lr": 8.985170532545622e-06, "epoch": 0.7154471544715447, "percentage": 72.13, "elapsed_time": "1:53:02", "remaining_time": "0:43:40", "throughput": 10203.82, "total_tokens": 69206016}
+{"current_steps": 45, "total_steps": 61, "loss": 0.5336, "lr": 8.018166527567672e-06, "epoch": 0.7317073170731707, "percentage": 73.77, "elapsed_time": "1:55:35", "remaining_time": "0:41:05", "throughput": 10205.18, "total_tokens": 70778880}
+{"current_steps": 46, "total_steps": 61, "loss": 0.5198, "lr": 7.096195292140173e-06, "epoch": 0.7479674796747967, "percentage": 75.41, "elapsed_time": "1:58:11", "remaining_time": "0:38:32", "throughput": 10203.27, "total_tokens": 72351744}
+{"current_steps": 47, "total_steps": 61, "loss": 0.5419, "lr": 6.221701728237009e-06, "epoch": 0.7642276422764228, "percentage": 77.05, "elapsed_time": "2:00:45", "remaining_time": "0:35:58", "throughput": 10202.64, "total_tokens": 73924608}
+{"current_steps": 48, "total_steps": 61, "loss": 0.5544, "lr": 5.397004835582242e-06, "epoch": 0.7804878048780488, "percentage": 78.69, "elapsed_time": "2:03:22", "remaining_time": "0:33:24", "throughput": 10198.78, "total_tokens": 75497472}
+{"current_steps": 49, "total_steps": 61, "loss": 0.5689, "lr": 4.624291562079719e-06, "epoch": 0.7967479674796748, "percentage": 80.33, "elapsed_time": "2:05:55", "remaining_time": "0:30:50", "throughput": 10200.12, "total_tokens": 77070336}
+{"current_steps": 50, "total_steps": 61, "loss": 0.517, "lr": 3.90561100442036e-06, "epoch": 0.8130081300813008, "percentage": 81.97, "elapsed_time": "2:08:29", "remaining_time": "0:28:16", "throughput": 10200.21, "total_tokens": 78643200}
+{"current_steps": 51, "total_steps": 61, "loss": 0.544, "lr": 3.2428689742461188e-06, "epoch": 0.8292682926829268, "percentage": 83.61, "elapsed_time": "2:11:03", "remaining_time": "0:25:41", "throughput": 10200.78, "total_tokens": 80216064}
+{"current_steps": 52, "total_steps": 61, "loss": 0.5265, "lr": 2.637822944280116e-06, "epoch": 0.8455284552845529, "percentage": 85.25, "elapsed_time": "2:13:40", "remaining_time": "0:23:08", "throughput": 10198.05, "total_tokens": 81788928}
+{"current_steps": 53, "total_steps": 61, "loss": 0.5301, "lr": 2.092077387824884e-06, "epoch": 0.8617886178861789, "percentage": 86.89, "elapsed_time": "2:16:14", "remaining_time": "0:20:33", "throughput": 10197.6, "total_tokens": 83361792}
+{"current_steps": 54, "total_steps": 61, "loss": 0.5357, "lr": 1.6070795239876618e-06, "epoch": 0.8780487804878049, "percentage": 88.52, "elapsed_time": "2:18:49", "remaining_time": "0:17:59", "throughput": 10196.24, "total_tokens": 84934656}
+{"current_steps": 55, "total_steps": 61, "loss": 0.522, "lr": 1.1841154799154374e-06, "epoch": 0.8943089430894309, "percentage": 90.16, "elapsed_time": "2:21:24", "remaining_time": "0:15:25", "throughput": 10196.15, "total_tokens": 86507520}
+{"current_steps": 56, "total_steps": 61, "loss": 0.5282, "lr": 8.243068802169906e-07, "epoch": 0.9105691056910569, "percentage": 91.8, "elapsed_time": "2:23:59", "remaining_time": "0:12:51", "throughput": 10194.76, "total_tokens": 88080384}
+{"current_steps": 57, "total_steps": 61, "loss": 0.5327, "lr": 5.286078726160549e-07, "epoch": 0.926829268292683, "percentage": 93.44, "elapsed_time": "2:26:33", "remaining_time": "0:10:17", "throughput": 10195.16, "total_tokens": 89653248}
+{"current_steps": 58, "total_steps": 61, "loss": 0.5237, "lr": 2.978025977230736e-07, "epoch": 0.943089430894309, "percentage": 95.08, "elapsed_time": "2:29:07", "remaining_time": "0:07:42", "throughput": 10195.52, "total_tokens": 91226112}
+{"current_steps": 59, "total_steps": 61, "loss": 0.5499, "lr": 1.3250310963527358e-07, "epoch": 0.959349593495935, "percentage": 96.72, "elapsed_time": "2:31:41", "remaining_time": "0:05:08", "throughput": 10195.79, "total_tokens": 92798976}
+{"current_steps": 60, "total_steps": 61, "loss": 0.5576, "lr": 3.314775287923677e-08, "epoch": 0.975609756097561, "percentage": 98.36, "elapsed_time": "2:34:16", "remaining_time": "0:02:34", "throughput": 10195.29, "total_tokens": 94371840}
+{"current_steps": 61, "total_steps": 61, "loss": 0.5487, "lr": 0.0, "epoch": 0.991869918699187, "percentage": 100.0, "elapsed_time": "2:36:50", "remaining_time": "0:00:00", "throughput": 10195.34, "total_tokens": 95944704}
+{"current_steps": 61, "total_steps": 61, "epoch": 0.991869918699187, "percentage": 100.0, "elapsed_time": "2:37:16", "remaining_time": "0:00:00", "throughput": 10167.7, "total_tokens": 95944704}

trainer_state.json

@@ -0,0 +1,531 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.991869918699187,
+  "eval_steps": 500,
+  "global_step": 61,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.016260162601626018,
+      "grad_norm": 1.3551084995269775,
+      "learning_rate": 4.9966852247120764e-05,
+      "loss": 0.8835,
+      "num_input_tokens_seen": 1572864,
+      "step": 1
+    },
+    {
+      "epoch": 0.032520325203252036,
+      "grad_norm": 1.0672752857208252,
+      "learning_rate": 4.9867496890364726e-05,
+      "loss": 0.8172,
+      "num_input_tokens_seen": 3145728,
+      "step": 2
+    },
+    {
+      "epoch": 0.04878048780487805,
+      "grad_norm": 0.6499503254890442,
+      "learning_rate": 4.970219740227693e-05,
+      "loss": 0.7415,
+      "num_input_tokens_seen": 4718592,
+      "step": 3
+    },
+    {
+      "epoch": 0.06504065040650407,
+      "grad_norm": 0.4817531704902649,
+      "learning_rate": 4.947139212738395e-05,
+      "loss": 0.7198,
+      "num_input_tokens_seen": 6291456,
+      "step": 4
+    },
+    {
+      "epoch": 0.08130081300813008,
+      "grad_norm": 0.33635902404785156,
+      "learning_rate": 4.9175693119783013e-05,
+      "loss": 0.6985,
+      "num_input_tokens_seen": 7864320,
+      "step": 5
+    },
+    {
+      "epoch": 0.0975609756097561,
+      "grad_norm": 0.27813318371772766,
+      "learning_rate": 4.881588452008456e-05,
+      "loss": 0.6642,
+      "num_input_tokens_seen": 9437184,
+      "step": 6
+    },
+    {
+      "epoch": 0.11382113821138211,
+      "grad_norm": 0.25561287999153137,
+      "learning_rate": 4.839292047601234e-05,
+      "loss": 0.6677,
+      "num_input_tokens_seen": 11010048,
+      "step": 7
+    },
+    {
+      "epoch": 0.13008130081300814,
+      "grad_norm": 0.25520581007003784,
+      "learning_rate": 4.790792261217512e-05,
+      "loss": 0.6451,
+      "num_input_tokens_seen": 12582912,
+      "step": 8
+    },
+    {
+      "epoch": 0.14634146341463414,
+      "grad_norm": 0.2094946950674057,
+      "learning_rate": 4.736217705571989e-05,
+      "loss": 0.6327,
+      "num_input_tokens_seen": 14155776,
+      "step": 9
+    },
+    {
+      "epoch": 0.16260162601626016,
+      "grad_norm": 0.21939203143119812,
+      "learning_rate": 4.6757131025753886e-05,
+      "loss": 0.6331,
+      "num_input_tokens_seen": 15728640,
+      "step": 10
+    },
+    {
+      "epoch": 0.17886178861788618,
+      "grad_norm": 0.20424292981624603,
+      "learning_rate": 4.609438899557964e-05,
+      "loss": 0.6219,
+      "num_input_tokens_seen": 17301504,
+      "step": 11
+    },
+    {
+      "epoch": 0.1951219512195122,
+      "grad_norm": 0.19756929576396942,
+      "learning_rate": 4.5375708437920284e-05,
+      "loss": 0.6205,
+      "num_input_tokens_seen": 18874368,
+      "step": 12
+    },
+    {
+      "epoch": 0.21138211382113822,
+      "grad_norm": 0.18154746294021606,
+      "learning_rate": 4.460299516441777e-05,
+      "loss": 0.601,
+      "num_input_tokens_seen": 20447232,
+      "step": 13
+    },
+    {
+      "epoch": 0.22764227642276422,
+      "grad_norm": 0.1677803248167038,
+      "learning_rate": 4.3778298271762995e-05,
+      "loss": 0.6077,
+      "num_input_tokens_seen": 22020096,
+      "step": 14
+    },
+    {
+      "epoch": 0.24390243902439024,
+      "grad_norm": 0.15535381436347961,
+      "learning_rate": 4.2903804707859835e-05,
+      "loss": 0.5958,
+      "num_input_tokens_seen": 23592960,
+      "step": 15
+    },
+    {
+      "epoch": 0.2601626016260163,
+      "grad_norm": 0.16249153017997742,
+      "learning_rate": 4.198183347243233e-05,
+      "loss": 0.5838,
+      "num_input_tokens_seen": 25165824,
+      "step": 16
+    },
+    {
+      "epoch": 0.2764227642276423,
+      "grad_norm": 0.145711287856102,
+      "learning_rate": 4.101482946745439e-05,
+      "loss": 0.5629,
+      "num_input_tokens_seen": 26738688,
+      "step": 17
+    },
+    {
+      "epoch": 0.2926829268292683,
+      "grad_norm": 0.12892590463161469,
+      "learning_rate": 4.000535701370921e-05,
+      "loss": 0.5848,
+      "num_input_tokens_seen": 28311552,
+      "step": 18
+    },
+    {
+      "epoch": 0.3089430894308943,
+      "grad_norm": 0.12926466763019562,
+      "learning_rate": 3.895609305067162e-05,
+      "loss": 0.5772,
+      "num_input_tokens_seen": 29884416,
+      "step": 19
+    },
+    {
+      "epoch": 0.3252032520325203,
+      "grad_norm": 0.12316538393497467,
+      "learning_rate": 3.7869820037745776e-05,
+      "loss": 0.5719,
+      "num_input_tokens_seen": 31457280,
+      "step": 20
+    },
+    {
+      "epoch": 0.34146341463414637,
+      "grad_norm": 0.12052515894174576,
+      "learning_rate": 3.6749418575683e-05,
+      "loss": 0.5445,
+      "num_input_tokens_seen": 33030144,
+      "step": 21
+    },
+    {
+      "epoch": 0.35772357723577236,
+      "grad_norm": 0.12180831283330917,
+      "learning_rate": 3.5597859767746524e-05,
+      "loss": 0.556,
+      "num_input_tokens_seen": 34603008,
+      "step": 22
+    },
+    {
+      "epoch": 0.37398373983739835,
+      "grad_norm": 0.12532484531402588,
+      "learning_rate": 3.4418197340879635e-05,
+      "loss": 0.5736,
+      "num_input_tokens_seen": 36175872,
+      "step": 23
+    },
+    {
+      "epoch": 0.3902439024390244,
+      "grad_norm": 0.12291638553142548,
+      "learning_rate": 3.321355954777087e-05,
+      "loss": 0.535,
+      "num_input_tokens_seen": 37748736,
+      "step": 24
+    },
+    {
+      "epoch": 0.4065040650406504,
+      "grad_norm": 0.11071067303419113,
+      "learning_rate": 3.1987140871290236e-05,
+      "loss": 0.5634,
+      "num_input_tokens_seen": 39321600,
+      "step": 25
+    },
+    {
+      "epoch": 0.42276422764227645,
+      "grad_norm": 0.09884963929653168,
+      "learning_rate": 3.07421935532949e-05,
+      "loss": 0.5648,
+      "num_input_tokens_seen": 40894464,
+      "step": 26
+    },
+    {
+      "epoch": 0.43902439024390244,
+      "grad_norm": 0.10612273216247559,
+      "learning_rate": 2.9482018970268393e-05,
+      "loss": 0.5467,
+      "num_input_tokens_seen": 42467328,
+      "step": 27
+    },
+    {
+      "epoch": 0.45528455284552843,
+      "grad_norm": 0.1015135645866394,
+      "learning_rate": 2.8209958878663778e-05,
+      "loss": 0.5567,
+      "num_input_tokens_seen": 44040192,
+      "step": 28
+    },
+    {
+      "epoch": 0.4715447154471545,
+      "grad_norm": 0.10951631516218185,
+      "learning_rate": 2.6929386553166164e-05,
+      "loss": 0.5847,
+      "num_input_tokens_seen": 45613056,
+      "step": 29
+    },
+    {
+      "epoch": 0.4878048780487805,
+      "grad_norm": 0.09695735573768616,
+      "learning_rate": 2.564369784137472e-05,
+      "loss": 0.5429,
+      "num_input_tokens_seen": 47185920,
+      "step": 30
+    },
+    {
+      "epoch": 0.5040650406504065,
+      "grad_norm": 0.09722153842449188,
+      "learning_rate": 2.4356302158625288e-05,
+      "loss": 0.5435,
+      "num_input_tokens_seen": 48758784,
+      "step": 31
+    },
+    {
+      "epoch": 0.5203252032520326,
+      "grad_norm": 0.0972503125667572,
+      "learning_rate": 2.3070613446833842e-05,
+      "loss": 0.5482,
+      "num_input_tokens_seen": 50331648,
+      "step": 32
+    },
+    {
+      "epoch": 0.5365853658536586,
+      "grad_norm": 0.09666866064071655,
+      "learning_rate": 2.1790041121336225e-05,
+      "loss": 0.5418,
+      "num_input_tokens_seen": 51904512,
+      "step": 33
+    },
+    {
+      "epoch": 0.5528455284552846,
+      "grad_norm": 0.09581635892391205,
+      "learning_rate": 2.0517981029731616e-05,
+      "loss": 0.532,
+      "num_input_tokens_seen": 53477376,
+      "step": 34
+    },
+    {
+      "epoch": 0.5691056910569106,
+      "grad_norm": 0.09590643644332886,
+      "learning_rate": 1.9257806446705116e-05,
+      "loss": 0.536,
+      "num_input_tokens_seen": 55050240,
+      "step": 35
+    },
+    {
+      "epoch": 0.5853658536585366,
+      "grad_norm": 0.0966959148645401,
+      "learning_rate": 1.8012859128709766e-05,
+      "loss": 0.5314,
+      "num_input_tokens_seen": 56623104,
+      "step": 36
+    },
+    {
+      "epoch": 0.6016260162601627,
+      "grad_norm": 0.09263308346271515,
+      "learning_rate": 1.6786440452229134e-05,
+      "loss": 0.5595,
+      "num_input_tokens_seen": 58195968,
+      "step": 37
+    },
+    {
+      "epoch": 0.6178861788617886,
+      "grad_norm": 0.08845654875040054,
+      "learning_rate": 1.558180265912037e-05,
+      "loss": 0.5418,
+      "num_input_tokens_seen": 59768832,
+      "step": 38
+    },
+    {
+      "epoch": 0.6341463414634146,
+      "grad_norm": 0.0915597528219223,
+      "learning_rate": 1.4402140232253486e-05,
+      "loss": 0.5438,
+      "num_input_tokens_seen": 61341696,
+      "step": 39
+    },
+    {
+      "epoch": 0.6504065040650406,
+      "grad_norm": 0.08968537300825119,
+      "learning_rate": 1.325058142431701e-05,
+      "loss": 0.5239,
+      "num_input_tokens_seen": 62914560,
+      "step": 40
+    },
+    {
+      "epoch": 0.6666666666666666,
+      "grad_norm": 0.0950733870267868,
+      "learning_rate": 1.213017996225424e-05,
+      "loss": 0.5459,
+      "num_input_tokens_seen": 64487424,
+      "step": 41
+    },
+    {
+      "epoch": 0.6829268292682927,
+      "grad_norm": 0.09233593195676804,
+      "learning_rate": 1.1043906949328387e-05,
+      "loss": 0.5373,
+      "num_input_tokens_seen": 66060288,
+      "step": 42
+    },
+    {
+      "epoch": 0.6991869918699187,
+      "grad_norm": 0.0939360037446022,
+      "learning_rate": 9.994642986290797e-06,
+      "loss": 0.5474,
+      "num_input_tokens_seen": 67633152,
+      "step": 43
+    },
+    {
+      "epoch": 0.7154471544715447,
+      "grad_norm": 0.08978823572397232,
+      "learning_rate": 8.985170532545622e-06,
+      "loss": 0.5236,
+      "num_input_tokens_seen": 69206016,
+      "step": 44
+    },
+    {
+      "epoch": 0.7317073170731707,
+      "grad_norm": 0.09436903893947601,
+      "learning_rate": 8.018166527567672e-06,
+      "loss": 0.5336,
+      "num_input_tokens_seen": 70778880,
+      "step": 45
+    },
+    {
+      "epoch": 0.7479674796747967,
+      "grad_norm": 0.088426373898983,
+      "learning_rate": 7.096195292140173e-06,
+      "loss": 0.5198,
+      "num_input_tokens_seen": 72351744,
+      "step": 46
+    },
+    {
+      "epoch": 0.7642276422764228,
+      "grad_norm": 0.09146667271852493,
+      "learning_rate": 6.221701728237009e-06,
+      "loss": 0.5419,
+      "num_input_tokens_seen": 73924608,
+      "step": 47
+    },
+    {
+      "epoch": 0.7804878048780488,
+      "grad_norm": 0.0876329094171524,
+      "learning_rate": 5.397004835582242e-06,
+      "loss": 0.5544,
+      "num_input_tokens_seen": 75497472,
+      "step": 48
+    },
+    {
+      "epoch": 0.7967479674796748,
+      "grad_norm": 0.09173890948295593,
+      "learning_rate": 4.624291562079719e-06,
+      "loss": 0.5689,
+      "num_input_tokens_seen": 77070336,
+      "step": 49
+    },
+    {
+      "epoch": 0.8130081300813008,
+      "grad_norm": 0.09023137390613556,
+      "learning_rate": 3.90561100442036e-06,
+      "loss": 0.517,
+      "num_input_tokens_seen": 78643200,
+      "step": 50
+    },
+    {
+      "epoch": 0.8292682926829268,
+      "grad_norm": 0.08453888446092606,
+      "learning_rate": 3.2428689742461188e-06,
+      "loss": 0.544,
+      "num_input_tokens_seen": 80216064,
+      "step": 51
+    },
+    {
+      "epoch": 0.8455284552845529,
+      "grad_norm": 0.09456021338701248,
+      "learning_rate": 2.637822944280116e-06,
+      "loss": 0.5265,
+      "num_input_tokens_seen": 81788928,
+      "step": 52
+    },
+    {
+      "epoch": 0.8617886178861789,
+      "grad_norm": 0.09165485203266144,
+      "learning_rate": 2.092077387824884e-06,
+      "loss": 0.5301,
+      "num_input_tokens_seen": 83361792,
+      "step": 53
+    },
+    {
+      "epoch": 0.8780487804878049,
+      "grad_norm": 0.0885196402668953,
+      "learning_rate": 1.6070795239876618e-06,
+      "loss": 0.5357,
+      "num_input_tokens_seen": 84934656,
+      "step": 54
+    },
+    {
+      "epoch": 0.8943089430894309,
+      "grad_norm": 0.09256651252508163,
+      "learning_rate": 1.1841154799154374e-06,
+      "loss": 0.522,
+      "num_input_tokens_seen": 86507520,
+      "step": 55
+    },
+    {
+      "epoch": 0.9105691056910569,
+      "grad_norm": 0.0941980704665184,
+      "learning_rate": 8.243068802169906e-07,
+      "loss": 0.5282,
+      "num_input_tokens_seen": 88080384,
+      "step": 56
+    },
+    {
+      "epoch": 0.926829268292683,
+      "grad_norm": 0.08401526510715485,
+      "learning_rate": 5.286078726160549e-07,
+      "loss": 0.5327,
+      "num_input_tokens_seen": 89653248,
+      "step": 57
+    },
+    {
+      "epoch": 0.943089430894309,
+      "grad_norm": 0.08636080473661423,
+      "learning_rate": 2.978025977230736e-07,
+      "loss": 0.5237,
+      "num_input_tokens_seen": 91226112,
+      "step": 58
+    },
+    {
+      "epoch": 0.959349593495935,
+      "grad_norm": 0.09349840134382248,
+      "learning_rate": 1.3250310963527358e-07,
+      "loss": 0.5499,
+      "num_input_tokens_seen": 92798976,
+      "step": 59
+    },
+    {
+      "epoch": 0.975609756097561,
+      "grad_norm": 0.08632226288318634,
+      "learning_rate": 3.314775287923677e-08,
+      "loss": 0.5576,
+      "num_input_tokens_seen": 94371840,
+      "step": 60
+    },
+    {
+      "epoch": 0.991869918699187,
+      "grad_norm": 0.09036959707736969,
+      "learning_rate": 0.0,
+      "loss": 0.5487,
+      "num_input_tokens_seen": 95944704,
+      "step": 61
+    },
+    {
+      "epoch": 0.991869918699187,
+      "num_input_tokens_seen": 95944704,
+      "step": 61,
+      "total_flos": 4.0703306623541576e+18,
+      "train_loss": 0.5785222493234228,
+      "train_runtime": 9437.3939,
+      "train_samples_per_second": 2.499,
+      "train_steps_per_second": 0.006
+    }
+  ],
+  "logging_steps": 1,
+  "max_steps": 61,
+  "num_input_tokens_seen": 95944704,
+  "num_train_epochs": 1,
+  "save_steps": 1000,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 4.0703306623541576e+18,
+  "train_batch_size": 16,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:444317418b08a04afeda97e23512c9220f7eff61887ad97eed28ba0f22a22c6f
+size 5624

training_args.yaml

@@ -0,0 +1,39 @@
+apollo_rank: 256
+apollo_scale: 1
+apollo_target: all
+apollo_update_interval: 200
+bf16: true
+cutoff_len: 4096
+dataset: codes_nlx_under8
+dataset_dir: data
+ddp_timeout: 180000000
+do_train: true
+enable_liger_kernel: true
+finetuning_type: freeze
+flash_attn: auto
+freeze_trainable_layers: 2
+freeze_trainable_modules: all
+gradient_accumulation_steps: 8
+include_num_input_tokens_seen: true
+learning_rate: 5.0e-05
+logging_steps: 1
+lr_scheduler_type: cosine
+max_grad_norm: 1.0
+max_samples: 50000000
+model_name_or_path: Qwen/Qwen2.5-Coder-7B-Instruct
+neat_packing: true
+num_train_epochs: 1.0
+output_dir: saves/Qwen2.5-Coder-7B-Instruct/freeze/qwen_under8_nlx
+packing: true
+per_device_train_batch_size: 16
+plot_loss: true
+preprocessing_num_workers: 16
+report_to: none
+rope_scaling: llama3
+save_steps: 1000
+stage: sft
+template: qwen
+trust_remote_code: true
+use_apollo: true
+use_llama_pro: true
+warmup_steps: 0