Upload folder using huggingface_hub

config.json

@@ -0,0 +1,62 @@
+{
+  "_attn_implementation_autoset": true,
+  "_name_or_path": "/home/azureuser/.cache/huggingface/hub/models--meta-llama--Llama-3.2-1B-Instruct/snapshots/9213176726f574b556790deb65791e0c5aa438b6",
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 128000,
+  "eos_token_id": [
+    128001,
+    128008,
+    128009
+  ],
+  "head_dim": 64,
+  "hidden_act": "silu",
+  "hidden_size": 2048,
+  "initializer_range": 0.02,
+  "intermediate_size": 8192,
+  "max_position_embeddings": 131072,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 16,
+  "num_key_value_heads": 8,
+  "pretraining_tp": 1,
+  "quantization_config": {
+    "bits": 4,
+    "checkpoint_format": "gptq",
+    "desc_act": false,
+    "dynamic": null,
+    "group_size": 32,
+    "lm_head": false,
+    "meta": {
+      "damp_auto_increment": 0.0025,
+      "damp_percent": 0.01,
+      "mse": 0.0,
+      "quantizer": [
+        "gptqmodel:1.5.1-dev"
+      ],
+      "static_groups": false,
+      "true_sequential": true,
+      "uri": "https://github.com/modelcloud/gptqmodel"
+    },
+    "quant_method": "gptq",
+    "sym": true
+  },
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": {
+    "factor": 32.0,
+    "high_freq_factor": 4.0,
+    "low_freq_factor": 1.0,
+    "original_max_position_embeddings": 8192,
+    "rope_type": "llama3"
+  },
+  "rope_theta": 500000.0,
+  "tie_word_embeddings": true,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.47.1",
+  "use_cache": true,
+  "vocab_size": 128256
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5435ed8cef43f02129c506f8b2e1dd08538eca737ad419c3ddfba23b81724499
+size 1614733472

quant_log.csv

@@ -0,0 +1,113 @@
+layer,module,loss,damp,time
+0,self_attn.k_proj,0.09601,0.01000,1.079
+0,self_attn.v_proj,0.00299,0.01000,0.841
+0,self_attn.q_proj,0.21113,0.01000,0.727
+0,self_attn.o_proj,0.00017,0.01000,0.727
+0,mlp.up_proj,0.14873,0.01000,0.763
+0,mlp.gate_proj,0.18908,0.01000,0.904
+0,mlp.down_proj,0.00190,0.01000,3.295
+1,self_attn.k_proj,0.16585,0.01000,0.711
+1,self_attn.v_proj,0.00992,0.01000,0.749
+1,self_attn.q_proj,0.32035,0.01000,0.690
+1,self_attn.o_proj,0.00039,0.01000,0.753
+1,mlp.up_proj,0.22062,0.01000,0.756
+1,mlp.gate_proj,0.30241,0.01000,0.732
+1,mlp.down_proj,0.16209,0.01000,3.030
+2,self_attn.k_proj,0.31228,0.01000,0.713
+2,self_attn.v_proj,0.02535,0.01000,0.677
+2,self_attn.q_proj,0.63971,0.01000,0.711
+2,self_attn.o_proj,0.00066,0.01000,0.730
+2,mlp.up_proj,0.28284,0.01000,0.746
+2,mlp.gate_proj,0.44825,0.01000,0.692
+2,mlp.down_proj,0.00335,0.01000,3.236
+3,self_attn.k_proj,0.19264,0.01000,0.758
+3,self_attn.v_proj,0.02762,0.01000,0.695
+3,self_attn.q_proj,0.42406,0.01000,0.738
+3,self_attn.o_proj,0.00111,0.01000,0.765
+3,mlp.up_proj,0.33513,0.01000,0.785
+3,mlp.gate_proj,0.66167,0.01000,0.750
+3,mlp.down_proj,0.00468,0.01000,3.254
+4,self_attn.k_proj,0.21151,0.01000,0.774
+4,self_attn.v_proj,0.02663,0.01000,0.746
+4,self_attn.q_proj,0.43425,0.01000,0.743
+4,self_attn.o_proj,0.00185,0.01000,0.801
+4,mlp.up_proj,0.33790,0.01000,0.835
+4,mlp.gate_proj,0.72013,0.01000,0.745
+4,mlp.down_proj,0.00553,0.01000,3.029
+5,self_attn.k_proj,0.30856,0.01000,0.716
+5,self_attn.v_proj,0.02561,0.01000,0.681
+5,self_attn.q_proj,0.57913,0.01000,0.702
+5,self_attn.o_proj,0.00256,0.01000,0.754
+5,mlp.up_proj,0.36168,0.01000,0.806
+5,mlp.gate_proj,0.66184,0.01000,0.734
+5,mlp.down_proj,0.00675,0.01000,3.046
+6,self_attn.k_proj,0.25328,0.01000,0.736
+6,self_attn.v_proj,0.02967,0.01000,0.681
+6,self_attn.q_proj,0.41055,0.01000,0.721
+6,self_attn.o_proj,0.00379,0.01000,0.724
+6,mlp.up_proj,0.37050,0.01000,0.751
+6,mlp.gate_proj,0.66534,0.01000,0.718
+6,mlp.down_proj,0.00709,0.01000,3.041
+7,self_attn.k_proj,0.25026,0.01000,0.748
+7,self_attn.v_proj,0.03261,0.01000,0.678
+7,self_attn.q_proj,0.47341,0.01000,0.708
+7,self_attn.o_proj,0.00349,0.01000,0.727
+7,mlp.up_proj,0.40085,0.01000,0.776
+7,mlp.gate_proj,0.65237,0.01000,0.749
+7,mlp.down_proj,0.00813,0.01000,3.014
+8,self_attn.k_proj,0.32256,0.01000,0.725
+8,self_attn.v_proj,0.03293,0.01000,0.690
+8,self_attn.q_proj,0.53772,0.01000,0.689
+8,self_attn.o_proj,0.00522,0.01000,0.755
+8,mlp.up_proj,0.47201,0.01000,0.770
+8,mlp.gate_proj,0.75260,0.01000,0.757
+8,mlp.down_proj,0.01079,0.01000,3.152
+9,self_attn.k_proj,0.24430,0.01000,0.727
+9,self_attn.v_proj,0.03725,0.01000,0.680
+9,self_attn.q_proj,0.60562,0.01000,0.724
+9,self_attn.o_proj,0.00523,0.01000,0.754
+9,mlp.up_proj,0.49043,0.01000,0.764
+9,mlp.gate_proj,0.80227,0.01000,0.715
+9,mlp.down_proj,0.01264,0.01000,3.103
+10,self_attn.k_proj,0.30944,0.01000,0.742
+10,self_attn.v_proj,0.04692,0.01000,0.668
+10,self_attn.q_proj,0.68625,0.01000,0.704
+10,self_attn.o_proj,0.00472,0.01000,0.736
+10,mlp.up_proj,0.57742,0.01000,0.771
+10,mlp.gate_proj,0.90810,0.01000,0.740
+10,mlp.down_proj,0.01584,0.01000,3.073
+11,self_attn.k_proj,0.36443,0.01000,0.712
+11,self_attn.v_proj,0.04592,0.01000,0.695
+11,self_attn.q_proj,0.66843,0.01000,0.697
+11,self_attn.o_proj,0.00372,0.01000,0.766
+11,mlp.up_proj,0.63095,0.01000,0.803
+11,mlp.gate_proj,0.97327,0.01000,0.726
+11,mlp.down_proj,0.01862,0.01000,3.065
+12,self_attn.k_proj,0.33120,0.01000,0.731
+12,self_attn.v_proj,0.04817,0.01000,0.694
+12,self_attn.q_proj,0.64928,0.01000,0.743
+12,self_attn.o_proj,0.00386,0.01000,0.726
+12,mlp.up_proj,0.66255,0.01000,0.791
+12,mlp.gate_proj,0.97375,0.01000,0.759
+12,mlp.down_proj,0.02400,0.01000,3.136
+13,self_attn.k_proj,0.36034,0.01000,0.763
+13,self_attn.v_proj,0.08636,0.01000,0.717
+13,self_attn.q_proj,0.78605,0.01000,0.696
+13,self_attn.o_proj,0.00579,0.01000,0.764
+13,mlp.up_proj,0.79367,0.01000,0.797
+13,mlp.gate_proj,1.07306,0.01000,0.757
+13,mlp.down_proj,0.03768,0.01000,3.024
+14,self_attn.k_proj,0.35830,0.01000,0.714
+14,self_attn.v_proj,0.16440,0.01000,0.697
+14,self_attn.q_proj,0.75050,0.01000,0.727
+14,self_attn.o_proj,0.01295,0.01000,0.740
+14,mlp.up_proj,0.92130,0.01000,0.780
+14,mlp.gate_proj,1.33264,0.01000,0.727
+14,mlp.down_proj,0.05907,0.01000,3.084
+15,self_attn.k_proj,0.35875,0.01000,0.726
+15,self_attn.v_proj,0.17096,0.01000,0.703
+15,self_attn.q_proj,0.67191,0.01000,0.680
+15,self_attn.o_proj,0.03647,0.01000,0.743
+15,mlp.up_proj,1.12717,0.01000,0.775
+15,mlp.gate_proj,1.49428,0.01000,0.742
+15,mlp.down_proj,0.22401,0.01000,2.976

quantize_config.json

@@ -0,0 +1,21 @@
+{
+  "bits": 4,
+  "dynamic": null,
+  "group_size": 32,
+  "desc_act": false,
+  "sym": true,
+  "lm_head": false,
+  "quant_method": "gptq",
+  "checkpoint_format": "gptq",
+  "meta": {
+    "quantizer": [
+      "gptqmodel:1.5.1-dev"
+    ],
+    "uri": "https://github.com/modelcloud/gptqmodel",
+    "damp_percent": 0.01,
+    "damp_auto_increment": 0.0025,
+    "static_groups": false,
+    "true_sequential": true,
+    "mse": 0.0
+  }
+}