Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+assets/tokenizer/vocabulary.spm filter=lfs diff=lfs merge=lfs -text

assets/tokenizer/vocabulary.spm

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dadfd56d766715c61d2ef780a525ab43b8e6da4de6865bda3d95fdef5e134055
+size 493443

config.json

@@ -0,0 +1,24 @@
+{
+    "module": "keras_hub.src.models.mixtral.mixtral_backbone",
+    "class_name": "MixtralBackbone",
+    "config": {
+        "name": "mixtral_backbone",
+        "trainable": true,
+        "vocabulary_size": 32000,
+        "num_layers": 32,
+        "num_query_heads": 32,
+        "hidden_dim": 4096,
+        "intermediate_dim": 14336,
+        "num_experts": 8,
+        "top_k": 2,
+        "router_jitter_noise": 0.0,
+        "rope_max_wavelength": 1000000.0,
+        "rope_scaling_factor": 1.0,
+        "num_key_value_heads": 8,
+        "router_aux_loss_coef": 0.02,
+        "sliding_window": null,
+        "layer_norm_epsilon": 1e-05,
+        "dropout": 0
+    },
+    "registered_name": "keras_hub>MixtralBackbone"
+}

metadata.json

@@ -0,0 +1,9 @@
+{
+    "keras_version": "3.10.0.dev2025061603",
+    "keras_hub_version": "0.22.0.dev202506160417",
+    "parameter_count": 46702792704,
+    "date_saved": "2025-06-16@22:25:46",
+    "tasks": [
+        "CausalLM"
+    ]
+}

model.weights.json

@@ -0,0 +1,336 @@
+{
+    "metadata": {
+        "total_size": 93405585408.0
+    },
+    "weight_map": {
+        "/layers/reversible_embedding/vars": [
+            "model_00000.weights.h5"
+        ],
+        "/layers/mixtral_transformer_decoder/_feedforward_layernorm/vars": "model_00000.weights.h5",
+        "/layers/mixtral_transformer_decoder/_self_attention_layer/key_dense/vars": "model_00000.weights.h5",
+        "/layers/mixtral_transformer_decoder/_self_attention_layer/output_dense/vars": "model_00000.weights.h5",
+        "/layers/mixtral_transformer_decoder/_self_attention_layer/query_dense/vars": "model_00000.weights.h5",
+        "/layers/mixtral_transformer_decoder/_self_attention_layer/value_dense/vars": "model_00000.weights.h5",
+        "/layers/mixtral_transformer_decoder/_self_attention_layernorm/vars": "model_00000.weights.h5",
+        "/layers/mixtral_transformer_decoder/_sparse_moe_block/expert_bank/vars": [
+            "model_00000.weights.h5"
+        ],
+        "/layers/mixtral_transformer_decoder/_sparse_moe_block/_sparse_feedforward_gate_dense/vars": "model_00000.weights.h5",
+        "/layers/mixtral_transformer_decoder_1/_feedforward_layernorm/vars": "model_00000.weights.h5",
+        "/layers/mixtral_transformer_decoder_1/_self_attention_layer/key_dense/vars": "model_00000.weights.h5",
+        "/layers/mixtral_transformer_decoder_1/_self_attention_layer/output_dense/vars": "model_00000.weights.h5",
+        "/layers/mixtral_transformer_decoder_1/_self_attention_layer/query_dense/vars": "model_00000.weights.h5",
+        "/layers/mixtral_transformer_decoder_1/_self_attention_layer/value_dense/vars": "model_00000.weights.h5",
+        "/layers/mixtral_transformer_decoder_1/_self_attention_layernorm/vars": "model_00000.weights.h5",
+        "/layers/mixtral_transformer_decoder_1/_sparse_moe_block/expert_bank/vars": [
+            "model_00000.weights.h5"
+        ],
+        "/layers/mixtral_transformer_decoder_1/_sparse_moe_block/_sparse_feedforward_gate_dense/vars": "model_00000.weights.h5",
+        "/layers/mixtral_transformer_decoder_2/_feedforward_layernorm/vars": "model_00000.weights.h5",
+        "/layers/mixtral_transformer_decoder_2/_self_attention_layer/key_dense/vars": "model_00000.weights.h5",
+        "/layers/mixtral_transformer_decoder_2/_self_attention_layer/output_dense/vars": "model_00000.weights.h5",
+        "/layers/mixtral_transformer_decoder_2/_self_attention_layer/query_dense/vars": "model_00000.weights.h5",
+        "/layers/mixtral_transformer_decoder_2/_self_attention_layer/value_dense/vars": "model_00000.weights.h5",
+        "/layers/mixtral_transformer_decoder_2/_self_attention_layernorm/vars": "model_00000.weights.h5",
+        "/layers/mixtral_transformer_decoder_2/_sparse_moe_block/expert_bank/vars": [
+            "model_00000.weights.h5"
+        ],
+        "/layers/mixtral_transformer_decoder_2/_sparse_moe_block/_sparse_feedforward_gate_dense/vars": "model_00000.weights.h5",
+        "/layers/mixtral_transformer_decoder_3/_feedforward_layernorm/vars": "model_00000.weights.h5",
+        "/layers/mixtral_transformer_decoder_3/_self_attention_layer/key_dense/vars": "model_00000.weights.h5",
+        "/layers/mixtral_transformer_decoder_3/_self_attention_layer/output_dense/vars": "model_00000.weights.h5",
+        "/layers/mixtral_transformer_decoder_3/_self_attention_layer/query_dense/vars": "model_00000.weights.h5",
+        "/layers/mixtral_transformer_decoder_3/_self_attention_layer/value_dense/vars": "model_00000.weights.h5",
+        "/layers/mixtral_transformer_decoder_3/_self_attention_layernorm/vars": "model_00000.weights.h5",
+        "/layers/mixtral_transformer_decoder_3/_sparse_moe_block/expert_bank/vars": [
+            "model_00000.weights.h5",
+            "model_00001.weights.h5"
+        ],
+        "/layers/mixtral_transformer_decoder_3/_sparse_moe_block/_sparse_feedforward_gate_dense/vars": "model_00001.weights.h5",
+        "/layers/mixtral_transformer_decoder_4/_feedforward_layernorm/vars": "model_00001.weights.h5",
+        "/layers/mixtral_transformer_decoder_4/_self_attention_layer/key_dense/vars": "model_00001.weights.h5",
+        "/layers/mixtral_transformer_decoder_4/_self_attention_layer/output_dense/vars": "model_00001.weights.h5",
+        "/layers/mixtral_transformer_decoder_4/_self_attention_layer/query_dense/vars": "model_00001.weights.h5",
+        "/layers/mixtral_transformer_decoder_4/_self_attention_layer/value_dense/vars": "model_00001.weights.h5",
+        "/layers/mixtral_transformer_decoder_4/_self_attention_layernorm/vars": "model_00001.weights.h5",
+        "/layers/mixtral_transformer_decoder_4/_sparse_moe_block/expert_bank/vars": [
+            "model_00001.weights.h5"
+        ],
+        "/layers/mixtral_transformer_decoder_4/_sparse_moe_block/_sparse_feedforward_gate_dense/vars": "model_00001.weights.h5",
+        "/layers/mixtral_transformer_decoder_5/_feedforward_layernorm/vars": "model_00001.weights.h5",
+        "/layers/mixtral_transformer_decoder_5/_self_attention_layer/key_dense/vars": "model_00001.weights.h5",
+        "/layers/mixtral_transformer_decoder_5/_self_attention_layer/output_dense/vars": "model_00001.weights.h5",
+        "/layers/mixtral_transformer_decoder_5/_self_attention_layer/query_dense/vars": "model_00001.weights.h5",
+        "/layers/mixtral_transformer_decoder_5/_self_attention_layer/value_dense/vars": "model_00001.weights.h5",
+        "/layers/mixtral_transformer_decoder_5/_self_attention_layernorm/vars": "model_00001.weights.h5",
+        "/layers/mixtral_transformer_decoder_5/_sparse_moe_block/expert_bank/vars": [
+            "model_00001.weights.h5"
+        ],
+        "/layers/mixtral_transformer_decoder_5/_sparse_moe_block/_sparse_feedforward_gate_dense/vars": "model_00001.weights.h5",
+        "/layers/mixtral_transformer_decoder_6/_feedforward_layernorm/vars": "model_00001.weights.h5",
+        "/layers/mixtral_transformer_decoder_6/_self_attention_layer/key_dense/vars": "model_00001.weights.h5",
+        "/layers/mixtral_transformer_decoder_6/_self_attention_layer/output_dense/vars": "model_00001.weights.h5",
+        "/layers/mixtral_transformer_decoder_6/_self_attention_layer/query_dense/vars": "model_00001.weights.h5",
+        "/layers/mixtral_transformer_decoder_6/_self_attention_layer/value_dense/vars": "model_00001.weights.h5",
+        "/layers/mixtral_transformer_decoder_6/_self_attention_layernorm/vars": "model_00001.weights.h5",
+        "/layers/mixtral_transformer_decoder_6/_sparse_moe_block/expert_bank/vars": [
+            "model_00001.weights.h5"
+        ],
+        "/layers/mixtral_transformer_decoder_6/_sparse_moe_block/_sparse_feedforward_gate_dense/vars": "model_00001.weights.h5",
+        "/layers/mixtral_transformer_decoder_7/_feedforward_layernorm/vars": "model_00001.weights.h5",
+        "/layers/mixtral_transformer_decoder_7/_self_attention_layer/key_dense/vars": "model_00001.weights.h5",
+        "/layers/mixtral_transformer_decoder_7/_self_attention_layer/output_dense/vars": "model_00001.weights.h5",
+        "/layers/mixtral_transformer_decoder_7/_self_attention_layer/query_dense/vars": "model_00001.weights.h5",
+        "/layers/mixtral_transformer_decoder_7/_self_attention_layer/value_dense/vars": "model_00001.weights.h5",
+        "/layers/mixtral_transformer_decoder_7/_self_attention_layernorm/vars": "model_00001.weights.h5",
+        "/layers/mixtral_transformer_decoder_7/_sparse_moe_block/expert_bank/vars": [
+            "model_00002.weights.h5"
+        ],
+        "/layers/mixtral_transformer_decoder_7/_sparse_moe_block/_sparse_feedforward_gate_dense/vars": "model_00002.weights.h5",
+        "/layers/mixtral_transformer_decoder_8/_feedforward_layernorm/vars": "model_00002.weights.h5",
+        "/layers/mixtral_transformer_decoder_8/_self_attention_layer/key_dense/vars": "model_00002.weights.h5",
+        "/layers/mixtral_transformer_decoder_8/_self_attention_layer/output_dense/vars": "model_00002.weights.h5",
+        "/layers/mixtral_transformer_decoder_8/_self_attention_layer/query_dense/vars": "model_00002.weights.h5",
+        "/layers/mixtral_transformer_decoder_8/_self_attention_layer/value_dense/vars": "model_00002.weights.h5",
+        "/layers/mixtral_transformer_decoder_8/_self_attention_layernorm/vars": "model_00002.weights.h5",
+        "/layers/mixtral_transformer_decoder_8/_sparse_moe_block/expert_bank/vars": [
+            "model_00002.weights.h5"
+        ],
+        "/layers/mixtral_transformer_decoder_8/_sparse_moe_block/_sparse_feedforward_gate_dense/vars": "model_00002.weights.h5",
+        "/layers/mixtral_transformer_decoder_9/_feedforward_layernorm/vars": "model_00002.weights.h5",
+        "/layers/mixtral_transformer_decoder_9/_self_attention_layer/key_dense/vars": "model_00002.weights.h5",
+        "/layers/mixtral_transformer_decoder_9/_self_attention_layer/output_dense/vars": "model_00002.weights.h5",
+        "/layers/mixtral_transformer_decoder_9/_self_attention_layer/query_dense/vars": "model_00002.weights.h5",
+        "/layers/mixtral_transformer_decoder_9/_self_attention_layer/value_dense/vars": "model_00002.weights.h5",
+        "/layers/mixtral_transformer_decoder_9/_self_attention_layernorm/vars": "model_00002.weights.h5",
+        "/layers/mixtral_transformer_decoder_9/_sparse_moe_block/expert_bank/vars": [
+            "model_00002.weights.h5"
+        ],
+        "/layers/mixtral_transformer_decoder_9/_sparse_moe_block/_sparse_feedforward_gate_dense/vars": "model_00002.weights.h5",
+        "/layers/mixtral_transformer_decoder_10/_feedforward_layernorm/vars": "model_00002.weights.h5",
+        "/layers/mixtral_transformer_decoder_10/_self_attention_layer/key_dense/vars": "model_00002.weights.h5",
+        "/layers/mixtral_transformer_decoder_10/_self_attention_layer/output_dense/vars": "model_00002.weights.h5",
+        "/layers/mixtral_transformer_decoder_10/_self_attention_layer/query_dense/vars": "model_00002.weights.h5",
+        "/layers/mixtral_transformer_decoder_10/_self_attention_layer/value_dense/vars": "model_00002.weights.h5",
+        "/layers/mixtral_transformer_decoder_10/_self_attention_layernorm/vars": "model_00002.weights.h5",
+        "/layers/mixtral_transformer_decoder_10/_sparse_moe_block/expert_bank/vars": [
+            "model_00002.weights.h5",
+            "model_00003.weights.h5"
+        ],
+        "/layers/mixtral_transformer_decoder_10/_sparse_moe_block/_sparse_feedforward_gate_dense/vars": "model_00003.weights.h5",
+        "/layers/mixtral_transformer_decoder_11/_feedforward_layernorm/vars": "model_00003.weights.h5",
+        "/layers/mixtral_transformer_decoder_11/_self_attention_layer/key_dense/vars": "model_00003.weights.h5",
+        "/layers/mixtral_transformer_decoder_11/_self_attention_layer/output_dense/vars": "model_00003.weights.h5",
+        "/layers/mixtral_transformer_decoder_11/_self_attention_layer/query_dense/vars": "model_00003.weights.h5",
+        "/layers/mixtral_transformer_decoder_11/_self_attention_layer/value_dense/vars": "model_00003.weights.h5",
+        "/layers/mixtral_transformer_decoder_11/_self_attention_layernorm/vars": "model_00003.weights.h5",
+        "/layers/mixtral_transformer_decoder_11/_sparse_moe_block/expert_bank/vars": [
+            "model_00003.weights.h5"
+        ],
+        "/layers/mixtral_transformer_decoder_11/_sparse_moe_block/_sparse_feedforward_gate_dense/vars": "model_00003.weights.h5",
+        "/layers/mixtral_transformer_decoder_12/_feedforward_layernorm/vars": "model_00003.weights.h5",
+        "/layers/mixtral_transformer_decoder_12/_self_attention_layer/key_dense/vars": "model_00003.weights.h5",
+        "/layers/mixtral_transformer_decoder_12/_self_attention_layer/output_dense/vars": "model_00003.weights.h5",
+        "/layers/mixtral_transformer_decoder_12/_self_attention_layer/query_dense/vars": "model_00003.weights.h5",
+        "/layers/mixtral_transformer_decoder_12/_self_attention_layer/value_dense/vars": "model_00003.weights.h5",
+        "/layers/mixtral_transformer_decoder_12/_self_attention_layernorm/vars": "model_00003.weights.h5",
+        "/layers/mixtral_transformer_decoder_12/_sparse_moe_block/expert_bank/vars": [
+            "model_00003.weights.h5"
+        ],
+        "/layers/mixtral_transformer_decoder_12/_sparse_moe_block/_sparse_feedforward_gate_dense/vars": "model_00003.weights.h5",
+        "/layers/mixtral_transformer_decoder_13/_feedforward_layernorm/vars": "model_00003.weights.h5",
+        "/layers/mixtral_transformer_decoder_13/_self_attention_layer/key_dense/vars": "model_00003.weights.h5",
+        "/layers/mixtral_transformer_decoder_13/_self_attention_layer/output_dense/vars": "model_00003.weights.h5",
+        "/layers/mixtral_transformer_decoder_13/_self_attention_layer/query_dense/vars": "model_00003.weights.h5",
+        "/layers/mixtral_transformer_decoder_13/_self_attention_layer/value_dense/vars": "model_00003.weights.h5",
+        "/layers/mixtral_transformer_decoder_13/_self_attention_layernorm/vars": "model_00003.weights.h5",
+        "/layers/mixtral_transformer_decoder_13/_sparse_moe_block/expert_bank/vars": [
+            "model_00003.weights.h5"
+        ],
+        "/layers/mixtral_transformer_decoder_13/_sparse_moe_block/_sparse_feedforward_gate_dense/vars": "model_00003.weights.h5",
+        "/layers/mixtral_transformer_decoder_14/_feedforward_layernorm/vars": "model_00003.weights.h5",
+        "/layers/mixtral_transformer_decoder_14/_self_attention_layer/key_dense/vars": "model_00003.weights.h5",
+        "/layers/mixtral_transformer_decoder_14/_self_attention_layer/output_dense/vars": "model_00003.weights.h5",
+        "/layers/mixtral_transformer_decoder_14/_self_attention_layer/query_dense/vars": "model_00003.weights.h5",
+        "/layers/mixtral_transformer_decoder_14/_self_attention_layer/value_dense/vars": "model_00003.weights.h5",
+        "/layers/mixtral_transformer_decoder_14/_self_attention_layernorm/vars": "model_00003.weights.h5",
+        "/layers/mixtral_transformer_decoder_14/_sparse_moe_block/expert_bank/vars": [
+            "model_00003.weights.h5",
+            "model_00004.weights.h5"
+        ],
+        "/layers/mixtral_transformer_decoder_14/_sparse_moe_block/_sparse_feedforward_gate_dense/vars": "model_00004.weights.h5",
+        "/layers/mixtral_transformer_decoder_15/_feedforward_layernorm/vars": "model_00004.weights.h5",
+        "/layers/mixtral_transformer_decoder_15/_self_attention_layer/key_dense/vars": "model_00004.weights.h5",
+        "/layers/mixtral_transformer_decoder_15/_self_attention_layer/output_dense/vars": "model_00004.weights.h5",
+        "/layers/mixtral_transformer_decoder_15/_self_attention_layer/query_dense/vars": "model_00004.weights.h5",
+        "/layers/mixtral_transformer_decoder_15/_self_attention_layer/value_dense/vars": "model_00004.weights.h5",
+        "/layers/mixtral_transformer_decoder_15/_self_attention_layernorm/vars": "model_00004.weights.h5",
+        "/layers/mixtral_transformer_decoder_15/_sparse_moe_block/expert_bank/vars": [
+            "model_00004.weights.h5"
+        ],
+        "/layers/mixtral_transformer_decoder_15/_sparse_moe_block/_sparse_feedforward_gate_dense/vars": "model_00004.weights.h5",
+        "/layers/mixtral_transformer_decoder_16/_feedforward_layernorm/vars": "model_00004.weights.h5",
+        "/layers/mixtral_transformer_decoder_16/_self_attention_layer/key_dense/vars": "model_00004.weights.h5",
+        "/layers/mixtral_transformer_decoder_16/_self_attention_layer/output_dense/vars": "model_00004.weights.h5",
+        "/layers/mixtral_transformer_decoder_16/_self_attention_layer/query_dense/vars": "model_00004.weights.h5",
+        "/layers/mixtral_transformer_decoder_16/_self_attention_layer/value_dense/vars": "model_00004.weights.h5",
+        "/layers/mixtral_transformer_decoder_16/_self_attention_layernorm/vars": "model_00004.weights.h5",
+        "/layers/mixtral_transformer_decoder_16/_sparse_moe_block/expert_bank/vars": [
+            "model_00004.weights.h5"
+        ],
+        "/layers/mixtral_transformer_decoder_16/_sparse_moe_block/_sparse_feedforward_gate_dense/vars": "model_00004.weights.h5",
+        "/layers/mixtral_transformer_decoder_17/_feedforward_layernorm/vars": "model_00004.weights.h5",
+        "/layers/mixtral_transformer_decoder_17/_self_attention_layer/key_dense/vars": "model_00004.weights.h5",
+        "/layers/mixtral_transformer_decoder_17/_self_attention_layer/output_dense/vars": "model_00004.weights.h5",
+        "/layers/mixtral_transformer_decoder_17/_self_attention_layer/query_dense/vars": "model_00004.weights.h5",
+        "/layers/mixtral_transformer_decoder_17/_self_attention_layer/value_dense/vars": "model_00004.weights.h5",
+        "/layers/mixtral_transformer_decoder_17/_self_attention_layernorm/vars": "model_00004.weights.h5",
+        "/layers/mixtral_transformer_decoder_17/_sparse_moe_block/expert_bank/vars": [
+            "model_00004.weights.h5"
+        ],
+        "/layers/mixtral_transformer_decoder_17/_sparse_moe_block/_sparse_feedforward_gate_dense/vars": "model_00004.weights.h5",
+        "/layers/mixtral_transformer_decoder_18/_feedforward_layernorm/vars": "model_00004.weights.h5",
+        "/layers/mixtral_transformer_decoder_18/_self_attention_layer/key_dense/vars": "model_00004.weights.h5",
+        "/layers/mixtral_transformer_decoder_18/_self_attention_layer/output_dense/vars": "model_00004.weights.h5",
+        "/layers/mixtral_transformer_decoder_18/_self_attention_layer/query_dense/vars": "model_00004.weights.h5",
+        "/layers/mixtral_transformer_decoder_18/_self_attention_layer/value_dense/vars": "model_00004.weights.h5",
+        "/layers/mixtral_transformer_decoder_18/_self_attention_layernorm/vars": "model_00004.weights.h5",
+        "/layers/mixtral_transformer_decoder_18/_sparse_moe_block/expert_bank/vars": [
+            "model_00005.weights.h5"
+        ],
+        "/layers/mixtral_transformer_decoder_18/_sparse_moe_block/_sparse_feedforward_gate_dense/vars": "model_00005.weights.h5",
+        "/layers/mixtral_transformer_decoder_19/_feedforward_layernorm/vars": "model_00005.weights.h5",
+        "/layers/mixtral_transformer_decoder_19/_self_attention_layer/key_dense/vars": "model_00005.weights.h5",
+        "/layers/mixtral_transformer_decoder_19/_self_attention_layer/output_dense/vars": "model_00005.weights.h5",
+        "/layers/mixtral_transformer_decoder_19/_self_attention_layer/query_dense/vars": "model_00005.weights.h5",
+        "/layers/mixtral_transformer_decoder_19/_self_attention_layer/value_dense/vars": "model_00005.weights.h5",
+        "/layers/mixtral_transformer_decoder_19/_self_attention_layernorm/vars": "model_00005.weights.h5",
+        "/layers/mixtral_transformer_decoder_19/_sparse_moe_block/expert_bank/vars": [
+            "model_00005.weights.h5"
+        ],
+        "/layers/mixtral_transformer_decoder_19/_sparse_moe_block/_sparse_feedforward_gate_dense/vars": "model_00005.weights.h5",
+        "/layers/mixtral_transformer_decoder_20/_feedforward_layernorm/vars": "model_00005.weights.h5",
+        "/layers/mixtral_transformer_decoder_20/_self_attention_layer/key_dense/vars": "model_00005.weights.h5",
+        "/layers/mixtral_transformer_decoder_20/_self_attention_layer/output_dense/vars": "model_00005.weights.h5",
+        "/layers/mixtral_transformer_decoder_20/_self_attention_layer/query_dense/vars": "model_00005.weights.h5",
+        "/layers/mixtral_transformer_decoder_20/_self_attention_layer/value_dense/vars": "model_00005.weights.h5",
+        "/layers/mixtral_transformer_decoder_20/_self_attention_layernorm/vars": "model_00005.weights.h5",
+        "/layers/mixtral_transformer_decoder_20/_sparse_moe_block/expert_bank/vars": [
+            "model_00005.weights.h5"
+        ],
+        "/layers/mixtral_transformer_decoder_20/_sparse_moe_block/_sparse_feedforward_gate_dense/vars": "model_00005.weights.h5",
+        "/layers/mixtral_transformer_decoder_21/_feedforward_layernorm/vars": "model_00005.weights.h5",
+        "/layers/mixtral_transformer_decoder_21/_self_attention_layer/key_dense/vars": "model_00005.weights.h5",
+        "/layers/mixtral_transformer_decoder_21/_self_attention_layer/output_dense/vars": "model_00005.weights.h5",
+        "/layers/mixtral_transformer_decoder_21/_self_attention_layer/query_dense/vars": "model_00005.weights.h5",
+        "/layers/mixtral_transformer_decoder_21/_self_attention_layer/value_dense/vars": "model_00005.weights.h5",
+        "/layers/mixtral_transformer_decoder_21/_self_attention_layernorm/vars": "model_00005.weights.h5",
+        "/layers/mixtral_transformer_decoder_21/_sparse_moe_block/expert_bank/vars": [
+            "model_00005.weights.h5",
+            "model_00006.weights.h5"
+        ],
+        "/layers/mixtral_transformer_decoder_21/_sparse_moe_block/_sparse_feedforward_gate_dense/vars": "model_00006.weights.h5",
+        "/layers/mixtral_transformer_decoder_22/_feedforward_layernorm/vars": "model_00006.weights.h5",
+        "/layers/mixtral_transformer_decoder_22/_self_attention_layer/key_dense/vars": "model_00006.weights.h5",
+        "/layers/mixtral_transformer_decoder_22/_self_attention_layer/output_dense/vars": "model_00006.weights.h5",
+        "/layers/mixtral_transformer_decoder_22/_self_attention_layer/query_dense/vars": "model_00006.weights.h5",
+        "/layers/mixtral_transformer_decoder_22/_self_attention_layer/value_dense/vars": "model_00006.weights.h5",
+        "/layers/mixtral_transformer_decoder_22/_self_attention_layernorm/vars": "model_00006.weights.h5",
+        "/layers/mixtral_transformer_decoder_22/_sparse_moe_block/expert_bank/vars": [
+            "model_00006.weights.h5"
+        ],
+        "/layers/mixtral_transformer_decoder_22/_sparse_moe_block/_sparse_feedforward_gate_dense/vars": "model_00006.weights.h5",
+        "/layers/mixtral_transformer_decoder_23/_feedforward_layernorm/vars": "model_00006.weights.h5",
+        "/layers/mixtral_transformer_decoder_23/_self_attention_layer/key_dense/vars": "model_00006.weights.h5",
+        "/layers/mixtral_transformer_decoder_23/_self_attention_layer/output_dense/vars": "model_00006.weights.h5",
+        "/layers/mixtral_transformer_decoder_23/_self_attention_layer/query_dense/vars": "model_00006.weights.h5",
+        "/layers/mixtral_transformer_decoder_23/_self_attention_layer/value_dense/vars": "model_00006.weights.h5",
+        "/layers/mixtral_transformer_decoder_23/_self_attention_layernorm/vars": "model_00006.weights.h5",
+        "/layers/mixtral_transformer_decoder_23/_sparse_moe_block/expert_bank/vars": [
+            "model_00006.weights.h5"
+        ],
+        "/layers/mixtral_transformer_decoder_23/_sparse_moe_block/_sparse_feedforward_gate_dense/vars": "model_00006.weights.h5",
+        "/layers/mixtral_transformer_decoder_24/_feedforward_layernorm/vars": "model_00006.weights.h5",
+        "/layers/mixtral_transformer_decoder_24/_self_attention_layer/key_dense/vars": "model_00006.weights.h5",
+        "/layers/mixtral_transformer_decoder_24/_self_attention_layer/output_dense/vars": "model_00006.weights.h5",
+        "/layers/mixtral_transformer_decoder_24/_self_attention_layer/query_dense/vars": "model_00006.weights.h5",
+        "/layers/mixtral_transformer_decoder_24/_self_attention_layer/value_dense/vars": "model_00006.weights.h5",
+        "/layers/mixtral_transformer_decoder_24/_self_attention_layernorm/vars": "model_00006.weights.h5",
+        "/layers/mixtral_transformer_decoder_24/_sparse_moe_block/expert_bank/vars": [
+            "model_00006.weights.h5"
+        ],
+        "/layers/mixtral_transformer_decoder_24/_sparse_moe_block/_sparse_feedforward_gate_dense/vars": "model_00006.weights.h5",
+        "/layers/mixtral_transformer_decoder_25/_feedforward_layernorm/vars": "model_00006.weights.h5",
+        "/layers/mixtral_transformer_decoder_25/_self_attention_layer/key_dense/vars": "model_00006.weights.h5",
+        "/layers/mixtral_transformer_decoder_25/_self_attention_layer/output_dense/vars": "model_00006.weights.h5",
+        "/layers/mixtral_transformer_decoder_25/_self_attention_layer/query_dense/vars": "model_00006.weights.h5",
+        "/layers/mixtral_transformer_decoder_25/_self_attention_layer/value_dense/vars": "model_00006.weights.h5",
+        "/layers/mixtral_transformer_decoder_25/_self_attention_layernorm/vars": "model_00006.weights.h5",
+        "/layers/mixtral_transformer_decoder_25/_sparse_moe_block/expert_bank/vars": [
+            "model_00006.weights.h5",
+            "model_00007.weights.h5"
+        ],
+        "/layers/mixtral_transformer_decoder_25/_sparse_moe_block/_sparse_feedforward_gate_dense/vars": "model_00007.weights.h5",
+        "/layers/mixtral_transformer_decoder_26/_feedforward_layernorm/vars": "model_00007.weights.h5",
+        "/layers/mixtral_transformer_decoder_26/_self_attention_layer/key_dense/vars": "model_00007.weights.h5",
+        "/layers/mixtral_transformer_decoder_26/_self_attention_layer/output_dense/vars": "model_00007.weights.h5",
+        "/layers/mixtral_transformer_decoder_26/_self_attention_layer/query_dense/vars": "model_00007.weights.h5",
+        "/layers/mixtral_transformer_decoder_26/_self_attention_layer/value_dense/vars": "model_00007.weights.h5",
+        "/layers/mixtral_transformer_decoder_26/_self_attention_layernorm/vars": "model_00007.weights.h5",
+        "/layers/mixtral_transformer_decoder_26/_sparse_moe_block/expert_bank/vars": [
+            "model_00007.weights.h5"
+        ],
+        "/layers/mixtral_transformer_decoder_26/_sparse_moe_block/_sparse_feedforward_gate_dense/vars": "model_00007.weights.h5",
+        "/layers/mixtral_transformer_decoder_27/_feedforward_layernorm/vars": "model_00007.weights.h5",
+        "/layers/mixtral_transformer_decoder_27/_self_attention_layer/key_dense/vars": "model_00007.weights.h5",
+        "/layers/mixtral_transformer_decoder_27/_self_attention_layer/output_dense/vars": "model_00007.weights.h5",
+        "/layers/mixtral_transformer_decoder_27/_self_attention_layer/query_dense/vars": "model_00007.weights.h5",
+        "/layers/mixtral_transformer_decoder_27/_self_attention_layer/value_dense/vars": "model_00007.weights.h5",
+        "/layers/mixtral_transformer_decoder_27/_self_attention_layernorm/vars": "model_00007.weights.h5",
+        "/layers/mixtral_transformer_decoder_27/_sparse_moe_block/expert_bank/vars": [
+            "model_00007.weights.h5"
+        ],
+        "/layers/mixtral_transformer_decoder_27/_sparse_moe_block/_sparse_feedforward_gate_dense/vars": "model_00007.weights.h5",
+        "/layers/mixtral_transformer_decoder_28/_feedforward_layernorm/vars": "model_00007.weights.h5",
+        "/layers/mixtral_transformer_decoder_28/_self_attention_layer/key_dense/vars": "model_00007.weights.h5",
+        "/layers/mixtral_transformer_decoder_28/_self_attention_layer/output_dense/vars": "model_00007.weights.h5",
+        "/layers/mixtral_transformer_decoder_28/_self_attention_layer/query_dense/vars": "model_00007.weights.h5",
+        "/layers/mixtral_transformer_decoder_28/_self_attention_layer/value_dense/vars": "model_00007.weights.h5",
+        "/layers/mixtral_transformer_decoder_28/_self_attention_layernorm/vars": "model_00007.weights.h5",
+        "/layers/mixtral_transformer_decoder_28/_sparse_moe_block/expert_bank/vars": [
+            "model_00007.weights.h5"
+        ],
+        "/layers/mixtral_transformer_decoder_28/_sparse_moe_block/_sparse_feedforward_gate_dense/vars": "model_00007.weights.h5",
+        "/layers/mixtral_transformer_decoder_29/_feedforward_layernorm/vars": "model_00007.weights.h5",
+        "/layers/mixtral_transformer_decoder_29/_self_attention_layer/key_dense/vars": "model_00007.weights.h5",
+        "/layers/mixtral_transformer_decoder_29/_self_attention_layer/output_dense/vars": "model_00007.weights.h5",
+        "/layers/mixtral_transformer_decoder_29/_self_attention_layer/query_dense/vars": "model_00007.weights.h5",
+        "/layers/mixtral_transformer_decoder_29/_self_attention_layer/value_dense/vars": "model_00007.weights.h5",
+        "/layers/mixtral_transformer_decoder_29/_self_attention_layernorm/vars": "model_00007.weights.h5",
+        "/layers/mixtral_transformer_decoder_29/_sparse_moe_block/expert_bank/vars": [
+            "model_00008.weights.h5"
+        ],
+        "/layers/mixtral_transformer_decoder_29/_sparse_moe_block/_sparse_feedforward_gate_dense/vars": "model_00008.weights.h5",
+        "/layers/mixtral_transformer_decoder_30/_feedforward_layernorm/vars": "model_00008.weights.h5",
+        "/layers/mixtral_transformer_decoder_30/_self_attention_layer/key_dense/vars": "model_00008.weights.h5",
+        "/layers/mixtral_transformer_decoder_30/_self_attention_layer/output_dense/vars": "model_00008.weights.h5",
+        "/layers/mixtral_transformer_decoder_30/_self_attention_layer/query_dense/vars": "model_00008.weights.h5",
+        "/layers/mixtral_transformer_decoder_30/_self_attention_layer/value_dense/vars": "model_00008.weights.h5",
+        "/layers/mixtral_transformer_decoder_30/_self_attention_layernorm/vars": "model_00008.weights.h5",
+        "/layers/mixtral_transformer_decoder_30/_sparse_moe_block/expert_bank/vars": [
+            "model_00008.weights.h5"
+        ],
+        "/layers/mixtral_transformer_decoder_30/_sparse_moe_block/_sparse_feedforward_gate_dense/vars": "model_00008.weights.h5",
+        "/layers/mixtral_transformer_decoder_31/_feedforward_layernorm/vars": "model_00008.weights.h5",
+        "/layers/mixtral_transformer_decoder_31/_self_attention_layer/key_dense/vars": "model_00008.weights.h5",
+        "/layers/mixtral_transformer_decoder_31/_self_attention_layer/output_dense/vars": "model_00008.weights.h5",
+        "/layers/mixtral_transformer_decoder_31/_self_attention_layer/query_dense/vars": "model_00008.weights.h5",
+        "/layers/mixtral_transformer_decoder_31/_self_attention_layer/value_dense/vars": "model_00008.weights.h5",
+        "/layers/mixtral_transformer_decoder_31/_self_attention_layernorm/vars": "model_00008.weights.h5",
+        "/layers/mixtral_transformer_decoder_31/_sparse_moe_block/expert_bank/vars": [
+            "model_00008.weights.h5"
+        ],
+        "/layers/mixtral_transformer_decoder_31/_sparse_moe_block/_sparse_feedforward_gate_dense/vars": "model_00008.weights.h5",
+        "/layers/mixtral_layer_normalization/vars": "model_00008.weights.h5"
+    }
+}

model_00000.weights.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0bd781f94920ae078bd6762fbb620d70f12c3e8dbde27fe4bdc6a7ac113fad15
+size 10255478248

model_00001.weights.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:914fe74f45ddc092a46e2d09d069966e2952c2d66d6c8a56d39d9585d4c676a5
+size 10670779248

model_00002.weights.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:61cc422ed9127b90f20b4b6008f5e8d19890b022835d277427e6aaa453d989f3
+size 10586780928

model_00003.weights.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:95cfc1aaf94aae7dc74c978f28466cdc036126857de4fda26aa0b551a9613af4
+size 10670779568

model_00004.weights.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:96940759d70b5d16d6023611ec96c339c0d77be50da001ac776509f0b21d2157
+size 10670779248

model_00005.weights.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9fd03008bbae40afbdb55332ebac6bd569cd715fe3e872c7e7e49564bd44e072
+size 10586780928

model_00006.weights.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:021da30ae6c89a9b9eeb433ccae95490609f4056834ada895519abb1a226fd26
+size 10670779568

model_00007.weights.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:220b921c709fbe8ec8c68c5379e3c18ec0b7e2ba0129567478fd1fa466848ace
+size 10670779248

model_00008.weights.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d5704809aa3aeb934ffe99f73833a03de47c0952b7b5098dca7f6cbedbbd19c1
+size 8623809544

preprocessor.json

@@ -0,0 +1,43 @@
+{
+    "module": "keras_hub.src.models.mixtral.mixtral_causal_lm_preprocessor",
+    "class_name": "MixtralCausalLMPreprocessor",
+    "config": {
+        "name": "mixtral_causal_lm_preprocessor_2",
+        "trainable": true,
+        "dtype": {
+            "module": "keras",
+            "class_name": "DTypePolicy",
+            "config": {
+                "name": "float32"
+            },
+            "registered_name": null
+        },
+        "tokenizer": {
+            "module": "keras_hub.src.models.mixtral.mixtral_tokenizer",
+            "class_name": "MixtralTokenizer",
+            "config": {
+                "name": "mixtral_tokenizer",
+                "trainable": true,
+                "dtype": {
+                    "module": "keras",
+                    "class_name": "DTypePolicy",
+                    "config": {
+                        "name": "int32"
+                    },
+                    "registered_name": null
+                },
+                "config_file": "tokenizer.json",
+                "proto": null,
+                "sequence_length": null,
+                "add_bos": false,
+                "add_eos": false
+            },
+            "registered_name": "keras_hub>MixtralTokenizer"
+        },
+        "config_file": "preprocessor.json",
+        "sequence_length": 1024,
+        "add_start_token": true,
+        "add_end_token": true
+    },
+    "registered_name": "keras_hub>MixtralCausalLMPreprocessor"
+}

task.json

@@ -0,0 +1,75 @@
+{
+    "module": "keras_hub.src.models.mixtral.mixtral_causal_lm",
+    "class_name": "MixtralCausalLM",
+    "config": {
+        "backbone": {
+            "module": "keras_hub.src.models.mixtral.mixtral_backbone",
+            "class_name": "MixtralBackbone",
+            "config": {
+                "name": "mixtral_backbone",
+                "trainable": true,
+                "vocabulary_size": 32000,
+                "num_layers": 32,
+                "num_query_heads": 32,
+                "hidden_dim": 4096,
+                "intermediate_dim": 14336,
+                "num_experts": 8,
+                "top_k": 2,
+                "router_jitter_noise": 0.0,
+                "rope_max_wavelength": 1000000.0,
+                "rope_scaling_factor": 1.0,
+                "num_key_value_heads": 8,
+                "router_aux_loss_coef": 0.02,
+                "sliding_window": null,
+                "layer_norm_epsilon": 1e-05,
+                "dropout": 0
+            },
+            "registered_name": "keras_hub>MixtralBackbone"
+        },
+        "preprocessor": {
+            "module": "keras_hub.src.models.mixtral.mixtral_causal_lm_preprocessor",
+            "class_name": "MixtralCausalLMPreprocessor",
+            "config": {
+                "name": "mixtral_causal_lm_preprocessor_2",
+                "trainable": true,
+                "dtype": {
+                    "module": "keras",
+                    "class_name": "DTypePolicy",
+                    "config": {
+                        "name": "float32"
+                    },
+                    "registered_name": null
+                },
+                "tokenizer": {
+                    "module": "keras_hub.src.models.mixtral.mixtral_tokenizer",
+                    "class_name": "MixtralTokenizer",
+                    "config": {
+                        "name": "mixtral_tokenizer",
+                        "trainable": true,
+                        "dtype": {
+                            "module": "keras",
+                            "class_name": "DTypePolicy",
+                            "config": {
+                                "name": "int32"
+                            },
+                            "registered_name": null
+                        },
+                        "config_file": "tokenizer.json",
+                        "proto": null,
+                        "sequence_length": null,
+                        "add_bos": false,
+                        "add_eos": false
+                    },
+                    "registered_name": "keras_hub>MixtralTokenizer"
+                },
+                "config_file": "preprocessor.json",
+                "sequence_length": 1024,
+                "add_start_token": true,
+                "add_end_token": true
+            },
+            "registered_name": "keras_hub>MixtralCausalLMPreprocessor"
+        },
+        "name": "mixtral_causal_lm"
+    },
+    "registered_name": "keras_hub>MixtralCausalLM"
+}

tokenizer.json

@@ -0,0 +1,22 @@
+{
+    "module": "keras_hub.src.models.mixtral.mixtral_tokenizer",
+    "class_name": "MixtralTokenizer",
+    "config": {
+        "name": "mixtral_tokenizer",
+        "trainable": true,
+        "dtype": {
+            "module": "keras",
+            "class_name": "DTypePolicy",
+            "config": {
+                "name": "int32"
+            },
+            "registered_name": null
+        },
+        "config_file": "tokenizer.json",
+        "proto": null,
+        "sequence_length": null,
+        "add_bos": false,
+        "add_eos": false
+    },
+    "registered_name": "keras_hub>MixtralTokenizer"
+}