Add INT4-W4A16 checkpoint

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

added_tokens.json

@@ -0,0 +1,38 @@
+{
+  "<begin_of_document>": 200034,
+  "<beginning_of_sentence>": 200019,
+  "<code_interpreter>": 200023,
+  "<commit_after>": 200018,
+  "<commit_before>": 200016,
+  "<commit_msg>": 200017,
+  "<empty_output>": 200015,
+  "<end_of_document>": 200021,
+  "<end_of_image>": 200030,
+  "<end_of_sentence>": 200020,
+  "<end_of_speech>": 200028,
+  "<end_of_video>": 200032,
+  "<filename>": 200006,
+  "<fim_middle>": 200002,
+  "<fim_pad>": 200004,
+  "<fim_prefix>": 200001,
+  "<fim_suffix>": 200003,
+  "<function_call>": 200022,
+  "<gh_stars>": 200007,
+  "<image>": 200025,
+  "<issue_closed>": 200010,
+  "<issue_comment>": 200009,
+  "<issue_start>": 200008,
+  "<jupyter_code>": 200013,
+  "<jupyter_error>": 200035,
+  "<jupyter_output>": 200014,
+  "<jupyter_start>": 200011,
+  "<jupyter_text>": 200012,
+  "<pad>": 200000,
+  "<reponame>": 200005,
+  "<speech>": 200024,
+  "<start_of_image>": 200029,
+  "<start_of_speech>": 200027,
+  "<start_of_video>": 200031,
+  "<video>": 200026,
+  "<vision_pad>": 200033
+}

chat_template.jinja

@@ -0,0 +1,24 @@
+{{ '<begin_of_document>' -}}{% set ns = namespace(system_prompt='') -%}{% for message in messages -%}{% if message['role'] == 'system' -%}{% set ns.system_prompt = ns.system_prompt + message['content'][0]['text'] -%}{% endif -%}{%- endfor -%}{% if ns.system_prompt != '' -%}{{ '<beginning_of_sentence>system ai_setting=assistant
+' + ns.system_prompt + '<end_of_sentence>
+' -}}{%- endif -%}{% if tools -%}{{ '<beginning_of_sentence>system tool_setting=tools
+You are provided with these tools:
+<tools>
+' -}}{% for tool in tools -%}{{ tool | tojson ~ '
+' -}}{%- endfor -%}{{ '</tools>
+
+If you need to call tools, please respond with <tool_calls></tool_calls> XML tags, and provide tool-name and json-object of arguments, following the format below:
+<tool_calls>
+{''name'': <tool-name-1>, ''arguments'': <args-json-object-1>}
+...
+</tool_calls><end_of_sentence>
+' -}}{%- endif -%}{% for message in messages -%}{% if message['role'] == 'user' -%}{{ '<beginning_of_sentence>user name=user
+' + message['content'][0]['text'] + '<end_of_sentence>
+' -}}{% elif message['role'] == 'assistant' -%}{{ '<beginning_of_sentence>ai name=assistant
+' -}}{% for content in message['content'] | selectattr('type', 'equalto', 'text') -%}{{ content['text'] -}}{%- endfor -%}{{ '<end_of_sentence>
+' -}}{% elif message['role'] == 'tool' -%}{{ '<beginning_of_sentence>tool name=tools
+' }} {%- for content in message['content'] -%}{{- 'tool name: ' + content['name'] + '
+' + 'tool result: ' + content['text'] + '
+
+' -}} {%- endfor -%}{{- '<end_of_sentence>
+' -}}{% endif -%}{%- endfor -%}{% if add_generation_prompt -%}{{ '<beginning_of_sentence>ai name=assistant
+' -}}{%- endif -%}

config.json

@@ -0,0 +1,486 @@
+{
+  "architectures": [
+    "MiniMaxM1ForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "attn_type_list": [
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    1,
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    1,
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    1,
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    1,
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    1,
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    1,
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    1,
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    1,
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    1,
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    1
+  ],
+  "auto_map": {
+    "AutoConfig": "MiniMaxAI/MiniMax-M1-80k--configuration_minimax_m1.MiniMaxM1Config",
+    "AutoModelForCausalLM": "MiniMaxAI/MiniMax-M1-80k--modeling_minimax_m1.MiniMaxM1ForCausalLM"
+  },
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 6144,
+  "initializer_range": 0.02,
+  "intermediate_size": 9216,
+  "layernorm_full_attention_alpha": 3.5565588200778455,
+  "layernorm_full_attention_beta": 1.0,
+  "layernorm_linear_attention_alpha": 3.5565588200778455,
+  "layernorm_linear_attention_beta": 1.0,
+  "layernorm_mlp_alpha": 3.5565588200778455,
+  "layernorm_mlp_beta": 1.0,
+  "max_position_embeddings": 10240000,
+  "model_type": "MiniMaxM1",
+  "num_attention_heads": 64,
+  "num_experts_per_tok": 2,
+  "num_hidden_layers": 80,
+  "num_key_value_heads": 8,
+  "num_local_experts": 32,
+  "output_router_logits": false,
+  "postnorm": true,
+  "quantization_config": {
+    "config_groups": {
+      "group_0": {
+        "input_activations": null,
+        "output_activations": null,
+        "targets": [
+          "Linear"
+        ],
+        "weights": {
+          "actorder": null,
+          "block_structure": null,
+          "dynamic": false,
+          "group_size": 128,
+          "num_bits": 4,
+          "observer": "minmax",
+          "observer_kwargs": {},
+          "strategy": "group",
+          "symmetric": true,
+          "type": "int"
+        }
+      }
+    },
+    "format": "pack-quantized",
+    "global_compression_ratio": null,
+    "ignore": [
+      "model.layers.0.self_attn.out_proj",
+      "model.layers.0.self_attn.qkv_proj",
+      "model.layers.0.self_attn.output_gate",
+      "model.layers.0.block_sparse_moe.gate",
+      "model.layers.1.self_attn.out_proj",
+      "model.layers.1.self_attn.qkv_proj",
+      "model.layers.1.self_attn.output_gate",
+      "model.layers.1.block_sparse_moe.gate",
+      "model.layers.2.self_attn.out_proj",
+      "model.layers.2.self_attn.qkv_proj",
+      "model.layers.2.self_attn.output_gate",
+      "model.layers.2.block_sparse_moe.gate",
+      "model.layers.3.self_attn.out_proj",
+      "model.layers.3.self_attn.qkv_proj",
+      "model.layers.3.self_attn.output_gate",
+      "model.layers.3.block_sparse_moe.gate",
+      "model.layers.4.self_attn.out_proj",
+      "model.layers.4.self_attn.qkv_proj",
+      "model.layers.4.self_attn.output_gate",
+      "model.layers.4.block_sparse_moe.gate",
+      "model.layers.5.self_attn.out_proj",
+      "model.layers.5.self_attn.qkv_proj",
+      "model.layers.5.self_attn.output_gate",
+      "model.layers.5.block_sparse_moe.gate",
+      "model.layers.6.self_attn.out_proj",
+      "model.layers.6.self_attn.qkv_proj",
+      "model.layers.6.self_attn.output_gate",
+      "model.layers.6.block_sparse_moe.gate",
+      "model.layers.7.self_attn.q_proj",
+      "model.layers.7.self_attn.k_proj",
+      "model.layers.7.self_attn.v_proj",
+      "model.layers.7.self_attn.o_proj",
+      "model.layers.7.block_sparse_moe.gate",
+      "model.layers.8.self_attn.out_proj",
+      "model.layers.8.self_attn.qkv_proj",
+      "model.layers.8.self_attn.output_gate",
+      "model.layers.8.block_sparse_moe.gate",
+      "model.layers.9.self_attn.out_proj",
+      "model.layers.9.self_attn.qkv_proj",
+      "model.layers.9.self_attn.output_gate",
+      "model.layers.9.block_sparse_moe.gate",
+      "model.layers.10.self_attn.out_proj",
+      "model.layers.10.self_attn.qkv_proj",
+      "model.layers.10.self_attn.output_gate",
+      "model.layers.10.block_sparse_moe.gate",
+      "model.layers.11.self_attn.out_proj",
+      "model.layers.11.self_attn.qkv_proj",
+      "model.layers.11.self_attn.output_gate",
+      "model.layers.11.block_sparse_moe.gate",
+      "model.layers.12.self_attn.out_proj",
+      "model.layers.12.self_attn.qkv_proj",
+      "model.layers.12.self_attn.output_gate",
+      "model.layers.12.block_sparse_moe.gate",
+      "model.layers.13.self_attn.out_proj",
+      "model.layers.13.self_attn.qkv_proj",
+      "model.layers.13.self_attn.output_gate",
+      "model.layers.13.block_sparse_moe.gate",
+      "model.layers.14.self_attn.out_proj",
+      "model.layers.14.self_attn.qkv_proj",
+      "model.layers.14.self_attn.output_gate",
+      "model.layers.14.block_sparse_moe.gate",
+      "model.layers.15.self_attn.q_proj",
+      "model.layers.15.self_attn.k_proj",
+      "model.layers.15.self_attn.v_proj",
+      "model.layers.15.self_attn.o_proj",
+      "model.layers.15.block_sparse_moe.gate",
+      "model.layers.16.self_attn.out_proj",
+      "model.layers.16.self_attn.qkv_proj",
+      "model.layers.16.self_attn.output_gate",
+      "model.layers.16.block_sparse_moe.gate",
+      "model.layers.17.self_attn.out_proj",
+      "model.layers.17.self_attn.qkv_proj",
+      "model.layers.17.self_attn.output_gate",
+      "model.layers.17.block_sparse_moe.gate",
+      "model.layers.18.self_attn.out_proj",
+      "model.layers.18.self_attn.qkv_proj",
+      "model.layers.18.self_attn.output_gate",
+      "model.layers.18.block_sparse_moe.gate",
+      "model.layers.19.self_attn.out_proj",
+      "model.layers.19.self_attn.qkv_proj",
+      "model.layers.19.self_attn.output_gate",
+      "model.layers.19.block_sparse_moe.gate",
+      "model.layers.20.self_attn.out_proj",
+      "model.layers.20.self_attn.qkv_proj",
+      "model.layers.20.self_attn.output_gate",
+      "model.layers.20.block_sparse_moe.gate",
+      "model.layers.21.self_attn.out_proj",
+      "model.layers.21.self_attn.qkv_proj",
+      "model.layers.21.self_attn.output_gate",
+      "model.layers.21.block_sparse_moe.gate",
+      "model.layers.22.self_attn.out_proj",
+      "model.layers.22.self_attn.qkv_proj",
+      "model.layers.22.self_attn.output_gate",
+      "model.layers.22.block_sparse_moe.gate",
+      "model.layers.23.self_attn.q_proj",
+      "model.layers.23.self_attn.k_proj",
+      "model.layers.23.self_attn.v_proj",
+      "model.layers.23.self_attn.o_proj",
+      "model.layers.23.block_sparse_moe.gate",
+      "model.layers.24.self_attn.out_proj",
+      "model.layers.24.self_attn.qkv_proj",
+      "model.layers.24.self_attn.output_gate",
+      "model.layers.24.block_sparse_moe.gate",
+      "model.layers.25.self_attn.out_proj",
+      "model.layers.25.self_attn.qkv_proj",
+      "model.layers.25.self_attn.output_gate",
+      "model.layers.25.block_sparse_moe.gate",
+      "model.layers.26.self_attn.out_proj",
+      "model.layers.26.self_attn.qkv_proj",
+      "model.layers.26.self_attn.output_gate",
+      "model.layers.26.block_sparse_moe.gate",
+      "model.layers.27.self_attn.out_proj",
+      "model.layers.27.self_attn.qkv_proj",
+      "model.layers.27.self_attn.output_gate",
+      "model.layers.27.block_sparse_moe.gate",
+      "model.layers.28.self_attn.out_proj",
+      "model.layers.28.self_attn.qkv_proj",
+      "model.layers.28.self_attn.output_gate",
+      "model.layers.28.block_sparse_moe.gate",
+      "model.layers.29.self_attn.out_proj",
+      "model.layers.29.self_attn.qkv_proj",
+      "model.layers.29.self_attn.output_gate",
+      "model.layers.29.block_sparse_moe.gate",
+      "model.layers.30.self_attn.out_proj",
+      "model.layers.30.self_attn.qkv_proj",
+      "model.layers.30.self_attn.output_gate",
+      "model.layers.30.block_sparse_moe.gate",
+      "model.layers.31.self_attn.q_proj",
+      "model.layers.31.self_attn.k_proj",
+      "model.layers.31.self_attn.v_proj",
+      "model.layers.31.self_attn.o_proj",
+      "model.layers.31.block_sparse_moe.gate",
+      "model.layers.32.self_attn.out_proj",
+      "model.layers.32.self_attn.qkv_proj",
+      "model.layers.32.self_attn.output_gate",
+      "model.layers.32.block_sparse_moe.gate",
+      "model.layers.33.self_attn.out_proj",
+      "model.layers.33.self_attn.qkv_proj",
+      "model.layers.33.self_attn.output_gate",
+      "model.layers.33.block_sparse_moe.gate",
+      "model.layers.34.self_attn.out_proj",
+      "model.layers.34.self_attn.qkv_proj",
+      "model.layers.34.self_attn.output_gate",
+      "model.layers.34.block_sparse_moe.gate",
+      "model.layers.35.self_attn.out_proj",
+      "model.layers.35.self_attn.qkv_proj",
+      "model.layers.35.self_attn.output_gate",
+      "model.layers.35.block_sparse_moe.gate",
+      "model.layers.36.self_attn.out_proj",
+      "model.layers.36.self_attn.qkv_proj",
+      "model.layers.36.self_attn.output_gate",
+      "model.layers.36.block_sparse_moe.gate",
+      "model.layers.37.self_attn.out_proj",
+      "model.layers.37.self_attn.qkv_proj",
+      "model.layers.37.self_attn.output_gate",
+      "model.layers.37.block_sparse_moe.gate",
+      "model.layers.38.self_attn.out_proj",
+      "model.layers.38.self_attn.qkv_proj",
+      "model.layers.38.self_attn.output_gate",
+      "model.layers.38.block_sparse_moe.gate",
+      "model.layers.39.self_attn.q_proj",
+      "model.layers.39.self_attn.k_proj",
+      "model.layers.39.self_attn.v_proj",
+      "model.layers.39.self_attn.o_proj",
+      "model.layers.39.block_sparse_moe.gate",
+      "model.layers.40.self_attn.out_proj",
+      "model.layers.40.self_attn.qkv_proj",
+      "model.layers.40.self_attn.output_gate",
+      "model.layers.40.block_sparse_moe.gate",
+      "model.layers.41.self_attn.out_proj",
+      "model.layers.41.self_attn.qkv_proj",
+      "model.layers.41.self_attn.output_gate",
+      "model.layers.41.block_sparse_moe.gate",
+      "model.layers.42.self_attn.out_proj",
+      "model.layers.42.self_attn.qkv_proj",
+      "model.layers.42.self_attn.output_gate",
+      "model.layers.42.block_sparse_moe.gate",
+      "model.layers.43.self_attn.out_proj",
+      "model.layers.43.self_attn.qkv_proj",
+      "model.layers.43.self_attn.output_gate",
+      "model.layers.43.block_sparse_moe.gate",
+      "model.layers.44.self_attn.out_proj",
+      "model.layers.44.self_attn.qkv_proj",
+      "model.layers.44.self_attn.output_gate",
+      "model.layers.44.block_sparse_moe.gate",
+      "model.layers.45.self_attn.out_proj",
+      "model.layers.45.self_attn.qkv_proj",
+      "model.layers.45.self_attn.output_gate",
+      "model.layers.45.block_sparse_moe.gate",
+      "model.layers.46.self_attn.out_proj",
+      "model.layers.46.self_attn.qkv_proj",
+      "model.layers.46.self_attn.output_gate",
+      "model.layers.46.block_sparse_moe.gate",
+      "model.layers.47.self_attn.q_proj",
+      "model.layers.47.self_attn.k_proj",
+      "model.layers.47.self_attn.v_proj",
+      "model.layers.47.self_attn.o_proj",
+      "model.layers.47.block_sparse_moe.gate",
+      "model.layers.48.self_attn.out_proj",
+      "model.layers.48.self_attn.qkv_proj",
+      "model.layers.48.self_attn.output_gate",
+      "model.layers.48.block_sparse_moe.gate",
+      "model.layers.49.self_attn.out_proj",
+      "model.layers.49.self_attn.qkv_proj",
+      "model.layers.49.self_attn.output_gate",
+      "model.layers.49.block_sparse_moe.gate",
+      "model.layers.50.self_attn.out_proj",
+      "model.layers.50.self_attn.qkv_proj",
+      "model.layers.50.self_attn.output_gate",
+      "model.layers.50.block_sparse_moe.gate",
+      "model.layers.51.self_attn.out_proj",
+      "model.layers.51.self_attn.qkv_proj",
+      "model.layers.51.self_attn.output_gate",
+      "model.layers.51.block_sparse_moe.gate",
+      "model.layers.52.self_attn.out_proj",
+      "model.layers.52.self_attn.qkv_proj",
+      "model.layers.52.self_attn.output_gate",
+      "model.layers.52.block_sparse_moe.gate",
+      "model.layers.53.self_attn.out_proj",
+      "model.layers.53.self_attn.qkv_proj",
+      "model.layers.53.self_attn.output_gate",
+      "model.layers.53.block_sparse_moe.gate",
+      "model.layers.54.self_attn.out_proj",
+      "model.layers.54.self_attn.qkv_proj",
+      "model.layers.54.self_attn.output_gate",
+      "model.layers.54.block_sparse_moe.gate",
+      "model.layers.55.self_attn.q_proj",
+      "model.layers.55.self_attn.k_proj",
+      "model.layers.55.self_attn.v_proj",
+      "model.layers.55.self_attn.o_proj",
+      "model.layers.55.block_sparse_moe.gate",
+      "model.layers.56.self_attn.out_proj",
+      "model.layers.56.self_attn.qkv_proj",
+      "model.layers.56.self_attn.output_gate",
+      "model.layers.56.block_sparse_moe.gate",
+      "model.layers.57.self_attn.out_proj",
+      "model.layers.57.self_attn.qkv_proj",
+      "model.layers.57.self_attn.output_gate",
+      "model.layers.57.block_sparse_moe.gate",
+      "model.layers.58.self_attn.out_proj",
+      "model.layers.58.self_attn.qkv_proj",
+      "model.layers.58.self_attn.output_gate",
+      "model.layers.58.block_sparse_moe.gate",
+      "model.layers.59.self_attn.out_proj",
+      "model.layers.59.self_attn.qkv_proj",
+      "model.layers.59.self_attn.output_gate",
+      "model.layers.59.block_sparse_moe.gate",
+      "model.layers.60.self_attn.out_proj",
+      "model.layers.60.self_attn.qkv_proj",
+      "model.layers.60.self_attn.output_gate",
+      "model.layers.60.block_sparse_moe.gate",
+      "model.layers.61.self_attn.out_proj",
+      "model.layers.61.self_attn.qkv_proj",
+      "model.layers.61.self_attn.output_gate",
+      "model.layers.61.block_sparse_moe.gate",
+      "model.layers.62.self_attn.out_proj",
+      "model.layers.62.self_attn.qkv_proj",
+      "model.layers.62.self_attn.output_gate",
+      "model.layers.62.block_sparse_moe.gate",
+      "model.layers.63.self_attn.q_proj",
+      "model.layers.63.self_attn.k_proj",
+      "model.layers.63.self_attn.v_proj",
+      "model.layers.63.self_attn.o_proj",
+      "model.layers.63.block_sparse_moe.gate",
+      "model.layers.64.self_attn.out_proj",
+      "model.layers.64.self_attn.qkv_proj",
+      "model.layers.64.self_attn.output_gate",
+      "model.layers.64.block_sparse_moe.gate",
+      "model.layers.65.self_attn.out_proj",
+      "model.layers.65.self_attn.qkv_proj",
+      "model.layers.65.self_attn.output_gate",
+      "model.layers.65.block_sparse_moe.gate",
+      "model.layers.66.self_attn.out_proj",
+      "model.layers.66.self_attn.qkv_proj",
+      "model.layers.66.self_attn.output_gate",
+      "model.layers.66.block_sparse_moe.gate",
+      "model.layers.67.self_attn.out_proj",
+      "model.layers.67.self_attn.qkv_proj",
+      "model.layers.67.self_attn.output_gate",
+      "model.layers.67.block_sparse_moe.gate",
+      "model.layers.68.self_attn.out_proj",
+      "model.layers.68.self_attn.qkv_proj",
+      "model.layers.68.self_attn.output_gate",
+      "model.layers.68.block_sparse_moe.gate",
+      "model.layers.69.self_attn.out_proj",
+      "model.layers.69.self_attn.qkv_proj",
+      "model.layers.69.self_attn.output_gate",
+      "model.layers.69.block_sparse_moe.gate",
+      "model.layers.70.self_attn.out_proj",
+      "model.layers.70.self_attn.qkv_proj",
+      "model.layers.70.self_attn.output_gate",
+      "model.layers.70.block_sparse_moe.gate",
+      "model.layers.71.self_attn.q_proj",
+      "model.layers.71.self_attn.k_proj",
+      "model.layers.71.self_attn.v_proj",
+      "model.layers.71.self_attn.o_proj",
+      "model.layers.71.block_sparse_moe.gate",
+      "model.layers.72.self_attn.out_proj",
+      "model.layers.72.self_attn.qkv_proj",
+      "model.layers.72.self_attn.output_gate",
+      "model.layers.72.block_sparse_moe.gate",
+      "model.layers.73.self_attn.out_proj",
+      "model.layers.73.self_attn.qkv_proj",
+      "model.layers.73.self_attn.output_gate",
+      "model.layers.73.block_sparse_moe.gate",
+      "model.layers.74.self_attn.out_proj",
+      "model.layers.74.self_attn.qkv_proj",
+      "model.layers.74.self_attn.output_gate",
+      "model.layers.74.block_sparse_moe.gate",
+      "model.layers.75.self_attn.out_proj",
+      "model.layers.75.self_attn.qkv_proj",
+      "model.layers.75.self_attn.output_gate",
+      "model.layers.75.block_sparse_moe.gate",
+      "model.layers.76.self_attn.out_proj",
+      "model.layers.76.self_attn.qkv_proj",
+      "model.layers.76.self_attn.output_gate",
+      "model.layers.76.block_sparse_moe.gate",
+      "model.layers.77.self_attn.out_proj",
+      "model.layers.77.self_attn.qkv_proj",
+      "model.layers.77.self_attn.output_gate",
+      "model.layers.77.block_sparse_moe.gate",
+      "model.layers.78.self_attn.out_proj",
+      "model.layers.78.self_attn.qkv_proj",
+      "model.layers.78.self_attn.output_gate",
+      "model.layers.78.block_sparse_moe.gate",
+      "model.layers.79.self_attn.q_proj",
+      "model.layers.79.self_attn.k_proj",
+      "model.layers.79.self_attn.v_proj",
+      "model.layers.79.self_attn.o_proj",
+      "model.layers.79.block_sparse_moe.gate",
+      "lm_head"
+    ],
+    "kv_cache_scheme": null,
+    "quant_method": "compressed-tensors",
+    "quantization_status": "compressed"
+  },
+  "rms_norm_eps": 1e-05,
+  "rope_theta": 10000000,
+  "rotary_dim": 64,
+  "router_aux_loss_coef": 0.001,
+  "router_jitter_noise": 0.0,
+  "shared_intermediate_size": 0,
+  "shared_moe_mode": "sigmoid",
+  "sliding_window": null,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.52.4",
+  "use_cache": true,
+  "vocab_size": 200064
+}

configuration_minimax_m1.py

@@ -0,0 +1,152 @@
+""" MiniMaxM1 model configuration"""
+
+from transformers.configuration_utils import PretrainedConfig
+from transformers.utils import logging
+
+
+logger = logging.get_logger(__name__)
+
+
+class MiniMaxM1Config(PretrainedConfig):
+    r"""
+    This is the configuration class to store the configuration of a [`MiniMaxM1Model`]. It is used to instantiate an
+    MiniMaxM1 model according to the specified arguments, defining the model architecture. Instantiating a configuration
+    with the defaults will yield a similar configuration to that of the MiniMaxM1.
+
+    Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+    documentation from [`PretrainedConfig`] for more information.
+
+
+    Args:
+        vocab_size (`int`, *optional*, defaults to 32000):
+            Vocabulary size of the MiniMaxM1 model. Defines the number of different tokens that can be represented by the
+            `inputs_ids` passed when calling [`MiniMaxM1Model`]
+        hidden_size (`int`, *optional*, defaults to 4096):
+            Dimension of the hidden representations.
+        intermediate_size (`int`, *optional*, defaults to 14336):
+            Dimension of the MLP representations.
+        num_hidden_layers (`int`, *optional*, defaults to 32):
+            Number of hidden layers in the Transformer encoder.
+        num_attention_heads (`int`, *optional*, defaults to 32):
+            Number of attention heads for each attention layer in the Transformer encoder.
+        num_key_value_heads (`int`, *optional*, defaults to 8):
+            This is the number of key_value heads that should be used to implement Grouped Query Attention. If
+            `num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if
+            `num_key_value_heads=1 the model will use Multi Query Attention (MQA) otherwise GQA is used. When
+            converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed
+            by meanpooling all the original heads within that group. For more details checkout [this
+            paper](https://arxiv.org/pdf/2305.13245.pdf). If it is not specified, will default to `8`.
+        hidden_act (`str` or `function`, *optional*, defaults to `"silu"`):
+            The non-linear activation function (function or string) in the decoder.
+        max_position_embeddings (`int`, *optional*, defaults to `4096*32`):
+            The maximum sequence length that this model might ever be used with. MiniMaxM1's sliding window attention
+            allows sequence of up to 4096*32 tokens.
+        initializer_range (`float`, *optional*, defaults to 0.02):
+            The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
+        rms_norm_eps (`float`, *optional*, defaults to 1e-05):
+            The epsilon used by the rms normalization layers.
+        use_cache (`bool`, *optional*, defaults to `True`):
+            Whether or not the model should return the last key/values attentions (not used by all models). Only
+            relevant if `config.is_decoder=True`.
+        pad_token_id (`int`, *optional*):
+            The id of the padding token.
+        bos_token_id (`int`, *optional*, defaults to 1):
+            The id of the "beginning-of-sequence" token.
+        eos_token_id (`int`, *optional*, defaults to 2):
+            The id of the "end-of-sequence" token.
+        tie_word_embeddings (`bool`, *optional*, defaults to `False`):
+            Whether the model's input and output word embeddings should be tied.
+        rope_theta (`float`, *optional*, defaults to 1000000.0):
+            The base period of the RoPE embeddings.
+        sliding_window (`int`, *optional*):
+            Sliding window attention window size. If not specified, will default to `4096`.
+        attention_dropout (`float`, *optional*, defaults to 0.0):
+            The dropout ratio for the attention probabilities.
+        num_experts_per_tok (`int`, *optional*, defaults to 2):
+            The number of experts to route per-token, can be also interpreted as the `top-k` routing
+            parameter
+        num_local_experts (`int`, *optional*, defaults to 8):
+            Number of experts per Sparse MLP layer.
+        output_router_logits (`bool`, *optional*, defaults to `False`):
+            Whether or not the router logits should be returned by the model. Enabeling this will also
+            allow the model to output the auxiliary loss. See [here]() for more details
+        router_aux_loss_coef (`float`, *optional*, defaults to 0.001):
+            The aux loss factor for the total loss.
+        router_jitter_noise (`float`, *optional*, defaults to 0.0):
+            Amount of noise to add to the router.
+
+    ```python
+    >>> from transformers import MiniMaxM1Model, MiniMaxM1Config
+
+    >>> # Initializing a MiniMaxM1 style configuration
+    >>> configuration = MiniMaxM1Config()
+
+    >>> # Initializing a model from the MiniMaxM1 style configuration
+    >>> model = MiniMaxM1Model(configuration)
+
+    >>> # Accessing the model configuration
+    >>> configuration = model.config
+    ```"""
+
+    model_type = "MiniMaxM1"
+    keys_to_ignore_at_inference = ["past_key_values"]
+
+    def __init__(
+        self,
+        vocab_size=32000,
+        hidden_size=4096,
+        intermediate_size=14336,
+        num_hidden_layers=32,
+        num_attention_heads=32,
+        num_key_value_heads=8,
+        hidden_act="silu",
+        max_position_embeddings=4096 * 32,
+        initializer_range=0.02,
+        rms_norm_eps=1e-5,
+        use_cache=True,
+        pad_token_id=None,
+        bos_token_id=None,
+        eos_token_id=None,
+        tie_word_embeddings=False,
+        rope_theta=1e6,
+        sliding_window=None,
+        attention_dropout=0.0,
+        num_experts_per_tok=2,
+        num_local_experts=8,
+        output_router_logits=False,
+        router_aux_loss_coef=0.001,
+        router_jitter_noise=0.0,
+        **kwargs,
+    ):
+        self.vocab_size = vocab_size
+        self.max_position_embeddings = max_position_embeddings
+        self.hidden_size = hidden_size
+        self.intermediate_size = intermediate_size
+        self.num_hidden_layers = num_hidden_layers
+        self.num_attention_heads = num_attention_heads
+        self.sliding_window = sliding_window
+
+        # for backward compatibility
+        if num_key_value_heads is None:
+            num_key_value_heads = num_attention_heads
+
+        self.num_key_value_heads = num_key_value_heads
+        self.hidden_act = hidden_act
+        self.initializer_range = initializer_range
+        self.rms_norm_eps = rms_norm_eps
+        self.use_cache = use_cache
+        self.rope_theta = rope_theta
+        self.attention_dropout = attention_dropout
+
+        self.num_experts_per_tok = num_experts_per_tok
+        self.num_local_experts = num_local_experts
+        self.output_router_logits = output_router_logits
+        self.router_aux_loss_coef = router_aux_loss_coef
+        self.router_jitter_noise = router_jitter_noise
+        super().__init__(
+            pad_token_id=pad_token_id,
+            bos_token_id=bos_token_id,
+            eos_token_id=eos_token_id,
+            tie_word_embeddings=tie_word_embeddings,
+            **kwargs,
+        )

generation_config.json

@@ -0,0 +1,4 @@
+{
+  "_from_model_config": true,
+  "transformers_version": "4.52.4"
+}

model-00001-of-00054.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f6204b0320fe59eec5b6fd1cfb61ef8f521a74039445f7a77d9777edaf57ca46
+size 4976684776

model-00002-of-00054.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b43d307508d0dad5f5559b91f5d3b38d4019f038440e49bdbf3b1db0ae3209ec
+size 4978252672

model-00003-of-00054.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d385b4fe4b83b11226c36fe85110eddc5c35032e2ea17e3d122ea918f1e2f2d6
+size 5000041624

model-00004-of-00054.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:94917e1b2eba7ed35c1159285a43a4bb35966328ac87cc1420ddd0f4bdcd4ea0
+size 4978252664

model-00005-of-00054.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:37da1e4c4db8b6c3e8cfcbf4bed1791b6698df15d78004b96131bd36a0df3fbd
+size 5000041632

model-00006-of-00054.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9d1c06dedb64a24c85e135c8279d3bcab1e20a6713eeff89c3d8d51276d8267c
+size 4993379088

model-00007-of-00054.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0684c79ae07e7140b526464657e3e3ebcee8e263749c5ffecb7bd0cd66df136f
+size 5000041664

model-00008-of-00054.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b7e393b8de5475c218afc7bfe1ad687692129a0e592a1d6322b08337fa4eb87e
+size 4978253008

model-00009-of-00054.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d9f52c14d62245cf17e1e6e2018864f91b6f9b5762f61b5ad799304409e8cdf4
+size 5000042144

model-00010-of-00054.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7f7aa27b214cecd8eefe675a56a2065849232faec54898ae4b2536c4fea14327
+size 4978253040

model-00011-of-00054.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eff19e899974d0b33bdf329b1229555084c8d135957739086a5c69868be811f3
+size 4927603232

model-00012-of-00054.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7dec8d6cb5ee1dd99737bef7cf2d183e91c364fb111eeff9073aaba0790bdf20
+size 4978228368

model-00013-of-00054.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1be20aff0461f959b8324f7cb35c80c72d4238922cc7151a589d2dce933defd0
+size 4941673432

model-00014-of-00054.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ef0832b98e8ccfd70bbac12d2f5f44a481af8d6279a54e0ebb8979ef928820c8
+size 4978228368

model-00015-of-00054.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1b1e999865cd32638fb5e773a4e2019f303187ae06fba32e2b660d8cdf4864dd
+size 4941673432

model-00016-of-00054.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7d7b9e20f911da59215688f1727151d4d73e6d64303fd3d145cd28f0af3e82f4
+size 4993354856

model-00017-of-00054.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:21dc9070d687e0cff8f34d100848213eef8e49ec366ffe335f568b851ff0e0d6
+size 4750386400

model-00018-of-00054.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e77ccabd5c15319224bc7318d89d719751ec8839891e7db3fb247bf195dbab8e
+size 4994335232

model-00019-of-00054.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f4853ffa3871c04b86d24d5b7755e02ca501e20836b7e2042539c8f4aa63a8b6
+size 4925566568

model-00020-of-00054.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:411fd9b8c00a277ce2f3f4b038a0ffa1d79d8c88062c0625cc50bcebdd9caf24
+size 4994335232

model-00021-of-00054.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c2075c8837306dc98661824c7fc4338729f62eae0611acb718c3459f80d201c0
+size 4950716128

model-00022-of-00054.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:439c726c259b4a55c2585e2957ffc91110e1d50f686305802d609a404d059daa
+size 4984312160

model-00023-of-00054.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:adfe1bf49e812a23ad5512e3f90edf80b0f173ed93ea7c53b1844bce0c0b755d
+size 4935589632

model-00024-of-00054.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9ae749f9f6db6da7523f0434172e55aa4cb574d5d4d53aef67db2322c88b84b4
+size 4984312168

model-00025-of-00054.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f95ae73c02f70f41d77551a5f0e698bfaed8b354a484f88804d1ed91fdabf77e
+size 4935589632

model-00026-of-00054.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aabe0e9d99453c3fe54b45da0327fe8d0d65bfac3912b7e05e3b1f9050727c9f
+size 4984312168

model-00027-of-00054.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3c34182f82678d954ea128d63ed703645b3e758ded2484b148758e472348d716
+size 4993379600

model-00028-of-00054.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4e01e79df76689d26508c3199a412afa5fd77b9608273e43213618a65098e1cb
+size 5000042080

model-00029-of-00054.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5000ec848fa1903220c8b4e08d10de6ea592c9d64fdef8b14c3b617d64885393
+size 4978253104

model-00030-of-00054.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:825c337593471bd22752f7373bfa61c8038867ba43b67589ddb7a6018f9cfae7
+size 5000042088

model-00031-of-00054.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1e0c294928eae7aa4ddf901c922cc5f5ab5932ca94e4d1cb7b9ee775918da617
+size 4978253096

model-00032-of-00054.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6e37371c304394aac49c96c8246b528d9c74f703511fd02e0ef09931ca0c9e49
+size 4985971888

model-00033-of-00054.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9ca5c4283ed6ff2eada41692e414ff6c96c4dee177f1a539c906f240429180fe
+size 4978253064

model-00034-of-00054.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d7170567c428cb1476fb68d5f3af02ad719663ca647376b78173b8f4584ea814
+size 5000042128

model-00035-of-00054.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a094ac609531b7821e778509f34f0287c8c6eb5b9821ec772006320f67262746
+size 4978253056

model-00036-of-00054.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fe18d3d032892d6e0a4c3d23c88f758c8e6b71bcb5de5d11e081421326e80a26
+size 5000042128

model-00037-of-00054.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b21934a416599ed2cc252f1b2155fc7371ccdfc331ccc5c6c94a550b224db101
+size 4993379536

model-00038-of-00054.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d42cf2eb086e0235ba81c82d5be68bcae07c1d1b2f9ab06ff8fe3da2e9b240f5
+size 5000042144

model-00039-of-00054.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0ee0fa9c72e2da22bbf15e4124f619da4490f697d0acf947868158ed8e503b35
+size 4978253040

model-00040-of-00054.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bdcd6dbe350d91526d73628c332a7648290b393928d969e927b33d6afcffefb4
+size 4970870128

model-00041-of-00054.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3d922d089da27024b9d5d63b7dff99549a2b19bd4a4b6497f646c2c375b4f45c
+size 4978228368

model-00042-of-00054.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8d60910c3eb7c8b2c235e05465a5bef8a6a2fa68caa8e6abc7da74032cb7e27f
+size 4941673432

model-00043-of-00054.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f4dfb3908bb4ac04da88b1333548ab20461e9e656a7d10242f584bd5a1aea9cb
+size 4993354856