Upload model

adapter_config.json

@@ -0,0 +1,32 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "./checkpoints/mcq_hf_model",
+  "bias": "none",
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_dropout": 0.05,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 16,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "attn.out_proj",
+    "classifier",
+    "ffn.proj_2",
+    "ffn.proj_1",
+    "attn.qkv_proj"
+  ],
+  "task_type": "CAUSAL_LM",
+  "use_dora": false,
+  "use_rslora": false
+}

adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:24b4c7db13193697e22635e38f93429c53087f5661d55ae54fdc474351a6603d
+size 24985072

model.py

@@ -0,0 +1,95 @@
+import torch
+import torch.nn as nn
+from transformers import (
+    PreTrainedModel,
+    PretrainedConfig,
+    AutoConfig,
+    AutoModel,
+    AutoModelForCausalLM,
+    AutoTokenizer,
+)
+
+import pandas as pd
+from datasets import load_dataset, Dataset, DatasetDict
+
+
+class MCQModel(nn.Module):
+    def __init__(self, name_model):
+        super(MCQModel, self).__init__()
+        self.model = AutoModelForCausalLM.from_pretrained(
+            name_model,
+            trust_remote_code=True,
+            output_hidden_states=True,
+        )
+        self.classifier = nn.Linear(
+            self.model.config.model_dim, 4
+        )  # 4 classes for 'A', 'B', 'C', 'D'
+
+    def forward(self, input_ids, attention_mask=None, labels=None,
+        position_ids = None,
+        past_key_values = None,
+        inputs_embeds = None,
+        use_cache  = None,
+        output_attentions  = None,
+        output_hidden_states = None,
+        return_dict  = None,
+        cache_position = None):
+
+        print("INSIDE CUSTOM MODEL LABELS: ", labels)
+        outputs = self.model(input_ids, attention_mask=attention_mask, position_ids= position_ids, past_key_values=past_key_values, inputs_embeds=inputs_embeds,
+                              output_attentions=output_attentions, cache_position=cache_position, use_cache=use_cache, return_dict=return_dict, output_hidden_states=output_hidden_states)
+        print("OUTPUT KEYS: " , outputs.keys())
+        # print(outputs.logits.shape)
+        # # print(outputs.hidden_states)
+        # print(outputs.hidden_states[0].shape)
+        # print(outputs.hidden_states[1].shape)
+        # print(len(outputs.hidden_states))
+
+        # hidden state is a tuple with all the hidden layer outputs from the attention,
+        # We are only interested in the last hidden layer and the last token
+        logits = self.classifier(outputs.hidden_states[-1][:, -1, :])
+        outputs.logits = logits
+        loss = None
+        if labels is not None:
+            loss_fct = nn.CrossEntropyLoss()
+            loss = loss_fct(
+                logits, labels
+            )  # labels [batch_size], logits [batch_size x num_classes]
+            # print("LOSS", loss)
+            outputs["loss"] = loss
+
+        print("===================")
+        print(loss)
+        print("===================")
+        # print("OUTPUTS KEY" ,outputs.keys())
+        return outputs
+
+
+class MyCustomConfig(PretrainedConfig):
+    model_type = "mcq_hf_model"
+
+    def __init__(self, name_model="apple/OpenELM-450M-Instruct", **kwargs):
+        super().__init__(**kwargs)
+        self.name_model = name_model
+
+
+class MCQHFModel(PreTrainedModel):
+    config_class = MyCustomConfig
+
+    def __init__(self, config):
+        super().__init__(config)
+        self.model = MCQModel(config.name_model)
+        self.lm_head = None
+
+    def forward(self, input_ids, attention_mask=None, labels=None,
+        position_ids = None,
+        past_key_values = None,
+        inputs_embeds = None,
+        use_cache  = None,
+        output_attentions  = None,
+        output_hidden_states = None,
+        return_dict  = None,
+        cache_position = None):
+
+        return self.model(input_ids, labels=labels, attention_mask=attention_mask, position_ids= position_ids, past_key_values=past_key_values, inputs_embeds=inputs_embeds,
+                            output_attentions=output_attentions, cache_position=cache_position, use_cache=use_cache, return_dict=return_dict, output_hidden_states=output_hidden_states)