Upload folder using huggingface_hub

added_tokens.json

@@ -0,0 +1,3 @@
+{
+  "[MASK]": 128000
+}

config.json

@@ -0,0 +1,6 @@
+{
+  "_attn_implementation_autoset": true,
+  "model_type": "ensemble-deberta",
+  "num_models": 3,
+  "transformers_version": "4.47.1"
+}

model_0/config.json

@@ -0,0 +1,46 @@
+{
+  "_name_or_path": "/content/drive/MyDrive/DeBERTa-final-65-10-25",
+  "architectures": [
+    "DebertaV2ForSequenceClassification"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "id2label": {
+    "0": "LABEL_0",
+    "1": "LABEL_1",
+    "2": "LABEL_2"
+  },
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "label2id": {
+    "LABEL_0": 0,
+    "LABEL_1": 1,
+    "LABEL_2": 2
+  },
+  "layer_norm_eps": 1e-07,
+  "legacy": true,
+  "max_position_embeddings": 512,
+  "max_relative_positions": -1,
+  "model_type": "deberta-v2",
+  "norm_rel_ebd": "layer_norm",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "pad_token_id": 0,
+  "pooler_dropout": 0,
+  "pooler_hidden_act": "gelu",
+  "pooler_hidden_size": 768,
+  "pos_att_type": [
+    "p2c",
+    "c2p"
+  ],
+  "position_biased_input": false,
+  "position_buckets": 256,
+  "relative_attention": true,
+  "share_att_key": true,
+  "torch_dtype": "float32",
+  "transformers_version": "4.47.1",
+  "type_vocab_size": 0,
+  "vocab_size": 128100
+}

model_0/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a9c7119d4c575e04ab37dae654818346d4ae5b8432eb33fccf16f681ba04595e
+size 737722356

model_1/config.json

@@ -0,0 +1,46 @@
+{
+  "_name_or_path": "/content/drive/MyDrive/DeBERTa-final-50-20-30",
+  "architectures": [
+    "DebertaV2ForSequenceClassification"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "id2label": {
+    "0": "LABEL_0",
+    "1": "LABEL_1",
+    "2": "LABEL_2"
+  },
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "label2id": {
+    "LABEL_0": 0,
+    "LABEL_1": 1,
+    "LABEL_2": 2
+  },
+  "layer_norm_eps": 1e-07,
+  "legacy": true,
+  "max_position_embeddings": 512,
+  "max_relative_positions": -1,
+  "model_type": "deberta-v2",
+  "norm_rel_ebd": "layer_norm",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "pad_token_id": 0,
+  "pooler_dropout": 0,
+  "pooler_hidden_act": "gelu",
+  "pooler_hidden_size": 768,
+  "pos_att_type": [
+    "p2c",
+    "c2p"
+  ],
+  "position_biased_input": false,
+  "position_buckets": 256,
+  "relative_attention": true,
+  "share_att_key": true,
+  "torch_dtype": "float32",
+  "transformers_version": "4.47.1",
+  "type_vocab_size": 0,
+  "vocab_size": 128100
+}

model_1/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:500d186382a08fe21b98b4ad1b92bcbdfc2f00cd3e61de5d421e516bc0b3ec97
+size 737722356

model_2/config.json

@@ -0,0 +1,46 @@
+{
+  "_name_or_path": "/content/drive/MyDrive/DeBERTa-final-33-33-33",
+  "architectures": [
+    "DebertaV2ForSequenceClassification"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "id2label": {
+    "0": "LABEL_0",
+    "1": "LABEL_1",
+    "2": "LABEL_2"
+  },
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "label2id": {
+    "LABEL_0": 0,
+    "LABEL_1": 1,
+    "LABEL_2": 2
+  },
+  "layer_norm_eps": 1e-07,
+  "legacy": true,
+  "max_position_embeddings": 512,
+  "max_relative_positions": -1,
+  "model_type": "deberta-v2",
+  "norm_rel_ebd": "layer_norm",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "pad_token_id": 0,
+  "pooler_dropout": 0,
+  "pooler_hidden_act": "gelu",
+  "pooler_hidden_size": 768,
+  "pos_att_type": [
+    "p2c",
+    "c2p"
+  ],
+  "position_biased_input": false,
+  "position_buckets": 256,
+  "relative_attention": true,
+  "share_att_key": true,
+  "torch_dtype": "float32",
+  "transformers_version": "4.47.1",
+  "type_vocab_size": 0,
+  "vocab_size": 128100
+}

model_2/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ac3e6a63390057c78f6eeef37db58396714ba022ddd62668eca0d61ea0302973
+size 737722356

modeling_ensemble_deberta.py

@@ -0,0 +1,83 @@
+class EnsembleDebertaConfig(PretrainedConfig):
+    model_type = "ensemble-deberta"
+    def __init__(self, num_models=3, **kwargs):
+        super().__init__(**kwargs)
+        self.num_models = num_models
+
+class EnsembleDeberta(PreTrainedModel):
+    config_class = EnsembleDebertaConfig
+    base_model_prefix = "ensemble-deberta"
+
+    def __init__(self, model_paths=None, config=None):
+        # If no config is provided, initialize a default one
+        if config is None:
+            config = EnsembleDebertaConfig()
+        super().__init__(config)
+
+        # Create a ModuleList to hold the individual models
+        # Can pass additional kwargs (e.g. num_labels) if needed
+        if model_paths is None:
+            raise ValueError("Please provide a list of model paths.")
+        # Load each model from the provided paths
+        self.models = nn.ModuleList([
+            DebertaV2ForSequenceClassification.from_pretrained(path)
+            for path in model_paths
+        ])
+        if len(self.models) != self.config.num_models:
+            # Update config if needed
+            self.config.num_models = len(self.models)
+
+        # Automatically load the tokenizer from the first submodel (as all submodels use the same tokenizer)
+        self.tokenizer = AutoTokenizer.from_pretrained(save_directory)
+
+    def forward(self, **kwargs):
+        '''
+        Forward pass that obtains the logits from each model, computes softmax probabilities and averages them (soft voting).
+        '''
+        # Collect softmax probabilities from each model
+        probs = []
+        for model in self.models:
+            # Each model is assumed to return a ModelOutput with a 'logits' attribute
+            outputs = model(**kwargs)
+            logits = outputs.logits  # shape: (batch_size, num_labels)
+            # Compute softmax for each model
+            probs.append(torch.nn.functional.softmax(logits, dim=-1))
+
+        # Stack and average the probabilities
+        avg_probs = torch.stack(probs, dim=0).mean(dim=0)
+        return avg_probs
+
+    def save_pretrained(self, save_directory, **kwargs):
+        '''
+        Saves the ensemble model and its tokenizer. Each submodel is saved in its own subdirectory.
+        '''
+        os.makedirs(save_directory, exist_ok=True)
+        # Save the ensemble configuration
+        self.config.save_pretrained(save_directory)
+
+        # Save each submodel to its own subdirectory
+        for idx, model in enumerate(self.models):
+            sub_dir = os.path.join(save_directory, f"model_{idx}")
+            model.save_pretrained(sub_dir)
+
+        # Save the tokenizer
+        self.tokenizer.save_pretrained(save_directory)
+
+        print(f"Ensemble saved to {save_directory}.")
+
+    @classmethod
+    def from_pretrained(cls, save_directory, **kwargs):
+        '''
+        Loads the ensemble model.
+        '''
+        # Load the ensemble configuration
+        config = EnsembleDebertaConfig.from_pretrained(save_directory)
+        num_models = config.num_models
+
+        # Determine the paths for each submodel
+        model_paths = [os.path.join(save_directory, f"model_{idx}") for idx in range(num_models)]
+
+        # Initialize the ensemble
+        ensemble = cls(model_paths=model_paths, config=config)
+
+        return ensemble

special_tokens_map.json

@@ -0,0 +1,51 @@
+{
+  "bos_token": {
+    "content": "[CLS]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "cls_token": {
+    "content": "[CLS]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "[SEP]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "mask_token": {
+    "content": "[MASK]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "[PAD]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "sep_token": {
+    "content": "[SEP]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "[UNK]",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  }
+}

spm.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c679fbf93643d19aab7ee10c0b99e460bdbc02fedf34b92b05af343b4af586fd
+size 2464616

tokenizer_config.json

@@ -0,0 +1,66 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "[PAD]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "[CLS]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "[SEP]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "[UNK]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128000": {
+      "content": "[MASK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "[CLS]",
+  "clean_up_tokenization_spaces": false,
+  "cls_token": "[CLS]",
+  "do_lower_case": false,
+  "eos_token": "[SEP]",
+  "extra_special_tokens": {},
+  "mask_token": "[MASK]",
+  "max_length": 256,
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_to_multiple_of": null,
+  "pad_token": "[PAD]",
+  "pad_token_type_id": 0,
+  "padding_side": "right",
+  "sep_token": "[SEP]",
+  "sp_model_kwargs": {},
+  "split_by_punct": false,
+  "stride": 0,
+  "tokenizer_class": "DebertaV2Tokenizer",
+  "truncation_side": "right",
+  "truncation_strategy": "longest_first",
+  "unk_token": "[UNK]",
+  "vocab_type": "spm"
+}

transformers.py

@@ -0,0 +1,6 @@
+from .modeling_ensemble_deberta import EnsembleDeberta, EnsembleDebertaConfig
+from transformers import AutoConfig, AutoModel
+
+# Register the custom config and model with the auto classes.
+AutoConfig.register("ensemble-deberta", EnsembleDebertaConfig)
+AutoModel.register(EnsembleDebertaConfig, EnsembleDeberta)