Upload 8 files

README.md

@@ -38,7 +38,7 @@ Here is a simple code for inference:
 from transformers import AutoModelForCausalLM, AutoTokenizer
 import torch
 
-model_name = 'plant-dnabert-6mer'
+model_name = 'plant-dnagemma-6mer'
 # load model and tokenizer
 model = AutoModelForCausalLM.from_pretrained(f'zhangtaolab/{model_name}', trust_remote_code=True)
 tokenizer = AutoTokenizer.from_pretrained(f'zhangtaolab/{model_name}', trust_remote_code=True)
@@ -71,4 +71,4 @@ Detailed training procedure can be found in our manuscript.
 
 
 #### Hardware
-Model was pre-trained on a NVIDIA RTX4090 GPU (24 GB).
+Model was pre-trained on a NVIDIA RTX4090 GPU (24 GB).

config.json

@@ -1,25 +1,29 @@
 {
-  "_name_or_path": "../model/PlantDna_BERT_6mer",
+  "_name_or_path": "Plant_DNAGemma",
   "architectures": [
-    "BertForMaskedLM"
+    "GemmaForCausalLM"
   ],
-  "attention_probs_dropout_prob": 0.1,
-  "classifier_dropout": null,
-  "hidden_act": "gelu",
-  "hidden_dropout_prob": 0.1,
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 2,
+  "eos_token_id": 1,
+  "head_dim": 256,
+  "hidden_act": "gelu_pytorch_tanh",
+  "hidden_activation": "gelu_pytorch_tanh",
   "hidden_size": 768,
   "initializer_range": 0.02,
   "intermediate_size": 3072,
-  "layer_norm_eps": 1e-12,
-  "max_position_embeddings": 512,
-  "model_type": "bert",
+  "max_position_embeddings": 1024,
+  "model_type": "gemma",
   "num_attention_heads": 12,
   "num_hidden_layers": 12,
+  "num_key_value_heads": 1,
   "pad_token_id": 0,
-  "position_embedding_type": "absolute",
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 10000.0,
   "torch_dtype": "float32",
   "transformers_version": "4.39.1",
-  "type_vocab_size": 2,
   "use_cache": true,
   "vocab_size": 4107
 }

generation_config.json

@@ -1,5 +1,7 @@
 {
   "_from_model_config": true,
+  "bos_token_id": 0,
+  "eos_token_id": 0,
   "pad_token_id": 0,
   "transformers_version": "4.39.1"
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3b5b40af2672337c6e25fde42af91a23fb3e4b37c03a349dd0392e6c4f2dfbf3
-size 356828260
+oid sha256:5c5437fe24770e71d4a8f4e3eedb4ef0f7309b9fbb96696e2f35bdc12fb121cf
+size 597811128

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2a0e12c47d51f3d2556da5976710c4c2fc05010a1677b147c1f561587df75af9
-size 356871026
+oid sha256:e1db4c29e12eef1f8e56e52f7d79f14c7ea92bc5c1c2f9f737d66e0dffb1aa1f
+size 597834498