Merge branch 'main' of into integrations/sentence_transformers

README.md

@@ -2666,12 +2666,15 @@ Training data to train the models is released in its entirety. For more details,
 
 ## Usage
 
+Note `nomic-embed-text` requires prefixes! We support the prefixes `[search_query, search_document, classification, clustering]`.
+For retrieval applications, you should prepend `search_document` for all your documents and `search_query` for your queries.
+
 ### Sentence Transformers
 ```python
 from sentence_transformers import SentenceTransformer
 
 model = SentenceTransformer("nomic-ai/nomic-embed-text-v1-unsupervised", trust_remote_code=True)
-sentences = ['What is TSNE?', 'Who is Laurens van der Maaten?']
+sentences = ['search_query: What is TSNE?', 'search_query: Who is Laurens van der Maaten?']
 embeddings = model.encode(sentences)
 print(embeddings)
 ```
@@ -2687,7 +2690,7 @@ def mean_pooling(model_output, attention_mask):
     input_mask_expanded = attention_mask.unsqueeze(-1).expand(token_embeddings.size()).float()
     return torch.sum(token_embeddings * input_mask_expanded, 1) / torch.clamp(input_mask_expanded.sum(1), min=1e-9)
 
-sentences = ['What is TSNE?', 'Who is Laurens van der Maaten?']
+sentences = ['search_query: What is TSNE?', 'search_query: Who is Laurens van der Maaten?']
 
 tokenizer = AutoTokenizer.from_pretrained('bert-base-uncased')
 model = AutoModel.from_pretrained('nomic-ai/nomic-embed-text-v1-unsupervised', trust_remote_code=True)

config.json

@@ -11,7 +11,7 @@
   "bos_token_id": null,
   "causal": false,
   "dense_seq_output": true,
-  "embd_pdrop": 0.0,
+  "embd_pdrop": 0.1,
   "eos_token_id": null,
   "fused_bias_fc": true,
   "fused_dropout_add_ln": true,
@@ -31,7 +31,7 @@
   "prenorm": false,
   "qkv_proj_bias": false,
   "reorder_and_upcast_attn": false,
-  "resid_pdrop": 0.0,
+  "resid_pdrop": 0.1,
   "rotary_emb_base": 1000,
   "rotary_emb_fraction": 1.0,
   "rotary_emb_interleaved": false,
@@ -40,7 +40,7 @@
   "scale_attn_by_inverse_layer_idx": false,
   "scale_attn_weights": true,
   "summary_activation": null,
-  "summary_first_dropout": 0.0,
+  "summary_first_dropout": 0.1,
   "summary_proj_to_labels": true,
   "summary_type": "cls_index",
   "summary_use_proj": true,