File size: 1,141 Bytes
024b2cf b5beca4 327897b b6d62ff 008c6be b6d62ff 327897b b6d62ff 008c6be 582af9b |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 |
8 bit bge-reranker-v2-m3.
!pip install "transformers==4.35" "bitsandbytes" "optimum" "datasets==2.13.0" "peft==0.9.0" "accelerate==0.27.1" "bitsandbytes==0.40.2" "trl==0.4.7" "safetensors>=0.3.1" "tiktoken"
# Load and compute scores with the quantized model
```
import torch
from transformers import AutoModelForSequenceClassification, AutoTokenizer, BitsAndBytesConfig
def load_and_compute_scores_with_quantized_model(model_path):
tokenizer = AutoTokenizer.from_pretrained(model_path)
model = AutoModelForSequenceClassification.from_pretrained(model_path)
def compute_score(pairs):
inputs = tokenizer(pairs, padding=True, truncation=True, return_tensors='pt')
with torch.no_grad():
outputs = model(**inputs)
return outputs.logits
scores = compute_score([['what is panda?', 'hi'], ['what is panda?', 'The giant panda (Ailuropoda melanoleuca), sometimes called a panda bear or simply panda, is a bear species endemic to China.']])
print("Scores:", scores)
quantized_model_path = "quantized_bge_reranker_v2_m3"
load_and_compute_scores_with_quantized_model(quantized_model_path)
``` |