File size: 1,245 Bytes
ea5cbbd a808e98 ea5cbbd a808e98 ea5cbbd a808e98 2c167ff a808e98 ea5cbbd a808e98 9d41ff0 ea5cbbd a808e98 ea5cbbd a808e98 2c167ff ea5cbbd a808e98 ea5cbbd 2c167ff a808e98 2c167ff f4dd327 2c167ff ea5cbbd a808e98 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 |
---
license: apache-2.0
---
## INFERENCE CODE
```bash
pip install transformers[torch]
```
```python
from transformers import AutoTokenizer, AutoModelForCausalLM, TextStreamer
import torch
import time
tokenizer = AutoTokenizer.from_pretrained("AquilaX-AI/DB-Summarizer")
model = AutoModelForCausalLM.from_pretrained("AquilaX-AI/DB-Summarizer")
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
question = "How many Vulnerability found today"
db_result = "243"
summ_inp = f"""<|im_start|>system
Generate a clear and accurate response based on the user's question and the database output.<|im_end|>
<|im_start|>user
user_question:
{question}
db_response:
{db_result}<|im_end|>
<|im_start|>assistant"""
import time
start = time.time()
encodeds = tokenizer(summ_inp, return_tensors="pt",truncation=True).input_ids.to(device)
model.to(device)
inputs = inputs.to(device)
text_streamer = TextStreamer(tokenizer, skip_prompt = True)
response = model.generate(
input_ids=encodeds,
streamer=text_streamer,
max_new_tokens=512,
use_cache=True,
pad_token_id=151645,
eos_token_id=151645,
num_return_sequences=1
)
end = time.time()
print(f"Time taken: {end - start}")
``` |