File size: 1,484 Bytes
38c936b 82f8eb5 38c936b d034337 ff740be 694ef91 8cd33e6 ff740be 38c936b 82f8eb5 38c936b 82f8eb5 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 |
#pip install fastapi
#uvicorn main:app --reload
#import gradio as gr
import torch
from transformers import pipeline
from fastapi import FastAPI
app = FastAPI()
#generator = pipeline('text-generation',model='gpt2')
#generator = pipeline('text-generation',model='Open-Orca/Mistral-7B-OpenOrca')
#generator = pipeline("text-generation", model="TheBloke/zephyr-7B-alpha-GGUF")
#model = AutoModel.from_pretrained("TheBloke/zephyr-7B-alpha-GGUF")
pipe = pipeline("text-generation", model="HuggingFaceH4/zephyr-7b-alpha", torch_dtype=torch.bfloat16, device_map="auto")
# We use the tokenizer's chat template to format each message - see https://huggingface.co/docs/transformers/main/en/chat_templating
messages = [
{
"role": "system",
"content": "You are a Spiritual Coach who always responds in the most profound and poetic style",
},
{"role": "user", "content": "What is Life?"},
]
prompt = pipe.tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
outputs = pipe(prompt, max_new_tokens=2560, do_sample=True, temperature=0.7, top_k=50, top_p=0.95)
print(outputs[0]["generated_text"])
@app.get("/")
async def root():
return {"message": "Hello World"}
#return generator('What is love',max_length=100, num_return_sequences=1)
@app.post("/predict")
async def root(text):
#return {"message": "Hello World"}
#return generator(text,max_length=2560, num_return_sequences=1)
return outputs[0]["generated_text"] |