solar

question_paper.py

@@ -1,25 +1,18 @@
 import transformers
 import torch
-import os
 
 from fastapi import FastAPI
 
-from llama_cpp import Llama
+from transformers import AutoModelForCausalLM, AutoTokenizer
 
 app = FastAPI()
 @app.get("/")
 def llama():
-    llm = Llama(
-      model_path="./llama-2-7b-chat.Q2_K.gguf"
-      # n_gpu_layers=-1, # Uncomment to use GPU acceleration
-      # seed=1337, # Uncomment to set a specific seed
-      # n_ctx=2048, # Uncomment to increase the context window
-    )
+    tokenizer = AutoTokenizer.from_pretrained("Upstage/SOLAR-10.7B-v1.0")
+    model = AutoModelForCausalLM.from_pretrained("Upstage/SOLAR-10.7B-v1.0", device_map="auto", torch_dtype=torch.float16,)
+    text = "Hi, my name is "
+    inputs = tokenizer(text, return_tensors="pt")
+    outputs = model.generate(**inputs, max_new_tokens=64)
+    print(tokenizer.decode(outputs[0], skip_special_tokens=True))
 
-    output = llm(
-      "Q: Name the planets in the solar system? A: ", # Prompt
-      max_tokens=32, # Generate up to 32 tokens, set to None to generate up to the end of the context window
-      echo=True # Echo the prompt back in the output
-    ) # Generate a completion, can also call create_completion
-
-    return output["choices"][0]["text"].strip()
+    return tokenizer.decode(outputs[0], skip_special_tokens=True)