Spaces:
Sleeping
Sleeping
File size: 2,706 Bytes
0df8e2c 3543359 0df8e2c 3543359 0df8e2c 3543359 0df8e2c 3543359 0df8e2c 3543359 0df8e2c 3543359 0df8e2c 3543359 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 |
import os
from fastapi import FastAPI
from pydantic import BaseModel
import gradio as gr
import threading
import uvicorn
# =======================
# Load Secrets
# =======================
SYSTEM_PROMPT = os.environ.get(
"prompt",
"You are a placeholder Sovereign. No secrets found in environment."
)
# =======================
# Initialize Unsloth-optimized Falcon-3B
# =======================
# Install via: pip install unsloth torch transformers
from unsloth import FastLanguageModel
from transformers import AutoTokenizer
MODEL_NAME = "tiiuae/Falcon3-3B-Instruct"
# 1) Load model and tokenizer with 4-bit quantization
model, tokenizer = FastLanguageModel.from_pretrained(
model_name=MODEL_NAME,
max_seq_length=2048,
load_in_4bit=True,
dtype=None,
)
# 2) Apply inference optimizations (fused kernels, streaming, etc.)
FastLanguageModel.for_inference(model)
# =======================
# Core Chat Function
# =======================
def chat_fn(user_input: str) -> str:
messages = [
{"role": "system", "content": SYSTEM_PROMPT},
{"role": "user", "content": f"User: {user_input}"}
]
prompt_text = "\n".join(f"{m['role'].capitalize()}: {m['content']}" for m in messages)
# Tokenize and run generation
inputs = tokenizer(prompt_text, return_tensors="pt").to(model.device)
output_ids = model.generate(
**inputs,
max_new_tokens=256,
do_sample=False,
eos_token_id=tokenizer.eos_token_id
)
# Decode only the newly generated tokens
gen_tokens = output_ids[0][inputs.input_ids.shape[-1]:]
generated_text = tokenizer.decode(gen_tokens, skip_special_tokens=True)
return generated_text.strip()
# =======================
# Gradio UI
# =======================
def gradio_chat(user_input: str) -> str:
return chat_fn(user_input)
iface = gr.Interface(
fn=gradio_chat,
inputs=gr.Textbox(lines=5, placeholder="Enter your prompt…"),
outputs="text",
title="Prompt cracking challenge",
description="Does he really think he is the king?"
)
# Run Gradio in a separate thread so FastAPI can also start
def run_gradio():
iface.launch(server_name="0.0.0.0", share=True)
# =======================
# FastAPI for API access
# =======================
app = FastAPI(title="Prompt cracking challenge API")
class Request(BaseModel):
prompt: str
@app.post("/generate")
def generate(req: Request):
return {"response": chat_fn(req.prompt)}
# =======================
# Launch Both Servers
# =======================
if __name__ == "__main__":
threading.Thread(target=run_gradio, daemon=True).start()
uvicorn.run(app, host="0.0.0.0", port=int(os.getenv("PORT", 8000))) |