Create app.py

app.py

@@ -0,0 +1,172 @@
+# Import necessary libraries
+from threading import Thread
+import argparse
+import torch
+import gradio as gr
+from transformers import AutoTokenizer, AutoModelForSeq2SeqLM, TextIteratorStreamer, AutoModelForCausalLM
+from peft import PeftConfig, PeftModel
+from utils import get_device  # Angenommen, diese Funktion existiert bereits
+
+# Create the parser
+parser = argparse.ArgumentParser(description='Check model usage.')
+
+# Add the arguments
+parser.add_argument('--baseonly', action='store_true', 
+                    help='A boolean switch to indicate base only mode')
+
+# Execute the parse_args() method
+args = parser.parse_args()
+
+# Define model and adapter names, data type, and quantization type
+model_name = "microsoft/Phi-3-mini-4k-instruct"
+adapters_name = "zurd46/eliAI"
+torch_dtype = torch.bfloat16  # Set the appropriate torch data type
+
+# Display device and CPU thread information
+device = get_device()
+print("Running on device:", device)
+print("CPU threads:", torch.get_num_threads())
+
+# Load tokenizer
+tokenizer = AutoTokenizer.from_pretrained(model_name)
+
+# Load base model
+model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch_dtype)
+model.resize_token_embeddings(len(tokenizer))
+
+# Load adapter if available and not baseonly
+usingAdapter = False
+if not args.baseonly:
+    usingAdapter = True
+    model = PeftModel.from_pretrained(model, adapters_name)
+
+model.to(device)
+
+print(f"Model {model_name} loaded successfully on {device}")
+
+# Function to run the text generation process
+def run_generation(user_text, top_p, temperature, top_k, max_new_tokens):
+    template = "\n{}\n"
+    model_inputs = tokenizer(template.format(user_text) if usingAdapter else user_text, return_tensors="pt")
+    model_inputs = model_inputs.to(device)
+
+    # Generate text in a separate thread
+    streamer = TextIteratorStreamer(tokenizer, timeout=10., skip_prompt=True, skip_special_tokens=True)
+    generate_kwargs = dict(
+        input_ids=model_inputs['input_ids'],
+        streamer=streamer,
+        max_new_tokens=max_new_tokens,
+        do_sample=True,
+        top_p=top_p,
+        temperature=float(temperature),
+        top_k=top_k,
+        pad_token_id=tokenizer.pad_token_id,
+        eos_token_id=tokenizer.eos_token_id
+    )
+    t = Thread(target=model.generate, kwargs=generate_kwargs)
+    t.start()
+
+    # Retrieve and yield the generated text
+    model_output = ""
+    for new_text in streamer:
+        model_output += new_text
+    return model_output
+
+# Gradio UI setup
+with gr.Blocks(css="""
+    div.svelte-sfqy0y {
+        display: flex;
+        flex-direction: inherit;
+        flex-wrap: wrap;
+        gap: var(--form-gap-width);
+        box-shadow: var(--block-shadow);
+        border: var(--block-border-width) solid var(--border-color-primary);
+        border-radius: var(--block-radius);
+        background: var(--block-background-fill);
+        overflow-y: hidden;
+        padding: 20px;
+    }
+    body {
+        font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif;
+        background-color: var(--body-background-fill);
+        color: #e0e0e0;
+        margin: 0;
+        padding: 0;
+        box-sizing: border-box;
+    }
+    .gradio-container {
+        max-width: 900px;
+        margin: auto;
+        padding: 20px;
+        border-radius: 8px;
+        box-shadow: 0 0 10px rgba(0,0,0,0.5);
+        background: var(--body-background-fill);
+    }
+    .gr-button {
+        background-color: var(--block-background-fill);
+        color: white;
+        border: none;
+        border-radius: 4px;
+        padding: 10px 24px;
+        cursor: pointer;
+    }
+    .gr-button:hover {
+        background-color: #3700b3;
+    }
+    .gr-slider input[type=range] {
+        -webkit-appearance: none;
+        width: 100%;
+        height: 8px;
+        border-radius: 5px;
+        background: #333;
+        outline: none;
+        opacity: 0.9;
+        -webkit-transition: .2s;
+        transition: opacity .2s;
+    }
+    .gr-slider input[type=range]:hover {
+        opacity: 1;
+    }
+    .gr-textbox {
+        background-color: var(--block-background-fill);
+        color: white;
+        border: none;
+        border-radius: 4px;
+        padding: 10px;
+    }
+    .chatbox {
+        max-height: 400px;
+        overflow-y: auto;
+        margin-bottom: 20px;
+    }
+""") as demo:
+    gr.Markdown(
+        """
+        <div style="text-align: center; padding: 20px;">
+            <h1>🌙 eliAI Text Generation Interface</h1>
+            <h3>Model: Phi-3-mini-4k-instruct</h3>
+            <h4>Developed by Daniel Zurmühle</h4>
+        </div>
+        """)
+    
+    with gr.Row():
+        with gr.Column(scale=3):
+            user_text = gr.Textbox(placeholder="Enter your question here", label="User Input", lines=3, elem_classes="gr-textbox")
+            button_submit = gr.Button(value="Submit", elem_classes="gr-button")
+
+            max_new_tokens = gr.Slider(minimum=1, maximum=1000, value=1000, step=1, label="Max New Tokens")
+            top_p = gr.Slider(minimum=0.05, maximum=1.0, value=0.95, step=0.05, label="Top-p (Nucleus Sampling)")
+            top_k = gr.Slider(minimum=1, maximum=50, value=50, step=1, label="Top-k")
+            temperature = gr.Slider(minimum=0.1, maximum=5.0, value=0.8, step=0.1, label="Temperature")
+        
+        with gr.Column(scale=7):
+            model_output = gr.Chatbot(label="Chatbot Output", height=566)
+
+    def handle_submit(text, top_p, temperature, top_k, max_new_tokens):
+        response = run_generation(text, top_p, temperature, top_k, max_new_tokens)
+        return [(text, response)]
+
+    button_submit.click(handle_submit, [user_text, top_p, temperature, top_k, max_new_tokens], model_output)
+    user_text.submit(handle_submit, [user_text, top_p, temperature, top_k, max_new_tokens], model_output)
+
+    demo.queue(max_size=32).launch(server_name="0.0.0.0", server_port=7860)