import re import gradio as gr from llama_cpp import Llama model = "ggml-org/gemma-3-1b-it-GGUF" llm = Llama.from_pretrained( repo_id=model, filename="gemma-3-1b-it-Q8_0.gguf", verbose=True, use_mmap=True, use_mlock=True, n_threads=4, n_threads_batch=4, n_ctx=8000, ) def respond( message, history: list[tuple[str, str]], system_message, max_tokens, temperature, top_p, ): if len(system_message) > 0: messages = [{"role": "system", "content": system_message}] else: messages = [] for val in history: if val[0]: messages.append({"role": "user", "content": val[0]}) if val[1]: messages.append({"role": "assistant", "content": val[1]}) messages.append({"role": "user", "content": message}) response = "" completion = llm.create_chat_completion( messages, max_tokens=max_tokens, stream=True, temperature=temperature, top_p=top_p ) for message in completion: delta = message['choices'][0]['delta'] if 'content' in delta: response += delta['content'] formatted_response = re.sub(r"\s*(.*?)\s*", r"*\1*", response, flags=re.DOTALL) yield formatted_response demo = gr.ChatInterface( respond, additional_inputs=[ gr.Textbox( value="", label="System message", ), gr.Slider(minimum=200, maximum=100000, value=4000, step=100, label="Max new tokens"), gr.Slider(minimum=0.1, maximum=4.0, value=0.6, step=0.1, label="Temperature"), gr.Slider( minimum=0.1, maximum=1.0, value=0.95, step=0.05, label="Top-p (nucleus sampling)", ), ], description=model, ) if __name__ == "__main__": demo.launch()