app.py

import gradio as gr
from functools import lru_cache
import random
import requests
import logging
import re
import config
import plotly.graph_objects as go
from typing import Dict
import json
import os
from leaderboard import (
    get_current_leaderboard,
    update_leaderboard, 
    start_backup_thread, 
    get_leaderboard, 
    get_elo_leaderboard,
    ensure_elo_ratings_initialized
)
import sys
import openai
import threading
import time
from collections import Counter
from release_notes import get_release_notes_html


# Update the logging format to redact URLs
logging.basicConfig(
    level=logging.WARNING,  # Only show warnings and errors
    format='%(asctime)s - %(levelname)s - %(message)s'
)

# Suppress verbose HTTP request logging
logging.getLogger("urllib3").setLevel(logging.CRITICAL)
logging.getLogger("httpx").setLevel(logging.CRITICAL)
logging.getLogger("openai").setLevel(logging.CRITICAL)

class RedactURLsFilter(logging.Filter):
    def filter(self, record):
        # Redact all URLs using regex pattern
        url_pattern = r'http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\\(\\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+'
        record.msg = re.sub(url_pattern, '[REDACTED_URL]', str(record.msg))
        
        # Remove HTTP status codes
        record.msg = re.sub(r'HTTP/\d\.\d \d+ \w+', '', record.msg)
        
        # Remove sensitive API references
        record.msg = record.msg.replace(config.API_URL, '[API]')
        record.msg = record.msg.replace(config.NEXTCLOUD_URL, '[CLOUD]')
        
        # Clean up residual artifacts
        record.msg = re.sub(r'\s+', ' ', record.msg).strip()
        record.msg = re.sub(r'("?) \1', '', record.msg)  # Remove empty quotes
        
        return True

# Apply the filter to all handlers
logger = logging.getLogger(__name__)
for handler in logging.root.handlers:
    handler.addFilter(RedactURLsFilter())

# Start the backup thread
start_backup_thread()

# Function to get available models (using predefined list)
def get_available_models():
    return [model[0] for model in config.get_approved_models()]

# Function to get recent opponents for a model
recent_opponents = {}

def update_recent_opponents(model_a, model_b):
    recent_opponents.setdefault(model_a, []).append(model_b)
    recent_opponents.setdefault(model_b, []).append(model_a)
    # Limit history to last 5 opponents
    recent_opponents[model_a] = recent_opponents[model_a][-5:]
    recent_opponents[model_b] = recent_opponents[model_b][-5:]

# Function to call Ollama API with caching
@lru_cache(maxsize=100)
def call_ollama_api(model, prompt):
    client = openai.OpenAI(
        api_key=config.API_KEY,
        base_url=config.API_URL
    )
    
    try:
        logger.info(f"Starting API call for model: {model}")
        response = client.chat.completions.create(
            model=model,
            messages=[
                {
                    "role": "system",
                    "content": "You are a helpful assistant. At no point should you reveal your name, identity or team affiliation to the user, especially if asked directly!"
                },
                {
                    "role": "user",
                    "content": prompt
                }
            ],
            timeout=180
        )
        logger.info(f"Received response for model: {model}")
        
        if not response or not response.choices:
            logger.error(f"Empty response received for model: {model}")
            return [
                {"role": "user", "content": prompt},
                {"role": "assistant", "content": "Error: Empty response from the model"}
            ]
            
        content = response.choices[0].message.content
        if not content:
            logger.error(f"Empty content received for model: {model}")
            return [
                {"role": "user", "content": prompt},
                {"role": "assistant", "content": "Error: Empty content from the model"}
            ]
        
        # Extract thinking part and main content using regex
        thinking_match = re.search(r'<think>(.*?)</think>', content, flags=re.DOTALL)
        
        if thinking_match:
            thinking_content = thinking_match.group(1).strip()
            main_content = re.sub(r'<think>.*?</think>', '', content, flags=re.DOTALL).strip()
            
            logger.info(f"Found thinking content for model: {model}")
            return [
                {"role": "user", "content": prompt},
                {"role": "assistant", "content": f"{main_content}\n\n<details><summary>🤔 View thinking process</summary>\n\n{thinking_content}\n\n</details>"}
            ]
        
        # If no thinking tags, return normal content
        logger.info(f"No thinking tags found for model: {model}")
        return [
            {"role": "user", "content": prompt},
            {"role": "assistant", "content": content.strip()}
        ]
        
    except requests.exceptions.Timeout:
        logger.error(f"Timeout error after 180 seconds for model: {model}")
        return [
            {"role": "user", "content": prompt},
            {"role": "assistant", "content": "Error: Model response timed out after 180 seconds"}
        ]
    except openai.BadRequestError as e:
        error_msg = str(e)
        logger.error(f"Bad request error for model: {model}. Error: {error_msg}")
        return [
            {"role": "user", "content": prompt},
            {"role": "assistant", "content": "Error: Unable to get response from the model"}
        ]
    except Exception as e:
        logger.error(f"Error calling Ollama API for model: {model}. Error: {str(e)}", exc_info=True)
        return [
            {"role": "user", "content": prompt},
            {"role": "assistant", "content": "Error: Unable to get response from the model"}
        ]

# Generate responses using two randomly selected models
def get_battle_counts():
    leaderboard = get_current_leaderboard()
    battle_counts = Counter()
    for model, data in leaderboard.items():
        battle_counts[model] = data['wins'] + data['losses']
    return battle_counts

def generate_responses(prompt):
    available_models = get_available_models()
    if len(available_models) < 2:
        return [
            {"role": "user", "content": prompt},
            {"role": "assistant", "content": "Error: Not enough models available"}
        ], [
            {"role": "user", "content": prompt},
            {"role": "assistant", "content": "Error: Not enough models available"}
        ], None, None
    
    battle_counts = get_battle_counts()
    
    # Sort models by battle count (ascending)
    sorted_models = sorted(available_models, key=lambda m: battle_counts.get(m, 0))
    
    # Select the first model (least battles)
    model_a = sorted_models[0]
    
    # Filter out recent opponents for model_a
    potential_opponents = [m for m in sorted_models[1:] if m not in recent_opponents.get(model_a, [])]
    
    # If no potential opponents left, reset recent opponents for model_a
    if not potential_opponents:
        recent_opponents[model_a] = []
        potential_opponents = sorted_models[1:]
    
    # For the second model, use weighted random selection
    weights = [1 / (battle_counts.get(m, 1) + 1) for m in potential_opponents]
    model_b = random.choices(potential_opponents, weights=weights, k=1)[0]
    
    # Update recent opponents
    update_recent_opponents(model_a, model_b)
    
    # Get responses from both models
    response_a = call_ollama_api(model_a, prompt)
    response_b = call_ollama_api(model_b, prompt)
    
    # Return responses directly (already formatted correctly)
    return response_a, response_b, model_a, model_b

def battle_arena(prompt):
    response_a, response_b, model_a, model_b = generate_responses(prompt)
    
    # Check for API errors in responses
    if any("Error: Unable to get response from the model" in msg["content"]
           for msg in response_a + response_b 
           if msg["role"] == "assistant"):
        return (
            [], [], None, None,
            gr.update(value=[]),
            gr.update(value=[]),
            gr.update(interactive=False, value="Voting Disabled - API Error"),
            gr.update(interactive=False, value="Voting Disabled - API Error"),
            gr.update(interactive=False, visible=False),
            prompt,
            0,
            gr.update(visible=False),
            gr.update(value="Error: Unable to get response from the model", visible=True)
        )
    
    nickname_a = random.choice(config.model_nicknames)
    nickname_b = random.choice(config.model_nicknames)
    
    # The responses are already in the correct format, no need to reformat
    if random.choice([True, False]):
        return (
            response_a, response_b, model_a, model_b,
            gr.update(label=nickname_a, value=response_a),
            gr.update(label=nickname_b, value=response_b),
            gr.update(interactive=True, value=f"Vote for {nickname_a}"),
            gr.update(interactive=True, value=f"Vote for {nickname_b}"),
            gr.update(interactive=True, visible=True),
            prompt,
            0,
            gr.update(visible=False),
            gr.update(value="Ready for your vote! 🗳️", visible=True)
        )
    else:
        return (
            response_b, response_a, model_b, model_a,
            gr.update(label=nickname_a, value=response_b),
            gr.update(label=nickname_b, value=response_a),
            gr.update(interactive=True, value=f"Vote for {nickname_a}"),
            gr.update(interactive=True, value=f"Vote for {nickname_b}"),
            gr.update(interactive=True, visible=True),
            prompt,
            0,
            gr.update(visible=False),
            gr.update(value="Ready for your vote! 🗳️", visible=True)
        )

def record_vote(prompt, left_response, right_response, left_model, right_model, choice):
    # Check if outputs are generated
    if not left_response or not right_response or not left_model or not right_model:
        return (
            "Please generate responses before voting.", 
            gr.update(), 
            gr.update(interactive=False), 
            gr.update(interactive=False), 
            gr.update(visible=False), 
            gr.update()
        )
    
    winner = left_model if choice == "Left is better" else right_model
    loser = right_model if choice == "Left is better" else left_model
    
    # Update the leaderboard
    battle_results = update_leaderboard(winner, loser)
    
    result_message = f"""
🎉 Vote recorded! You're awesome! 🌟
🔵 In the left corner: {get_human_readable_name(left_model)}
🔴 In the right corner: {get_human_readable_name(right_model)}
🏆 And the champion you picked is... {get_human_readable_name(winner)}! 🥇
    """
    
    return (
        gr.update(value=result_message, visible=True),  # Show result as Markdown
        get_leaderboard(),                              # Update leaderboard
        get_elo_leaderboard(),                         # Update ELO leaderboard
        gr.update(interactive=False),                   # Disable left vote button
        gr.update(interactive=False),                   # Disable right vote button
        gr.update(interactive=False),                   # Disable tie button
        gr.update(visible=True)                         # Show model names
    )

def get_leaderboard_chart():
    battle_results = get_current_leaderboard()
    
    # Calculate scores and sort results
    for model, results in battle_results.items():
        total_battles = results["wins"] + results["losses"]
        if total_battles > 0:
            win_rate = results["wins"] / total_battles
            results["score"] = win_rate * (1 - 1 / (total_battles + 1))
        else:
            results["score"] = 0
    
    sorted_results = sorted(
        battle_results.items(), 
        key=lambda x: (x[1]["score"], x[1]["wins"] + x[1]["losses"]), 
        reverse=True
    )

    models = [get_human_readable_name(model) for model, _ in sorted_results]
    wins = [results["wins"] for _, results in sorted_results]
    losses = [results["losses"] for _, results in sorted_results]
    scores = [results["score"] for _, results in sorted_results]

    fig = go.Figure()

    # Stacked Bar chart for Wins and Losses
    fig.add_trace(go.Bar(
        x=models,
        y=wins,
        name='Wins',
        marker_color='#22577a'
    ))
    fig.add_trace(go.Bar(
        x=models,
        y=losses,
        name='Losses',
        marker_color='#38a3a5'
    ))

    # Line chart for Scores
    fig.add_trace(go.Scatter(
        x=models,
        y=scores,
        name='Score',
        yaxis='y2',
        line=dict(color='#ff7f0e', width=2)
    ))

    # Update layout for full-width, increased height, and secondary y-axis
    fig.update_layout(
        title='Model Performance',
        xaxis_title='Models',
        yaxis_title='Number of Battles',
        yaxis2=dict(
            title='Score',
            overlaying='y',
            side='right'
        ),
        barmode='stack',
        height=800,
        width=1450,
        autosize=True,
        legend=dict(
            orientation='h',
            yanchor='bottom',
            y=1.02,
            xanchor='right',
            x=1
        )
    )

    chart_data = fig.to_json()
    return fig

def new_battle():
    nickname_a = random.choice(config.model_nicknames)
    nickname_b = random.choice(config.model_nicknames)
    return (
        "", # Reset prompt_input
        gr.update(value=[], label=nickname_a),  # Reset left Chatbot
        gr.update(value=[], label=nickname_b),  # Reset right Chatbot
        None,
        None,
        gr.update(interactive=False, value=f"Vote for {nickname_a}"),
        gr.update(interactive=False, value=f"Vote for {nickname_b}"),
        gr.update(interactive=False, visible=False),  # Reset Tie button
        gr.update(value="", visible=False),
        gr.update(),
        gr.update(visible=False),
        gr.update(),
        0  # Reset tie_count
    )

# Add this new function
def get_human_readable_name(model_name: str) -> str:
    model_dict = dict(config.get_approved_models())
    return model_dict.get(model_name, model_name)

# Add this new function to randomly select a prompt
def random_prompt():
    return random.choice(config.example_prompts)

# Modify the continue_conversation function
def continue_conversation(prompt, left_chat, right_chat, left_model, right_model, previous_prompt, tie_count):
    # Check if the prompt is empty or the same as the previous one
    if not prompt or prompt == previous_prompt:
        prompt = random.choice(config.example_prompts)
    
    # Get responses (which are lists of messages)
    left_response = call_ollama_api(left_model, prompt)
    right_response = call_ollama_api(right_model, prompt)
    
    # Append messages from the response lists
    left_chat.extend(left_response)
    right_chat.extend(right_response)
    
    tie_count += 1
    tie_button_state = gr.update(interactive=True) if tie_count < 3 else gr.update(interactive=False, value="Max ties reached. Please vote!")
    
    return (
        gr.update(value=left_chat),
        gr.update(value=right_chat),
        gr.update(value=""),  # Clear the prompt input
        tie_button_state,
        prompt,  # Return the new prompt
        tie_count
    )

def normalize_parameter_size(param_size: str) -> str:
    """Convert parameter size to billions (B) format."""
    try:
        # Remove any spaces and convert to uppercase for consistency
        param_size = param_size.replace(" ", "").upper()
        
        # Extract the number and unit
        if 'M' in param_size:
            # Convert millions to billions
            number = float(param_size.replace('M', '').replace(',', ''))
            return f"{number/1000:.2f}B"
        elif 'B' in param_size:
            # Already in billions, just format consistently
            number = float(param_size.replace('B', '').replace(',', ''))
            return f"{number:.2f}B"
        else:
            # If no unit or unrecognized format, try to convert the raw number
            number = float(param_size.replace(',', ''))
            if number >= 1000000000:
                return f"{number/1000000000:.2f}B"
            elif number >= 1000000:
                return f"{number/1000000000:.2f}B"
            else:
                return f"{number/1000000000:.2f}B"
    except:
        return param_size  # Return original if conversion fails

def load_latest_model_stats():
    """Load model stats from the model_stats.json file."""
    try:
        # Read directly from model_stats.json in root directory
        with open('model_stats.json', 'r') as f:
            stats = json.load(f)
            
        # Convert stats to table format
        table_data = []
        headers = ["Model", "VRAM (GB)", "Size", "Parameters", "Quantization", "Tokens/sec", "Gen Tokens/sec", "Total Tokens", "Response Time (s)"]
        
        for model in stats:
            if not model.get("success", False):  # Skip failed tests
                continue
                
            perf = model.get("performance", {})
            info = model.get("model_info", {})
            
            try:
                # Format numeric values with 2 decimal places
                model_size = float(info.get("size", 0))  # Get raw size
                vram_gb = round(model_size/1024/1024/1024, 2)  # Convert to GB
                tokens_per_sec = round(float(perf.get("tokens_per_second", 0)), 2)
                gen_tokens_per_sec = round(float(perf.get("generation_tokens_per_second", 0)), 2)
                total_tokens = perf.get("total_tokens", 0)
                response_time = round(float(perf.get("response_time", 0)), 2)
                
                # Normalize parameter size to billions format
                param_size = normalize_parameter_size(info.get("parameter_size", "Unknown"))
                
                row = [
                    model.get("model_name", "Unknown"),      # String
                    vram_gb,                                 # Number (2 decimals)
                    model_size,                              # Number (bytes)
                    param_size,                              # String (normalized to B)
                    info.get("quantization_level", "Unknown"),  # String
                    tokens_per_sec,                          # Number (2 decimals)
                    gen_tokens_per_sec,                      # Number (2 decimals)
                    total_tokens,                            # Number (integer)
                    response_time                            # Number (2 decimals)
                ]
                table_data.append(row)
            except Exception as row_error:
                logger.warning(f"Skipping model {model.get('model_name', 'Unknown')}: {str(row_error)}")
                continue
            
        if not table_data:
            return None, "No valid model stats found"
            
        # Sort by tokens per second (numerically)
        table_data.sort(key=lambda x: float(x[5]) if isinstance(x[5], (int, float)) else 0, reverse=True)
        
        return headers, table_data
    except Exception as e:
        logger.error(f"Error in load_latest_model_stats: {str(e)}")
        return None, f"Error loading model stats: {str(e)}"

# Initialize Gradio Blocks
with gr.Blocks(css="""
    #dice-button {
        min-height: 90px;
        font-size: 35px;
    }
    .sponsor-button {
        background-color: #30363D;
        color: white;
        border: none;
        padding: 10px 20px;
        border-radius: 6px;
        cursor: pointer;
        display: inline-flex;
        align-items: center;
        gap: 8px;
        font-weight: bold;
    }
    .sponsor-button:hover {
        background-color: #2D333B;
    }
""") as demo:
    gr.Markdown(config.ARENA_NAME)
    
    # Main description with sponsor button
    with gr.Row():
        with gr.Column(scale=8):
            gr.Markdown("""
                **Step right up to the arena where frugal meets fabulous in the world of AI!**
                Watch as our compact contenders (maxing out at 14B parameters) duke it out in a battle of wits and words.
                
                What started as a simple experiment has grown into a popular platform for evaluating compact language models.
                As the arena continues to expand with more models, features, and battles, it requires computational resources to maintain and improve.
                If you find this project valuable and would like to support its development, consider sponsoring:
            """)
        with gr.Column(scale=2):
            gr.Button(
                "Sponsor on GitHub",
                link="https://github.com/sponsors/k-mktr",
                elem_classes="sponsor-button"
            )
    
    # Instructions in an accordion
    with gr.Accordion("📖 How to Use", open=False):
        gr.Markdown("""
            1. To start the battle, go to the 'Battle Arena' tab.
            2. Type your prompt into the text box. Alternatively, click the "🎲" button to receive a random prompt.
            3. Click the "Generate Responses" button to view the models' responses.
            4. Cast your vote for the model that provided the better response. In the event of a Tie, enter a new prompt before continuing the battle.
            5. Check out the Leaderboard to see how models rank against each other.
            
            More info: [README.md](https://huggingface.co/spaces/k-mktr/gpu-poor-llm-arena/blob/main/README.md)
        """)
    
    # Leaderboard Tab (now first)
    with gr.Tab("Leaderboard"):
        gr.Markdown("""
        ### Main Leaderboard
        This leaderboard uses a scoring system that balances win rate and total battles. The score is calculated using the formula:
        **Score = Win Rate * (1 - 1 / (Total Battles + 1))**
        
        This formula rewards models with higher win rates and more battles. As the number of battles increases, the score approaches the win rate.
        """)
        leaderboard = gr.Dataframe(
            headers=["#", "Model", "Score", "Wins", "Losses", "Total Battles", "Win Rate"],
            row_count=10,
            col_count=7,
            interactive=True,
            label="Leaderboard"
        )
    
    # Battle Arena Tab (now second)
    with gr.Tab("Battle Arena"):
        with gr.Row():
            prompt_input = gr.Textbox(
                label="Enter your prompt", 
                placeholder="Type your prompt here...",
                scale=20
            )
            random_prompt_btn = gr.Button("🎲", scale=1, elem_id="dice-button")
        
        gr.Markdown("<br>")
        
        # Add the random prompt button functionality
        random_prompt_btn.click(
            random_prompt,
            outputs=prompt_input
        )
        
        submit_btn = gr.Button("Generate Responses", variant="primary")
        
        with gr.Row():
            left_output = gr.Chatbot(label=random.choice(config.model_nicknames), type="messages")
            right_output = gr.Chatbot(label=random.choice(config.model_nicknames), type="messages")
        
        with gr.Row():
            left_vote_btn = gr.Button(f"Vote for {left_output.label}", interactive=False)
            tie_btn = gr.Button("Tie 🙈 Continue with a new prompt", interactive=False, visible=False)
            right_vote_btn = gr.Button(f"Vote for {right_output.label}", interactive=False)
        
        result = gr.Textbox(
            label="Status", 
            interactive=False, 
            value="Generate responses to start the battle! 🚀",
            visible=True  # Always visible
        )
        
        with gr.Row(visible=False) as model_names_row:
            left_model = gr.Textbox(label="🔵 Left Model", interactive=False)
            right_model = gr.Textbox(label="🔴 Right Model", interactive=False)
        
        previous_prompt = gr.State("")  # Add this line to store the previous prompt
        tie_count = gr.State(0)  # Add this line to keep track of tie count
        
        new_battle_btn = gr.Button("New Battle")
    
    # ELO Leaderboard Tab
    with gr.Tab("ELO Leaderboard"):
        gr.Markdown("""
        ### ELO Rating System
        This leaderboard uses a modified ELO rating system that takes into account both the performance and size of the models.
        Initial ratings are based on model size, with larger models starting at higher ratings.
        The ELO rating is calculated based on wins and losses, with adjustments made based on the relative strengths of opponents.
        """)
        elo_leaderboard = gr.Dataframe(
            headers=["#", "Model", "ELO Rating", "Wins", "Losses", "Total Battles", "Win Rate"],
            row_count=10,
            col_count=7,
            interactive=True,
            label="ELO Leaderboard"
        )
    
    # Latest Updates Tab
    with gr.Tab("Latest Updates"):
        release_notes = gr.HTML(get_release_notes_html())
        refresh_notes_btn = gr.Button("Refresh Updates")
        
        refresh_notes_btn.click(
            get_release_notes_html,
            outputs=[release_notes]
        )
    
    # Model Stats Tab
    with gr.Tab("Model Stats"):
        gr.Markdown("""
        ### Model Performance Statistics
        
        This tab shows detailed performance metrics for each model, tested using a creative writing prompt.
        The tests were performed on an **AMD Radeon RX 7600 XT 16GB GPU**.
        
        For detailed information about the testing methodology, parameters, and hardware setup, please refer to the 
        [README_model_stats.md](https://huggingface.co/spaces/k-mktr/gpu-poor-llm-arena/blob/main/README_model_stats.md).
        
        """)
        
        headers, table_data = load_latest_model_stats()
        if headers:
            model_stats_table = gr.Dataframe(
                headers=headers,
                value=table_data,
                row_count=len(table_data),
                col_count=len(headers),
                interactive=True,
                label="Model Performance Statistics"
            )
        else:
            gr.Markdown(f"⚠️ {table_data}")  # Show error message if loading failed
    
    # Define interactions
    submit_btn.click(
        battle_arena,
        inputs=prompt_input,
        outputs=[
            left_output, right_output, left_model, right_model, 
            left_output, right_output, left_vote_btn, right_vote_btn,
            tie_btn, previous_prompt, tie_count, model_names_row, result
        ]
    )
    
    left_vote_btn.click(
        lambda *args: record_vote(*args, "Left is better"),
        inputs=[prompt_input, left_output, right_output, left_model, right_model],
        outputs=[result, leaderboard, elo_leaderboard, left_vote_btn, 
                 right_vote_btn, tie_btn, model_names_row]
    )
    
    right_vote_btn.click(
        lambda *args: record_vote(*args, "Right is better"),
        inputs=[prompt_input, left_output, right_output, left_model, right_model],
        outputs=[result, leaderboard, elo_leaderboard, left_vote_btn, 
                 right_vote_btn, tie_btn, model_names_row]
    )
    
    tie_btn.click(
        continue_conversation,
        inputs=[prompt_input, left_output, right_output, left_model, right_model, previous_prompt, tie_count],
        outputs=[left_output, right_output, prompt_input, tie_btn, previous_prompt, tie_count]
    )
    
    new_battle_btn.click(
        new_battle,
        outputs=[prompt_input, left_output, right_output, left_model, 
                right_model, left_vote_btn, right_vote_btn, tie_btn,
                result, leaderboard, model_names_row, tie_count]
    )
    
    # Update leaderboard on launch
    demo.load(get_leaderboard, outputs=leaderboard)
    demo.load(get_elo_leaderboard, outputs=elo_leaderboard)

if __name__ == "__main__":
    # Initialize ELO ratings before launching the app
    ensure_elo_ratings_initialized()
    # Start the model refresh thread
    config.start_model_refresh_thread()
    demo.launch(show_api=False)