Push main

app.py

@@ -2,57 +2,36 @@ import json
 import re
 import random
 from collections import defaultdict
-from datetime import datetime
+from datetime import datetime, timezone
 import hashlib
-import gradio as gr
 
 from dotenv import load_dotenv
-load_dotenv()
 
-from gen_api_answer import (
-    get_model_response, 
-    parse_model_response,
-    prometheus_parse_model_response,
-    atla_parse_model_response,
-    flow_judge_parse_model_response
-)
+load_dotenv()
 
-from random_sample_generation import (
-    get_random_human_ai_pair,
-    get_random_human_ai_ground_truth_pair,
-    generate_ai_response
-)   
+import gradio as gr
+from gen_api_answer import get_model_response, parse_model_response, get_random_human_ai_pair
 from db import add_vote, create_db_connection, get_votes
-
 from utils import Vote
-
 from common import (
     POLICY_CONTENT,
     ACKNOWLEDGEMENTS,
+    DEFAULT_EVAL_PROMPT,
+    DEFAULT_INPUT,
+    DEFAULT_RESPONSE,
     CSS_STYLES,
     MAIN_TITLE,
     HOW_IT_WORKS,
+    BATTLE_RULES,
+    EVAL_DESCRIPTION,
+    VOTING_HEADER,
 )
-from prompts import (
-    DEFAULT_EVAL_PROMPT,
-    DEFAULT_EVAL_PROMPT_EDITABLE,
-    FIXED_EVAL_SUFFIX,
-    DEFAULT_EVAL_CRITERIA,
-    DEFAULT_SCORE_1,
-    DEFAULT_SCORE_2,
-    DEFAULT_SCORE_3,
-    DEFAULT_SCORE_4,
-    DEFAULT_SCORE_5,
-)
-from leaderboard import (
-    get_leaderboard,
-    get_leaderboard_stats,
-    get_model_rankings,
-    DEFAULT_ELO,
-    K_FACTOR
-)
+from example_metrics import EXAMPLE_METRICS
 
 
+# Model and ELO score data
+DEFAULT_ELO = 1200  # Starting ELO for new models
+K_FACTOR = 32  # Standard chess K-factor, adjust as needed
 elo_scores = defaultdict(lambda: DEFAULT_ELO)
 vote_counts = defaultdict(int)
 
@@ -73,7 +52,6 @@ def load_model_data():
                     "organization": model["organization"],
                     "license": model["license"],
                     "api_model": model["api_model"],
-                    "active": model["active"]  
                 }
     except FileNotFoundError:
         print("Warning: models.jsonl not found")
@@ -84,11 +62,9 @@ def load_model_data():
 model_data = load_model_data()
 
 def store_vote_data(prompt, response_a, response_b, model_a, model_b, winner, judge_id):
-    prompt_value = prompt.value if hasattr(prompt, 'value') else prompt
-    
     vote = Vote(
         timestamp=datetime.now().isoformat(),
-        prompt=prompt_value,
+        prompt=prompt,
         response_a=response_a,
         response_b=response_b,
         model_a=model_a,
@@ -117,6 +93,40 @@ def get_final_prompt(eval_prompt, variable_values):
     return eval_prompt
 
 
+def submit_prompt(eval_prompt, *variable_values):
+    try:
+        variables = parse_variables(eval_prompt)
+        variable_values_dict = {var: val for var, val in zip(variables, variable_values)}
+        final_prompt = get_final_prompt(eval_prompt, variable_values_dict)
+
+        models = list(model_data.keys())
+        model1, model2 = random.sample(models, 2)
+        model_a, model_b = (model1, model2) if random.random() < 0.5 else (model2, model1)
+
+        response_a = get_model_response(model_a, model_data.get(model_a), final_prompt)
+        response_b = get_model_response(model_b, model_data.get(model_b), final_prompt)
+
+        return (
+            response_a,
+            response_b,
+            gr.update(visible=True),
+            gr.update(visible=True),
+            model_a,
+            model_b,
+            final_prompt,
+        )
+    except Exception as e:
+        print(f"Error in submit_prompt: {str(e)}")
+        return (
+            "Error generating response",
+            "Error generating response",
+            gr.update(visible=False),
+            gr.update(visible=False),
+            None,
+            None,
+            None,
+        )
+
 
 def get_ip(request: gr.Request) -> str:
     """Get and hash the IP address from the request."""
@@ -133,26 +143,6 @@ def get_ip(request: gr.Request) -> str:
     return hashlib.sha256(ip.encode()).hexdigest()[:16]
 
 
-def get_vote_message(choice: str, model_a: str, model_b: str) -> tuple[str, str]:
-    """Generate appropriate message based on vote and model rankings.
-    Returns (title, message) tuple."""
-    # Get current rankings
-    voting_data = get_current_votes()
-    leaderboard = get_leaderboard(model_data, voting_data, show_preliminary=True)
-    rankings = get_model_rankings(leaderboard)
-    pos_a = rankings.get(model_a, 0)
-    pos_b = rankings.get(model_b, 0)
-    
-    if choice == "Tie":
-        return "It's a tie!", "Keep voting responsibly 🤗"
-    
-    # Check if vote aligns with leaderboard
-    if (choice == "A" and pos_a < pos_b) or (choice == "B" and pos_b < pos_a):
-        return "The favourite wins!", "Keep voting responsibly 🤗"
-    else:
-        return "The underdog wins!", "Keep voting responsibly 🤗"
-
-
 def vote(
     choice,
     model_a,
@@ -202,39 +192,16 @@ def vote(
     store_vote_data(
         final_prompt, response_a, response_b, model_a, model_b, choice, judge_id
     )
-    
-    # Get model positions for display
-    voting_data = get_current_votes()
-    leaderboard = get_leaderboard(model_data, voting_data, show_preliminary=True)
-    rankings = get_model_rankings(leaderboard)
-    pos_a = rankings.get(model_a, 0)
-    pos_b = rankings.get(model_b, 0)
-    
-    # Format model names with positions and win/loss indicators
-    if choice == "Tie":
-        model_a_display = f"*Model: {model_a} (Position #{pos_a})*"
-        model_b_display = f"*Model: {model_b} (Position #{pos_b})*"
-    else:
-        winner = model_a if choice == "A" else model_b
-        loser = model_b if choice == "A" else model_a
-        winner_pos = pos_a if choice == "A" else pos_b
-        loser_pos = pos_b if choice == "A" else pos_a
-        
-        model_a_display = f"*Model: {model_a} {'✅' if choice == 'A' else '❌'} (Position #{pos_a})*"
-        model_b_display = f"*Model: {model_b} {'✅' if choice == 'B' else '❌'} (Position #{pos_b})*"
-    
-    # Generate vote message
-    title, message = get_vote_message(choice, model_a, model_b)
-    
+
+    # Return updates for UI components
     return [
-        gr.update(interactive=False, variant="primary" if choice == "A" else "secondary"),  # vote_a
-        gr.update(interactive=False, variant="primary" if choice == "B" else "secondary"),  # vote_b
-        gr.update(interactive=False, variant="primary" if choice == "Tie" else "secondary"),  # vote_tie
-        gr.update(value=model_a_display),  # model_name_a
-        gr.update(value=model_b_display),  # model_name_b
-        gr.update(interactive=True, value="Regenerate judges", variant="secondary"),  # send_btn
-        gr.update(value="🎲 New round", variant="primary"),  # random_btn
-        gr.Info(message, title=title),  # success message
+        gr.update(visible=False),  # vote_a
+        gr.update(visible=False),  # vote_b
+        gr.update(visible=False),  # tie_button_row
+        gr.update(value=f"*Model: {model_a}*"),  # model_name_a
+        gr.update(value=f"*Model: {model_b}*"),  # model_name_b
+        gr.update(interactive=True, value="Run the evaluators", variant="primary"),  # send_btn
+        gr.update(visible=True),  # spacing_div
     ]
 
 
@@ -243,24 +210,150 @@ def get_current_votes():
     return get_votes(db)
 
 
-# Update the refresh_leaderboard function
-def refresh_leaderboard(show_preliminary):
-    """Refresh the leaderboard data and stats."""
+def get_leaderboard(show_preliminary=True):
+    """Generate leaderboard data using fresh votes from MongoDB."""
+    # Get fresh voting data
     voting_data = get_current_votes()
-    leaderboard = get_leaderboard(model_data, voting_data, show_preliminary)
-    data = [
-        [
-            entry["Model"],
-            float(entry["ELO Score"]),
-            entry["95% CI"],
-            entry["# Votes"],
-            entry["Organization"],
-            entry["License"],
-        ]
-        for entry in leaderboard
-    ]
-    stats = get_leaderboard_stats(model_data, voting_data)
-    return [gr.update(value=data), gr.update(value=stats)]
+    print(f"Fetched {len(voting_data)} votes from database")  # Debug log
+
+    # Initialize dictionaries for tracking
+    ratings = defaultdict(lambda: DEFAULT_ELO)
+    matches = defaultdict(int)
+
+    # Process each vote
+    for vote in voting_data:
+        try:
+            model_a = vote.get("model_a")
+            model_b = vote.get("model_b")
+            winner = vote.get("winner")
+
+            # Skip if models aren't in current model_data
+            if (
+                not all([model_a, model_b, winner])
+                or model_a not in model_data
+                or model_b not in model_data
+            ):
+                continue
+
+            # Update match counts
+            matches[model_a] += 1
+            matches[model_b] += 1
+
+            # Calculate ELO changes
+            elo_a = ratings[model_a]
+            elo_b = ratings[model_b]
+
+            # Expected scores
+            expected_a = 1 / (1 + 10 ** ((elo_b - elo_a) / 400))
+            expected_b = 1 - expected_a
+
+            # Actual scores
+            score_a = 1 if winner == "A" else 0 if winner == "B" else 0.5
+            score_b = 1 - score_a
+
+            # Update ratings
+            ratings[model_a] += K_FACTOR * (score_a - expected_a)
+            ratings[model_b] += K_FACTOR * (score_b - expected_b)
+
+        except Exception as e:
+            print(f"Error processing vote: {e}")
+            continue
+
+    # Generate leaderboard data
+    leaderboard = []
+    for model in model_data.keys():
+        votes = matches[model]
+        # Skip models with < 500 votes if show_preliminary is False
+        if not show_preliminary and votes < 500:
+            continue
+            
+        elo = ratings[model]
+        ci = 1.96 * (400 / (votes + 1) ** 0.5) if votes > 0 else 0
+        data = {
+            "Model": model,
+            "ELO Score": f"{int(elo)}",
+            "95% CI": f"±{int(ci)}",
+            "# Votes": votes,
+            "Organization": model_data[model]["organization"],
+            "License": model_data[model]["license"],
+        }
+        leaderboard.append(data)
+
+    # Sort leaderboard by ELO score in descending order
+    leaderboard.sort(key=lambda x: float(x["ELO Score"]), reverse=True)
+
+    return leaderboard
+
+
+def calculate_elo_change(rating_a, rating_b, winner):
+    """Calculate ELO rating changes for both players."""
+    expected_a = 1 / (1 + 10 ** ((rating_b - rating_a) / 400))
+    expected_b = 1 - expected_a
+
+    if winner == "A":
+        score_a, score_b = 1, 0
+    elif winner == "B":
+        score_a, score_b = 0, 1
+    else:  # Handle ties
+        score_a, score_b = 0.5, 0.5
+
+    change_a = K_FACTOR * (score_a - expected_a)
+    change_b = K_FACTOR * (score_b - expected_b)
+
+    return change_a, change_b
+
+
+def update_leaderboard():
+    """Generate leaderboard DataFrame using fresh votes from MongoDB."""
+    # Get fresh voting data
+    voting_data = get_current_votes()
+    print(f"Found {len(voting_data)} votes in database")
+    matches = defaultdict(int)
+
+    # Process each vote chronologically
+    for vote in voting_data:
+        # Extract model names from the vote document
+        try:
+            model_a = vote.get("model_a")
+            model_b = vote.get("model_b")
+            winner = vote.get("winner")
+
+            print(f"Processing vote: {model_a} vs {model_b}, winner: {winner}")
+
+            # Skip if any required field is missing or models aren't in current model_data
+            if not all([model_a, model_b, winner]):
+                print(f"Missing required fields in vote: {vote}")
+                continue
+
+            if model_a not in model_data:
+                print(f"Model A '{model_a}' not found in model_data")
+                continue
+
+            if model_b not in model_data:
+                print(f"Model B '{model_b}' not found in model_data")
+                continue
+
+            # Update match counts
+            matches[model_a] += 1
+            matches[model_b] += 1
+            print(
+                f"Updated matches - {model_a}: {matches[model_a]}, {model_b}: {matches[model_b]}"
+            )
+        except Exception as e:
+            print(f"Error processing vote: {e}")
+            print(f"Problematic vote data: {vote}")
+            continue
+
+
+# Update the display_leaderboard function
+def display_leaderboard():
+    df = update_leaderboard()
+    return gr.DataFrame(
+        value=df,
+        headers=["Model", "ELO", "95% CI", "Matches", "Organization", "License"],
+        datatype=["str", "number", "str", "number", "str", "str", "str"],
+        row_count=(len(df) + 1, "dynamic"),
+    )
 
 
 # Update the leaderboard table definition in the UI
@@ -270,30 +363,63 @@ leaderboard_table = gr.Dataframe(
 )
 
 
-def populate_random_example(request: gr.Request, compatible_mode: bool):
-    """Generate a random human-AI conversation example and reset judge outputs."""
-    if compatible_mode:
-        # Generate all three components when compatible mode is enabled
-        human_msg, ai_msg, ground_truth_msg = get_random_human_ai_ground_truth_pair()
-    else:
-        # Generate only human and AI messages when compatible mode is disabled
-        human_msg, ai_msg = get_random_human_ai_pair()
-        ground_truth_msg = ""
-    
+def get_leaderboard_stats():
+    """Get summary statistics for the leaderboard."""
+    now = datetime.now(timezone.utc)
+    total_votes = len(get_current_votes())
+    total_models = len(model_data)
+    last_updated = now.replace(minute=0, second=0, microsecond=0).strftime(
+        "%B %d, %Y at %H:00 UTC"
+    )
+
+    return f"""
+### Leaderboard Stats
+- **Total Models**: {total_models}
+- **Total Votes**: {total_votes}
+- **Last Updated**: {last_updated}
+"""
+
+
+#def set_example_metric(metric_name):
+#    if metric_name == "Custom":
+#        variables = parse_variables(DEFAULT_EVAL_PROMPT)
+#        variable_values = []
+#        for var in variables:
+#            if var == "input":
+#                variable_values.append(DEFAULT_INPUT)
+#            elif var == "response":
+#                variable_values.append(DEFAULT_RESPONSE)
+#            else:
+#                variable_values.append("")  # Default empty value
+        # Pad variable_values to match the length of variable_rows
+#        while len(variable_values) < len(variable_rows):
+#            variable_values.append("")
+#        return [DEFAULT_EVAL_PROMPT] + variable_values
+
+#    metric_data = EXAMPLE_METRICS[metric_name]
+#    variables = parse_variables(metric_data["prompt"])
+#    variable_values = []
+#    for var in variables:
+#        value = metric_data.get(var, "")  # Default to empty string if not found
+#        variable_values.append(value)
+    # Pad variable_values to match the length of variable_rows
+#    while len(variable_values) < len(variable_rows):
+#        variable_values.append("")
+#    return [metric_data["prompt"]] + variable_values
+
+
+# Select random metric at startup
+#  def get_random_metric():
+#    metrics = list(EXAMPLE_METRICS.keys())
+#    return set_example_metric(random.choice(metrics))
+
+
+def populate_random_example(request: gr.Request):
+    """Generate a random human-AI conversation example."""
+    human_msg, ai_msg = get_random_human_ai_pair()
     return [
         gr.update(value=human_msg),
-        gr.update(value=ai_msg),
-        gr.update(value="🎲", variant="secondary"),  # Reset random button appearance
-        gr.update(value=""),  # Clear score A
-        gr.update(value=""),  # Clear critique A
-        gr.update(value=""),  # Clear score B
-        gr.update(value=""),  # Clear critique B
-        gr.update(interactive=False, variant="primary"),  # Reset vote A
-        gr.update(interactive=False, variant="primary"),  # Reset vote B
-        gr.update(interactive=False, variant="primary"),  # Reset vote tie
-        gr.update(value="*Model: Hidden*"),  # Reset model name A
-        gr.update(value="*Model: Hidden*"),  # Reset model name B
-        gr.update(value=ground_truth_msg, visible=compatible_mode),  # Set ground truth and visibility
+        gr.update(value=ai_msg)
     ]
 
 
@@ -309,43 +435,27 @@ with gr.Blocks(theme="default", css=CSS_STYLES) as demo:
 
     with gr.Tabs():
         with gr.TabItem("Judge Arena"):
+            random_btn = gr.Button("🎲", scale=0)
             with gr.Row():
                 # Left side - Input section
                 with gr.Column(scale=1):
                     with gr.Group():
                         human_input = gr.TextArea(
-                            label="👩 User Input",
-                            lines=10,
+                            label="👩 Human Input",
+                            lines=12,
                             placeholder="Enter the human message here..."
                         )
-                        with gr.Row():
-                            generate_btn = gr.Button(
-                                "Generate AI Response",
-                                size="sm",
-                                interactive=False
-                            )
                         
                         ai_response = gr.TextArea(
                             label="🤖 AI Response", 
-                            lines=15,
-                            placeholder="Enter the AI response here..."
-                        )
-                        
-                        # Ground truth response (initially hidden)
-                        ground_truth = gr.TextArea(
-                            label="🎯 Ground truth response",
                             lines=12,
-                            placeholder="Enter the ground truth response here...",
-                            visible=False
+                            placeholder="Enter the AI response here..."
                         )
                         
-                    with gr.Row():
-                        random_btn = gr.Button("🎲", scale=2)
                         send_btn = gr.Button(
-                            value="Run judges",
+                            value="Run the evaluators",
                             variant="primary",
-                            size="lg",
-                            scale=8
+                            size="lg"
                         )
 
                 # Right side - Model outputs
@@ -355,15 +465,18 @@ with gr.Blocks(theme="default", css=CSS_STYLES) as demo:
                         model_name_a = gr.Markdown("*Model: Hidden*")
                         with gr.Row():
                             with gr.Column(scale=1, min_width=100):  # Fixed narrow width for score
-                                score_a = gr.Textbox(label="Score", lines=6, interactive=False)
-                                vote_a = gr.Button("Vote A", variant="primary", interactive=False)
+                                score_a = gr.Textbox(label="Score", lines=5, interactive=False)
+                                vote_a = gr.Button("Vote A", variant="primary", visible=False)
                             with gr.Column(scale=9, min_width=400):  # Wider width for critique
-                                critique_a = gr.TextArea(label="Critique", lines=8, interactive=False)
+                                critique_a = gr.TextArea(label="Critique", lines=7, interactive=False)
+                
+                    # Spacing div that's visible only when tie button is hidden
+                    spacing_div = gr.HTML('<div style="height: 42px;"></div>', visible=True, elem_id="spacing-div")
                 
                     # Tie button row
-                    with gr.Row() as tie_button_row:
+                    with gr.Row(visible=False) as tie_button_row:
                         with gr.Column():
-                            vote_tie = gr.Button("Tie", variant="primary", interactive=False)
+                            vote_tie = gr.Button("Tie", variant="secondary")
                     
                 
                     gr.Markdown("### 🧑‍⚖️ Judge B")
@@ -371,90 +484,16 @@ with gr.Blocks(theme="default", css=CSS_STYLES) as demo:
                         model_name_b = gr.Markdown("*Model: Hidden*")
                         with gr.Row():
                             with gr.Column(scale=1, min_width=100):  # Fixed narrow width for score
-                                score_b = gr.Textbox(label="Score", lines=6, interactive=False)
-                                vote_b = gr.Button("Vote B", variant="primary", interactive=False)
+                                score_b = gr.Textbox(label="Score", lines=5, interactive=False)
+                                vote_b = gr.Button("Vote B", variant="primary", visible=False)
                             with gr.Column(scale=9, min_width=400):  # Wider width for critique
-                                critique_b = gr.TextArea(label="Critique", lines=8, interactive=False)
-                        # Place Vote B button directly under Judge B
+                                critique_b = gr.TextArea(label="Critique", lines=7, interactive=False)
+                    # Place Vote B button directly under Judge B
                 
             gr.Markdown("<br>")
-            
-
-            # Replace the "Edit Judge Prompt" Accordion section with:
-            with gr.Accordion("📝 Edit Judge Prompt", open=False) as prompt_accordion:
-                gr.Markdown("<br>")
-                use_reference_toggle = gr.Checkbox(
-                    label="Use a reference response",
-                    value=False
-                )
-                
-                # Hide the default prompt editor
-                with gr.Column(visible=False) as default_prompt_editor:
-                    eval_prompt_editable = gr.TextArea(
-                        value=DEFAULT_EVAL_PROMPT_EDITABLE,
-                        label="Evaluation Criteria",
-                        lines=12
-                    )
 
-                    with gr.Row(visible=False) as edit_buttons_row:
-                        cancel_prompt_btn = gr.Button("Cancel")
-                        save_prompt_btn = gr.Button("Save", variant="primary")
-                    gr.Markdown("*The sample being evaluated is always appended as:*")
-                    gr.Markdown(f"```{FIXED_EVAL_SUFFIX}")
-                
-                # Show the compatible mode editor
-                with gr.Column(visible=True) as compatible_prompt_editor:
-                    with gr.Row():
-                        # Left column - Evaluation Criteria
-                        with gr.Column(scale=1):
-                            eval_criteria_text = gr.TextArea(
-                                label="Evaluation Criteria",
-                                lines=12,
-                                value=DEFAULT_EVAL_CRITERIA,
-                                placeholder="Enter the evaluation criteria..."
-                            )
-                            prometheus_reference = gr.Markdown(
-                                "<br> *By default, we use the Prometheus absolute grading prompt template - see [here](https://huggingface.co/prometheus-eval/prometheus-7b-v2.0).*",
-                                visible=True 
-                            )
-                        
-                        # Right column - Score Descriptions
-                        with gr.Column(scale=1):
-                            score1_description = gr.TextArea(
-                                label="Score 1",
-                                value=DEFAULT_SCORE_1,
-                                placeholder="Description for score 1",
-                                lines=2
-                            )
-                            score2_description = gr.TextArea(
-                                label="Score 2", 
-                                value=DEFAULT_SCORE_2,
-                                placeholder="Description for score 2",
-                                lines=2
-                            )
-                            score3_description = gr.TextArea(
-                                label="Score 3",
-                                value=DEFAULT_SCORE_3,
-                                placeholder="Description for score 3",
-                                lines=2
-                            )
-                            score4_description = gr.TextArea(
-                                label="Score 4",
-                                value=DEFAULT_SCORE_4,
-                                placeholder="Description for score 4",
-                                lines=2
-                            )
-                            score5_description = gr.TextArea(
-                                label="Score 5",
-                                value=DEFAULT_SCORE_5,
-                                placeholder="Description for score 5",
-                                lines=2
-                            )
-
-                    # Add save/cancel buttons for compatible mode
-                    with gr.Row(visible=False) as compatible_edit_buttons_row:
-                        compatible_cancel_btn = gr.Button("Cancel")
-                        compatible_save_btn = gr.Button("Save", variant="primary")
+            # Add spacing and acknowledgements at the bottom
+            gr.Markdown(ACKNOWLEDGEMENTS)
 
         with gr.TabItem("Leaderboard"):
             with gr.Row():
@@ -462,7 +501,7 @@ with gr.Blocks(theme="default", css=CSS_STYLES) as demo:
                     show_preliminary = gr.Checkbox(
                         label="Reveal preliminary results",
                         value=True,  # Checked by default
-                        info="Show all models, including models with less human ratings (< 300 votes)",
+                        info="Show all models, including models with less few human ratings (< 500 votes)",
                         interactive=True
                     )
             stats_display = gr.Markdown()
@@ -470,13 +509,24 @@ with gr.Blocks(theme="default", css=CSS_STYLES) as demo:
                 headers=["Model", "ELO", "95% CI", "Matches", "Organization", "License"],
                 datatype=["str", "number", "str", "number", "str", "str", "str"],
             )
-            
-            gr.Markdown("""<br>
-                        <br>
-                        Judge Arena uses Together AI for inference of open-source models. FP8 models are named as -- "Turbo" where the performance of the FP16 reference models is closely matched:
 
-                        [*"Together Turbo achieves this performance while maintaining full accuracy compared to Meta's reference implementation across all models. Llama-3.1-405B-Instruct-Turbo matches the accuracy of Meta reference models."*](https://www.together.ai/blog/together-inference-engine-2)
-            """)
+            # Update refresh_leaderboard to use the checkbox value
+            def refresh_leaderboard(show_preliminary):
+                """Refresh the leaderboard data and stats."""
+                leaderboard = get_leaderboard(show_preliminary)
+                data = [
+                    [
+                        entry["Model"],
+                        float(entry["ELO Score"]),
+                        entry["95% CI"],
+                        entry["# Votes"],
+                        entry["Organization"],
+                        entry["License"],
+                    ]
+                    for entry in leaderboard
+                ]
+                stats = get_leaderboard_stats()
+                return [gr.update(value=data), gr.update(value=stats)]
 
             # Add change handler for checkbox
             show_preliminary.change(
@@ -494,15 +544,67 @@ with gr.Blocks(theme="default", css=CSS_STYLES) as demo:
 
         with gr.TabItem("Policy"):
             gr.Markdown(POLICY_CONTENT)
-            gr.Markdown(ACKNOWLEDGEMENTS)
 
     # Define state variables for model tracking
     model_a_state = gr.State()
     model_b_state = gr.State()
     final_prompt_state = gr.State()
-    eval_prompt_previous = gr.State(value=DEFAULT_EVAL_PROMPT_EDITABLE)  # Initialize with default value
-    is_editing = gr.State(False)  # Track editing state
-    compatible_mode_state = gr.State(False)  # Track compatible mode state
+
+    # Update variable inputs based on the eval prompt
+    def update_variables(eval_prompt):
+        variables = parse_variables(eval_prompt)
+        updates = []
+
+        for i in range(len(variable_rows)):
+            var_row, var_input = variable_rows[i]
+            if i < len(variables):
+                var_name = variables[i]
+                # Set the number of lines based on the variable name
+                if var_name == "response":
+                    lines = 4  # Adjust this number as needed
+                else:
+                    lines = 1  # Default to single line for other variables
+                updates.extend(
+                    [
+                        gr.update(visible=True),  # Show the variable row
+                        gr.update(
+                            label=var_name, visible=True, lines=lines
+                        ),  # Update label and lines
+                    ]
+                )
+            else:
+                updates.extend(
+                    [
+                        gr.update(visible=False),  # Hide the variable row
+                        gr.update(value="", visible=False),  # Clear value when hidden
+                    ]
+                )
+        return updates
+
+    #eval_prompt.change(
+    #    fn=update_variables,
+    #    inputs=eval_prompt,
+    #    outputs=[item for sublist in variable_rows for item in sublist],
+    #)
+
+    # Regenerate button functionality
+    #regenerate_button.click(
+    #    fn=regenerate_prompt,
+    #    inputs=[model_a_state, model_b_state, eval_prompt, human_input, ai_response],
+    #    outputs=[
+    #        score_a,
+    #        critique_a,
+    #        score_b,
+    #        critique_b,
+    #        vote_a,
+    #        vote_b,
+    #        tie_button_row,
+    #        model_name_a,
+    #        model_name_b,
+    #        model_a_state,
+    #        model_b_state,
+    #    ],
+    #)
 
     # Update model names after responses are generated
     def update_model_names(model_a, model_b):
@@ -517,7 +619,7 @@ with gr.Blocks(theme="default", css=CSS_STYLES) as demo:
     vote_a.click(
         fn=vote,
         inputs=[
-            gr.State("A"),
+            gr.State("A"),  # Choice
             model_a_state,
             model_b_state,
             final_prompt_state,
@@ -529,19 +631,18 @@ with gr.Blocks(theme="default", css=CSS_STYLES) as demo:
         outputs=[
             vote_a,
             vote_b,
-            vote_tie,
+            tie_button_row,
             model_name_a,
             model_name_b,
             send_btn,
-            random_btn,
-            gr.State(),  # placeholder for success message
+            spacing_div,
         ],
     )
 
     vote_b.click(
         fn=vote,
         inputs=[
-            gr.State("B"),
+            gr.State("B"),  # Choice
             model_a_state,
             model_b_state,
             final_prompt_state,
@@ -553,19 +654,18 @@ with gr.Blocks(theme="default", css=CSS_STYLES) as demo:
         outputs=[
             vote_a,
             vote_b,
-            vote_tie,
+            tie_button_row,
             model_name_a,
             model_name_b,
             send_btn,
-            random_btn,
-            gr.State(),  # placeholder for success message
+            spacing_div,
         ],
     )
 
     vote_tie.click(
         fn=vote,
         inputs=[
-            gr.State("Tie"),
+            gr.State("Tie"),  # Choice
             model_a_state,
             model_b_state,
             final_prompt_state,
@@ -577,250 +677,66 @@ with gr.Blocks(theme="default", css=CSS_STYLES) as demo:
         outputs=[
             vote_a,
             vote_b,
-            vote_tie,
+            tie_button_row,
             model_name_a,
             model_name_b,
             send_btn,
-            random_btn,
-            gr.State(),  # placeholder for success message
+            spacing_div,
         ],
     )
 
-    # Add handlers for save/cancel buttons
-    def save_prompt(new_prompt, previous_prompt):
-        return [
-            gr.update(value=new_prompt),  # Update the prompt
-            new_prompt,  # Update the previous prompt state
-            gr.update(visible=False)  # Hide the buttons
-        ]
-
-    def cancel_prompt(previous_prompt):
-        return [
-            gr.update(value=previous_prompt),  # Revert to previous prompt
-            previous_prompt,  # Keep the previous prompt state
-            gr.update(visible=False)  # Hide the buttons
-        ]
+    # Update the send button handler to store the submitted inputs
+    def submit_and_store(prompt, *variables):
+        # Create a copy of the current submission
+        current_submission = {"prompt": prompt, "variables": variables}
+
+        # Get the responses
+        (
+            response_a,
+            response_b,
+            buttons_visible,
+            regen_visible,
+            model_a,
+            model_b,
+            final_prompt,
+        ) = submit_prompt(prompt, *variables)
 
-    def show_edit_buttons(current_value, previous_value):
-        # Show buttons only if the current value differs from the previous value
-        return gr.update(visible=current_value != previous_value)
+        # Parse the responses
+        score_a, critique_a = parse_model_response(response_a)
+        score_b, critique_b = parse_model_response(response_b)
 
-    # Add handlers for save/cancel buttons and prompt changes
-    save_prompt_btn.click(
-        fn=save_prompt,
-        inputs=[eval_prompt_editable, eval_prompt_previous],
-        outputs=[eval_prompt_editable, eval_prompt_previous, edit_buttons_row]
-    )
+        # Format scores with "/ 5"
+        score_a = f"{score_a} / 5"
+        score_b = f"{score_b} / 5"
 
-    cancel_prompt_btn.click(
-        fn=cancel_prompt,
-        inputs=[eval_prompt_previous],
-        outputs=[eval_prompt_editable, eval_prompt_previous, edit_buttons_row]
-    )
-
-    eval_prompt_editable.change(
-        fn=show_edit_buttons,
-        inputs=[eval_prompt_editable, eval_prompt_previous],
-        outputs=edit_buttons_row
-    )
+        # Update the last_submission state with the current values
+        last_submission.value = current_submission
 
-    # Function to toggle visibility based on compatible mode
-    def toggle_use_reference(checked):
-        if checked:
-            # Get new random samples with ground truth when enabling reference mode
-            human_msg, ai_msg, ground_truth_msg = get_random_human_ai_ground_truth_pair()
-            return {
-                ground_truth: gr.update(visible=True, value=ground_truth_msg),
-                human_input: gr.update(value=human_msg),
-                ai_response: gr.update(value=ai_msg),
-                # Reset other UI elements
-                score_a: gr.update(value=""),
-                critique_a: gr.update(value=""),
-                score_b: gr.update(value=""),
-                critique_b: gr.update(value=""),
-                vote_a: gr.update(interactive=False, variant="primary"),
-                vote_b: gr.update(interactive=False, variant="primary"),
-                vote_tie: gr.update(interactive=False, variant="primary"),
-                model_name_a: gr.update(value="*Model: Hidden*"),
-                model_name_b: gr.update(value="*Model: Hidden*"),
-                random_btn: gr.update(value="🎲", variant="secondary"),
-            }
-        else:
-            # Just hide ground truth when disabling reference mode
-            return {
-                ground_truth: gr.update(visible=False)
-            }
-
-    # Update the change handler to include all necessary outputs
-    use_reference_toggle.change(
-        fn=toggle_use_reference,
-        inputs=[use_reference_toggle],
-        outputs=[
-            ground_truth,
-            human_input,
-            ai_response,
+        return (
             score_a,
             critique_a,
             score_b,
             critique_b,
-            vote_a,
-            vote_b,
-            vote_tie,
-            model_name_a,
-            model_name_b,
-            random_btn,
-        ]
-    )
-
-    # Add a new state variable to track first game
-    first_game_state = gr.State(True)  # Initialize as True
-
-    # Update the submit function to use the state variable
-    def submit_and_store(
-        use_reference,
-        eval_criteria_text_input,
-        human_input,
-        ai_response,
-        ground_truth_input,
-        score1_description,
-        score2_description,
-        score3_description,
-        score4_description,
-        score5_description,
-    ):
-        # Build prompt data dictionary
-        prompt_data = {
-            'human_input': human_input,
-            'ai_response': ai_response,
-            'ground_truth_input': ground_truth_input,
-            'eval_criteria': eval_criteria_text_input,
-            'score1_desc': score1_description,
-            'score2_desc': score2_description,
-            'score3_desc': score3_description,
-            'score4_desc': score4_description,
-            'score5_desc': score5_description,
-        }
-
-        # Get list of active models only for matches
-        active_models = [name for name, info in model_data.items() 
-                        if info.get("active", True) is True]  # Explicitly check for True
-        
-        # Define new models list
-        new_models = ["Atla Selene 1 Mini", "SFR-LLaMA-3.1-70B-Judge"]
-        
-        # New models appear 40% of the time
-        if random.random() < 0.4:
-            # Randomly choose between new models
-            new_model = random.choice(new_models)
-            other_models = [m for m in active_models if m not in new_models]
-            other_model = random.choice(other_models)
-            
-            if random.random() < 0.5:
-                model_a, model_b = new_model, other_model
-            else:
-                model_a, model_b = other_model, new_model
-        else:
-            # For other cases, exclude new models
-            non_special_models = [m for m in active_models if m not in new_models]
-            model1, model2 = random.sample(non_special_models, 2)
-            model_a, model_b = (model1, model2) if random.random() < 0.5 else (model2, model1)
-
-        # Get responses from models
-        response_a = get_model_response(
+            gr.update(visible=True),  # vote_a
+            gr.update(visible=True),  # vote_b
+            gr.update(visible=True),  # tie_button_row
             model_a,
-            model_data.get(model_a),
-            prompt_data,
-            use_reference=use_reference
-        )
-        response_b = get_model_response(
             model_b,
-            model_data.get(model_b),
-            prompt_data,
-            use_reference=use_reference
-        )
-
-        
-        is_prometheus_a = model_data.get(model_a, {}).get('organization') == 'Prometheus'
-        is_prometheus_b = model_data.get(model_b, {}).get('organization') == 'Prometheus'
-        is_atla_a = model_data.get(model_a, {}).get('organization') == 'Atla'
-        is_atla_b = model_data.get(model_b, {}).get('organization') == 'Atla'
-        is_flow_judge_a = model_data.get(model_a, {}).get('organization') == 'Flow AI'
-        is_flow_judge_b = model_data.get(model_b, {}).get('organization') == 'Flow AI'
-        is_salesforce_a = model_data.get(model_a, {}).get('organization') == 'Salesforce'
-        is_salesforce_b = model_data.get(model_b, {}).get('organization') == 'Salesforce'
-
-        # Parse the responses based on model, using appropriate parsing for different models
-        if is_prometheus_a:
-            score_a_val, critique_a_val = prometheus_parse_model_response(response_a)
-            score_a_val = f"{score_a_val} / 5"
-        elif is_atla_a or is_salesforce_a:  # Same parser for Atla and Salesforce
-            score_a_val, critique_a_val = atla_parse_model_response(response_a)
-            score_a_val = f"{score_a_val} / 5"
-        elif is_flow_judge_a:
-            score_a_val, critique_a_val = flow_judge_parse_model_response(response_a)
-            score_a_val = f"{score_a_val} / 5"
-        else:
-            score_a_val, critique_a_val = parse_model_response(response_a)
-            score_a_val = f"{score_a_val} / 5"
-
-        if is_prometheus_b:
-            score_b_val, critique_b_val = prometheus_parse_model_response(response_b)
-            score_b_val = f"{score_b_val} / 5"
-        elif is_atla_b or is_salesforce_b:  # Same parser for Atla and Salesforce
-            score_b_val, critique_b_val = atla_parse_model_response(response_b)
-            score_b_val = f"{score_b_val} / 5"
-        elif is_flow_judge_b:
-            score_b_val, critique_b_val = flow_judge_parse_model_response(response_b)
-            score_b_val = f"{score_b_val} / 5"
-        else:
-            score_b_val, critique_b_val = parse_model_response(response_b)
-            score_b_val = f"{score_b_val} / 5"
-
-        return (
-            score_a_val,
-            critique_a_val,
-            score_b_val,
-            critique_b_val,
-            gr.update(interactive=True, variant="primary"),  # vote_a
-            gr.update(interactive=True, variant="primary"),  # vote_b
-            gr.update(interactive=True, variant="primary"),  # vote_tie
-            model_a,
-            model_b,
-            eval_prompt,
+            final_prompt,  # Add final_prompt to state
             gr.update(value="*Model: Hidden*"),
             gr.update(value="*Model: Hidden*"),
-            gr.update(value="Regenerate judges", variant="secondary", interactive=True),
-            gr.update(value="🎲"),  # random_btn
-            False,  # Set first_game_state to False after first submission
+            # Change the button to "Regenerate" mode after evaluation
+            gr.update(
+                value="Regenerate with different models",
+                variant="secondary",
+                interactive=True
+            ),
+            gr.update(visible=False),  # spacing_div
         )
 
-    # Update the click handler to use False for is_first_game after first submission
-    def create_submit_handler():
-        first_game = True
-        
-        def handler(*args):
-            nonlocal first_game
-            result = submit_and_store(*args)
-            first_game = False  # Set to False after first submission
-            return result
-        
-        return handler
-
-    # Update the send_btn click handler
     send_btn.click(
         fn=submit_and_store,
-        inputs=[
-            use_reference_toggle,
-            eval_criteria_text,
-            human_input,
-            ai_response,
-            ground_truth,
-            score1_description,
-            score2_description,
-            score3_description,
-            score4_description,
-            score5_description,
-        ],
+        inputs=[eval_prompt, human_input, ai_response],
         outputs=[
             score_a,
             critique_a,
@@ -828,225 +744,101 @@ with gr.Blocks(theme="default", css=CSS_STYLES) as demo:
             critique_b,
             vote_a,
             vote_b,
-            vote_tie,
+            tie_button_row,
             model_a_state,
             model_b_state,
             final_prompt_state,
             model_name_a,
             model_name_b,
             send_btn,
-            random_btn,
+            spacing_div,
         ],
     )
 
+    # Update the input change handlers to also disable regenerate button
+    def handle_input_changes(prompt, *variables):
+        """Enable send button and manage regenerate button based on input changes"""
+        last_inputs = last_submission.value
+        current_inputs = {"prompt": prompt, "variables": variables}
+        inputs_changed = last_inputs != current_inputs
+        return [
+            gr.update(interactive=True),  # send button always enabled
+            gr.update(
+                interactive=not inputs_changed
+            ),  # regenerate button disabled if inputs changed
+        ]
+
+    # Update the change handlers for prompt and variables
+    #eval_prompt.change(
+    #    fn=handle_input_changes,
+    #    inputs=[eval_prompt] + [var_input for _, var_input in variable_rows],
+    #    outputs=[send_btn, regenerate_button],
+    #)
+
+    # for _, var_input in variable_rows:
+    #    var_input.change(
+    #        fn=handle_input_changes,
+    #        inputs=[eval_prompt] + [var_input for _, var_input in variable_rows],
+    #        outputs=[send_btn, regenerate_button],
+    #    )
+
+    # Add click handlers for metric buttons
+    #outputs_list = [eval_prompt] + [var_input for _, var_input in variable_rows]
+
+    #custom_btn.click(fn=lambda: set_example_metric("Custom"), outputs=outputs_list)
+
+    #hallucination_btn.click(
+    #    fn=lambda: set_example_metric("Hallucination"), outputs=outputs_list
+    #)
+
+    #precision_btn.click(fn=lambda: set_example_metric("Precision"), outputs=outputs_list)
+
+    #recall_btn.click(fn=lambda: set_example_metric("Recall"), outputs=outputs_list)
+
+    #coherence_btn.click(
+    #    fn=lambda: set_example_metric("Logical_Coherence"), outputs=outputs_list
+    #)
+
+    #faithfulness_btn.click(
+    #    fn=lambda: set_example_metric("Faithfulness"), outputs=outputs_list
+    #)
+
+    # Set default metric at startup
+    demo.load(
+        #fn=lambda: set_example_metric("Hallucination"),
+        #outputs=[eval_prompt] + [var_input for _, var_input in variable_rows],
+    )
+
     # Add random button handler
     random_btn.click(
         fn=populate_random_example,
-        inputs=[use_reference_toggle],  # Use compatible mode toggle to decide behavior
-        outputs=[
-            human_input, 
-            ai_response,
-            random_btn,
-            score_a,
-            critique_a,
-            score_b,
-            critique_b,
-            vote_a,
-            vote_b,
-            vote_tie,
-            model_name_a,
-            model_name_b,
-            ground_truth,  # Set ground truth
-        ]
+        inputs=[],
+        outputs=[human_input, ai_response]
     )
 
     # Add new input change handlers
     def handle_input_change():
-        """Reset UI state when inputs are changed"""
-        return [
-            gr.update(interactive=False),  # vote_a
-            gr.update(interactive=False),  # vote_b
-            gr.update(interactive=False),  # vote_tie
-            gr.update(value="Run judges", variant="primary"),  # send_btn
-            gr.update(value="🎲", variant="secondary"),  # random_btn
-        ]
+        return gr.update(value="Run the evaluators", variant="primary")
 
     # Update the change handlers for inputs
     human_input.change(
         fn=handle_input_change,
         inputs=[],
-        outputs=[vote_a, vote_b, vote_tie, send_btn, random_btn]
+        outputs=[send_btn]
     )
 
     ai_response.change(
         fn=handle_input_change,
         inputs=[],
-        outputs=[vote_a, vote_b, vote_tie, send_btn, random_btn]
-    )
-
-    generate_btn.click(
-        fn=lambda msg: (
-            generate_ai_response(msg)[0],  # Only take the response text
-            gr.update(
-                value="Generate AI Response",  # Keep the label
-                interactive=False  # Disable the button
-            )
-        ),
-        inputs=[human_input],
-        outputs=[ai_response, generate_btn]
-    )
-
-    human_input.change(
-        fn=lambda x: gr.update(interactive=bool(x.strip())),
-        inputs=[human_input],
-        outputs=[generate_btn]
+        outputs=[send_btn]
     )
 
     # Update the demo.load to include the random example population
     demo.load(
-        fn=lambda: populate_random_example(None, False),  # Pass False for initial compatible_mode
+        fn=populate_random_example,
         inputs=[],
-        outputs=[
-            human_input,
-            ai_response,
-            random_btn,
-            score_a,
-            critique_a,
-            score_b,
-            critique_b,
-            vote_a,
-            vote_b,
-            vote_tie,
-            model_name_a,
-            model_name_b,
-            ground_truth,
-        ]
+        outputs=[human_input, ai_response]
     )
 
-    # Add new state variables for compatible mode
-    eval_criteria_previous = gr.State(value=DEFAULT_EVAL_CRITERIA)
-    score1_previous = gr.State(value=DEFAULT_SCORE_1)
-    score2_previous = gr.State(value=DEFAULT_SCORE_2)
-    score3_previous = gr.State(value=DEFAULT_SCORE_3)
-    score4_previous = gr.State(value=DEFAULT_SCORE_4)
-    score5_previous = gr.State(value=DEFAULT_SCORE_5)
-
-    # Add new functions to handle compatible mode saves/cancels
-    def save_compatible_prompt(criteria, score1, score2, score3, score4, score5):
-        return [
-            gr.update(value=criteria),  # Update criteria
-            criteria,  # Update previous criteria state
-            gr.update(value=score1),
-            score1,
-            gr.update(value=score2),
-            score2,
-            gr.update(value=score3),
-            score3,
-            gr.update(value=score4),
-            score4,
-            gr.update(value=score5),
-            score5,
-            gr.update(visible=False)  # Hide buttons
-        ]
-
-    def cancel_compatible_prompt(prev_criteria, prev_score1, prev_score2, prev_score3, prev_score4, prev_score5):
-        return [
-            gr.update(value=prev_criteria),
-            prev_criteria,
-            gr.update(value=prev_score1),
-            prev_score1,
-            gr.update(value=prev_score2),
-            prev_score2,
-            gr.update(value=prev_score3),
-            prev_score3,
-            gr.update(value=prev_score4),
-            prev_score4,
-            gr.update(value=prev_score5),
-            prev_score5,
-            gr.update(visible=False)
-        ]
-
-    def show_compatible_edit_buttons(*current_values):
-        previous_values = current_values[1::2]  # Get previous values
-        current_values = current_values[::2]    # Get current values
-        return gr.update(visible=any(curr != prev for curr, prev in zip(current_values, previous_values)))
-
-    # Add click handlers for compatible mode buttons
-    compatible_save_btn.click(
-        fn=save_compatible_prompt,
-        inputs=[
-            eval_criteria_text,
-            score1_description,
-            score2_description,
-            score3_description,
-            score4_description,
-            score5_description
-        ],
-        outputs=[
-            eval_criteria_text,
-            eval_criteria_previous,
-            score1_description,
-            score1_previous,
-            score2_description,
-            score2_previous,
-            score3_description,
-            score3_previous,
-            score4_description,
-            score4_previous,
-            score5_description,
-            score5_previous,
-            compatible_edit_buttons_row
-        ]
-    )
-
-    compatible_cancel_btn.click(
-        fn=cancel_compatible_prompt,
-        inputs=[
-            eval_criteria_previous,
-            score1_previous,
-            score2_previous,
-            score3_previous,
-            score4_previous,
-            score5_previous
-        ],
-        outputs=[
-            eval_criteria_text,
-            eval_criteria_previous,
-            score1_description,
-            score1_previous,
-            score2_description,
-            score2_previous,
-            score3_description,
-            score3_previous,
-            score4_description,
-            score4_previous,
-            score5_description,
-            score5_previous,
-            compatible_edit_buttons_row
-        ]
-    )
-
-    # Add change handlers for all compatible mode inputs
-    for component in [eval_criteria_text, score1_description, score2_description, 
-                     score3_description, score4_description, score5_description]:
-        component.change(
-            fn=show_compatible_edit_buttons,
-            inputs=[
-                eval_criteria_text,
-                eval_criteria_previous,
-                score1_description,
-                score1_previous,
-                score2_description,
-                score2_previous,
-                score3_description,
-                score3_previous,
-                score4_description,
-                score4_previous,
-                score5_description,
-                score5_previous
-            ],
-            outputs=compatible_edit_buttons_row
-        )
-
 if __name__ == "__main__":
     demo.launch()

common.py

@@ -37,7 +37,7 @@ CSS_STYLES = """
         gap: 8px;
     }
 """
-
+    
 # Default Eval Prompt
 EVAL_DESCRIPTION = """
 ## 📝 Tips
@@ -47,6 +47,27 @@ EVAL_DESCRIPTION = """
 - Examples (Optional)
 """
 
+DEFAULT_EVAL_PROMPT = """You are assessing a chat bot response to a user's input based on how well it follows the user's instructions. Your evaluation should consider factors such as the helpfulness, relevance, accuracy, depth, creativity, and level of detail of the response. Do not allow the length of the response to influence your evaluation. Be objective as possible and give a brief explanation for your score.
+
+Score:
+Score 1: The response ignores or misinterprets instructions, providing irrelevant or inaccurate content that fails to address the request.
+Score 2: The response follows instructions partially but misses key elements, lacking depth or precision while containing minor inaccuracies.
+Score 3: The response follows main instructions adequately, providing correct and relevant information with reasonable depth.
+Score 4: The response follows instructions thoroughly with strong attention to detail, offering accurate, well-developed content that thoughtfully addresses needs.
+Score 5: The response demonstrates exceptional instruction following with precise, comprehensive content that shows both insight and perfect alignment with the request. 
+
+[User Query]: {{input}}
+
+[Response]: {{response}}"""
+
+# Default Variable Values
+DEFAULT_INPUT = """Which of these animals is least likely to be found in a rainforest?"
+A) Jaguar
+B) Toucan
+C) Polar Bear
+D) Sloth"""
+DEFAULT_RESPONSE = "C) Polar Bear"
+
 # Voting Section Header
 VOTING_HEADER = """
 # Start Voting Now
@@ -68,50 +89,55 @@ POLICY_CONTENT = """
 
 Atla is an applied research organization that trains models as evaluators to capture human preferences. We're a team of researchers, engineers, and operational leaders, with experience spanning a variety of disciplines, all working together to build reliable and understandable AI systems. Our research is informed by our experiences conducting AI safety research at the UK AI Task Force, OpenAI and the Stanford Existential Risks Initiative.
 <br><br>
-# [Our Mission](https://www.atla-ai.com/company)
+# Our Mission
 
-By creating advanced evaluation models, we enable AI developers to identify and fix risks, leading to safer, more reliable AI that can be trusted and widely used. Our aim is to surpass the current state-of-the-art evaluation methods by training models specifically for evaluation. AIs will probably become very powerful, and perform tasks that are difficult for us to verify. We want to enable humans to oversee AI systems that are solving tasks too difficult for humans to evaluate. 
-Read more about [our approach to scalable oversight](https://www.atla-ai.com/post/scaling-alignment) on our blog.
+By creating advanced evaluation models, we enable AI developers to identify and fix risks, leading to safer, more reliable AI that can be trusted and widely used. Our aim is to surpass the current state-of-the-art evaluation methods by training models specifically for evaluation. AIs will probably become very powerful, and perform tasks that are difficult for us to verify. We want to enable humans to oversee AI systems that are solving tasks too difficult for humans to evaluate. We have written more about [our approach to scalable oversight](https://www.atla-ai.com/post/scaling-alignment) on our blog.
 <br><br>
 # Judge Arena Policy
 
 ## Overview
 
-Judge Arena is an open-source platform dedicated to determining which models make the best judges. Users can run evals and assess anonymized responses from two competing model judges, choosing the better judgement or declaring a tie. This policy outlines our commitments to maintain a fair and open environment :)
+Judge Arena is an open-source platform dedicated to improving the standard of evaluation of generative AI models in their role as judges. Users can run evals and assess anonymized responses from two competing model judges, choosing the better judgement or declaring a tie. This policy outlines our commitments to maintain a fair, open, and collaborative environment :)
 
 ## Transparency
 
 - **Open-Source**: Judge Arena's code is open-source and available on GitHub. We encourage contributions from the community and anyone can replicate or modify the platform to suit their needs. We use proprietary model provider APIs where provided and Together AI's API to serve leading open-source models.
-- **Methodology**: All processes related to model evaluation, rating calculations, and model selection are openly documented. 
+- **Methodology**: All processes related to model evaluation, rating calculations, and model selection are openly documented. We'd like to ensure that our ranking system is understandable and reproducible by others!
 - **Data Sharing**: Periodically, we'll share 20% of the collected evaluation data with the community. The data collected from Judge Arena is restricted to an anonymized user ID, the final prompt sent, the model responses, the user vote, and the timestamp.
 
 ## Model Inclusion Criteria
 
 Judge Arena is specifically designed to assess AI models that function as evaluators (a.k.a judges). This includes but is not limited to powerful general-purpose models and the latest language models designed for evaluation tasks. Models are eligible for inclusion if they meet the following criteria:
 
-- **Judge Capability**: The model should possess the ability to score AND critique other models' outputs effectively.
-- **Promptable:** The model must be promptable to be evaluate in different scoring formats, for different criteria.
+- **Judge Capability**: The model should possess the ability to score AND critique responses, content, or other models' outputs effectively.
+- **Adaptable:** The model must be prompt-able to be evaluate in different scoring formats, for different criteria.
 - **Accessibility**:
    - **Public API Access**: Models accessible through public APIs without restrictive barriers.
    - **Open-Source Models**: Models with publicly available weights that can be downloaded and run by the community.
 
 ## Leaderboard Management
 
-- **ELO Ranking System**: Models are ranked on a public leaderboard based on aggregated user evaluations. We use an ELO rating system to rank AI judges on the public leaderboard. Each model begins with an initial rating of 1200, and we use a K-factor of 32 to determine the maximum rating adjustment after each evaluation.
+- **ELO Ranking System**: Models are ranked on a public leaderboard based on aggregated user evaluations. We use an ELO rating system to rank AI judges on the public leaderboard. Each model begins with an initial rating of 1500 (as is used by the International Chess Federation), and we use a K-factor of 32 to determine the maximum rating adjustment after each evaluation.
 - **Minimum Period**: Listed models remain accessible on Judge Arena for a minimum period of two weeks so they can be comprehensively evaluated.
 - **Deprecation Policy**: Models may be removed from the leaderboard if they become inaccessible or are no longer publicly available.
 
-*This policy might be updated to reflect changes in our practices or in response to community feedback.*
-<br><br>
+This policy might be updated to reflect changes in our practices or in response to community feedback.
+
 # FAQ
 
 **Isn't this the same as Chatbot Arena?**
 
 We are big fans of what the LMSYS team have done with Chatbot Arena and fully credit them for the inspiration to develop this. We were looking for a dynamic leaderboard that graded on AI judge capabilities and didn't manage to find one, so we created Judge Arena. This UI is designed especially for evals; to match the format of the model-based eval prompts that you would use in your LLM evaluation / monitoring tool.
 
+**What are the Evaluator Prompt Templates based on?**
+
+As a quick start, we've set up templates that cover the most popular evaluation metrics out there on LLM evaluation / monitoring tools, often known as 'base metrics'. The data samples used in these were randomly picked from popular datasets from academia - [ARC](https://huggingface.co/datasets/allenai/ai2_arc), [Preference Collection](https://huggingface.co/datasets/prometheus-eval/Preference-Collection), [RewardBench](https://huggingface.co/datasets/allenai/reward-bench), [RAGTruth](https://arxiv.org/abs/2401.00396).
+
+These templates are designed as a starting point to showcase how to interact with the Judge Arena, especially for those less familiar with using LLM judges.
+
 **Why should I trust this leaderboard?**
 
-We have listed out our efforts to be fully transparent in the policies above. All of the code for this leaderboard is open-source and can be found on our [Github](https://github.com/atla-ai/judge-arena). Check out our [blog](https://www.atla-ai.com/blog) to stay up to date as we analyse the results from the leaderboard.
+We have listed out our efforts to be fully transparent in the policies above. All of the code for this leaderboard is open-source and can be found on our [Github](https://github.com/atla-ai/judge-arena).
 
 **Who funds this effort?**
 
@@ -122,5 +148,4 @@ Atla currently funds this out of our own pocket. We are looking for API credits
 We are training a general-purpose evaluator that you will soon be able to run in this Judge Arena. Our next step will be to open-source a powerful model that the community can use to run fast and accurate evaluations.
 <br><br>
 # Get in touch
-We’d love to hear your feedback! For general feature requests or to submit / suggest new models to add to the arena, please open up a discussion in the [community](https://huggingface.co/spaces/AtlaAI/judge-arena/discussions) tab. You can also contact us directly on [X](https://x.com/Atla_AI) or [Discord](https://discord.com/invite/qFCMgkGwUK).
-\nPlease file any issues on our [Github](https://github.com/atla-ai/judge-arena)."""
+Feel free to email us at [support@atla-ai.com](mailto:support@atla-ai.com) or leave feedback on our [Github](https://github.com/atla-ai/judge-arena)!"""

data/models.jsonl

@@ -1,27 +1,16 @@
-{"name": "Meta Llama 3.1 70B Instruct Turbo", "organization": "Meta", "license": "Open Source", "api_model": "meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo", "active": false}
-{"name": "Meta Llama 3.1 405B Instruct Turbo", "organization": "Meta", "license": "Open Source", "api_model": "meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo", "active": true}
-{"name": "Gemma 2 27B", "organization": "Google", "license": "Open Source", "api_model": "google/gemma-2-27b-it", "active": true}
-{"name": "Gemma 2 9B", "organization": "Google", "license": "Open Source", "api_model": "google/gemma-2-9b-it", "active": true}
-{"name": "Qwen 2 Instruct (72B)", "organization": "Alibaba", "license": "Open Source", "api_model": "Qwen/Qwen2-72B-Instruct", "active": true}
-{"name": "Mistral (7B) Instruct v0.3", "organization": "Mistral AI", "license": "Open Source", "api_model": "mistralai/Mistral-7B-Instruct-v0.3", "active": true}
-{"name": "GPT-4o", "organization": "OpenAI", "license": "Proprietary", "api_model": "gpt-4o", "active": true}
-{"name": "GPT-4 Turbo", "organization": "OpenAI", "license": "Proprietary", "api_model": "gpt-4-turbo", "active": true}
-{"name": "GPT-3.5 Turbo", "organization": "OpenAI", "license": "Proprietary", "api_model": "gpt-3.5-turbo", "active": true}
-{"name": "Claude 3 Haiku", "organization": "Anthropic", "license": "Proprietary", "api_model": "claude-3-haiku-20240307", "active": true}
-{"name": "Claude 3 Sonnet", "organization": "Anthropic", "license": "Proprietary", "api_model": "claude-3-sonnet-20240229", "active": true}
-{"name": "Claude 3 Opus", "organization": "Anthropic", "license": "Proprietary", "api_model": "claude-3-opus-latest", "active": true}
-{"name": "Meta Llama 3.1 8B Instruct Turbo", "organization": "Meta", "license": "Open Source", "api_model": "meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo", "active": true}
-{"name": "Qwen 2.5 72B Instruct Turbo", "organization": "Alibaba", "license": "Open Source", "api_model": "Qwen/Qwen2.5-72B-Instruct-Turbo", "active": true}
-{"name": "Qwen 2.5 7B Instruct Turbo", "organization": "Alibaba", "license": "Open Source", "api_model": "Qwen/Qwen2.5-7B-Instruct-Turbo", "active": true}
-{"name": "Mistral (7B) Instruct v0.1", "organization": "Mistral AI", "license": "Open Source", "api_model": "mistralai/Mistral-7B-Instruct-v0.1", "active": true}
-{"name": "Claude 3.5 Sonnet", "organization": "Anthropic", "license": "Proprietary", "api_model": "claude-3-5-sonnet-latest", "active": true}
-{"name": "Claude 3.5 Haiku", "organization": "Anthropic", "license": "Proprietary", "api_model": "claude-3-5-haiku-latest", "active": true}
-{"name": "Prometheus-7b v2", "organization": "Prometheus", "license": "Open Source", "api_model": "prometheus/prometheus-7b-v2", "active": false}
-{"name": "Command-R", "organization": "Cohere", "license": "Proprietary", "api_model": "command-r", "active": true}
-{"name": "Command-R Plus", "organization": "Cohere", "license": "Proprietary", "api_model": "command-r-plus", "active": true}
-{"name": "Atla-8B-preview", "organization": "Atla", "license": "Open Source", "api_model": "Atla-8B-preview", "active": false}
-{"name": "Meta Llama 3.3 70B Instruct Turbo", "organization": "Meta", "license": "Open Source", "api_model": "meta-llama/Llama-3.3-70B-Instruct-Turbo", "active": true}
-{"name": "QwQ 32B Preview", "organization": "Qwen", "license": "Open Source", "api_model": "Qwen/QwQ-32B-Preview", "active": true}
-{"name": "Flow-Judge-v0.1", "organization": "Flow AI", "license": "Open Source", "api_model": "Flow-Judge-v0.1-4.65bpw-exl2", "active": false}
-{"name": "SFR-LLaMA-3.1-70B-Judge", "organization": "Salesforce", "license": "Proprietary", "api_model": "sfr-llama-3.1-70b-judge", "active": true}
-{"name": "Atla Selene 1 Mini", "organization": "Atla", "license": "Open Source", "api_model": "Atla-Selene-Mini", "active": true}
+{"name": "Meta Llama 3.1 70B Instruct Turbo", "organization": "Meta", "license": "Open Source", "api_model": "meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo"}
+{"name": "Meta Llama 3.1 405B Instruct Turbo", "organization": "Meta", "license": "Open Source", "api_model": "meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo"}
+{"name": "Gemma 2 27B", "organization": "Google", "license": "Open Source", "api_model": "google/gemma-2-27b-it"}
+{"name": "Gemma 2 9B", "organization": "Google", "license": "Open Source", "api_model": "google/gemma-2-9b-it"}
+{"name": "Qwen 2 Instruct (72B)", "organization": "Alibaba", "license": "Open Source", "api_model": "Qwen/Qwen2-72B-Instruct"}
+{"name": "Mistral (7B) Instruct v0.3", "organization": "Mistral AI", "license": "Open Source", "api_model": "mistralai/Mistral-7B-Instruct-v0.3"}
+{"name": "GPT-4o", "organization": "OpenAI", "license": "Proprietary", "api_model": "gpt-4o"}
+{"name": "GPT-4 Turbo", "organization": "OpenAI", "license": "Proprietary", "api_model": "gpt-4-turbo"}
+{"name": "GPT-3.5 Turbo", "organization": "OpenAI", "license": "Proprietary", "api_model": "gpt-3.5-turbo"}
+{"name": "Claude 3 Haiku", "organization": "Anthropic", "license": "Proprietary", "api_model": "claude-3-haiku-20240307"}
+{"name": "Claude 3 Sonnet", "organization": "Anthropic", "license": "Proprietary", "api_model": "claude-3-sonnet-20240229"}
+{"name": "Claude 3 Opus", "organization": "Anthropic", "license": "Proprietary", "api_model": "claude-3-opus-20240229"}
+{"name": "Meta Llama 3.1 8B Instruct Turbo", "organization": "Meta", "license": "Open Source", "api_model": "meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo"}
+{"name": "Qwen 2.5 72B Instruct Turbo", "organization": "Alibaba", "license": "Open Source", "api_model": "Qwen/Qwen2.5-72B-Instruct-Turbo"}
+{"name": "Qwen 2.5 7B Instruct Turbo", "organization": "Alibaba", "license": "Open Source", "api_model": "Qwen/Qwen2.5-7B-Instruct-Turbo"}
+{"name": "Mistral (7B) Instruct v0.1", "organization": "Mistral AI", "license": "Open Source", "api_model": "mistralai/Mistral-7B-Instruct-v0.1"}

gen_api_answer.py

@@ -1,238 +1,95 @@
 from openai import OpenAI
 import anthropic
 from together import Together
-import cohere
 import json
 import re
-import os
-import requests
-from prompts import (
-    JUDGE_SYSTEM_PROMPT,
-    PROMETHEUS_PROMPT,
-    PROMETHEUS_PROMPT_WITH_REFERENCE,
-    ATLA_PROMPT,
-    ATLA_PROMPT_WITH_REFERENCE,
-     FLOW_JUDGE_PROMPT
-)
-from transformers import AutoTokenizer
 
 # Initialize clients
 anthropic_client = anthropic.Anthropic()
 openai_client = OpenAI()
 together_client = Together()
-hf_api_key = os.getenv("HF_API_KEY")
-flow_judge_api_key = os.getenv("FLOW_JUDGE_API_KEY")
-cohere_client = cohere.ClientV2(os.getenv("CO_API_KEY"))
-salesforce_api_key = os.getenv("SALESFORCE_API_KEY")
-def get_openai_response(model_name, prompt, system_prompt=JUDGE_SYSTEM_PROMPT, max_tokens=500, temperature=0):
+
+# Initialize OpenAI client
+
+EXAMPLE_GENERATION_PROMPT_SYSTEM = """You are an assistant that generates random conversations between a human and an AI assistant for testing purposes."""
+EXAMPLE_GENERATION_PROMPT_USER = """Please provide a random human message and an appropriate AI response in the format of an academic benchmark dataset e.g.,. User: "Hi, I'm trying to solve a crossword puzzle, but I've never done one of these before. Can you help me out?" / AI Response: "Absolutely! I'd be delighted to help you with your crossword puzzle. Just tell me the clues and the number of letters needed for each answer (and any letters you may have already filled in), and I'll do my best to help you find the solutions. If you have any specific questions about how to approach solving crossword puzzles in general, feel free to ask those as well!". Format the output as JSON:\n\n{\"human\": \"<human message>\", \"ai\": \"<AI assistant response>\"}"""
+
+def get_random_human_ai_pair():
+    # Use GPT-3.5 to generate a random conversation
+    completion = openai_client.chat.completions.create(
+        model="gpt-3.5-turbo",
+        messages=[
+            {"role": "system", "content": EXAMPLE_GENERATION_PROMPT_SYSTEM},
+            {"role": "user", "content": EXAMPLE_GENERATION_PROMPT_USER},
+        ],
+        max_completion_tokens=300,
+        temperature=1,
+    )
+    
+    # Parse the response to get the human input and AI response
+    raw_response = completion.choices[0].message.content.strip()
+    
+    try:
+        data = json.loads(raw_response)
+        human_message = data.get("human", "Hello, how are you?")
+        ai_message = data.get("ai", "I'm doing well, thank you!")
+    except json.JSONDecodeError:
+        # If parsing fails, set default messages
+        human_message = "Hello, how are you?"
+        ai_message = "I'm doing well, thank you!"
+    
+    return human_message, ai_message
+
+SYSTEM_PROMPT = """Please act as an impartial judge and evaluate based on the user's instruction. Your output format should strictly adhere to JSON as follows: {"feedback": "<write feedback>", "result": <numerical score>}. Ensure the output is valid JSON, without additional formatting or explanations."""
+
+
+def get_openai_response(model_name, prompt):
     """Get response from OpenAI API"""
     try:
         response = openai_client.chat.completions.create(
             model=model_name,
             messages=[
-                {"role": "system", "content": system_prompt},
+                {"role": "system", "content": SYSTEM_PROMPT},
                 {"role": "user", "content": prompt},
             ],
-            max_completion_tokens=max_tokens,
-            temperature=temperature,
         )
         return response.choices[0].message.content
     except Exception as e:
         return f"Error with OpenAI model {model_name}: {str(e)}"
 
-def get_anthropic_response(model_name, prompt, system_prompt=JUDGE_SYSTEM_PROMPT, max_tokens=500, temperature=0):
+
+def get_anthropic_response(model_name, prompt):
     """Get response from Anthropic API"""
     try:
         response = anthropic_client.messages.create(
             model=model_name,
-            max_tokens=max_tokens,
-            temperature=temperature,
-            system=system_prompt,
+            max_tokens=1000,
+            temperature=0,
+            system=SYSTEM_PROMPT,
             messages=[{"role": "user", "content": [{"type": "text", "text": prompt}]}],
         )
         return response.content[0].text
     except Exception as e:
         return f"Error with Anthropic model {model_name}: {str(e)}"
 
-def get_together_response(model_name, prompt, system_prompt=JUDGE_SYSTEM_PROMPT, max_tokens=500, temperature=0):
+
+def get_together_response(model_name, prompt):
     """Get response from Together API"""
     try:
         response = together_client.chat.completions.create(
             model=model_name,
             messages=[
-                {"role": "system", "content": system_prompt},
+                {"role": "system", "content": SYSTEM_PROMPT},
                 {"role": "user", "content": prompt},
             ],
-            max_tokens=max_tokens,
-            temperature=temperature,
             stream=False,
         )
         return response.choices[0].message.content
     except Exception as e:
         return f"Error with Together model {model_name}: {str(e)}"
 
-def get_prometheus_response(model_name, prompt, system_prompt=None, max_tokens=500, temperature=0.01):
-    """Get response from Hugging Face model"""
-    try:
-        headers = {
-            "Accept": "application/json",
-            "Authorization": f"Bearer {hf_api_key}",
-            "Content-Type": "application/json"
-        }
-        
-        # Create messages list for chat template
-        messages = []
-        if system_prompt:
-            messages.append({"role": "system", "content": system_prompt})
-        messages.append({"role": "user", "content": prompt})
-        
-        # Apply chat template
-        model_id = "prometheus-eval/prometheus-7b-v2.0"
-        tokenizer = AutoTokenizer.from_pretrained(model_id, token=hf_api_key)
-        formatted_prompt = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
-        
-        payload = {
-            "inputs": formatted_prompt,
-            "parameters": {
-                "max_new_tokens": max_tokens,
-                "return_full_text": False,
-                "temperature": temperature
-            }
-        }
-        
-        response = requests.post(
-            "https://otb7jglxy6r37af6.us-east-1.aws.endpoints.huggingface.cloud",
-            headers=headers,
-            json=payload
-        )
-        return response.json()[0]["generated_text"]
-    except Exception as e:
-        return f"Error with Hugging Face model {model_name}: {str(e)}"
-
-def get_atla_response(model_name, prompt, system_prompt=None, max_tokens=500, temperature=0.01):
-    """Get response from HF endpoint for Atla model"""
-    try:
-        headers = {
-            "Accept": "application/json",
-            "Authorization": f"Bearer {hf_api_key}",
-            "Content-Type": "application/json"
-        }
-        
-        # Create messages list for chat template
-        messages = []
-        if system_prompt:
-            messages.append({"role": "system", "content": system_prompt})
-        messages.append({"role": "user", "content": prompt})
-        
-        # Apply chat template
-        model_id = "AtlaAI/Selene-1-Mini-Llama-3.1-8B"
-        tokenizer = AutoTokenizer.from_pretrained(model_id, token=hf_api_key)
-        formatted_prompt = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
-        
-        payload = {
-            "inputs": formatted_prompt,
-            "parameters": {
-                "max_new_tokens": max_tokens,
-                "return_full_text": False,
-                "temperature": temperature,
-                "seed": 42,
-                "add_generation_prompt": True
-            }
-        }
-        
-        response = requests.post(
-            "https://bkp9p28gri93egqh.us-east-1.aws.endpoints.huggingface.cloud",
-            headers=headers,
-            json=payload
-        )
-        return response.json()[0]["generated_text"]
-    except Exception as e:
-        return f"Error with Atla model {model_name}: {str(e)}"
-
-def get_flow_judge_response(model_name, prompt, max_tokens=2048, temperature=0.1, top_p=0.95) -> str:
-    """Get response from Flow Judge"""
-    try:
-        response = requests.post(
-            "https://arena.flow-ai.io/v1/chat/completions",
-            headers={
-                "Content-Type": "application/json",
-                "Authorization": f"Bearer {flow_judge_api_key}"
-            },
-            json={
-                "model": model_name,
-                "messages": [
-                    {"role": "user", "content": prompt}
-                ],
-                "max_tokens": max_tokens,
-                "temperature": temperature,
-                "top_p": top_p,
-                "stop": None
-            }
-        )
-        response.raise_for_status()
-        return response.json()["choices"][0]['message']['content']
-    except Exception as e:
-        return f"Error with Flow Judge completions model {model_name}: {str(e)}"
-
-def get_cohere_response(model_name, prompt, system_prompt=JUDGE_SYSTEM_PROMPT, max_tokens=500, temperature=0):
-    """Get response from Cohere API"""
-    try:
-        response = cohere_client.chat(
-            model=model_name,
-            messages=[
-                {"role": "system", "content": system_prompt},
-                {"role": "user", "content": prompt}
-            ],
-            max_tokens=max_tokens,
-            temperature=temperature
-        )
-        # Extract the text from the content items
-        content_items = response.message.content
-        if isinstance(content_items, list):
-            # Get the text from the first content item
-            return content_items[0].text
-        return str(content_items)  # Fallback if it's not a list
-    except Exception as e:
-        return f"Error with Cohere model {model_name}: {str(e)}"
-
-def get_salesforce_response(model_name, prompt, system_prompt=None, max_tokens=2048, temperature=0):
-    """Get response from Salesforce Research API"""
-    try:
-        headers = {
-            'accept': 'application/json',
-            "content-type": "application/json",
-            "X-Api-Key": salesforce_api_key,
-        }
-
-        # Create messages list
-        messages = []
-        messages.append({"role": "user", "content": prompt})
-
-        json_data = {
-            "prompts": messages,
-            "temperature": temperature,
-            "top_p": 1,
-            "max_tokens": max_tokens,
-        }
-
-        response = requests.post(
-            'https://gateway.salesforceresearch.ai/sfr-judge/process',
-            headers=headers,
-            json=json_data
-        )
-        response.raise_for_status()
-        return response.json()['result'][0]
-    except Exception as e:
-        return f"Error with Salesforce model {model_name}: {str(e)}"
 
-def get_model_response(
-    model_name,
-    model_info,
-    prompt_data,
-    use_reference=False,
-    max_tokens=500,
-    temperature=0
-):
+def get_model_response(model_name, model_info, prompt):
     """Get response from appropriate API based on model organization"""
     if not model_info:
         return "Model not found or unsupported."
@@ -240,250 +97,37 @@ def get_model_response(
     api_model = model_info["api_model"]
     organization = model_info["organization"]
 
-    # Determine if model is Prometheus, Atla, Flow Judge, or Salesforce
-    is_prometheus = (organization == "Prometheus")
-    is_atla = (organization == "Atla")
-    is_flow_judge = (organization == "Flow AI")
-    is_salesforce = (organization == "Salesforce")  
-    
-    # For non-Prometheus/Atla/Flow Judge/Salesforce models, use the Judge system prompt
-    system_prompt = None if (is_prometheus or is_atla or is_flow_judge or is_salesforce) else JUDGE_SYSTEM_PROMPT
-
-    # Select the appropriate base prompt
-    if is_atla or is_salesforce:  # Use same prompt for Atla and Salesforce
-        base_prompt = ATLA_PROMPT_WITH_REFERENCE if use_reference else ATLA_PROMPT
-    elif is_flow_judge:
-        base_prompt = FLOW_JUDGE_PROMPT
-    else:
-        base_prompt = PROMETHEUS_PROMPT_WITH_REFERENCE if use_reference else PROMETHEUS_PROMPT
-    
-    # For non-Prometheus/non-Atla/non-Salesforce models, use Prometheus but replace the output format with JSON
-    if not (is_prometheus or is_atla or is_flow_judge or is_salesforce):
-        base_prompt = base_prompt.replace(
-            '3. The output format should look as follows: "Feedback: (write a feedback for criteria) [RESULT] (an integer number between 1 and 5)"',
-            '3. Your output format should strictly adhere to JSON as follows: {{"feedback": "<write feedback>", "result": <numerical score>}}. Ensure the output is valid JSON, without additional formatting or explanations.'
-        )
-
-    try:
-        if not is_flow_judge:
-            # Format the prompt with the provided data, only using available keys
-            final_prompt = base_prompt.format(
-                human_input=prompt_data['human_input'],
-                ai_response=prompt_data['ai_response'],
-                ground_truth_input=prompt_data.get('ground_truth_input', ''),
-                eval_criteria=prompt_data['eval_criteria'],
-                score1_desc=prompt_data['score1_desc'],
-                score2_desc=prompt_data['score2_desc'],
-                score3_desc=prompt_data['score3_desc'],
-                score4_desc=prompt_data['score4_desc'],
-                score5_desc=prompt_data['score5_desc']
-            )
-        else:
-            human_input = f"<user_input>\n{prompt_data['human_input']}\n</user_input>"
-            ai_response = f"<response>\n{prompt_data['ai_response']}\n</response>"
-            ground_truth=prompt_data.get('ground_truth_input', '')
-            if ground_truth:
-                response_reference = f"<response_reference>\n{ground_truth}\n</response_reference>"
-            else:
-                response_reference = ""
-            eval_criteria = prompt_data['eval_criteria']
-            score1_desc = f"- Score 1: {prompt_data['score1_desc']}\n"
-            score2_desc = f"- Score 2: {prompt_data['score2_desc']}\n"
-            score3_desc = f"- Score 3: {prompt_data['score3_desc']}\n"
-            score4_desc = f"- Score 4: {prompt_data['score4_desc']}\n"
-            score5_desc = f"- Score 5: {prompt_data['score5_desc']}"
-            rubric = score1_desc + score2_desc + score3_desc + score4_desc + score5_desc
-            if response_reference:
-                inputs = human_input + "\n"+ response_reference
-            else:
-                inputs = human_input
-            final_prompt = base_prompt.format(
-                INPUTS=inputs,
-                OUTPUT=ai_response,
-                EVALUATION_CRITERIA=eval_criteria,
-                RUBRIC=rubric
-            )
-        
-    except KeyError as e:
-        return f"Error formatting prompt: Missing required field {str(e)}"
-
     try:
         if organization == "OpenAI":
-            return get_openai_response(
-                api_model, final_prompt, system_prompt, max_tokens, temperature
-            )
+            return get_openai_response(api_model, prompt)
         elif organization == "Anthropic":
-            return get_anthropic_response(
-                api_model, final_prompt, system_prompt, max_tokens, temperature
-            )
-        elif organization == "Prometheus":
-            return get_prometheus_response(
-                api_model, final_prompt, system_prompt, max_tokens, temperature = 0.01
-            )
-        elif organization == "Atla":
-            return get_atla_response(
-                api_model, final_prompt, system_prompt, max_tokens, temperature = 0.01
-            )
-        elif organization == "Cohere":
-            return get_cohere_response(
-                api_model, final_prompt, system_prompt, max_tokens, temperature
-            )
-        elif organization == "Flow AI":
-            return get_flow_judge_response(
-                api_model, final_prompt
-            )
-        elif organization == "Salesforce":
-            response = get_salesforce_response(
-                api_model, final_prompt, system_prompt, max_tokens, temperature
-            )           
-            return response
+            return get_anthropic_response(api_model, prompt)
         else:
             # All other organizations use Together API
-            return get_together_response(
-                api_model, final_prompt, system_prompt, max_tokens, temperature
-            )
+            return get_together_response(api_model, prompt)
     except Exception as e:
         return f"Error with {organization} model {model_name}: {str(e)}"
 
+
 def parse_model_response(response):
     try:
         # Debug print
         print(f"Raw model response: {response}")
 
-        # If response is already a dictionary, use it directly
-        if isinstance(response, dict):
-            return str(response.get("result", "N/A")), response.get("feedback", "N/A")
-
         # First try to parse the entire response as JSON
         try:
             data = json.loads(response)
             return str(data.get("result", "N/A")), data.get("feedback", "N/A")
         except json.JSONDecodeError:
-            # If that fails, check if this is a Salesforce response (which uses ATLA format)
-            if "**Reasoning:**" in response or "**Result:**" in response:
-                # Use ATLA parser for Salesforce responses
-                return atla_parse_model_response(response)
-                
-            # Otherwise try to find JSON within the response
-            json_match = re.search(r"{.*}", response, re.DOTALL)
+            # If that fails (typically for smaller models), try to find JSON within the response
+            json_match = re.search(r"{.*}", response)
             if json_match:
                 data = json.loads(json_match.group(0))
                 return str(data.get("result", "N/A")), data.get("feedback", "N/A")
             else:
-                return "Error", f"Invalid response format returned - here is the raw model response: {response}"
+                return "Error", f"Failed to parse response: {response}"
 
     except Exception as e:
         # Debug print for error case
         print(f"Failed to parse response: {str(e)}")
-        
-        # If the error message itself contains valid JSON, try to parse that
-        try:
-            error_json_match = re.search(r"{.*}", str(e), re.DOTALL)
-            if error_json_match:
-                data = json.loads(error_json_match.group(0))
-                return str(data.get("result", "N/A")), data.get("feedback", "N/A")
-        except:
-            pass
-            
         return "Error", f"Failed to parse response: {response}"
-    
-def prometheus_parse_model_response(output):
-    try:
-        print(f"Raw model response: {output}")
-        output = output.strip()
-
-        # Remove "Feedback:" prefix if present (case insensitive)
-        output = re.sub(r'^feedback:\s*', '', output, flags=re.IGNORECASE)
-        
-        # New pattern to match [RESULT] X at the beginning
-        begin_result_pattern = r'^\[RESULT\]\s*(\d+)\s*\n*(.*?)$'
-        begin_match = re.search(begin_result_pattern, output, re.DOTALL | re.IGNORECASE)
-        if begin_match:
-            score = int(begin_match.group(1))
-            feedback = begin_match.group(2).strip()
-            return str(score), feedback
-
-        # Existing patterns for end-of-string results...
-        pattern = r"(.*?)\s*\[RESULT\]\s*[\(\[]?(\d+)[\)\]]?"
-        match = re.search(pattern, output, re.DOTALL | re.IGNORECASE)
-        if match:
-            feedback = match.group(1).strip()
-            score = int(match.group(2))
-            return str(score), feedback
-
-        # If no match, try to match "... Score: X"
-        pattern = r"(.*?)\s*(?:Score|Result)\s*:\s*[\(\[]?(\d+)[\)\]]?"
-        match = re.search(pattern, output, re.DOTALL | re.IGNORECASE)
-        if match:
-            feedback = match.group(1).strip()
-            score = int(match.group(2))
-            return str(score), feedback
-
-        # Pattern to handle [Score X] at the end
-        pattern = r"(.*?)\s*\[(?:Score|Result)\s*[\(\[]?(\d+)[\)\]]?\]$"
-        match = re.search(pattern, output, re.DOTALL)
-        if match:
-            feedback = match.group(1).strip()
-            score = int(match.group(2))
-            return str(score), feedback
-
-        # Final fallback attempt
-        pattern = r"[\(\[]?(\d+)[\)\]]?\s*\]?$"
-        match = re.search(pattern, output)
-        if match:
-            score = int(match.group(1))
-            feedback = output[:match.start()].rstrip()
-            # Remove any trailing brackets from feedback
-            feedback = re.sub(r'\s*\[[^\]]*$', '', feedback).strip()
-            return str(score), feedback
-
-        return "Error", f"Failed to parse response: {output}"
-
-    except Exception as e:
-        print(f"Failed to parse response: {str(e)}")
-        return "Error", f"Exception during parsing: {str(e)}"
-
-def atla_parse_model_response(output):
-    """Parse response from ATLA model"""
-    try:
-        print(f"Raw Atla model response: {output}")
-        output = output.strip()
-        
-        # Look for the Reasoning and Result sections
-        reasoning_match = re.search(r'\*\*Reasoning:\*\*(.*?)(?=\*\*Result:|$)', output, re.DOTALL)
-        result_match = re.search(r'\*\*Result:\*\*\s*(\d+)', output)
-        
-        if reasoning_match and result_match:
-            feedback = reasoning_match.group(1).strip()
-            score = result_match.group(1)
-            return str(score), feedback
-            
-        return "Error", f"Failed to parse ATLA response format: {output}"
-
-    except Exception as e:
-        print(f"Failed to parse ATLA response: {str(e)}")
-        return "Error", f"Exception during parsing: {str(e)}"
-    
-def flow_judge_parse_model_response(output):
-    try:
-        print(f"Raw model response: {output}")
-        # Convert multiple line breaks to single ones and strip whitespace
-        output = re.sub(r'\n{2,}', '\n', output.strip())
-        
-        # Compile regex patterns
-        feedback_pattern = re.compile(r"<feedback>\s*(.*?)\s*</feedback>", re.DOTALL)
-        score_pattern = re.compile(r"<score>\s*(\d+)\s*</score>", re.DOTALL)
-
-        feedback_match = feedback_pattern.search(output)
-        score_match = score_pattern.search(output)
-
-        if feedback_match or not score_match:
-            feedback = feedback_match.group(1).strip()
-            score = int(score_match.group(1).strip())
-            return str(score), feedback
-            
-        return "Error", f"Failed to parse response: {output}"
-        
-    except Exception as e:
-        print(f"Failed to parse response: {str(e)}")
-        return "Error", f"Exception during parsing: {str(e)}"

leaderboard.py

@@ -1,116 +0,0 @@
-from collections import defaultdict
-from datetime import datetime, timezone
-from typing import Dict, List
-
-# Constants
-DEFAULT_ELO = 1200  # Starting ELO for new models
-K_FACTOR = 32  # Standard chess K-factor
-
-def get_leaderboard(model_data: Dict, voting_data: List, show_preliminary=True):
-    """Generate leaderboard data using votes from MongoDB."""
-    # Initialize dictionaries for tracking
-    ratings = defaultdict(lambda: DEFAULT_ELO)
-    matches = defaultdict(int)
-
-    # Process each vote
-    for vote in voting_data:
-        try:
-            model_a = vote.get("model_a")
-            model_b = vote.get("model_b")
-            winner = vote.get("winner")
-
-            # Skip if models aren't in current model_data
-            if (
-                not all([model_a, model_b, winner])
-                or model_a not in model_data
-                or model_b not in model_data
-            ):
-                continue
-
-            # Update match counts
-            matches[model_a] += 1
-            matches[model_b] += 1
-
-            # Calculate ELO changes
-            elo_a = ratings[model_a]
-            elo_b = ratings[model_b]
-
-            # Expected scores
-            expected_a = 1 / (1 + 10 ** ((elo_b - elo_a) / 400))
-            expected_b = 1 - expected_a
-
-            # Actual scores
-            score_a = 1 if winner == "A" else 0 if winner == "B" else 0.5
-            score_b = 1 - score_a
-
-            # Update ratings
-            ratings[model_a] += K_FACTOR * (score_a - expected_a)
-            ratings[model_b] += K_FACTOR * (score_b - expected_b)
-
-        except Exception as e:
-            print(f"Error processing vote: {e}")
-            continue
-
-    # Generate leaderboard data
-    leaderboard = []
-    for model in model_data.keys():
-        votes = matches[model]
-        # Skip models with < 300 votes if show_preliminary is False
-        if not show_preliminary and votes < 300:
-            continue
-            
-        elo = ratings[model]
-        ci = 1.96 * (400 / (votes + 1) ** 0.5) if votes > 0 else 0
-        data = {
-            "Model": model,
-            "ELO Score": f"{int(elo)}",
-            "95% CI": f"±{int(ci)}",
-            "# Votes": votes,
-            "Organization": model_data[model]["organization"],
-            "License": model_data[model]["license"],
-        }
-        leaderboard.append(data)
-
-    # Sort leaderboard by ELO score in descending order
-    leaderboard.sort(key=lambda x: float(x["ELO Score"]), reverse=True)
-
-    return leaderboard
-
-def get_leaderboard_stats(model_data: Dict, voting_data: List) -> str:
-    """Get summary statistics for the leaderboard."""
-    now = datetime.now(timezone.utc)
-    total_votes = len(voting_data)
-    total_models = len(model_data)
-    # last_updated = now.strftime("%B %d, %Y at %H:%M:%S UTC")
-
-    last_updated = now.replace(minute=0, second=0, microsecond=0).strftime(
-        "%B %d, %Y at %H:00 UTC"
-    )
-
-    return f"""
-### Leaderboard Stats
-- **Total Models**: {total_models}
-- **Total Votes**: {total_votes}
-- **Last Updated**: {last_updated}
-"""
-
-def calculate_elo_change(rating_a: float, rating_b: float, winner: str) -> tuple[float, float]:
-    """Calculate ELO rating changes for both players."""
-    expected_a = 1 / (1 + 10 ** ((rating_b - rating_a) / 400))
-    expected_b = 1 - expected_a
-
-    if winner == "A":
-        score_a, score_b = 1, 0
-    elif winner == "B":
-        score_a, score_b = 0, 1
-    else:  # Handle ties
-        score_a, score_b = 0.5, 0.5
-
-    change_a = K_FACTOR * (score_a - expected_a)
-    change_b = K_FACTOR * (score_b - expected_b)
-
-    return change_a, change_b
-
-def get_model_rankings(leaderboard: List[Dict]) -> Dict[str, int]:
-    """Get current rankings of all models from leaderboard data."""
-    return {entry["Model"]: idx + 1 for idx, entry in enumerate(leaderboard)}

prompts.py

@@ -1,210 +0,0 @@
-# Default values for compatible mode
-DEFAULT_EVAL_CRITERIA = """Does the model provide relevant and useful responses to the user's needs or questions?"""
-
-DEFAULT_SCORE_1 = "The model's responses are irrelevant or unhelpful to the user's needs or queries."
-DEFAULT_SCORE_2 = "The model sometimes provides helpful information, but often fails to address the user's actual needs or questions."
-DEFAULT_SCORE_3 = "The model generally provides helpful responses that address the user's needs, though it may occasionally miss the mark."
-DEFAULT_SCORE_4 = "The model regularly provides helpful responses that are well-aligned with the user's inquiries, with only rare inaccuracies."
-DEFAULT_SCORE_5 = "The model consistently offers highly relevant and useful responses that perfectly cater to the user's needs and inquiries."
-
-# Default Eval Prompt
-DEFAULT_EVAL_PROMPT = """Does the model provide relevant and useful responses to the user's needs or questions?
-
-Scoring Rubric:
-Score 1: The model's responses are irrelevant or unhelpful to the user's needs or queries.
-Score 2: The model sometimes provides helpful information, but often fails to address the user's actual needs or questions.
-Score 3: The model generally provides helpful responses that address the user's needs, though it may occasionally miss the mark.
-Score 4: The model regularly provides helpful responses that are well-aligned with the user's inquiries, with only rare inaccuracies.
-Score 5: The model consistently offers highly relevant and useful responses that perfectly cater to the user's needs and inquiries.
-
-[User Query]: {{input}}
-
-[AI Response]: {{response}}"""
-
-# Split the eval prompt into editable and fixed parts
-DEFAULT_EVAL_PROMPT_EDITABLE = """Does the model provide relevant and useful responses to the user's needs or questions?
-
-Scoring Rubric:
-Score 1: The model's responses are irrelevant or unhelpful to the user's needs or queries.
-Score 2: The model sometimes provides helpful information, but often fails to address the user's actual needs or questions.
-Score 3: The model generally provides helpful responses that address the user's needs, though it may occasionally miss the mark.
-Score 4: The model regularly provides helpful responses that are well-aligned with the user's inquiries, with only rare inaccuracies.
-Score 5: The model consistently offers highly relevant and useful responses that perfectly cater to the user's needs and inquiries."""
-
-# Fixed suffix that will always be appended
-FIXED_EVAL_SUFFIX = """
-[User Query]: {{human_input}}
-
-[AI Response]: {{ai_response}}"""
-
-# Define the Prometheus prompt used by default (without reference)
-PROMETHEUS_PROMPT = """###Task Description:
-An instruction (might include an Input inside it) and a response to evaluate are given.
-1. Write a detailed feedback that assesses the quality of the response strictly based on the given score rubric, not evaluating in general.
-2. After writing the feedback, write a score that is an integer between 1 and 5.
-3. The output format should look as follows: "Feedback: (write a feedback for criteria) [RESULT] (an integer number between 1 and 5)"
-4. Please do not generate any other openings, closings, or explanations.
-
-###The instruction to evaluate:
-{human_input}
-
-###Response to evaluate:
-{ai_response}
-
-###Score Rubrics:
-[{eval_criteria}]
-Score 1: {score1_desc}
-Score 2: {score2_desc}
-Score 3: {score3_desc}
-Score 4: {score4_desc}
-Score 5: {score5_desc}
-
-###Feedback:
-"""
-
-# Define the Prometheus prompt with reference response
-PROMETHEUS_PROMPT_WITH_REFERENCE = """###Task Description:
-An instruction (might include an Input inside it), a response to evaluate, a reference answer that gets a score of 5, and a score rubric representing an evaluation criteria are given.
-1. Write a detailed feedback that assesses the quality of the response strictly based on the given score rubric, not evaluating in general.
-2. After writing the feedback, write a score that is an integer between 1 and 5.
-3. The output format should look as follows: "Feedback: (write a feedback for criteria) [RESULT] (an integer number between 1 and 5)"
-4. Please do not generate any other openings, closings, or explanations.
-
-###The instruction to evaluate:
-{human_input}
-
-###Response to evaluate:
-{ai_response}
-
-###Reference Answer (Score 5):
-{ground_truth_input}
-
-###Score Rubrics:
-[{eval_criteria}]
-Score 1: {score1_desc}
-Score 2: {score2_desc}
-Score 3: {score3_desc}
-Score 4: {score4_desc}
-Score 5: {score5_desc}
-
-###Feedback:
-"""
-
-# Judge system prompt for non-Prometheus models
-JUDGE_SYSTEM_PROMPT = """Please act as an impartial judge and evaluate based on the user's instruction. Your output format should strictly adhere to JSON as follows: {"feedback": "<write feedback>", "result": <numerical score>}. Ensure the output is valid JSON, without additional formatting or explanations.""" 
-
-ATLA_PROMPT = """You are tasked with evaluating a response based on a given instruction (which may contain an Input) and a scoring rubric that serve as the evaluation standard. Provide a comprehensive feedback on the response quality strictly adhering to the scoring rubric, without any general evaluation. Follow this with a score between 1 and 5, referring to the scoring rubric. Avoid generating any additional opening, closing, or explanations.
-  Here are some rules of the evaluation:
-  (1) You should prioritize evaluating whether the response satisfies the provided rubric. The basis of your score should depend exactly on the rubric. However, the response does not need to explicitly address points raised in the rubric. Rather, evaluate the response based on the criteria outlined in the rubric.
-
-  Your reply should strictly follow this format:
-  **Reasoning:** <Your feedback>
-
-  **Result:** <an integer between 1 and 5>
-
-  Here is the data:
-
-  Instruction:
-  ```
-  {human_input}
-  ```
-
-  Response:
-  ```
-  {ai_response}
-  ```
-
-  Score Rubrics:
-  [{eval_criteria}]
-  Score 1: {score1_desc}
-  Score 2: {score2_desc}
-  Score 3: {score3_desc}
-  Score 4: {score4_desc}
-  Score 5: {score5_desc}"""
-
-ATLA_PROMPT_WITH_REFERENCE = """You are tasked with evaluating a response based on a given instruction (which may contain an Input) and a scoring rubric and reference answer that serve as the evaluation standard. Provide a comprehensive feedback on the response quality strictly adhering to the scoring rubric, without any general evaluation. Follow this with a score between 1 and 5, referring to the scoring rubric. Avoid generating any additional opening, closing, or explanations.
-
-  Here are some rules of the evaluation:
-  (1) You should prioritize evaluating whether the response satisfies the provided rubric. The basis of your score should depend exactly on the rubric. However, the response does not need to explicitly address points raised in the rubric. Rather, evaluate the response based on the criteria outlined in the rubric.
-
-  Your reply should strictly follow this format:
-  **Reasoning:** <Your feedback>
-
-  **Result:** <an integer between 1 and 5>
-
-  Here is the data:
-
-  Instruction:
-  ```
-  {human_input}
-  ```
-
-  Response:
-  ```
-  {ai_response}
-  ```
-
-  Score Rubrics:
-  [{eval_criteria}]
-  Score 1: {score1_desc}
-  Score 2: {score2_desc}
-  Score 3: {score3_desc}
-  Score 4: {score4_desc}
-  Score 5: {score5_desc}
-
-  Reference answer:
-  {ground_truth_input}"""
-
-# Define the Flow Judge prompt
-FLOW_JUDGE_PROMPT = """# GOAL
-Your job is to evaluate a task carried out by an AI system powered by a large \
-language model.
-
-You will be provided with the inputs and output of the task, as well as the evaluation criteria \
-and scoring rubric. Your task is to evaluate the output of the AI system based on the evaluation \
-criteria and scoring rubric provided.
-
-# INPUT
-Below are the inputs required for performing the task:
-<inputs>
-{INPUTS}
-</inputs>
-
-# OUTPUT
-Below is the output of the task:
-<output>
-{OUTPUT}
-</output>
-
-# EVALUATION CRITERIA AND SCORING RUBRIC
-Here are the evaluation criteria and the rubric that you need to use for evaluating the task:
-<evaluation_criteria>
-{EVALUATION_CRITERIA}
-</evaluation_criteria>
-
-<scoring_rubric>
-{RUBRIC}
-</scoring_rubric>
-
-# INSTRUCTIONS FOR THE EVALUATION
-1. Understand the task and criteria: Familiarize yourself with the task to be evaluated. \
-Review the evaluation criteria and scoring rubric to understand the different levels of \
-performance and the descriptions for each score.
-2. Review the inputs and output: Look at the inputs provided for the task. Examine the output \
-generated from completing the task.
-3. Compare output to score descriptions: Compare the output against the criteria and score \
-descriptions in the scoring rubric. For each criterion,decide which description best matches the \
-output.
-4. After comparing the output to the score descriptions, pay attention to the small details that \
-might impact the final score that you assign. Sometimes a small difference can dictate the final \
-score.
-5. Write verbal feedback justifying your evaluation that includes a detailed rationale, referring \
-to specific aspects of the output and comparing them to the rubric.
-6. Assign a final score based on the scoring rubric.
-
-## FORMAT FOR THE EVALUATION
-- Write the verbal feedback inside <feedback> tags without any additional surrounding text.
-- Write the numeric score inside <score> tags, without any additional surrounding text and always \
-after the feedback.
-
-Please accurately evaluate the task. Strictly adhere to the evaluation criteria and rubric."""

random_sample_generation.py

@@ -1,183 +0,0 @@
-from openai import OpenAI
-import anthropic
-import json
-import re
-import random
-import os
-from gen_api_answer import get_openai_response, get_anthropic_response
-
-# Initialize clients
-anthropic_client = anthropic.Anthropic()
-openai_client = OpenAI()
-
-GOOD_SYSTEM_PROMPT = """You are an assistant that generates random conversations between a human and an AI assistant for testing purposes. The AI response generated should be a few sentences long. Format your output as JSON: {"human": "<human message>", "ai": <AI assistant response>}. Ensure the output is valid JSON, without additional formatting or explanations."""
-BAD_SYSTEM_PROMPT = """You are an assistant that generates random conversations between a human and an AI assistant for testing purposes. The response should contain incorrect information, logical fallacies, or misleading explanations. It should sound plausible but be fundamentally wrong. The AI response generated should be a few sentences long. Format your output as JSON: {"human": "<human message>", "ai": <AI assistant response>}. Ensure the output is valid JSON, without additional formatting or explanations."""
-AMBIGUOUS_SYSTEM_PROMPT = """You are an assistant that generates random conversations between a human and an AI assistant for testing purposes. The response should mix correct and incorrect information - it should contain some accurate points but also include nuanced, questionable claims or exaggerations. The AI response generated should be a few sentences long. Format your output as JSON: {"human": "<human message>", "ai": <AI assistant response>}. Ensure the output is valid JSON, without additional formatting or explanations."""
-
-GOOD_SYSTEM_PROMPT_WITH_GROUND_TRUTH = """You are an assistant that generates random conversations between a human and an AI assistant for testing purposes, along with an ideal reference answer. The AI response generated should be a few sentences long and contain accurate information. The ground truth response should be a perfect, comprehensive answer that would score 5/5. Format your output as JSON: {"human": "<human message>", "ai": "<AI assistant response>", "ground_truth": "<perfect reference answer>"}. Ensure the output is valid JSON, without additional formatting or explanations."""
-BAD_SYSTEM_PROMPT_WITH_GROUND_TRUTH = """You are an assistant that generates random conversations between a human and an AI assistant for testing purposes, along with an ideal reference answer. The AI response should be a few sentences long and contain incorrect information, logical fallacies, or misleading explanations. It should sound plausible but be fundamentally wrong. The ground truth response should be a perfect, comprehensive answer that would score 5/5. Format your output as JSON: {"human": "<human message>", "ai": "<AI assistant response>", "ground_truth": "<perfect reference answer>"}. Ensure the output is valid JSON, without additional formatting or explanations."""
-AMBIGUOUS_SYSTEM_PROMPT_WITH_GROUND_TRUTH = """You are an assistant that generates random conversations between a human and an AI assistant for testing purposes, along with an ideal reference answer. The AI response should be a few sentences long and mix correct and incorrect information - it should contain some accurate points but also include nuanced, questionable claims or exaggerations. The ground truth response should be a perfect, comprehensive answer that would score 5/5. Format your output as JSON: {"human": "<human message>", "ai": "<AI assistant response>", "ground_truth": "<perfect reference answer>"}. Ensure the output is valid JSON, without additional formatting or explanations."""
-
-GENERATION_PROMPT = """Please generate a random human message and an AI response in the format of a QA dataset. The human input should not be a one-word answer question like "What is the capital of France?". The AI response generated should be a few sentences long."""
-GENERATION_PROMPT_WITH_GROUND_TRUTH = """Please generate:
-1. A random human message (not a simple one-word answer question)
-2. An AI response (a few sentences long)
-3. A perfect reference answer that would score 5/5 on all criteria (e.g., concise, helpful, and accurate)
-
-Format as JSON with "human", "ai", and "ground_truth" fields."""
-
-RESPONSE_GENERATION_SYSTEM_PROMPT = "You are an assistant that generates random responses to human messages for testing purposes. Generate bad responses (with a mix of correct and incorrect information) 60% of the time and good responses 40% of the time. Do not say which type of response you are generating, just generate the response."
-
-def get_random_human_ai_pair():
-    # Select system prompt with specified probabilities
-    system_prompt = random.choices(
-        [GOOD_SYSTEM_PROMPT, BAD_SYSTEM_PROMPT, AMBIGUOUS_SYSTEM_PROMPT],
-        weights=[0.2, 0.2, 0.6]  # 20% good, 20% bad, 60% ambiguous
-    )[0]
-    
-    # Log which type of response is being generated
-    prompt_type = {
-        GOOD_SYSTEM_PROMPT: "good",
-        BAD_SYSTEM_PROMPT: "bad", 
-        AMBIGUOUS_SYSTEM_PROMPT: "ambiguous"
-    }[system_prompt]
-    print(f"Generating {prompt_type} response")
-    
-    # Randomly choose between GPT-3.5 and Claude with 65%/35% weights
-    model_choice = random.choices([
-        ("gpt-3.5-turbo", get_openai_response),
-        ("claude-3-5-haiku-latest", get_anthropic_response)
-    ], weights=[0.5, 0.5])[0]
-    model_name, api_func = model_choice
-    
-    # Generate response using selected model
-    response = api_func(
-        model_name=model_name,
-        prompt=GENERATION_PROMPT,
-        system_prompt=system_prompt,
-        max_tokens=500,
-        temperature=1
-    )
-    
-    # Define default messages
-    default_human = "How do muscles grow?"
-    default_ai = """Muscles grow through a process called skeletal muscle hypertrophy, which adds more myosin filaments to each muscle fiber, making the engine of the cell bigger and stronger over time. This is achieved through increased muscle tension and physical stress, breaking down muscle fiber. Muscle growth is also a direct consequence of resistance training and nutrition. People build muscle at different rates depending on their age, sex, and genetics, but muscle development significantly increases if exercise is done correctly and the body stores more protein through a process called protein synthesis."""
-    
-    try:
-        # Clean the response by replacing newlines with spaces
-        cleaned_response = response.replace('\n', ' ').replace('\r', '')
-        data = json.loads(cleaned_response)
-        
-        # Extract messages with fallbacks
-        human_message = data.get("human", default_human)
-        ai_message = data.get("ai", default_ai)
-        
-        # Debug logging
-        print(f"Parsed response: human='{human_message}', ai='{ai_message[:50]}...'")
-        
-    except Exception as e:
-        print(f"Failed to parse response: {str(e)}\n {response}")
-        human_message = default_human
-        ai_message = default_ai
-    
-    return human_message, ai_message
-
-def get_random_human_ai_ground_truth_pair():
-    # Select system prompt with specified probabilities
-    system_prompts = {
-        "good": GOOD_SYSTEM_PROMPT_WITH_GROUND_TRUTH,
-        "bad": BAD_SYSTEM_PROMPT_WITH_GROUND_TRUTH,
-        "ambiguous": AMBIGUOUS_SYSTEM_PROMPT_WITH_GROUND_TRUTH
-    }
-    
-    prompt_type = random.choices(
-        ["good", "bad", "ambiguous"],
-        weights=[0.2, 0.2, 0.6]  # 20% good, 20% bad, 60% ambiguous
-    )[0]
-    
-    system_prompt = system_prompts[prompt_type]
-    print(f"Generating {prompt_type} response with ground truth")
-    
-    # Randomly choose between GPT-3.5 and Claude with 50/50 weights
-    model_choice = random.choices([
-        ("gpt-3.5-turbo", get_openai_response),
-        ("claude-3-5-haiku-latest", get_anthropic_response)
-    ], weights=[0.5, 0.5])[0]
-    model_name, api_func = model_choice
-    
-    # Define default messages
-    defaults = {
-        "human": "How do muscles grow?",
-        "ai": """Muscles grow through a process called skeletal muscle hypertrophy, which adds more myosin filaments to each muscle fiber, making the engine of the cell bigger and stronger over time. This is achieved through increased muscle tension and physical stress, breaking down muscle fiber. Muscle growth is also a direct consequence of resistance training and nutrition. People build muscle at different rates depending on their age, sex, and genetics, but muscle development significantly increases if exercise is done correctly and the body stores more protein through a process called protein synthesis.""",
-        "ground_truth": """Muscle growth (hypertrophy) occurs through a complex biological process involving several key mechanisms:
-
-1. Mechanical Tension: Resistance training creates mechanical tension in muscle fibers, triggering molecular and cellular responses that promote growth.
-
-2. Metabolic Stress: The depletion of energy resources and accumulation of metabolic byproducts during exercise contributes to muscle growth signaling.
-
-3. Muscle Damage: Exercise-induced micro-damage to muscle fibers activates satellite cells, which help repair and build new muscle tissue.
-
-4. Protein Synthesis: After exercise, increased protein synthesis rates exceed protein breakdown, leading to net muscle protein accretion.
-
-5. Hormonal Response: Exercise triggers the release of growth-promoting hormones like testosterone, growth hormone, and IGF-1.
-
-6. Recovery: Adequate rest between training sessions allows for repair and growth, supported by proper nutrition, particularly protein intake (1.6-2.2g/kg/day).
-
-This process is influenced by factors including genetics, age, sex, nutrition, sleep quality, and training variables. Optimal muscle growth requires a structured resistance training program, adequate protein intake, sufficient calories, and proper recovery."""
-    }
-    
-    # Generate response using selected model
-    response = api_func(
-        model_name=model_name,
-        prompt=GENERATION_PROMPT_WITH_GROUND_TRUTH,
-        system_prompt=system_prompt,
-        max_tokens=1000,  # Increased token limit to accommodate ground truth
-        temperature=1
-    )
-    
-    # Parse the response to get all three components
-    try:
-        # Clean the response by replacing newlines with spaces
-        cleaned_response = response.replace('\n', ' ').replace('\r', '')
-        data = json.loads(cleaned_response)
-        
-        # Extract messages with fallbacks
-        human_message = data.get("human", defaults["human"])
-        ai_message = data.get("ai", defaults["ai"])
-        ground_truth = data.get("ground_truth", defaults["ground_truth"])
-        
-        # Debug logging
-        print(f"Parsed response: human='{human_message}', ai='{ai_message[:50]}...', ground_truth='{ground_truth[:50]}...'")
-        
-    except Exception as e:
-        print(f"Failed to parse response: {str(e)}\n {response}")
-        human_message = defaults["human"]
-        ai_message = defaults["ai"]
-        ground_truth = defaults["ground_truth"]
-    
-    return human_message, ai_message, ground_truth
-
-def generate_ai_response(human_msg):
-    """Generate AI response using GPT-3.5-turbo"""
-    if not human_msg.strip():
-        return "", False
-        
-    try:
-        response = get_openai_response(
-            "gpt-3.5-turbo", 
-            human_msg,
-            system_prompt=RESPONSE_GENERATION_SYSTEM_PROMPT,
-            max_tokens=1000,
-            temperature=1
-        )
-        # Extract just the response content since we don't need JSON format here
-        if isinstance(response, str):
-            # Clean up any JSON formatting if present
-            try:
-                data = json.loads(response)
-                response = data.get("content", response)
-            except json.JSONDecodeError:
-                pass
-        return response, False  # Return response and button interactive state
-    except Exception as e:
-        return f"Error generating response: {str(e)}", False

requirements.txt

@@ -4,5 +4,3 @@ python-dotenv
 openai
 anthropic
 together
-cohere
-transformers