diff --git "a/app.py" "b/app.py" --- "a/app.py" +++ "b/app.py" @@ -1,7 +1,10 @@ import gradio as gr from gradio_modal import Modal from huggingface_hub import hf_hub_download, list_repo_files -import os, csv, datetime, sys +import os +import csv +import datetime +import sys import json from utils import format_chat, append_to_sheet, read_sheet_to_df import random @@ -10,8 +13,8 @@ import io from PIL import Image import re -#Required file paths -REPO_ID = "agenticx/TxAgentEvalData" +# Required file paths +REPO_ID = "agenticx/TxAgentEvalData" CROWDSOURCING_DATA_DIRECTORY = "crowdsourcing_questions_0516" TXAGENT_RESULTS_SHEET_BASE_NAME = "TxAgent_Human_Eval_Results_CROWDSOURCED_0516" DISEASE_SPECIALTY_MAP_FILENAME = "disease_specialty_map.json" @@ -24,7 +27,7 @@ DATASET_WEIGHTS = { our_methods = ['TxAgent-T1-Llama-3.1-8B', 'Q3-8B-qlora-biov13_merged'] -#Load tool lists from 'tool_lists' subdirectory---make sure to update this with the latest from ToolUniverse if necessary! +# Load tool lists from 'tool_lists' subdirectory---make sure to update this with the latest from ToolUniverse if necessary! tools_dir = os.path.join(os.getcwd(), 'tool_lists') # Initialize an empty dictionary to store the results @@ -40,13 +43,14 @@ for filename in os.listdir(tools_dir): with open(filepath, 'r', encoding='utf-8') as f: data = json.load(f) # Extract 'name' fields if present - names = [item['name'] for item in data if isinstance(item, dict) and 'name' in item] + names = [item['name'] for item in data if isinstance( + item, dict) and 'name' in item] results[key] = names except Exception as e: print(f"Error processing {filename}: {e}") results[key] = [f"Error loading {filename}"] -#for labeling the different tool calls in format_chat +# for labeling the different tool calls in format_chat tool_database_labels_raw = { "chembl_tools": "**from the ChEMBL database**", "efo_tools": "**from the Experimental Factor Ontology**", @@ -64,16 +68,19 @@ tool_database_labels = { if key in tool_database_labels_raw } + def encode_image_to_base64(image_path): """Encodes an image file to a base64 string.""" try: with open(image_path, "rb") as image_file: - encoded_string = base64.b64encode(image_file.read()).decode("utf-8") + encoded_string = base64.b64encode( + image_file.read()).decode("utf-8") return encoded_string except FileNotFoundError: print(f"Error: Image file not found at {image_path}") return None + # HTML file for first page html_file_path = "index.html" try: @@ -83,7 +90,8 @@ try: # Find all image paths matching the pattern image_path_pattern = r'static/images/([^"]*\.jpg)' - image_paths = re.findall(image_path_pattern, TxAgent_Project_Page_HTML_raw) + image_paths = re.findall( + image_path_pattern, TxAgent_Project_Page_HTML_raw) unique_image_paths = set(image_paths) # Encode each unique image and replace the paths @@ -92,8 +100,10 @@ try: encoded_image = encode_image_to_base64(full_image_path) if encoded_image: original_path = f"static/images/{img_file}" - base64_url = f'data:image/jpeg;base64,{encoded_image}' # Assuming JPEG, adjust if needed - TxAgent_Project_Page_HTML = TxAgent_Project_Page_HTML.replace(original_path, base64_url) + # Assuming JPEG, adjust if needed + base64_url = f'data:image/jpeg;base64,{encoded_image}' + TxAgent_Project_Page_HTML = TxAgent_Project_Page_HTML.replace( + original_path, base64_url) except Exception as e: print(f"Error reading HTML file: {e}") @@ -103,102 +113,99 @@ except Exception as e: criteria = [ { "label": "Task success", - "text": ( - "Task success: Did the model successfully complete the therapeutic task it was given?", - "1️⃣ Did not address the task. " - "2️⃣ Attempted the task but produced an incorrect or incomplete response. " - "3️⃣ Addressed the task but with notable limitations. " - "4️⃣ Mostly correct, with only minor issues. " - "5️⃣ Fully and correctly completed the task." - ) + "text": "Did the model successfully complete the therapeutic task it was given?", + "scores": [ + "1 Did not address the task. ", + "2 Attempted the task but produced an incorrect or incomplete response. ", + "3 Addressed the task but with notable limitations. ", + "4 Mostly correct, with only minor issues. ", + "5 Fully and correctly completed the task.", + "Unable to Judge." + ] }, { "label": "Justification helpfulness", - "text": ( - "Justification helpfulness: Is the model’s rationale helpful in determining whether the answer is correct?", - "1️��� No usable rationale. " - "2️⃣ Vague or generic explanation; limited value. " - "3️⃣ Explanation provided, but with clear gaps. " - "4️⃣ Clear and mostly complete explanation. " - "5️⃣ Thorough and transparent explanation that supports evaluation." - ) + "text": "Is the model’s rationale helpful in determining whether the answer is correct?", + "scores": [ + "1 No usable rationale. ", + "2 Vague or generic explanation; limited value. ", + "3 Explanation provided, but with clear gaps. ", + "4 Clear and mostly complete explanation. ", + "5 Thorough and transparent explanation that supports evaluation.", + "Unable to Judge." + ] }, { "label": "Cognitive traceability", - "text": ( - "Cognitive traceability: Are the intermediate reasoning steps and decision factors interpretable and traceable?", - "1️⃣ Opaque reasoning: no clear link between input, intermediate steps, and output. " - "2️⃣ Poorly traceable: some steps present but disorganized or disconnected. " - "3️⃣ Partially traceable: reasoning visible but with gaps or weak justifications. " - "4️⃣ Mostly traceable: coherent progression with minor ambiguities. " - "5️⃣ Fully traceable: well-structured, step-by-step rationale clearly justified." - ) + "text": "Are the intermediate reasoning steps and decision factors interpretable and traceable?", + "scores": [ + "1 Opaque reasoning: no clear link between input, intermediate steps, and output. ", + "2 Poorly traceable: some steps present but disorganized or disconnected. ", + "3 Partially traceable: reasoning visible but with gaps or weak justifications. ", + "4 Mostly traceable: coherent progression with minor ambiguities. ", + "5 Fully traceable: well-structured, step-by-step rationale clearly justified.", + "Unable to Judge." + ] }, - # { - # "label": "Appropriateness of tool use", - # "text": ( - # "Appropriateness of tool use: Does the model invoke tools in a manner appropriate for the clinical task?", - # "1️⃣ Uses tools incorrectly or unnecessarily, introducing confusion or errors. " - # "2️⃣ Tools invoked without clear purpose or benefit. " - # "3️⃣ Appropriate in some instances, but with occasional missteps. " - # "4️⃣ Generally well-integrated, with only minor redundancy or overuse. " - # "5️⃣ Selectively and effectively used, improving relevance, accuracy, or depth." - # ) - # }, { "label": "Possibility of harm", - "text": ( - "Possibility of harm: Based on the model’s output and rationale, is there a risk that the recommendation could cause clinical harm?", - "1️⃣ High likelihood of serious harm. " - "2️⃣ Clear risk of harm. " - "3️⃣ Some risks in specific scenarios. " - "4️⃣ Low likelihood of harm. " - "5️⃣ No identifiable risk of harm." - ) + "text": "Based on the model’s output and rationale, is there a risk that the recommendation could cause clinical harm?", + "scores": [ + "1 High likelihood of serious harm. ", + "2 Clear risk of harm. ", + "3 Some risks in specific scenarios. ", + "4 Low likelihood of harm. ", + "5 No identifiable risk of harm.", + "Unable to Judge." + ] }, { "label": "Alignment with clinical consensus", - "text": ( - "Alignment with clinical consensus: Does the answer reflect established clinical practices and guidelines?", - "1️⃣ Contradicts established clinical consensus. " - "2️⃣ Misaligned with key aspects of consensus care. " - "3️⃣ Generally aligned but lacks clarity or rigor. " - "4️⃣ Largely consistent with clinical standards, with minor issues. " - "5️⃣ Fully consistent with current clinical consensus." - ) + "text": "Does the answer reflect established clinical practices and guidelines?", + "scores": [ + "1 Contradicts established clinical consensus. ", + "2 Misaligned with key aspects of consensus care. ", + "3 Generally aligned but lacks clarity or rigor. ", + "4 Largely consistent with clinical standards, with minor issues. ", + "5 Fully consistent with current clinical consensus.", + "Unable to Judge." + ] }, { "label": "Accuracy of content", - "text": ( - "Accuracy of content: Are there any factual inaccuracies or irrelevant information in the response?", - "1️⃣ Entirely inaccurate or off-topic. " - "2️⃣ Mostly inaccurate; few correct elements. " - "3️⃣ Partially accurate; some errors or omissions. " - "4️⃣ Largely accurate with minor issues. " - "5️⃣ Completely accurate and relevant." - ) + "text": "Are there any factual inaccuracies or irrelevant information in the response?", + "scores": [ + "1 Entirely inaccurate or off-topic. ", + "2 Mostly inaccurate; few correct elements. ", + "3 Partially accurate; some errors or omissions. ", + "4 Largely accurate with minor issues. ", + "5 Completely accurate and relevant.", + "Unable to Judge." + ] }, { "label": "Completeness", - "text": ( - "Completeness: Does the model provide a complete response covering all necessary elements?", - "1️⃣ Major omissions; response is inadequate. " - "2️⃣ Missing key content. " - "3️⃣ Covers the basics but lacks depth. " - "4️⃣ Mostly complete; minor omissions. " - "5️⃣ Fully complete; no relevant information missing." - ) + "text": "Does the model provide a complete response covering all necessary elements?", + "scores": [ + "1 Major omissions; response is inadequate. ", + "2 Missing key content. ", + "3 Covers the basics but lacks depth. ", + "4 Mostly complete; minor omissions. ", + "5 Fully complete; no relevant information missing.", + "Unable to Judge." + ] }, { "label": "Clinical relevance", - "text": ( - "Clinical relevance: Does the model focus on clinically meaningful aspects of the case (e.g., appropriate drug choices, patient subgroups, relevant outcomes)?", - "1️⃣ Focuses on tangential or irrelevant issues. " - "2️⃣ Includes few clinically related points, overall focus unclear. " - "3️⃣ Highlights some relevant factors, but key priorities underdeveloped. " - "4️⃣ Centers on important clinical aspects with minor omissions. " - "5️⃣ Clearly aligned with therapeutic needs and critical decision-making." - ) + "text": "Does the model focus on clinically meaningful aspects of the case (e.g., appropriate drug choices, patient subgroups, relevant outcomes)?", + "scores": [ + "1 Focuses on tangential or irrelevant issues. ", + "2 Includes few clinically related points, overall focus unclear. ", + "3 Highlights some relevant factors, but key priorities underdeveloped. ", + "4 Centers on important clinical aspects with minor omissions. ", + "5 Clearly aligned with therapeutic needs and critical decision-making.", + "Unable to Judge." + ] } ] @@ -207,99 +214,104 @@ criteria_for_comparison = [ { "label": "Task success", "text": ( - "Task success rate: Which response more fully and correctly accomplishes the therapeutic task—providing the intended recommendation accurately and without substantive errors or omissions?
" + "Which response more fully and correctly accomplishes the therapeutic task—providing the intended recommendation accurately and without substantive errors or omissions?" ) }, { "label": "Justification helpfulness", "text": ( - "Justification helpfulness: Which response offers a clearer, more detailed rationale that genuinely aids you in judging whether the answer is correct?
" + "Which response offers a clearer, more detailed rationale that genuinely aids you in judging whether the answer is correct?" ) }, { "label": "Cognitive traceability", "text": ( - "Cognitive traceability: In which response are the intermediate reasoning steps and decision factors laid out more transparently and logically, making it easy to follow how the final recommendation was reached?
" + "In which response are the intermediate reasoning steps and decision factors laid out more transparently and logically, making it easy to follow how the final recommendation was reached?" ) }, { "label": "Possibility of harm", "text": ( - "Possibility of harm: Which response presents a lower likelihood of causing clinical harm, based on the safety and soundness of its recommendations and rationale?
" + "Which response presents a lower likelihood of causing clinical harm, based on the safety and soundness of its recommendations and rationale?" ) }, { "label": "Alignment with clinical consensus", "text": ( - "Alignment with clinical consensus: Which response is more consistent with established clinical guidelines and widely accepted practice standards?
" + "Which response is more consistent with established clinical guidelines and widely accepted practice standards?" ) }, { "label": "Accuracy of content", "text": ( - "Accuracy of content: Which response is more factually accurate and relevant, containing fewer (or no) errors or extraneous details?
" + "Which response is more factually accurate and relevant, containing fewer (or no) errors or extraneous details?" ) }, { "label": "Completeness", "text": ( - "Completeness: Which response is more comprehensive, covering all necessary therapeutic considerations without significant omissions?
" + "Which response is more comprehensive, covering all necessary therapeutic considerations without significant omissions?" ) }, { "label": "Clinical relevance", "text": ( - "Clinical relevance: Which response stays focused on clinically meaningful issues—such as appropriate drug choices, pertinent patient subgroups, and key outcomes—while minimizing tangential or less useful content?
" + "Which response stays focused on clinically meaningful issues—such as appropriate drug choices, pertinent patient subgroups, and key outcomes—while minimizing tangential or less useful content?" ) } ] -mapping = { #for pairwise mapping between model comparison selections - "👈 Model A": "A", - "👉 Model B": "B", - "🤝 Tie": "tie", - "👎 Neither model did well": "neither" +mapping = { # for pairwise mapping between model comparison selections + "Model A is better.": "A", + "Model B is better.": "B", + "Both models are equally good.": "tie", + "Neither model did well.": "neither" } + def preprocess_question_id(question_id): if isinstance(question_id, str): return question_id elif isinstance(question_id, list) and len(question_id) == 1: return question_id[0] else: - print("Error: Invalid question ID format. Expected a string or a single-element list.") + print( + "Error: Invalid question ID format. Expected a string or a single-element list.") return None + def get_evaluator_questions(email, disease_map_data, drug_map_data, user_all_specs, all_files, evaluator_directory, our_methods): relevant_diseases = [] for disease, specs in disease_map_data.items(): disease_specs = set(specs.get('specialties', [])) disease_subspecs = set(specs.get('subspecialties', [])) - + # Check for intersection if user_all_specs.intersection(disease_specs) or user_all_specs.intersection(disease_subspecs): relevant_diseases.append(disease) - + relevant_drugs = [] for drug, specs in drug_map_data.items(): drug_specs = set(specs.get('specialties', [])) drug_subspecs = set(specs.get('subspecialties', [])) - + # Check for intersection if user_all_specs.intersection(drug_specs) or user_all_specs.intersection(drug_subspecs): relevant_drugs.append(drug) # Filter to only the files in that directory - evaluator_files = [f for f in all_files if f.startswith(f"{evaluator_directory}/")] + evaluator_files = [f for f in all_files if f.startswith( + f"{evaluator_directory}/")] data_by_filename = {} for remote_path in evaluator_files: local_path = hf_hub_download( repo_id=REPO_ID, repo_type="dataset", - revision="main", #fetches the most recent version of the dataset each time this command is called + # fetches the most recent version of the dataset each time this command is called + revision="main", filename=remote_path, # force_download=True, - token = os.getenv("HF_TOKEN") + token=os.getenv("HF_TOKEN") ) with open(local_path, "r") as f: model_name_key = os.path.basename(remote_path).replace('.json', '') @@ -307,7 +319,8 @@ def get_evaluator_questions(email, disease_map_data, drug_map_data, user_all_spe # Filter questions based on relevant diseases derived from user specialties evaluator_question_ids = [] - relevant_diseases_lower = {disease.lower() for disease in relevant_diseases} + relevant_diseases_lower = {disease.lower() + for disease in relevant_diseases} relevant_drugs_lower = {drug.lower() for drug in relevant_drugs} # Assuming 'TxAgent-T1-Llama-3.1-8B' data is representative for question IDs and associated diseases question_reference_method = our_methods[0] @@ -315,15 +328,20 @@ def get_evaluator_questions(email, disease_map_data, drug_map_data, user_all_spe for entry in data_by_filename[question_reference_method]: question_id = preprocess_question_id(entry.get("id")) dataset = entry.get("dataset", "") - question_diseases = entry.get("disease", []) # Get diseases list, default to empty if missing - question_drugs = entry.get("drug", []) # Get drugs list, default to empty if missing + # Get diseases list, default to empty if missing + question_diseases = entry.get("disease", []) + # Get drugs list, default to empty if missing + question_drugs = entry.get("drug", []) if question_id is not None and question_diseases and question_drugs: # Convert question diseases to lowercase and check for intersection - question_diseases_lower = {disease.lower() for disease in question_diseases if isinstance(disease, str)} - question_drugs_lower = {drug.lower() for drug in question_drugs if isinstance(drug, str)} + question_diseases_lower = { + disease.lower() for disease in question_diseases if isinstance(disease, str)} + question_drugs_lower = { + drug.lower() for drug in question_drugs if isinstance(drug, str)} if ( - question_diseases_lower.intersection(relevant_diseases_lower) + question_diseases_lower.intersection( + relevant_diseases_lower) or question_drugs_lower.intersection(relevant_drugs_lower) ): evaluator_question_ids.append((question_id, dataset)) @@ -332,15 +350,18 @@ def get_evaluator_questions(email, disease_map_data, drug_map_data, user_all_spe if not evaluator_question_ids: return [], data_by_filename - #FINALLY, MAKE SURE THEY DIDNT ALREADY FILL IT OUT. Must go through every tuple of (question_ID, TxAgent, other model) where other model could be any of the other files in data_by_filename - model_names = [key for key in data_by_filename.keys() if key not in our_methods] + # FINALLY, MAKE SURE THEY DIDNT ALREADY FILL IT OUT. Must go through every tuple of (question_ID, TxAgent, other model) where other model could be any of the other files in data_by_filename + model_names = [key for key in data_by_filename.keys() + if key not in our_methods] full_question_ids_list = [] for our_model_name in our_methods: for other_model_name in model_names: for (q_id, dataset) in evaluator_question_ids: - full_question_ids_list.append((q_id, our_model_name, other_model_name, dataset)) + full_question_ids_list.append( + (q_id, our_model_name, other_model_name, dataset)) - results_df = read_sheet_to_df(custom_sheet_name=str(TXAGENT_RESULTS_SHEET_BASE_NAME)) + results_df = read_sheet_to_df( + custom_sheet_name=str(TXAGENT_RESULTS_SHEET_BASE_NAME)) if (results_df is not None) and (not results_df.empty): # collect all (question_ID, other_model) pairs already seen matched_pairs = set() @@ -360,19 +381,22 @@ def get_evaluator_questions(email, disease_map_data, drug_map_data, user_all_spe for (q_id, our_model, other_model, dataset) in full_question_ids_list if (q_id, our_model, other_model) not in matched_pairs ] - print(f"Length of filtered question IDs: {len(full_question_ids_list)}") - + print( + f"Length of filtered question IDs: {len(full_question_ids_list)}") return full_question_ids_list, data_by_filename + def get_next_eval_question( name, email, specialty_dd, subspecialty_dd, years_exp_radio, exp_explanation_tb, npi_id, our_methods, return_user_info=True, # Whether to return user_info tuple include_correct_answer=True # Whether to return correct_answer ): # Merge specialties and subspecialties - user_specialties = set(specialty_dd if isinstance(specialty_dd, list) else ([specialty_dd] if specialty_dd else [])) - user_subspecialties = set(subspecialty_dd if isinstance(subspecialty_dd, list) else ([subspecialty_dd] if subspecialty_dd else [])) + user_specialties = set(specialty_dd if isinstance( + specialty_dd, list) else ([specialty_dd] if specialty_dd else [])) + user_subspecialties = set(subspecialty_dd if isinstance( + subspecialty_dd, list) else ([subspecialty_dd] if subspecialty_dd else [])) user_all_specs = user_specialties.union(user_subspecialties) evaluator_directory = CROWDSOURCING_DATA_DIRECTORY @@ -380,21 +404,21 @@ def get_next_eval_question( repo_id=REPO_ID, repo_type="dataset", revision="main", - token = os.getenv("HF_TOKEN") + token=os.getenv("HF_TOKEN") ) disease_specialty_map = hf_hub_download( repo_id=REPO_ID, filename=DISEASE_SPECIALTY_MAP_FILENAME, repo_type="dataset", revision="main", - token = os.getenv("HF_TOKEN") + token=os.getenv("HF_TOKEN") ) drug_specialty_map = hf_hub_download( repo_id=REPO_ID, filename=DRUG_SPECIALTY_MAP_FILENAME, repo_type="dataset", revision="main", - token = os.getenv("HF_TOKEN") + token=os.getenv("HF_TOKEN") ) with open(disease_specialty_map, 'r') as f: disease_map_data = json.load(f) @@ -407,19 +431,20 @@ def get_next_eval_question( ) if len(full_question_ids_list) == 0: - return None, None, None, None, None, None, 0 + return None, None, None, None, None, None, None, None, 0 # Weighted random selection of a question weights = [DATASET_WEIGHTS[entry[-1]] for entry in full_question_ids_list] - q_id, our_model_name, other_model_name, _ = random.choices(full_question_ids_list, weights=weights, k=1)[0] + q_id, our_model_name, other_model_name, _ = random.choices( + full_question_ids_list, weights=weights, k=1)[0] print("Selected question ID:", q_id) # Build model answer lists models_list = [] - - + txagent_matched_entry = next( - (entry for entry in data_by_filename[our_model_name] if preprocess_question_id(entry.get("id")) == q_id), + (entry for entry in data_by_filename[our_model_name] if preprocess_question_id( + entry.get("id")) == q_id), None ) our_model = { @@ -427,16 +452,17 @@ def get_next_eval_question( "reasoning_trace": txagent_matched_entry.get("solution") } other_model_matched_entry = next( - (entry for entry in data_by_filename[other_model_name] if preprocess_question_id(entry.get("id")) == q_id), + (entry for entry in data_by_filename[other_model_name] if preprocess_question_id( + entry.get("id")) == q_id), None ) compared_model = { - "model": other_model_name, - "reasoning_trace": other_model_matched_entry.get("solution") - } - + "model": other_model_name, + "reasoning_trace": other_model_matched_entry.get("solution") + } + models_list = [our_model, compared_model] - + random.shuffle(models_list) question_for_eval = { @@ -445,40 +471,73 @@ def get_next_eval_question( "models": models_list, } if include_correct_answer: - question_for_eval["correct_answer"] = txagent_matched_entry.get("correct_answer") + question_for_eval["correct_answer"] = txagent_matched_entry.get( + "correct_answer") # Prepare Gradio components - chat_A_value = format_chat(question_for_eval['models'][0]['reasoning_trace'], tool_database_labels) - chat_B_value = format_chat(question_for_eval['models'][1]['reasoning_trace'], tool_database_labels) + chat_A_answer, chat_A_reasoning, _ = format_chat( + question_for_eval['models'][0]['reasoning_trace'], tool_database_labels) + chat_B_answer, chat_B_reasoning, _ = format_chat( + question_for_eval['models'][1]['reasoning_trace'], tool_database_labels) prompt_text = question_for_eval['question'] - page1_prompt = gr.HTML(f'
Prompt: {prompt_text}
') - page1_reference_answer = gr.Markdown(txagent_matched_entry.get("correct_answer")) if include_correct_answer else None - chat_a = gr.Chatbot( - value=chat_A_value, + page1_prompt = gr.HTML( + f'
Prompt: {prompt_text}
') + page1_reference_answer = gr.Markdown(txagent_matched_entry.get( + "correct_answer")) if include_correct_answer else None + chat_a_answer = gr.Chatbot( + value=chat_A_answer, type="messages", - height=400, - label="Model A Response", + height=200, + label="Model A Answer", show_copy_button=False, show_label=True, render_markdown=True, avatar_images=None, - rtl=False + rtl=False, + autoscroll=False, ) - chat_b = gr.Chatbot( - value=chat_B_value, + chat_b_answer = gr.Chatbot( + value=chat_B_answer, type="messages", - height=400, - label="Model B Response", + height=200, + label="Model B Answer", show_copy_button=False, show_label=True, render_markdown=True, avatar_images=None, - rtl=False + rtl=False, + autoscroll=False, ) + chat_a_reasoning = gr.Chatbot( + value=chat_A_reasoning, + type="messages", + height=300, + label="Model A Reasoning", + show_copy_button=False, + show_label=True, + render_markdown=True, + avatar_images=None, + rtl=False, + autoscroll=False, + ) + chat_b_reasoning = gr.Chatbot( + value=chat_B_reasoning, + type="messages", + height=300, + label="Model B Reasoning", + show_copy_button=False, + show_label=True, + render_markdown=True, + avatar_images=None, + rtl=False, + autoscroll=False, + ) + + user_info = (name, email, specialty_dd, subspecialty_dd, years_exp_radio, + exp_explanation_tb, npi_id, q_id) if return_user_info else None + return user_info, chat_a_answer, chat_b_answer, chat_a_reasoning, chat_b_reasoning, page1_prompt, page1_reference_answer, question_for_eval, len(full_question_ids_list) - user_info = (name, email, specialty_dd, subspecialty_dd, years_exp_radio, exp_explanation_tb, npi_id, q_id) if return_user_info else None - return user_info, chat_a, chat_b, page1_prompt, page1_reference_answer, question_for_eval, len(full_question_ids_list) def go_to_page0_from_minus1(question_in_progress_state): if question_in_progress_state == 1: @@ -491,19 +550,26 @@ def go_to_page0_from_minus1(question_in_progress_state): # If no question is in progress, show the initial page 0 return gr.update(visible=False), gr.update(visible=True), gr.update(visible=False), gr.update(visible=False) + def go_to_eval_progress_modal(name, email, specialty_dd, subspecialty_dd, years_exp_radio, exp_explanation_tb, npi_id, our_methods=our_methods): # 校验用户信息 if not name or not email or not specialty_dd or not years_exp_radio: - return gr.update(visible=True), gr.update(visible=False), None, "Please fill out all the required fields (name, email, specialty, years of experience). If you are not a licensed physician with a specific specialty, please choose the specialty that most closely aligns with your biomedical expertise.", gr.Chatbot(), gr.Chatbot(), gr.HTML(),gr.Markdown(),gr.State(),gr.update(visible=False), "" + gr.Info("Please fill out all the required fields (name, email, specialty, years of experience). If you are not a licensed physician with a specific specialty, please choose the specialty that most closely aligns with your biomedical expertise.", duration=5) + return gr.update(visible=True), gr.update(visible=False), None, "Please fill out all the required fields (name, email, specialty, years of experience). If you are not a licensed physician with a specific specialty, please choose the specialty that most closely aligns with your biomedical expertise.", gr.Chatbot(), gr.Chatbot(), gr.Chatbot(), gr.Chatbot(), gr.HTML(), gr.State() - user_info, chat_a, chat_b, page1_prompt, page1_reference_answer, question_for_eval, remaining_count = get_next_eval_question( + gr.Info("Loading the data...", duration=3) + user_info, chat_a_answer, chat_b_answer, chat_a_reasoning, chat_b_reasoning, page1_prompt, page1_reference_answer, question_for_eval, remaining_count = get_next_eval_question( name, email, specialty_dd, subspecialty_dd, years_exp_radio, exp_explanation_tb, npi_id, our_methods ) if remaining_count == 0: - return gr.update(visible=True), gr.update(visible=False), None, "Based on your submitted data, you have no more questions to evaluate. You may exit the page; we will follow-up if we require anything else from you. Thank you!", gr.Chatbot(), gr.Chatbot(), gr.HTML(),gr.Markdown(),gr.State(),gr.update(visible=False),"" - return gr.update(visible=True), gr.update(visible=False), user_info,"", chat_a, chat_b, page1_prompt, page1_reference_answer, question_for_eval, gr.update(visible=True), f"You are about to evaluate the next question." + gr.Info("Based on your submitted data, you have no more questions to evaluate. You may exit the page; we will follow-up if we require anything else from you. Thank you!", duration=5) + return gr.update(visible=True), gr.update(visible=False), None, "Based on your submitted data, you have no more questions to evaluate. You may exit the page; we will follow-up if we require anything else from you. Thank you!", gr.Chatbot(), gr.Chatbot(), gr.Chatbot(), gr.Chatbot(), gr.HTML(), gr.State() + gr.Info(f"You are about to evaluate the next question.", duration=3) + return gr.update(visible=False), gr.update(visible=True), user_info, "", chat_a_answer, chat_b_answer, chat_a_reasoning, chat_b_reasoning, page1_prompt, question_for_eval + +# goes to page 1 from confirmation modal that tells users how many questions they have left to evaluate + -#goes to page 1 from confirmation modal that tells users how many questions they have left to evaluate def go_to_page1(show_page_1): """ Shows page 1 if user requests it, otherwise shows page 0 @@ -512,155 +578,49 @@ def go_to_page1(show_page_1): # Return updates to hide modal, hide page 0, show page 1, populate page 1, and set final state if show_page_1: updates = [ - gr.update(visible=False), #hide modal - gr.update(visible=False), #hide page 0 - gr.update(visible=True), #show page 1 + gr.update(visible=False), # hide modal + gr.update(visible=False), # hide page 0 + gr.update(visible=True), # show page 1 ] else: updates = [ - gr.update(visible=False), #hide modal - gr.update(visible=True), #show page 0 - gr.update(visible=False), #hide page 1 + gr.update(visible=False), # hide modal + gr.update(visible=True), # show page 0 + gr.update(visible=False), # hide page 1 ] return updates -# Callback to transition from Page 1 to Page 2. -def go_to_page2(data_subset_state,*pairwise_values): - # pairwise_values is a tuple of values from each radio input. - criteria_count = len(criteria_for_comparison) - pairwise_list = list(pairwise_values[:criteria_count]) - comparison_reasons_list = list(pairwise_values[criteria_count:]) - - #gradio components to display previous page results on next page - pairwise_results_for_display = [gr.Markdown(f"***As a reminder, your pairwise comparison answer for this criterion was: {pairwise_list[i]}. Your answer choices will be restricted based on your comparison answer, but you may go back and change the comparison answer if you wish.***") for i in range(len(criteria))] - - if any(answer is None for answer in pairwise_list): - # Instead of using *pairwise_results_for_display in a tuple, use list concatenation and convert to tuple - gr.Info("Please select an option for every pairwise comparison.") - return (gr.update(visible=True), gr.update(visible=False), None, None, "Error: Please select an option for every pairwise comparison.", gr.Chatbot(type="messages"), gr.Chatbot(type="messages"), gr.HTML(), gr.Markdown()) + tuple(pairwise_results_for_display) - - chat_A_value = format_chat(data_subset_state['models'][0]['reasoning_trace'], tool_database_labels) - chat_B_value = format_chat(data_subset_state['models'][1]['reasoning_trace'], tool_database_labels) - prompt_text = data_subset_state['question'] - - # Construct the question-specific elements of the rating page (page 2) - chat_A_rating = gr.Chatbot( - value=chat_A_value, - type="messages", - height=400, - label="Model A Response", - show_copy_button=False, - render_markdown=True - ) - - chat_B_rating = gr.Chatbot( - value=chat_B_value, - type="messages", - height=400, - label="Model B Response", - show_copy_button=False, - render_markdown=True - ) - - page2_prompt = gr.HTML(f'
Prompt: {prompt_text}
') - page2_reference_answer = gr.Markdown(data_subset_state['correct_answer']) - - # Instead of using *pairwise_results_for_display in a tuple, use list concatenation and convert to tuple - return (gr.update(visible=False), gr.update(visible=True), pairwise_list, comparison_reasons_list, "", chat_A_rating, chat_B_rating, page2_prompt, page2_reference_answer) + tuple(pairwise_results_for_display) - - -# Callback to store scores for Response A. -def store_A_scores(*args): - # Unpack the arguments: first half are scores, second half are checkboxes. - num = len(args) // 2 - scores = list(args[:num]) - unquals = list(args[num:]) - return scores, unquals - -# Callback to transition from Page 2 to Page 3. -def go_to_page3(): - return gr.update(visible=False), gr.update(visible=True) - -# Updated validation callback that ignores criteria with 'Unable to Judge' -def validate_ratings(pairwise_choices, *args): - num_criteria = len(criteria) - ratings_A_list = list(args[:num_criteria]) - ratings_B_list = list(args[num_criteria:]) - if any(r is None for r in ratings_A_list) or any(r is None for r in ratings_B_list): - return "Error: Please provide ratings for both responses for every criterion.", "Error: Please provide ratings for both responses for every criterion." - error_msgs = [] - for i, choice in enumerate(pairwise_choices): - score_a = ratings_A_list[i] - score_b = ratings_B_list[i] - # Skip criteria if either rating is "Unable to Judge" - if score_a == "Unable to Judge" or score_b == "Unable to Judge": - continue - # Convert string scores to integers for comparison. - score_a = int(score_a) - score_b = int(score_b) - if choice == "👈 Model A" and score_a < score_b: - error_msgs.append(f"Criterion {i+1} ({criteria[i]['label']}): You selected A as better but scored A lower than B.") - elif choice == "👉 Model B" and score_b < score_a: - error_msgs.append(f"Criterion {i+1} ({criteria[i]['label']}): You selected B as better but scored B lower than A.") - elif choice == "🤝 Tie" and score_a != score_b: - error_msgs.append(f"Criterion {i+1} ({criteria[i]['label']}): You selected Tie but scored A and B differently.") - - if error_msgs: - err_str = "\n".join(error_msgs) - return err_str, err_str - else: - return "No errors in responses; feel free to submit!", "No errors in responses; feel free to submit!" - -# # Additional callback to handle submission results. -def toggle_slider(is_unqualified): - # When the checkbox is checked (True), set interactive to False to disable the slider. - return gr.update(interactive=not is_unqualified) - -#show reference answer -def toggle_reference(selection): - if selection == "Show Reference Answer": - return gr.update(visible=True) - else: - return gr.update(visible=False) - -#nonsense button helper -# def mark_invalid_question(btn_clicked_status): -# new_status = not btn_clicked_status -# if new_status == True: -# # Show skip modal when marking as Wrong Question -# return new_status, gr.update(value="Undo: Correct Question", variant="primary"), gr.update(visible=True) -# else: -# return new_status, gr.update(value="Wrong Question", variant="stop"), gr.update(visible=False) - # --- Skip Question Modal Callbacks --- def skip_question_and_load_new(user_info_state, our_methods): # user_info_state is a tuple: (name, email, specialty_dd, subspecialty_dd, years_exp_radio, exp_explanation_tb, npi_id, q_id) if user_info_state is None: # Defensive: just close modal if no user info - return gr.update(visible=False), gr.update(visible=False), None, "", gr.Chatbot(), gr.Chatbot(), gr.HTML(), gr.Markdown(), gr.State() + return gr.update(visible=False), gr.update(visible=False), None, "", gr.Chatbot(), gr.Chatbot(), gr.Chatbot(), gr.Chatbot(), gr.HTML(), gr.Markdown(), gr.State() # Unpack user_info_state name, email, specialty_dd, subspecialty_dd, years_exp_radio, exp_explanation_tb, npi_id, _ = user_info_state - user_info, chat_a, chat_b, page1_prompt, page1_reference_answer, question_for_eval, remaining_count = get_next_eval_question( + user_info, chat_a_answer, chat_b_answer, chat_a_reasoning, chat_b_reasoning, page1_prompt, page1_reference_answer, question_for_eval, remaining_count = get_next_eval_question( name, email, specialty_dd, subspecialty_dd, years_exp_radio, exp_explanation_tb, npi_id, our_methods ) if remaining_count == 0: # No more questions, go to final page - return gr.update(visible=False), gr.update(visible=False), None, "Based on your submitted data, you have no more questions to evaluate. You may exit the page; we will follow-up if we require anything else from you. Thank you!", gr.Chatbot(), gr.Chatbot(), gr.HTML(), gr.Markdown(), gr.State() - return gr.update(visible=False), gr.update(visible=True), user_info, "", chat_a, chat_b, page1_prompt, page1_reference_answer, question_for_eval + return gr.update(visible=False), gr.update(visible=False), None, "Based on your submitted data, you have no more questions to evaluate. You may exit the page; we will follow-up if we require anything else from you. Thank you!", gr.Chatbot(), gr.Chatbot(), gr.Chatbot(), gr.Chatbot(), gr.HTML(), gr.Markdown(), gr.State() + return gr.update(visible=False), gr.update(visible=True), user_info, "", chat_a_answer, chat_b_answer, chat_a_reasoning, chat_b_reasoning, page1_prompt, page1_reference_answer, question_for_eval # --- Skip‑question handler for the "Wrong Question?" button ------------------- + + def skip_current_question(user_info_state, our_methods: list = our_methods): # Guard: user clicked before session started gr.Info("Skipping this question and loading the next one…", duration=5) if user_info_state is None: return ( None, - gr.update(value="Please start the evaluation before skipping questions."), + gr.update( + value="Please start the evaluation before skipping questions."), gr.update(value=[]), # Chatbot A history gr.update(value=[]), # Chatbot B history gr.update(value=""), # Prompt HTML - gr.update(value=""), # Reference answer gr.State() # data_subset_state ) @@ -670,8 +630,10 @@ def skip_current_question(user_info_state, our_methods: list = our_methods): # Pull the next unused question ( user_info_new, - _chat_a_comp, # we will rebuild the values directly - _chat_b_comp, + _chat_a_answer, + _chat_b_answer, + _chat_a_reasoning, + _chat_b_reasoning, _prompt_comp, _ref_comp, question_for_eval, @@ -692,14 +654,17 @@ def skip_current_question(user_info_state, our_methods: list = our_methods): gr.update(value=final_msg), gr.update(value=[]), gr.update(value=[]), - gr.update(value=""), + gr.update(value=[]), + gr.update(value=[]), gr.update(value=""), gr.State() ) # --- Build fresh values for the existing UI components --- - chat_a_value = format_chat(question_for_eval['models'][0]['reasoning_trace'], tool_database_labels) - chat_b_value = format_chat(question_for_eval['models'][1]['reasoning_trace'], tool_database_labels) + chat_a_answer, chat_a_reasoning, _ = format_chat( + question_for_eval['models'][0]['reasoning_trace'], tool_database_labels) + chat_b_answer, chat_b_reasoning, _ = format_chat( + question_for_eval['models'][1]['reasoning_trace'], tool_database_labels) prompt_html = ( f"

TxAgent Evaluation Portal

-

Welcome to the TxAgent Evaluation Portal.

""") - with gr.Row(elem_classes=["center-row"]): - # 第一行：并排放两个按钮 - with gr.Column(scale=1): - participate_eval_btn = gr.Button( - value="Click to 🌟Participate in TxAgent Evaluation 🌟", - variant="primary", - size="lg", - elem_id="participate-btn" - ) - with gr.Column(scale=1): - submit_questions_btn = gr.Button( - value="Click to 🚀 Submit Questions for TxAgent Evaluation 🚀", - variant="primary", - size="lg", - elem_id="submit-btn" - ) + # with gr.Row(elem_classes=["center-row"]): + # 第一行：并排放两个按钮 + with gr.Column(scale=1): + participate_eval_btn = gr.Button( + value="Evaluate TxAgent", + variant="primary", + size="lg", + elem_id="participate-btn" + ) + with gr.Column(scale=1): + gr.Markdown( + """ + Joining the **Evaluate TxAgent**, you will: + - See how the model responds to a variety of prompts. + - Provide instant thumbs-up / thumbs-down ratings. + - Shape the roadmap for upcoming releases. - with gr.Row(elem_classes=["center-row"]): + _Thank you for helping us improve!_ + """ + ) + with gr.Column(scale=1): + submit_questions_btn = gr.Button( + value="Submit Therapeutic Questions", + variant="primary", + size="lg", + elem_id="submit-btn" + ) + + # with gr.Row(elem_classes=["center-row"]): # 第二行：分别放两段说明文字 - with gr.Column(scale=1): - gr.Markdown( - """ - ### Why participate in the evaluation? - Joining the **TxAgent human evaluation** lets you experience the model in real-time and give immediate feedback. + with gr.Column(scale=1): + gr.Markdown( + """ + By submitting **Submit Therapeutic Questions**, you will: + - Highlight edge-cases and identify blind spots. + - Push TxAgent to reason in new domains. + - Directly influence future model improvements. + + _We're excited to see what you come up with!_ + """ + ) - **By clicking the button above, you will:** - - See how the model responds to a variety of prompts. - - Provide instant thumbs-up / thumbs-down ratings. - - Shape the roadmap for upcoming releases. + # Add contact information in Markdown format + contact_info_markdown = """ + ## Contact - _Thank you for helping us improve!_ - """ - ) - with gr.Column(scale=1): - gr.Markdown( - """ - ### Why submit questions? - Help us build a richer evaluation set by sending unique, challenging prompts. + If you have any questions or suggestions, please email [Shanghua Gao](mailto:shanghuagao@gmail.com) and [Marinka Zitnik](mailto:marinka@hms.harvard.edu). + """ - **By clicking the button above, you will:** - - Highlight edge-cases and identify blind spots. - - Push TxAgent to reason in new domains. - - Directly influence future model improvements. + gr.Markdown(contact_info_markdown) - _We're excited to see what you come up with!_ - """ - ) gr.HTML(TxAgent_Project_Page_HTML) # Define actions for the new buttons @@ -911,372 +890,338 @@ with gr.Blocks(css=centered_col_css) as demo: # Page 0: Welcome / Informational page. with gr.Column(visible=False, elem_id="page0") as page0: - with gr.Row(): - with gr.Column(): - gr.Markdown("## Welcome to the TxAgent Evalution Study!") - gr.Markdown("Please read the following instructions and then enter your information to begin:") - # Existing informational markdown... - gr.Markdown(""" - - Each session requires a minimum commitment of 5-10 minutes to complete one question. - - If you wish to evaluate multiple questions, you may do so; you will never be asked to re-evaluate questions you have already seen. - - When evaluating a question, you will be asked to compare the responses of two different models to the question and then rate each model's response on a scale of 1-5. - - If you feel that a question does not make sense or is not biomedically relevant, there is a RED BUTTON at the top of the first model comparison page to indicate this - - You may use the Back and Next buttons at the bottom of each page to edit any of your responses before submitting. - - You may use the Home Page button at the bottom of each page to the home page. Your progress will be saved but not submitted. - - You must submit your answers to the current question before moving on to evaluate the next question. - - You may stop in between questions and return at a later time; however, you must submit your answers to the current question if you would like them saved. - - Please review the example question and LLM model response below: - - """) - with open("anatomyofAgentResponse.jpg", "rb") as image_file: - img = Image.open(image_file) - new_size = (int(img.width * 0.5), int(img.height * 0.5)) - img = img.resize(new_size, Image.LANCZOS) - buffer = io.BytesIO() - img.save(buffer, format="PNG") - encoded_string = base64.b64encode(buffer.getvalue()).decode("utf-8") - #encoded_string = base64.b64encode(image_file.read()).decode("utf-8") - - image_html = f'
Your Image
' - ReasoningTraceExampleHTML = f""" -
- {image_html} -
- """ - gr.HTML(ReasoningTraceExampleHTML) - with gr.Column(): - gr.Markdown("## Please enter your information to get a question to evaluate. Please use the same email every time you log onto this evaluation portal, as we use your email to prevent showing repeat questions.") - name = gr.Textbox(label="Name (required)") - email = gr.Textbox(label="Email (required). Please use the same email every time you log onto this evaluation portal, as we use your email to prevent showing repeat questions.") - specialty_dd = gr.Dropdown(choices=specialties_list, label="Primary Medical Specialty (required). Go to https://www.abms.org/member-boards/specialty-subspecialty-certificates/ for categorization)", multiselect=True) - subspecialty_dd = gr.Dropdown(choices=subspecialties_list, label="Subspecialty (if applicable). Go to https://www.abms.org/member-boards/specialty-subspecialty-certificates/ for categorization)", multiselect=True) - npi_id = gr.Textbox(label="National Provider Identifier ID (optional). Got to https://npiregistry.cms.hhs.gov/search to search for your NPI ID. If you do not have an NPI ID, please leave this blank.") - years_exp_radio = gr.Radio( - choices=["0-2 years", "3-5 years", "6-10 years", "11-20 years", "20+ years", "Not Applicable"], - label="How many years have you been involved in clinical and/or research activities related to your biomedical area of expertise? (required)" - ) - exp_explanation_tb = gr.Textbox(label="Please briefly explain your expertise/experience relevant to evaluating AI for clinical decision support (optional)") - - page0_error_box = gr.Markdown("") - with gr.Row(): - next_btn_0 = gr.Button("Next") - with gr.Row(): - home_btn_0 = gr.Button("Home") # (your registration info will be saved) - gr.Markdown("""By clicking 'Next', you will start the study, with your progress saved after submitting each question. If you have any other questions or concerns, please contact us directly. Thank you for your participation! - """) + gr.Markdown("## Information:") + name = gr.Textbox(label="Name (required)") + email = gr.Textbox( + label="Email (required). Please use the same email every time you log onto this evaluation portal, as we use your email to prevent showing repeat questions.") + specialty_dd = gr.Dropdown( + choices=specialties_list, label="Primary Medical Specialty (required). Go to https://www.abms.org/member-boards/specialty-subspecialty-certificates/ for categorization)", multiselect=True) + subspecialty_dd = gr.Dropdown( + choices=subspecialties_list, label="Subspecialty (if applicable). Go to https://www.abms.org/member-boards/specialty-subspecialty-certificates/ for categorization)", multiselect=True) + npi_id = gr.Textbox( + label="National Provider Identifier ID (optional). Got to https://npiregistry.cms.hhs.gov/search to search for your NPI ID. If you do not have an NPI ID, please leave this blank.") + years_exp_radio = gr.Radio( + choices=["0-2 years", "3-5 years", "6-10 years", + "11-20 years", "20+ years", "Not Applicable"], + label="How many years have you been involved in clinical and/or research activities related to your biomedical area of expertise? (required)" + ) + exp_explanation_tb = gr.Textbox( + label="Please briefly explain your expertise/experience relevant to evaluating AI for clinical decision support (optional)") - with Modal(visible=False, elem_id="confirm_modal") as eval_progress_modal: - eval_progress_text = gr.Markdown("You have X questions remaining.") - cancel_and_edit_user_info_btn = gr.Button("Cancel, I would like to keep editing my medical info") - eval_progress_proceed_btn = gr.Button("OK, proceed to question evaluation") + page0_error_box = gr.Markdown("") + with gr.Row(): + next_btn_0 = gr.Button("Next") + gr.Markdown("""By clicking 'Next', you will start the study, with your progress saved after submitting each question. If you have any other questions or concerns, please contact us directly. Thank you for your participation! + """) + + gr.Markdown(""" + ## Instructions: + Please review these instructions and enter your information to begin: + + - Each session requires at least 5-10 minutes per question. + - You can evaluate multiple questions; you will not repeat evaluations. + - For each question, compare responses from two models and rate them (scale: 1-5). + - If a question is unclear or irrelevant to biomedicine, click the RED BUTTON at the top of the comparison page. + - Use the Back and Next buttons to edit responses before submission. + - Use the Home Page button to return to the homepage; progress will save but not submit. + - Submit answers to the current question before moving to the next. + - You can pause between questions and return later; ensure current answers are submitted to save them. + """) + with open("anatomyofAgentResponse.jpg", "rb") as image_file: + img = Image.open(image_file) + new_size = (int(img.width * 0.5), int(img.height * 0.5)) + img = img.resize(new_size, Image.LANCZOS) + buffer = io.BytesIO() + img.save(buffer, format="PNG") + encoded_string = base64.b64encode( + buffer.getvalue()).decode("utf-8") + + image_html = f'
Your Image
' + ReasoningTraceExampleHTML = f""" +
+ {image_html} +
+ """ + gr.HTML(ReasoningTraceExampleHTML) # Page 1: Pairwise Comparison. with gr.Column(visible=False) as page1: - gr.Markdown("## Part 1/2: Pairwise Comparison") #Make the number controlled by question indexing! + # Make the number controlled by question indexing! + gr.Markdown("Comparison") + # Add small red button and comments text box in the same row page1_prompt = gr.HTML() - - with gr.Accordion("Click to reveal a reference answer.", open=False, elem_id="answer-reference-btn"): - note_reference_answer = gr.Markdown( - """ - Warning: This answer has been generated automatically and may be incomplete or one of several correct solutions—please use it for reference only. - """, - elem_classes="reference-box" - ) - page1_reference_answer = gr.Markdown( - """ - **Reference Answer:** - - This is the reference answer content. - """, - elem_classes="reference-box" - ) - - - # Add small red button under the prompt with gr.Row(): nonsense_btn = gr.Button( - "Wrong Question?", + "Skip Question", size="sm", variant="stop", # red variant elem_id="invalid-question-btn", - elem_classes=["short-btn"] + elem_classes=["short-btn"], + scale=1 ) - gr.Markdown( - "Click the button if you think this question does not make sense or is not biomedically-relevant", - render=True - ) - with gr.Row(): - unfamiliar_btn = gr.Button( - "Unfamiliar Question?", - size="sm", - variant="stop", # red variant - elem_id="invalid-question-btn", - elem_classes=["short-btn"] - ) - gr.Markdown( - "Click the button if you are not familiar with this area", - render=True + skip_comments = gr.Textbox( + placeholder="(Optional) Comments about why you're skipping this question...", + show_label=False, + scale=3, + container=False, ) page1_error_box = gr.Markdown("") # ADDED: display validation errors - # --- Define chat_a and chat_b before using them in outputs --- + # --- Define four chat components: answer and reasoning for each model --- with gr.Row(): - # ADDED: Use gr.Chatbot to display the scrollable chat window for Response A. + # Model A components with gr.Column(): - gr.Markdown("**Model A Response:**") # Already bold label. - chat_a = gr.Chatbot( - value=[], # Placeholder for chat history + gr.Markdown("**Model A Response:**") + chat_a_answer = gr.Chatbot( + value=[], # Placeholder for chat history type="messages", - height=400, - label="Model A Response", + height=200, + label="Model A Answer", show_copy_button=False, show_label=True, - render_markdown=True, # Required for markdown/HTML support in messages - avatar_images=None, # Optional: omit user/assistant icons + render_markdown=True, + avatar_images=None, rtl=False ) - # ADDED: Use gr.Chatbot to display the scrollable chat window for Response B. - with gr.Column(): - gr.Markdown("**Model B Response:**") - chat_b = gr.Chatbot( + # gr.Markdown("**Model A Reasoning:**") + chat_a_reasoning = gr.Chatbot( value=[], type="messages", - height=400, - label="Model B Response", + height=300, + label="Model A Reasoning", show_copy_button=False, show_label=True, - render_markdown=True, # Required for markdown/HTML support in messages - avatar_images=None, # Optional: omit user/assistant icons + render_markdown=True, + avatar_images=None, rtl=False ) - gr.Markdown("

") - gr.Markdown("### For each criterion, select which response did better:") - comparison_reasons_inputs = [] # ADDED: list to store the free-text inputs - pairwise_inputs = [] - for crit in criteria_for_comparison: - with gr.Row(): - with gr.Column(scale=1): - gr.Markdown(crit['text'], elem_classes="criteria-font-large") # <--- add class here - with gr.Column(scale=1): - radio = gr.Radio( - choices=[ - "👈 Model A", # A - "👉 Model B", # B - "🤝 Tie", # tie - "👎 Neither model did well" # neither - ], - label="Which is better?", - elem_classes="criteria-radio-label" # <--- add class here - ) - pairwise_inputs.append(radio) - # ADDED: free text under each comparison - text_input = gr.Textbox(label=f"Reasons for your selection (optional)") - comparison_reasons_inputs.append(text_input) - - - with gr.Row(): - # back_btn_0 = gr.Button("Back") #removed button to go back to page 0 from page 1 to avoid page 1 + 2 question changing after this action - next_btn_1 = gr.Button("Next: Rate Responses") - - with gr.Row(): - home_btn_1 = gr.Button("Home Page (your progress on this question will be saved but not submitted)") # ADDED: Home button on page11 - - # Page 2: Combined Rating Page for both responses. - with gr.Column(visible=False) as page2: - gr.Markdown("## Part 2/2: Rate Model Responses") - # ### EDIT: Show a highlighted prompt as on previous pages. - page2_prompt = gr.HTML() - with gr.Accordion("Click to reveal a reference answer.", open=False, elem_id="answer-reference-btn"): - note2_reference_answer = gr.Markdown( - """ - Warning: This answer has been generated automatically and may be incomplete or one of several correct solutions—please use it for reference only. - """, - elem_classes="reference-box" - ) - page2_reference_answer = gr.Markdown( - """ - **Reference Answer:** - - This is the reference answer content. - """, - elem_classes="reference-box" - ) - # ### EDIT: Display both responses side-by-side using Chatbot windows. - with gr.Row(): + # Model B components with gr.Column(): - gr.Markdown("**Model A Response:**") - chat_a_rating = gr.Chatbot( + gr.Markdown("**Model B Response:**") + chat_b_answer = gr.Chatbot( value=[], type="messages", - height=400, - label="Model A Response", + height=200, + label="Model B Answer", show_copy_button=False, - render_markdown=True + show_label=True, + render_markdown=True, + avatar_images=None, + rtl=False ) - with gr.Column(): - gr.Markdown("**Model B Response:**") - chat_b_rating = gr.Chatbot( + # gr.Markdown("**Model B Reasoning:**") + chat_b_reasoning = gr.Chatbot( value=[], type="messages", - height=400, - label="Model B Response", + height=300, + label="Model B Reasoning", show_copy_button=False, - render_markdown=True + show_label=True, + render_markdown=True, + avatar_images=None, + rtl=False ) - gr.Markdown("

For each criterion, select your ratings for each model response:

") - # ### EDIT: For each criterion, create a row with two multiple-choice sets (left: Response A, right: Response B) separated by a border. - ratings_A = [] # to store the radio components for response A - ratings_B = [] # to store the radio components for response B - - def restrict_choices(pairwise_list, index, score_a, score_b): - """ - Returns (update_for_A, update_for_B). - Enforces rating constraints based on the pairwise choice for the given criterion index. - """ - # Get the specific pairwise choice for this criterion using the index - # Add error handling in case the state/list is not ready or index is wrong - if not pairwise_list or index >= len(pairwise_list): - pairwise_choice = None - else: - pairwise_choice = pairwise_list[index] - - base = ["1","2","3","4","5","Unable to Judge"] - # Default: no restrictions unless explicitly set - upd_A = gr.update(choices=base) - upd_B = gr.update(choices=base) - - # Skip if no meaningful pairwise choice or either score is "Unable to Judge" - if pairwise_choice is None or pairwise_choice == "👎 Neither model did well" or (score_a is None and score_b is None): - # If one score is UJ but the other isn't, AND it's a Tie, we might still want to restrict the non-UJ one later? - # For now, keep it simple: if either is UJ or choice is Neither/None, don't restrict. - return upd_A, upd_B - - # Helper to parse int safely - def to_int(x): - try: return int(x) - except (ValueError, TypeError): return None - - a_int = to_int(score_a) - b_int = to_int(score_b) - - # --- Apply Restrictions --- - if pairwise_choice == "👈 Model A": - # B must be ≤ A (if A is numeric) - if a_int is not None: #it is None if unable to judge - allowed_b_choices = [str(i) for i in range(1, a_int + 1)] + ["Unable to Judge"] - current_b = score_b if score_b in allowed_b_choices else None # Keep current valid choice - upd_B = gr.update(choices=allowed_b_choices, value=current_b) - # If A is UJ or non-numeric, B is unrestricted by this rule - # else: upd_B remains gr.update(choices=base) - if b_int is not None: - # A must be >= B (if B is numeric) - allowed_a_choices = [str(i) for i in range(b_int, 6)] + ["Unable to Judge"] - current_a = score_a if score_a in allowed_a_choices else None # Keep current valid choice - upd_A = gr.update(choices=allowed_a_choices, value=current_a) - # If B is UJ or non-numeric, A is unrestricted by this rule - # else: upd_A remains gr.update(choices=base) - - elif pairwise_choice == "👉 Model B": - # A must be ≤ B (if B is numeric) - if b_int is not None: - allowed_a_choices = [str(i) for i in range(1, b_int + 1)] + ["Unable to Judge"] - current_a = score_a if score_a in allowed_a_choices else None # Keep current valid choice - upd_A = gr.update(choices=allowed_a_choices, value=current_a) - # If B is UJ or non-numeric, A is unrestricted by this rule - # else: upd_A remains gr.update(choices=base) - if a_int is not None: - # B must be >= A (if A is numeric) - allowed_b_choices = [str(i) for i in range(a_int, 6)] + ["Unable to Judge"] - current_b = score_b if score_b in allowed_b_choices else None # Keep current valid choice - upd_B = gr.update(choices=allowed_b_choices, value=current_b) - # If A is UJ or non-numeric, B is unrestricted by this rule - # else: upd_B remains gr.update(choices=base) - - elif pairwise_choice == "🤝 Tie": - # If both are numeric, they must match. Enforce based on the one that *just changed*. - # If one changes to numeric, force the other (if also numeric) to match. - # If one changes to UJ, the other is unrestricted. - if a_int is not None: - upd_B = gr.update(choices=[score_a]) - elif score_a == "Unable to Judge": - upd_B = gr.update(choices=["Unable to Judge"]) - if b_int is not None: - upd_A = gr.update(choices=[score_b]) - elif score_b == "Unable to Judge": - upd_A = gr.update(choices=["Unable to Judge"]) - - return upd_A, upd_B - - def clear_selection(): - return None, None - - pairwise_results_for_display = [gr.Markdown(render=False) for _ in range(len(criteria))] - indices_for_change = [] - for i, crit in enumerate(criteria): - index_component = gr.Number(value=i, visible=False, interactive=False) - indices_for_change.append(index_component) + # gr.Markdown("

") + # gr.Markdown("### For each criterion, select which response did better:") + comparison_reasons_inputs = [] # ADDED: list to store the free-text inputs + pairwise_inputs = [] + ratings_A_page1 = [] # Store rating components for page 1 + ratings_B_page1 = [] # Store rating components for page 1 + + for i, crit_comp in enumerate(criteria_for_comparison): + # for crit in criteria_for_comparison: + crit_score = criteria[i] # Get the corresponding score criterion + + # Define restrict function for each criterion + def make_restrict_function(base_choices): + def restrict_choices_page1(radio_choice, score_a, score_b): + """ + Returns (update_for_A, update_for_B). + Enforces rating constraints based on the radio choice for page 1. + """ + # Helper to parse int safely + def to_int(x): + try: + # Extract number from "1 text..." format + return int(x.split()[0]) + except (ValueError, TypeError, AttributeError): + return None + + # Default: no restrictions, but ensure current values are valid + upd_A = gr.update(choices=base_choices, value=score_a if score_a in base_choices else None) + upd_B = gr.update(choices=base_choices, value=score_b if score_b in base_choices else None) + + # Skip if no meaningful pairwise choice + if radio_choice is None or radio_choice == "Neither model did well.": + return upd_A, upd_B + + a_int = to_int(score_a) + b_int = to_int(score_b) + + # Apply Restrictions based on radio choice + if radio_choice == "Model A is better.": + # Rule: A >= B + if a_int is not None and b_int is not None: + # Both are numeric, enforce A >= B + if a_int < b_int: + # Violation: A < B, reset the one that doesn't match the constraint + upd_A = gr.update(choices=base_choices, value=None) + upd_B = gr.update(choices=base_choices, value=None) + else: + # Valid: A >= B, apply mutual restrictions + allowed_a_choices = [choice for choice in base_choices if to_int(choice) is None or to_int(choice) >= b_int] + allowed_b_choices = [choice for choice in base_choices if to_int(choice) is None or to_int(choice) <= a_int] + upd_A = gr.update(choices=allowed_a_choices, value=score_a if score_a in allowed_a_choices else None) + upd_B = gr.update(choices=allowed_b_choices, value=score_b if score_b in allowed_b_choices else None) + elif a_int is not None: + # Only A is numeric, B must be <= A + allowed_b_choices = [choice for choice in base_choices if to_int(choice) is None or to_int(choice) <= a_int] + upd_B = gr.update(choices=allowed_b_choices, value=score_b if score_b in allowed_b_choices else None) + elif b_int is not None: + # Only B is numeric, A must be >= B + allowed_a_choices = [choice for choice in base_choices if to_int(choice) is None or to_int(choice) >= b_int] + upd_A = gr.update(choices=allowed_a_choices, value=score_a if score_a in allowed_a_choices else None) + # If both are "Unable to Judge", no restrictions needed + + elif radio_choice == "Model B is better.": + # Rule: B >= A + if a_int is not None and b_int is not None: + # Both are numeric, enforce B >= A + if b_int < a_int: + # Violation: B < A, reset both + upd_A = gr.update(choices=base_choices, value=None) + upd_B = gr.update(choices=base_choices, value=None) + else: + # Valid: B >= A, apply mutual restrictions + allowed_a_choices = [choice for choice in base_choices if to_int(choice) is None or to_int(choice) <= b_int] + allowed_b_choices = [choice for choice in base_choices if to_int(choice) is None or to_int(choice) >= a_int] + upd_A = gr.update(choices=allowed_a_choices, value=score_a if score_a in allowed_a_choices else None) + upd_B = gr.update(choices=allowed_b_choices, value=score_b if score_b in allowed_b_choices else None) + elif a_int is not None: + # Only A is numeric, B must be >= A + allowed_b_choices = [choice for choice in base_choices if to_int(choice) is None or to_int(choice) >= a_int] + upd_B = gr.update(choices=allowed_b_choices, value=score_b if score_b in allowed_b_choices else None) + elif b_int is not None: + # Only B is numeric, A must be <= B + allowed_a_choices = [choice for choice in base_choices if to_int(choice) is None or to_int(choice) <= b_int] + upd_A = gr.update(choices=allowed_a_choices, value=score_a if score_a in allowed_a_choices else None) + + elif radio_choice == "Both models are equally good.": + # Rule: A == B + if a_int is not None and b_int is not None: + # Both are numeric + if a_int == b_int: + # Valid: A == B, restrict both to the same value + upd_A = gr.update(choices=[score_a], value=score_a) + upd_B = gr.update(choices=[score_b], value=score_b) + else: + # Invalid: A != B, reset both + upd_A = gr.update(choices=base_choices, value=None) + upd_B = gr.update(choices=base_choices, value=None) + elif a_int is not None: + # A is numeric, B must match A + upd_B = gr.update(choices=[score_a], value=score_a) + elif b_int is not None: + # B is numeric, A must match B + upd_A = gr.update(choices=[score_b], value=score_b) + elif score_a == "Unable to Judge." and score_b == "Unable to Judge.": + # Both are "Unable to Judge", restrict both to that + upd_A = gr.update(choices=["Unable to Judge."], value="Unable to Judge.") + upd_B = gr.update(choices=["Unable to Judge."], value="Unable to Judge.") + elif score_a == "Unable to Judge.": + # A is "Unable to Judge", B must match + upd_B = gr.update(choices=["Unable to Judge."], value="Unable to Judge.") + elif score_b == "Unable to Judge.": + # B is "Unable to Judge", A must match + upd_A = gr.update(choices=["Unable to Judge."], value="Unable to Judge.") + # If neither has a value, no restrictions needed + + return upd_A, upd_B + return restrict_choices_page1 + + restrict_fn = make_restrict_function(sorted(crit_score["scores"])) + + # Add bold formatting + gr.Markdown(f"**{crit_comp['label']}**", + elem_classes="criteria-font-large") + radio = gr.Radio( + choices=[ + "Model A is better.", + "Model B is better.", + "Both models are equally good.", + "Neither model did well." + ], + # Remove duplicate label since we have markdown above + label=crit_comp['text'], + elem_classes="criteria-radio-label" # <--- add class here + ) + pairwise_inputs.append(radio) + # ADDED: free text under each comparison + + # for i, crit in enumerate(criteria): + index_component = gr.Number( + value=i, visible=False, interactive=False) + # indices_for_change.append(index_component) - with gr.Column(elem_id="centered-column"): - gr.Markdown(f'
{crit["text"][0]}
', elem_classes="criteria-font-large") - gr.Markdown(f'
{crit["text"][1]}
', elem_classes="criteria-font-large") - pairwise_results_for_display[i].render() with gr.Row(): with gr.Column(scale=1): - rating_a = gr.Radio(choices=["1", "2", "3", "4", "5", "Unable to Judge"], - label=f"Score for Response A - {crit['label']}", - interactive=True, - elem_classes="criteria-radio-label") + rating_a = gr.Radio(choices=sorted(crit_score["scores"]), # ["1", "2", "3", "4", "5", "Unable to Judge"], + label=f"Response A - {crit_score['text']}", + interactive=True, + elem_classes="criteria-radio-label") with gr.Column(scale=1): - rating_b = gr.Radio(choices=["1", "2", "3", "4", "5", "Unable to Judge"], - label=f"Score for Response B - {crit['label']}", - interactive=True, - elem_classes="criteria-radio-label") - with gr.Row(): - clear_btn = gr.Button("Clear Selection", size="sm", elem_id="clear_btn") - clear_btn.click(fn=clear_selection, outputs=[rating_a,rating_b]) + rating_b = gr.Radio(choices=sorted(crit_score["scores"]), # ["1", "2", "3", "4", "5", "Unable to Judge"], + label=f"Response B - {crit_score['text']}", + interactive=True, + elem_classes="criteria-radio-label") + # Add clear button and wire up the restrictions + with gr.Row(): # wire each to re‐restrict the other on change + radio.change( + fn=restrict_fn, + inputs=[radio, rating_a, rating_b], + outputs=[rating_a, rating_b] + ) rating_a.change( - fn=restrict_choices, - inputs=[ pairwise_state, index_component, rating_a, rating_b ], - outputs=[ rating_a, rating_b ] + fn=restrict_fn, + inputs=[radio, rating_a, rating_b], + outputs=[rating_a, rating_b] ) rating_b.change( - fn=restrict_choices, - inputs=[ pairwise_state, index_component, rating_a, rating_b ], - outputs=[ rating_a, rating_b ] + fn=restrict_fn, + inputs=[radio, rating_a, rating_b], + outputs=[rating_a, rating_b] ) - ratings_A.append(rating_a) - ratings_B.append(rating_b) - with gr.Row(): - back_btn_2 = gr.Button("Back") - submit_btn = gr.Button("Submit (Note: Once submitted, you cannot edit your responses)", elem_id="submit_btn") + + ratings_A_page1.append(rating_a) + ratings_B_page1.append(rating_b) + + text_input = gr.Textbox( + # Remove label since we have markdown above + placeholder="Comments for your selection (optional)", + show_label=False, + # elem_classes="textbox-bold-label" + ) + comparison_reasons_inputs.append(text_input) with gr.Row(): - home_btn_2 = gr.Button("Home Page (your progress on this question will be saved but not submitted)") + submit_btn_1 = gr.Button( + "Submit Evaluation", variant="primary", elem_id="submit_btn") - result_text = gr.Textbox(label="Validation Result") - # Final Page: Thank you message. with gr.Column(visible=False, elem_id="final_page") as final_page: - gr.Markdown("## You have no questions left to evaluate. Thank you for your participation!") - + gr.Markdown( + "## You have no questions left to evaluate. Thank you for your participation!") + # Error Modal: For displaying validation errors. with Modal("Error", visible=False, elem_id="error_modal") as error_modal: error_message_box = gr.Markdown() ok_btn = gr.Button("OK") # Clicking OK hides the modal. ok_btn.click(lambda: gr.update(visible=False), None, error_modal) - - # Confirmation Modal: Ask for final submission confirmation. - with Modal("Confirm Submission", visible=False, elem_id="confirm_modal") as confirm_modal: - gr.Markdown("Are you sure you want to submit? Once submitted, you cannot edit your responses.") - with gr.Row(): - yes_btn = gr.Button("Yes, please submit") - cancel_btn = gr.Button("Cancel") - # --- Define Callback Functions for Confirmation Flow --- def build_row_dict(data_subset_state, user_info, pairwise, comparisons_reasons, nonsense_btn_clicked, *args): @@ -1317,56 +1262,52 @@ with gr.Blocks(css=centered_col_css) as demo: def final_submit(data_subset_state, user_info, pairwise, comparisons_reasons, nonsense_btn_clicked, *args): # --- Part 1: Submit the current results (Existing Logic) --- - row_dict = build_row_dict(data_subset_state, user_info, pairwise, comparisons_reasons, nonsense_btn_clicked, *args) - append_to_sheet(user_data=None, custom_row_dict=row_dict, custom_sheet_name=str(TXAGENT_RESULTS_SHEET_BASE_NAME), add_header_when_create_sheet=True) + row_dict = build_row_dict(data_subset_state, user_info, + pairwise, comparisons_reasons, nonsense_btn_clicked, *args) + append_to_sheet(user_data=None, custom_row_dict=row_dict, custom_sheet_name=str( + TXAGENT_RESULTS_SHEET_BASE_NAME), add_header_when_create_sheet=True) # --- Part 2: Recalculate remaining questions (Existing Logic + Modified Error Handling) --- name, email, specialty, subspecialty, years_exp_radio, exp_explanation_tb, npi_id, _ = user_info - user_info_new, chat_a, chat_b, page1_prompt, page1_reference_answer, question_for_eval, remaining_count = get_next_eval_question( + user_info_new, chat_a_answer, chat_b_answer, chat_a_reasoning, chat_b_reasoning, page1_prompt, page1_reference_answer, question_for_eval, remaining_count = get_next_eval_question( name, email, specialty, subspecialty, years_exp_radio, exp_explanation_tb, npi_id, our_methods ) if remaining_count == 0: return ( - gr.update(visible=False), # page0 (Hide) - gr.update(visible=False), # page2 (Hide) - gr.update(visible=False), # confirm_modal - gr.update(visible=False), - "", + "", # page1_error_box + gr.update(visible=False), # page1 (Hide) gr.update(visible=True), # final_page (Show) - "", - None, - None, - None, - None, - None, - user_info_new, + "", # page0_error_box + None, # chat_a_answer + None, # chat_b_answer + None, # chat_a_reasoning + None, # chat_b_reasoning + None, # page1_prompt + None, # data_subset_state + user_info_new, # user_info_state ) return ( - gr.update(visible=False), # page0 (Hide) - gr.update(visible=False), # page2 (Hide) - gr.update(visible=False), # confirm_modal (Hide) - gr.update(visible=True), # eval_progress_modal (Show) - f"Submission successful! There are more questions to evaluate. You may exit the page and return later if you wish.", + "", # page1_error_box + gr.update(visible=True), # page1 (Show for next question) gr.update(visible=False), # final_page (Hide) - "", - chat_a, - chat_b, - page1_prompt, - page1_reference_answer, - question_for_eval, - user_info_new + "", # page0_error_box + chat_a_answer, # chat_a_answer + chat_b_answer, # chat_b_answer + chat_a_reasoning, # chat_a_reasoning + chat_b_reasoning, # chat_b_reasoning + page1_prompt, # page1_prompt + question_for_eval, # data_subset_state + user_info_new # user_info_state ) - def cancel_submission(): - # Cancel final submission: just hide the confirmation modal. - return gr.update(visible=False) - def reset_everything_except_user_info(): # 3) Reset all pairwise radios & textboxes - reset_pairwise_radios = [gr.update(value=None) for i in range(len(criteria))] - reset_pairwise_reasoning_texts = [gr.update(value=None) for i in range(len(criteria))] + reset_pairwise_radios = [gr.update(value=None) + for i in range(len(criteria))] + reset_pairwise_reasoning_texts = [ + gr.update(value=None) for i in range(len(criteria))] # 4) Reset all rating radios reset_ratings_A = [gr.update(value=None) for i in range(len(criteria))] @@ -1375,7 +1316,7 @@ with gr.Blocks(css=centered_col_css) as demo: return ( # pages gr.update(visible=True), # page0 - gr.update(visible=False), # final_page + gr.update(visible=False), # final_page # states # gr.update(value=None), # user_info_state @@ -1383,208 +1324,151 @@ with gr.Blocks(css=centered_col_css) as demo: gr.update(value=None), # scores_A_state gr.update(value=None), # comparison_reasons gr.update(value=None), # unqualified_A_state - gr.update(value=0), # question_in_progress + gr.update(value=0), # question_in_progress # gr.update(value=None), # data_subset_state - #page0 elements that need to be reset - gr.update(value=""), #page0_error_box + # page0 elements that need to be reset + gr.update(value=""), # page0_error_box # page1 elements that need to be reset - # gr.update(value=""), #page1_prompt - # gr.update(value=[]), #chat_a - # gr.update(value=[]), #chat_b - gr.update(value=""), #page1_error_box - - # page2 elements that need to be reset - gr.update(value=""), #page2_prompt - gr.update(value=""), #page2_reference_answer - gr.update(value=[]), #chat_a_rating - gr.update(value=[]), #chat_b_rating - gr.update(value=""), #result_text - - #lists of gradio elements that need to be unrolled + gr.update(value=""), # page1_error_box + + # result text element + gr.update(value=""), # result_text + + # lists of gradio elements that need to be unrolled *reset_pairwise_radios, *reset_pairwise_reasoning_texts, *reset_ratings_A, *reset_ratings_B ) - - # --- Define Transitions Between Pages --- + # --- Define Transitions Between Pages --- # For the "Participate in Evaluation" button, transition to page0 participate_eval_btn.click( fn=go_to_page0_from_minus1, inputs=[question_in_progress], - outputs=[page_minus1, page0, page1, page2] + # Removed page2 reference + outputs=[page_minus1, page0, page1, final_page] ) - # Transition from Page 0 (Welcome) to Page 1. next_btn_0.click( fn=go_to_eval_progress_modal, - inputs=[name, email, specialty_dd, subspecialty_dd, years_exp_radio, exp_explanation_tb, npi_id], - outputs=[page0, page1, user_info_state, page0_error_box, chat_a, chat_b, page1_prompt, page1_reference_answer, data_subset_state,eval_progress_modal,eval_progress_text], - scroll_to_output=True - ) - - cancel_and_edit_user_info_btn.click( - fn=go_to_page1, - inputs=gr.State(False), - outputs=[eval_progress_modal, page0, page1], + inputs=[name, email, specialty_dd, subspecialty_dd, + years_exp_radio, exp_explanation_tb, npi_id], + outputs=[page0, page1, user_info_state, page0_error_box, chat_a_answer, + chat_b_answer, chat_a_reasoning, chat_b_reasoning, page1_prompt, data_subset_state], scroll_to_output=True ) - - eval_progress_proceed_btn.click( - fn=go_to_page1, - inputs=gr.State(True), - outputs=[eval_progress_modal, page0, page1], - scroll_to_output=True - ) - - #Home page buttons to simply show page-1 (but if we move there from page 1 or 2, we must indicate a question is in progress) - home_btn_0.click(lambda: (gr.update(visible=True), gr.update(visible=False), 0), None, [page_minus1, page0, question_in_progress]) - home_btn_1.click(lambda: (gr.update(visible=True), gr.update(visible=False), 1), None, [page_minus1, page1, question_in_progress]) - home_btn_2.click(lambda: (gr.update(visible=True), gr.update(visible=False), 2), None, [page_minus1, page2, question_in_progress]) - # Skip the current question and load a new one when the evaluator flags it nonsense_btn.click( fn=flag_nonsense_and_skip, - inputs=[user_info_state], - outputs=[user_info_state, page1_error_box, chat_a, chat_b, - page1_prompt, page1_reference_answer, data_subset_state], + inputs=[user_info_state, skip_comments], + outputs=[user_info_state, page1_error_box, chat_a_answer, chat_b_answer, chat_a_reasoning, chat_b_reasoning, + page1_prompt, data_subset_state], scroll_to_output=True ) - unfamiliar_btn.click( - fn=skip_current_question, - inputs=[user_info_state], - outputs=[user_info_state, page1_error_box, chat_a, chat_b, - page1_prompt, page1_reference_answer, data_subset_state], - scroll_to_output=True - ) - - - # Transition from Page 1 (Pairwise) to the combined Rating Page (Page 2). - next_btn_1.click( - fn=go_to_page2, # ### EDIT: Rename or update the function to simply pass the pairwise inputs if needed. - inputs=[data_subset_state,*pairwise_inputs,*comparison_reasons_inputs], - outputs=[page1, page2, pairwise_state, comparison_reasons, page1_error_box, chat_a_rating, chat_b_rating, page2_prompt, page2_reference_answer,*pairwise_results_for_display], - scroll_to_output=True - ) - - # Transition from Rating Page (Page 2) back to Pairwise page. - back_btn_2.click( - fn=lambda: (gr.update(visible=True), gr.update(visible=False)), - inputs=None, - outputs=[page1, page2], - scroll_to_output=True - ) - - # --- Submission: Validate the Ratings and then Process the Result --- - def process_result(result): - # If validation passed, show the confirmation modal and proceed. - if result == "No errors in responses; feel free to submit!": + # Function to validate page1 inputs and directly submit if valid + def validate_and_submit_page1(data_subset_state, user_info, *combined_values): + # combined_values contains pairwise choices + comparison reasons + ratings + criteria_count = len(criteria_for_comparison) + pairwise_list = list(combined_values[:criteria_count]) + comparison_reasons_list = list( + combined_values[criteria_count:criteria_count*2]) + ratings_A_list = list( + combined_values[criteria_count*2:criteria_count*3]) + ratings_B_list = list(combined_values[criteria_count*3:]) + + # Check if all pairwise comparisons are filled + if any(answer is None for answer in pairwise_list): + missing_comparisons = [] + for i, answer in enumerate(pairwise_list): + if answer is None: + missing_comparisons.append(criteria_for_comparison[i]['label']) + + missing_text = ", ".join(missing_comparisons) + error_msg = f"Please select an option for the following pairwise comparison(s): {missing_text}" + gr.Info(error_msg) return ( - gr.update(), # Show page 3 - gr.update(), # Hide final page - gr.update(visible=True), # Show confirmation modal - gr.update(visible=False), # Hide error modal - gr.update(value="") # EDIT: Clear the error_message_box + gr.update(value=f"Error: {error_msg}"), + gr.update(visible=True), # Keep page1 visible + gr.update(visible=False), # Keep final_page hidden + gr.update(), # page0_error_box - keep unchanged + gr.update(), # chat_a - keep unchanged + gr.update(), # chat_b - keep unchanged + gr.update(), # chat_a - keep unchanged + gr.update(), # chat_b - keep unchanged + gr.update(), # page1_prompt - keep unchanged + gr.update(), # data_subset_state - keep unchanged + gr.update(), # user_info_state - keep unchanged + # Keep form fields unchanged on validation error + *combined_values ) - else: - # If validation fails, show the error modal and display the error in error_message_box. + + # Check if all ratings are filled + if any(r is None for r in ratings_A_list) or any(r is None for r in ratings_B_list): + missing_ratings = [] + for i in range(len(criteria)): + missing_parts = [] + if ratings_A_list[i] is None: + missing_parts.append("Response A") + if ratings_B_list[i] is None: + missing_parts.append("Response B") + if missing_parts: + missing_ratings.append(f"{criteria[i]['label']} ({', '.join(missing_parts)})") + + missing_text = "; ".join(missing_ratings) + error_msg = f"Please provide ratings for: {missing_text}" + gr.Info(error_msg) return ( - gr.update(), # Keep page3 as is - gr.update(), # Keep final page unchanged - gr.update(visible=False), # Hide confirmation modal - gr.update(visible=True), # Show error modal - gr.update(value=result) # EDIT: Update error_message_box with the validation error + gr.update(value=f"Error: {error_msg}"), + gr.update(visible=True), # Keep page1 visible + gr.update(visible=False), # Keep final_page hidden + gr.update(), # page0_error_box - keep unchanged + gr.update(), # chat_a - keep unchanged + gr.update(), # chat_b - keep unchanged + gr.update(), # chat_a - keep unchanged + gr.update(), # chat_b - keep unchanged + gr.update(), # page1_prompt - keep unchanged + gr.update(), # data_subset_state - keep unchanged + gr.update(), # user_info_state - keep unchanged + # Keep form fields unchanged on validation error + *combined_values ) - - # ### EDIT: Update the submission callback to use the new radio inputs. - submit_btn.click( - fn=validate_ratings, - inputs=[pairwise_state, *ratings_A, *ratings_B], - outputs=[error_message_box, result_text] - ).then( - fn=process_result, - inputs=error_message_box, - outputs=[page2, final_page, confirm_modal, error_modal, error_message_box], - scroll_to_output=True - ) - - # Finalize submission if user confirms. - question_submission_event = yes_btn.click( - fn=final_submit, - inputs=[data_subset_state, user_info_state, pairwise_state, comparison_reasons, nonsense_btn_clicked, *ratings_A, *ratings_B], - outputs=[ - page0, # Controlled by final_submit return value 1 - page2, # Controlled by final_submit return value 2 - confirm_modal, # Controlled by final_submit return value 3 - eval_progress_modal, # Controlled by final_submit return value 4 - eval_progress_text, # Controlled by final_submit return value 5 - final_page, # Controlled by final_submit return value 6 - page0_error_box, - chat_a, - chat_b, - page1_prompt, - page1_reference_answer, - data_subset_state, - user_info_state, - ], + gr.Info("Submitting your evaluation and loading the next question...") + # If validation passes, call final_submit and handle form reset + submit_result = final_submit(data_subset_state, user_info, pairwise_list, + comparison_reasons_list, False, *ratings_A_list, *ratings_B_list) + + # Check if there are more questions by looking at the page1 update dict + # submit_result[1] is the page1 update, submit_result[2] is the final_page update + page1_update = submit_result[1] + page1_visible = page1_update.get('visible', False) if isinstance( + page1_update, dict) else False + gr.Info(f"You are about to evaluate the next question...") + # If there are more questions (page1 is visible after submit), reset the form + if page1_visible: # page1 is visible, meaning there's a next question + # Reset form fields for next question + reset_values = [] + for _ in range(len(combined_values)): + reset_values.append(None) + return submit_result + tuple(reset_values) + else: + # Final page is shown, keep current form values (though they won't be visible) + return submit_result + tuple(combined_values) + + # Transition from Page 1 to direct submission (no confirmation modal) + submit_btn_1.click( + fn=validate_and_submit_page1, + inputs=[data_subset_state, user_info_state, *pairwise_inputs, + *comparison_reasons_inputs, *ratings_A_page1, *ratings_B_page1], + outputs=[page1_error_box, page1, final_page, page0_error_box, chat_a_answer, chat_b_answer, chat_a_reasoning, chat_b_reasoning, + page1_prompt, data_subset_state, user_info_state, *pairwise_inputs, *comparison_reasons_inputs, *ratings_A_page1, *ratings_B_page1], scroll_to_output=True ) - - # Cancel final submission. - cancel_btn.click( - fn=cancel_submission, - inputs=None, - outputs=confirm_modal - ) - - # Reset everything and evaluate another question button - question_submission_event.then( - fn=reset_everything_except_user_info, - inputs=[], - outputs=[ - # pages - page0, - final_page, - - # states - # user_info_state, - pairwise_state, - scores_A_state, - comparison_reasons, - unqualified_A_state, - question_in_progress, - # data_subset_state, - - #page0 elements that need to be reset - page0_error_box, - - # # page1 elements that need to be reset - # page1_prompt, - # chat_a, - # chat_b, - page1_error_box, - - # page2 elements that need to be reset - page2_prompt, - page2_reference_answer, - chat_a_rating, - chat_b_rating, - result_text, - - #lists of gradio elements that need to be unrolled - *pairwise_inputs, - *comparison_reasons_inputs, - *ratings_A, - *ratings_B - ] - ) - -demo.launch(share=True, allowed_paths = ["."]) +demo.launch(share=True, allowed_paths=["."])