Sync local directory to agenticx/ExpertEval

app.py

@@ -43,20 +43,6 @@ except Exception as e:
     print(f"Error reading HTML file: {e}")
     TxAgent_Project_Page_HTML = "<p>Error: Project page content could not be loaded.</p>"
 
-# # Assume 'your_image.png' is in the same directory
-# with open("txagent.jpg", "rb") as image_file:
-#     encoded_string = base64.b64encode(image_file.read()).decode("utf-8")
-
-# image_html = f'<img src="data:image/png;base64,{encoded_string}" alt="Your Image">'
-
-# TxAgent_Project_Page_HTML = f"""
-# <div>
-#     <h2>Project Information</h2>
-#     <p>Here's some information about the TxAgent project.</p>
-#     {image_html}
-# </div>
-# """
-
 # Load tool lists
 fda_drug_labeling_tools_path = "fda_drug_labeling_tools.json"
 monarch_tools_path = "monarch_tools.json"
@@ -175,18 +161,11 @@ mapping = {   #for pairwise mapping between model comparison selections
 #Prepare data
 REPO_ID  = "RichardZhu52/TxAgent_human_eval"
 CROWDSOURCING_DATA_DIRECTORY = "crowdsourcing_eval_data_0430"
-TXAGENT_RESULTS_SHEET_BASE_NAME = "TxAgent_Human_Eval_Results_CROWDSOURCED"
+TXAGENT_RESULTS_SHEET_BASE_NAME = "TxAgent_Human_Eval_Results"
 DISEASE_SPECIALTY_MAP_FILENAME = "disease_specialty_map.json"
+QUESTION_MAP_FILENAME = "question_map.json"
 
-def get_evaluator_questions(email, disease_map_data, user_all_specs, all_files, evaluator_directory):
-    relevant_diseases = []
-    for disease, specs in disease_map_data.items():
-        disease_specs = set(specs.get('specialties', []))
-        disease_subspecs = set(specs.get('subspecialties', []))
-        
-        # Check for intersection
-        if user_all_specs.intersection(disease_specs) or user_all_specs.intersection(disease_subspecs):
-            relevant_diseases.append(disease)
+def get_evaluator_questions(evaluator_id, all_files, evaluator_directory, question_map):
 
     # Filter to only the files in that directory
     evaluator_files = [f for f in all_files if f.startswith(f"{evaluator_directory}/")]
@@ -203,46 +182,26 @@ def get_evaluator_questions(email, disease_map_data, user_all_specs, all_files,
             model_name_key = os.path.basename(remote_path).replace('.json', '')
             data_by_filename[model_name_key] = json.load(f)
 
-    # Filter questions based on relevant diseases derived from user specialties
-    evaluator_question_ids = []
-    relevant_diseases_lower = {disease.lower() for disease in relevant_diseases} # Convert relevant diseases to lowercase set for efficient lookup
-
-    # Assuming 'txagent' data is representative for question IDs and associated diseases
-    if 'txagent' in data_by_filename:
-        for entry in data_by_filename['txagent']:
-            question_id = entry.get("question_ID")
-            question_diseases = entry.get("disease", []) # Get diseases list, default to empty if missing
-            if question_id is not None and question_diseases:
-                # Convert question diseases to lowercase and check for intersection
-                question_diseases_lower = {disease.lower() for disease in question_diseases if isinstance(disease, str)}
-                if question_diseases_lower.intersection(relevant_diseases_lower):
-                    evaluator_question_ids.append(question_id)
-
-    # Handle case where no relevant questions are found based on specialty
-    if not evaluator_question_ids:
-        return [], data_by_filename
-
     #FINALLY, MAKE SURE THEY DIDNT ALREADY FILL IT OUT. Must go through every tuple of (question_ID, TxAgent, other model) where other model could be any of the other files in data_by_filename
     model_names = [key for key in data_by_filename.keys() if key != 'txagent']
-    # evaluator_question_ids = question_map.get(email).get('question_ids')
+    evaluator_question_ids = question_map.get(evaluator_id).get('question_ids')
     full_question_ids_list = []
     for other_model_name in model_names:
         for q_id in evaluator_question_ids:
             full_question_ids_list.append((q_id, other_model_name))
 
-    results_df = read_sheet_to_df(custom_sheet_name=str(TXAGENT_RESULTS_SHEET_BASE_NAME))
+    results_df = read_sheet_to_df(custom_sheet_name=str(TXAGENT_RESULTS_SHEET_BASE_NAME + f"_{str(evaluator_id)}"))
     if (results_df is not None) and (not results_df.empty):
         # collect all (question_ID, other_model) pairs already seen
         matched_pairs = set()
         for _, row in results_df.iterrows():
-            if row["Email"] == email:
-                q = row["Question ID"]
-                # pick whichever response isn’t 'txagent'
-                a, b = row["ResponseA_Model"], row["ResponseB_Model"]
-                if a == "txagent" and b != "txagent":
-                    matched_pairs.add((q, b))
-                elif b == "txagent" and a != "txagent":
-                    matched_pairs.add((q, a))
+            q = row["Question ID"]
+            # pick whichever response isn’t 'txagent'
+            a, b = row["ResponseA_Model"], row["ResponseB_Model"]
+            if a == "txagent" and b != "txagent":
+                matched_pairs.add((q, b))
+            elif b == "txagent" and a != "txagent":
+                matched_pairs.add((q, a))
 
         # filter out any tuple whose (q_id, other_model) was already matched
         full_question_ids_list = [
@@ -253,26 +212,31 @@ def get_evaluator_questions(email, disease_map_data, user_all_specs, all_files,
         print(f"Filtered question IDs: {full_question_ids_list}")
         print(f"Length of filtered question IDs: {len(full_question_ids_list)}")
 
-    
     return full_question_ids_list, data_by_filename
 
 def go_to_page0_from_minus1():
     return gr.update(visible=False), gr.update(visible=True)
 
-def go_to_eval_progress_modal(name, email, specialty_dd, subspecialty_dd, years_exp_radio, exp_explanation_tb, npi_id):
+def go_to_eval_progress_modal(name, email, evaluator_id, specialty_dd, subspecialty_dd, years_exp_radio, exp_explanation_tb, npi_id):
 
     # ADDED: Validate that name and email are non-empty before proceeding
-    if not name or not email or not specialty_dd or not years_exp_radio:
-        return gr.update(visible=True), gr.update(visible=False), None, "Please fill out all the required fields (name, email, specialty, years of experience). If you are not a licensed physician with a specific specialty, please choose the specialty that most closely aligns with your biomedical expertise.", gr.Chatbot(), gr.Chatbot(), gr.HTML(),gr.State(),gr.update(visible=False), ""
+    if not name or not email or not evaluator_id or not specialty_dd or not years_exp_radio:
+        return gr.update(visible=True), gr.update(visible=False), None, "Please fill out all the required fields (name, email, evaluator ID, specialty, years of experience). If you are not a licensed physician with a specific specialty, please choose the specialty that most closely aligns with your biomedical expertise.", gr.Chatbot(), gr.Chatbot(), gr.HTML(),gr.State(),gr.update(visible=False), ""
 
-    # Combine user's selected specialties and subspecialties into a set for efficient lookup
-    # Ensure inputs are lists, even if None or single strings are passed (though Dropdown with multiselect=True should return lists)
-    user_specialties = set(specialty_dd if isinstance(specialty_dd, list) else ([specialty_dd] if specialty_dd else []))
-    user_subspecialties = set(subspecialty_dd if isinstance(subspecialty_dd, list) else ([subspecialty_dd] if subspecialty_dd else []))
-    user_all_specs = user_specialties.union(user_subspecialties)
+    question_map_path = hf_hub_download(
+        repo_id=REPO_ID,
+        filename=QUESTION_MAP_FILENAME,
+        repo_type="dataset",       # or omit if it's a Model/Space
+        # force_download=True,       # ← always fetch new copy
+        revision="main"            # branch/tag/commit, fetches the most recent version of the dataset each time this command is called
+    )
+
+    # Load the question map from the downloaded file
+    with open(question_map_path, 'r') as f:
+        question_map = json.load(f)
 
     #retrieve data from HF
-    evaluator_directory = CROWDSOURCING_DATA_DIRECTORY
+    evaluator_directory = question_map.get(evaluator_id, {}).get('evaluator_name', None)
     if evaluator_directory is None:
         return gr.update(visible=True), gr.update(visible=False), None, "Invalid Evaluator ID, please try again.", gr.Chatbot(), gr.Chatbot(), gr.HTML(),gr.State(),gr.update(visible=False),"" 
     all_files = list_repo_files(
@@ -281,17 +245,7 @@ def go_to_eval_progress_modal(name, email, specialty_dd, subspecialty_dd, years_
         revision="main",
     )
 
-    disease_specialty_map = hf_hub_download(
-        repo_id=REPO_ID,
-        filename=DISEASE_SPECIALTY_MAP_FILENAME,
-        repo_type="dataset",
-        revision="main",
-    )
-
-    with open(disease_specialty_map, 'r') as f:
-        disease_map_data = json.load(f)
-
-    full_question_ids_list, data_by_filename = get_evaluator_questions(email, disease_map_data, user_all_specs, all_files, evaluator_directory)
+    full_question_ids_list, data_by_filename = get_evaluator_questions(evaluator_id, all_files, evaluator_directory, question_map)
 
     if len(full_question_ids_list) == 0:
         return gr.update(visible=True), gr.update(visible=False), None, "Based on your submitted data, you have no more questions to evaluate. You may exit the page; we will follow-up if we require anything else from you. Thank you!", gr.Chatbot(), gr.Chatbot(), gr.HTML(),gr.State(),gr.update(visible=False),""
@@ -329,7 +283,7 @@ def go_to_eval_progress_modal(name, email, specialty_dd, subspecialty_dd, years_
     }
 
     #update user_info
-    user_info = (name, email, specialty_dd, subspecialty_dd, years_exp_radio, exp_explanation_tb, npi_id, q_id)
+    user_info = (name, email, specialty_dd, subspecialty_dd, years_exp_radio, exp_explanation_tb, npi_id, q_id, evaluator_id)
     chat_A_value = format_chat(question_for_eval['models'][0]['reasoning_trace'], tool_database_labels)
     chat_B_value = format_chat(question_for_eval['models'][1]['reasoning_trace'], tool_database_labels)
     prompt_text = question_for_eval['question']
@@ -358,7 +312,7 @@ def go_to_eval_progress_modal(name, email, specialty_dd, subspecialty_dd, years_
                     avatar_images=None,    # Optional: omit user/assistant icons
                     rtl=False
                 )
-    return gr.update(visible=True), gr.update(visible=False), user_info,"", chat_a, chat_b, page1_prompt, question_for_eval, gr.update(visible=True), f"You are about to evaluate the next question."#f"You have {len(full_question_ids_list)} question(s) remaining to evaluate."
+    return gr.update(visible=True), gr.update(visible=False), user_info,"", chat_a, chat_b, page1_prompt, question_for_eval, gr.update(visible=True), f"You are about to evaluate the next question. You have {len(full_question_ids_list)} question(s) remaining to evaluate."
 
 #goes to page 1 from confirmation modal that tells users how many questions they have left to evaluate
 def go_to_page1():
@@ -470,6 +424,16 @@ centered_col_css = """
     max-width: 800px; /* Adjust this width as desired */
     width: 100%;
 }
+#participate-btn {
+    background-color: purple !important;
+    color: white !important;
+    border-color: purple !important;
+}
+#clear_btn {
+    background-color: #F08080 !important;
+    color: white !important;
+    border-color: #F08080 !important;
+}
 """
 with gr.Blocks(css=centered_col_css) as demo:
     # States to save information between pages.
@@ -508,21 +472,14 @@ with gr.Blocks(css=centered_col_css) as demo:
         </div>
         """)
         with gr.Row():
-            submit_questions_btn = gr.Button("Submit Questions for TxAgent Evaluation")
-            participate_eval_btn = gr.Button("Participate in TxAgent Evaluation")
+            participate_eval_btn = gr.Button(
+            value="🌟 Participate in TxAgent Evaluation 🌟",
+            variant="primary",
+            size="lg",
+            elem_id="participate-btn"
+            )
         gr.HTML(TxAgent_Project_Page_HTML)
 
-        # Define actions for the new buttons
-        # For the Google Form button, we'll use JavaScript to open a new tab.
-        # The URL for the Google Form should be replaced with the actual link.
-        google_form_url = "https://forms.gle/pYvyvEQQwS5gdupQA"  # Replace with your actual Google Form link
-        submit_questions_btn.click(
-            fn=None,
-            inputs=None,
-            outputs=None,
-            js=f"() => {{ window.open('{google_form_url}', '_blank'); }}"
-        )
-
     # Page 0: Welcome / Informational page.
     with gr.Column(visible=False, elem_id="page0") as page0:
         gr.Markdown("## Welcome to the TxAgent Evalution Study!")
@@ -542,6 +499,7 @@ with gr.Blocks(css=centered_col_css) as demo:
         gr.Markdown("## Please enter your information to get a question to evaluate. Please use the same email every time you log onto this evaluation portal, as we use your email to prevent showing repeat questions.")
         name = gr.Textbox(label="Name (required)")
         email = gr.Textbox(label="Email (required). Please use the same email every time you log onto this evaluation portal, as we use your email to prevent showing repeat questions.")
+        evaluator_id = gr.Textbox(label="Evaluator ID (required). This is the four-digit ID you received from us for the evaluation study. If you do not have an Evaluator ID or are unsure about your Evaluator ID, please contact us.")
         specialty_dd = gr.Dropdown(choices=specialties_list, label="Primary Medical Specialty (required). Go to https://www.abms.org/member-boards/specialty-subspecialty-certificates/ for categorization)", multiselect=True)
         subspecialty_dd = gr.Dropdown(choices=subspecialties_list, label="Subspecialty (if applicable). Go to https://www.abms.org/member-boards/specialty-subspecialty-certificates/ for categorization)", multiselect=True)
         npi_id = gr.Textbox(label="National Provider Identifier ID (optional). Got to https://npiregistry.cms.hhs.gov/search to search for your NPI ID. If you do not have an NPI ID, please leave this blank.")
@@ -759,7 +717,7 @@ with gr.Blocks(css=centered_col_css) as demo:
                                             label=f"Score for Response B - {crit['label']}",
                                             interactive=True)
             with gr.Row():
-                clear_btn = gr.Button("Clear Selection", size="sm")
+                clear_btn = gr.Button("Clear Selection", size="sm",elem_id="clear_btn")
                 clear_btn.click(fn=clear_selection, outputs=[rating_a,rating_b])
 
                 # wire each to re‐restrict the other on change
@@ -787,7 +745,6 @@ with gr.Blocks(css=centered_col_css) as demo:
     # Final Page: Thank you message.
     with gr.Column(visible=False, elem_id="final_page") as final_page:
         gr.Markdown("## You have no questions left to evaluate. Thank you for your participation!")
-        eval_again_btn = gr.Button("Evaluate Another Question")
     
     # Error Modal: For displaying validation errors.
     with Modal("Error", visible=False, elem_id="error_modal") as error_modal:
@@ -818,6 +775,7 @@ with gr.Blocks(css=centered_col_css) as demo:
             "Timestamp": timestamp,
             "Name": user_info[0],
             "Email": user_info[1],
+            "Evaluator ID": user_info[8],
             "Specialty": str(user_info[2]),
             "Subspecialty": str(user_info[3]),
             "Years of Experience": user_info[4],
@@ -849,23 +807,32 @@ with gr.Blocks(css=centered_col_css) as demo:
     def final_submit(data_subset_state, user_info, pairwise, comparisons_reasons, *args):
         # --- Part 1: Submit the current results (Existing Logic) ---
         row_dict = build_row_dict(data_subset_state, user_info, pairwise, comparisons_reasons, *args)
-        append_to_sheet(user_data=None, custom_row_dict=row_dict, custom_sheet_name=str(TXAGENT_RESULTS_SHEET_BASE_NAME), add_header_when_create_sheet=True)
+        _, _, _, _, _, _, _, _, evaluator_id = user_info
+        append_to_sheet(user_data=None, custom_row_dict=row_dict, custom_sheet_name=str(TXAGENT_RESULTS_SHEET_BASE_NAME + f"_{evaluator_id}"), add_header_when_create_sheet=True)
 
         # --- Part 2: Recalculate remaining questions (Existing Logic + Modified Error Handling) ---
         # try:
 
         # --- Re-fetch data and filter questions (Same logic as before) ---
-        evaluator_directory = CROWDSOURCING_DATA_DIRECTORY
-        all_files = list_repo_files(repo_id=REPO_ID, repo_type="dataset", revision="main")
-        disease_specialty_map = hf_hub_download(repo_id=REPO_ID, filename=DISEASE_SPECIALTY_MAP_FILENAME, repo_type="dataset", revision="main")
-        with open(disease_specialty_map, 'r') as f: disease_map_data = json.load(f)
+        question_map_path = hf_hub_download(
+            repo_id=REPO_ID,
+            filename=QUESTION_MAP_FILENAME,
+            repo_type="dataset",       # or omit if it's a Model/Space
+            # force_download=True,       # ← always fetch new copy
+            revision="main"            # branch/tag/commit, fetches the most recent version of the dataset each time this command is called
+        )
+
+        with open(question_map_path, 'r') as f:
+            question_map = json.load(f)
 
-        _, email, specialty, subspecialty, _, _, _, _ = user_info
-        user_specialties = set(specialty if isinstance(specialty, list) else ([specialty] if specialty else []))
-        user_subspecialties = set(subspecialty if isinstance(subspecialty, list) else ([subspecialty] if subspecialty else []))
-        user_all_specs = user_specialties.union(user_subspecialties)
+        evaluator_directory = question_map.get(evaluator_id, {}).get('evaluator_name', None)
+        all_files = list_repo_files(
+            repo_id=REPO_ID,
+            repo_type="dataset",
+            revision="main",
+        )
 
-        full_question_ids_list, data_by_filename = get_evaluator_questions(email, disease_map_data, user_all_specs, all_files, evaluator_directory)
+        full_question_ids_list, data_by_filename = get_evaluator_questions(evaluator_id, all_files, evaluator_directory, question_map)
         remaining_count = len(full_question_ids_list)
 
         # --- Part 3: Determine UI updates based on remaining count ---
@@ -952,7 +919,7 @@ with gr.Blocks(css=centered_col_css) as demo:
             gr.update(visible=False),  # page2 (Hide)
             gr.update(visible=False),  # confirm_modal (Hide)
             gr.update(visible=True),   # eval_progress_modal (Show)
-            f"Submission successful! There are more questions to evaluate. You may exit the page and return later if you wish.",#f"Submission successful! You have {remaining_count} question(s) remaining to evaluate. You may exit the page and return later if you wish.", # eval_progress_text
+            f"Submission successful! You have {remaining_count} question(s) remaining to evaluate. You may exit the page and return later if you wish.", # eval_progress_text
             gr.update(visible=False),  # final_page (Hide)
             "",         
             chat_a,
@@ -989,9 +956,6 @@ with gr.Blocks(css=centered_col_css) as demo:
         reset_ratings_B = [gr.update(value=None) for i in range(len(criteria))]
 
         return (
-            # pages
-            gr.update(visible=True),  # page0
-            gr.update(visible=False), # final_page
 
             # states
             # gr.update(value=None),  # user_info_state
@@ -1037,7 +1001,7 @@ with gr.Blocks(css=centered_col_css) as demo:
     # Transition from Page 0 (Welcome) to Page 1.
     next_btn_0.click(
         fn=go_to_eval_progress_modal,
-        inputs=[name, email, specialty_dd, subspecialty_dd, years_exp_radio, exp_explanation_tb, npi_id],
+        inputs=[name, email, evaluator_id, specialty_dd, subspecialty_dd, years_exp_radio, exp_explanation_tb, npi_id],
         outputs=[page0, page1, user_info_state, page0_error_box, chat_a, chat_b, page1_prompt, data_subset_state,eval_progress_modal,eval_progress_text],
         scroll_to_output=True
     )
@@ -1147,9 +1111,6 @@ with gr.Blocks(css=centered_col_css) as demo:
         fn=reset_everything_except_user_info,
         inputs=[],
         outputs=[
-            # pages
-            page0,
-            final_page,
 
             # states
             # user_info_state,

utils.py

@@ -161,7 +161,11 @@ def append_to_sheet(user_data=None, custom_row_dict=None, custom_sheet_name=None
     # Access the first worksheet
     sheet = spreadsheet.sheet1
 
-    if is_new and add_header_when_create_sheet:
+    # Check if the sheet has any rows yet
+    existing_values = sheet.get_all_values()
+    is_empty = (existing_values == [[]]) #indicates empty spreadsheet that was cleared in the past
+
+    if (is_new or is_empty) and add_header_when_create_sheet:
         # headers come from the keys of our row dict
         if custom_row_dict is not None:
             headers = list(custom_row_dict.keys())