files

.gitattributes

@@ -33,3 +33,6 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+*.json filter=lfs diff=lfs merge=lfs -text
+*.jsonl filter=lfs diff=lfs merge=lfs -text
+

app.py

@@ -0,0 +1,687 @@
+# import gradio as gr
+# import os
+# import json
+# from openai import AzureOpenAI
+# from trialgpt_retrieval.retriever import retrieve_trials
+# from nltk.tokenize import sent_tokenize
+# import nltk
+
+# # Download required NLTK data
+# try:
+#     nltk.download('punkt')
+#     nltk.download('punkt_tab')
+# except Exception as e:
+#     print(f"Warning: Could not download NLTK data: {e}")
+#     print("The app may still work but tokenization might fail.")
+
+# # Check environment variables
+# if not os.getenv("OPENAI_ENDPOINT") or not os.getenv("OPENAI_API_KEY"):
+#     print("Warning: OPENAI_ENDPOINT and OPENAI_API_KEY environment variables are required.")
+#     print("Please set these environment variables before running the app.")
+
+# # Initialize the Azure OpenAI client
+# client = AzureOpenAI(
+#     api_version="2023-09-01-preview",
+#     azure_endpoint=os.getenv("OPENAI_ENDPOINT"),
+#     api_key=os.getenv("OPENAI_API_KEY"),
+# )
+
+# def generate_keywords(note):
+#     system = 'You are a helpful assistant and your task is to help search relevant clinical trials for a given patient description. Please first summarize the main medical problems of the patient. Then generate up to 32 key conditions for searching relevant clinical trials for this patient. The key condition list should be ranked by priority. Please output only a JSON dict formatted as Dict{{"summary": Str(summary), "conditions": List[Str(condition)]}}.'
+#     prompt = f"Here is the patient description: \\n{note}\\n\\nJSON output:"
+#     messages = [
+#         {"role": "system", "content": system},
+#         {"role": "user", "content": prompt}
+#     ]
+    
+#     try:
+#         response = client.chat.completions.create(
+#             model="gpt-35-turbo", 
+#             messages=messages,
+#             temperature=0,
+#             max_tokens=4086,
+#         )
+
+#         output = response.choices[0].message.content
+
+#         if output is None:
+#             return "Error: Received empty response from the model.", []
+
+#         try:
+#             start = output.find('{')
+#             end = output.rfind('}')
+#             if start != -1 and end > start:
+#                 json_output = output[start:end+1]
+#                 data = json.loads(json_output)
+#                 return data["summary"], data["conditions"]
+#             else:
+#                 return "Error: No JSON found in model response.", []
+#         except (json.JSONDecodeError, KeyError) as e:
+#             return f"Error parsing model output: {e}", []
+            
+#     except Exception as e:
+#         return f"Error calling OpenAI API: {e}", []
+
+# def parse_criteria(criteria):
+#     output = ""
+#     criteria = criteria.split("\\n\\n")
+    
+#     idx = 0
+#     for criterion in criteria:
+#         criterion = criterion.strip()
+#         if "inclusion criteria" in criterion.lower() or "exclusion criteria" in criterion.lower():
+#             continue
+#         if len(criterion) < 5:
+#             continue
+#         output += f"{idx}. {criterion}\\n" 
+#         idx += 1
+#     return output
+
+# def print_trial(trial_info: dict, inc_exc: str) -> str:
+#     trial = f"Title: {trial_info.get('title', 'N/A')}\\n"
+    
+#     # Safely get metadata fields
+#     metadata = trial_info.get('metadata', {})
+#     diseases_list = metadata.get('diseases_list', [])
+#     drugs_list = metadata.get('drugs_list', [])
+    
+#     trial += f"Target diseases: {', '.join(diseases_list) if diseases_list else 'N/A'}\\n"
+#     trial += f"Interventions: {', '.join(drugs_list) if drugs_list else 'N/A'}\\n"
+#     trial += f"Summary: {trial_info.get('text', 'N/A')}\\n"
+    
+#     if inc_exc == "inclusion":
+#         inclusion_criteria = metadata.get('inclusion_criteria', 'No inclusion criteria available')
+#         trial += "Inclusion criteria:\\n %s\\n" % parse_criteria(inclusion_criteria)
+#     elif inc_exc == "exclusion":
+#         exclusion_criteria = metadata.get('exclusion_criteria', 'No exclusion criteria available')
+#         trial += "Exclusion criteria:\\n %s\\n" % parse_criteria(exclusion_criteria) 
+#     return trial
+
+# def get_matching_prompt(trial_info: dict, inc_exc: str, patient: str) -> tuple[str, str]:
+# 	"""Output the prompt."""
+# 	prompt = f"You are a helpful assistant for clinical trial recruitment. Your task is to compare a given patient note and the {inc_exc} criteria of a clinical trial to determine the patient's eligibility at the criterion level.\\n"
+
+# 	if inc_exc == "inclusion":
+# 		prompt += "The factors that allow someone to participate in a clinical study are called inclusion criteria. They are based on characteristics such as age, gender, the type and stage of a disease, previous treatment history, and other medical conditions.\\n"
+	
+# 	elif inc_exc == "exclusion":
+# 		prompt += "The factors that disqualify someone from participating are called exclusion criteria. They are based on characteristics such as age, gender, the type and stage of a disease, previous treatment history, and other medical conditions.\\n"
+
+# 	prompt += f"You should check the {inc_exc} criteria one-by-one, and output the following three elements for each criterion:\\n"
+# 	prompt += f"\\tElement 1. For each {inc_exc} criterion, briefly generate your reasoning process: First, judge whether the criterion is not applicable (not very common), where the patient does not meet the premise of the criterion. Then, check if the patient note contains direct evidence. If so, judge whether the patient meets or does not meet the criterion. If there is no direct evidence, try to infer from existing evidence, and answer one question: If the criterion is true, is it possible that a good patient note will miss such information? If impossible, then you can assume that the criterion is not true. Otherwise, there is not enough information.\\n"
+# 	prompt += f"\\tElement 2. If there is relevant information, you must generate a list of relevant sentence IDs in the patient note. If there is no relevant information, you must annotate an empty list.\\n" 
+# 	prompt += f"\\tElement 3. Classify the patient eligibility for this specific {inc_exc} criterion: "
+	
+# 	if inc_exc == "inclusion":
+# 		prompt += 'the label must be chosen from {"not applicable", "not enough information", "included", "not included"}. "not applicable" should only be used for criteria that are not applicable to the patient. "not enough information" should be used where the patient note does not contain sufficient information for making the classification. Try to use as less "not enough information" as possible because if the note does not mention a medically important fact, you can assume that the fact is not true for the patient. "included" denotes that the patient meets the inclusion criterion, while "not included" means the reverse.\\n'
+# 	elif inc_exc == "exclusion":
+# 		prompt += 'the label must be chosen from {"not applicable", "not enough information", "excluded", "not excluded"}. "not applicable" should only be used for criteria that are not applicable to the patient. "not enough information" should be used where the patient note does not contain sufficient information for making the classification. Try to use as less "not enough information" as possible because if the note does not mention a medically important fact, you can assume that the fact is not true for the patient. "excluded" denotes that the patient meets the exclusion criterion and should be excluded in the trial, while "not excluded" means the reverse.\\n'
+	
+# 	prompt += "You should output only a JSON dict exactly formatted as: dict{str(criterion_number): list[str(element_1_brief_reasoning), list[int(element_2_sentence_id)], str(element_3_eligibility_label)]}."
+	
+# 	user_prompt = f"Here is the patient note, each sentence is led by a sentence_id:\\n{patient}\\n\\n" 
+# 	user_prompt += f"Here is the clinical trial:\\n{print_trial(trial_info, inc_exc)}\\n\\n"
+# 	user_prompt += f"Plain JSON output:"
+
+# 	return prompt, user_prompt
+
+
+# def trialgpt_matching(trial: dict, patient: str, model: str):
+# 	results = {}
+# 	for inc_exc in ["inclusion", "exclusion"]:
+# 		system_prompt, user_prompt = get_matching_prompt(trial, inc_exc, patient)
+# 		messages = [
+# 			{"role": "system", "content": system_prompt},
+# 			{"role": "user", "content": user_prompt},
+# 		]
+# 		response = client.chat.completions.create(
+# 			model=model,
+# 			messages=messages,
+# 			temperature=0,
+# 		)
+# 		message = response.choices[0].message.content.strip()
+# 		start = message.find('{')
+# 		end = message.rfind('}')
+# 		if start != -1 and end > start:
+# 			message = message[start:end+1]
+# 		try:
+# 			results[inc_exc] = json.loads(message)
+# 		except (json.JSONDecodeError, TypeError):
+# 			results[inc_exc] = {"error": "Failed to parse model output", "raw": message}
+# 	return results
+
+# def load_corpus(corpus="sigir"):
+#     corpus_path = f"dataset/{corpus}/corpus.jsonl"
+#     corpus_map = {}
+#     with open(corpus_path, "r") as f:
+#         for line in f:
+#             entry = json.loads(line)
+#             corpus_map[entry["_id"]] = entry
+#     return corpus_map
+
+# corpus_data = load_corpus()
+
+# # --- Trial Ranking (from trialgpt_ranking/rank_results.py) ---
+
+# eps = 1e-9
+
+# def get_matching_score(matching):
+# 	included, not_inc, na_inc, no_info_inc = 0, 0, 0, 0
+# 	excluded, not_exc, na_exc, no_info_exc = 0, 0, 0, 0
+	
+# 	if "inclusion" in matching:
+# 		for criteria, info in matching["inclusion"].items():
+# 			if len(info) == 3:
+# 				if info[2] == "included": included += 1	
+# 				elif info[2] == "not included": not_inc += 1
+# 				elif info[2] == "not applicable": na_inc += 1
+# 				elif info[2] == "not enough information": no_info_inc += 1
+	
+# 	if "exclusion" in matching:
+# 		for criteria, info in matching["exclusion"].items():
+# 			if len(info) == 3:
+# 				if info[2] == "excluded": excluded += 1	
+# 				elif info[2] == "not excluded": not_exc += 1
+# 				elif info[2] == "not applicable": na_exc += 1
+# 				elif info[2] == "not enough information": no_info_exc += 1
+
+# 	score = 0
+# 	if (included + not_inc + no_info_inc) > 0:
+# 		score += included / (included + not_inc + no_info_inc + eps)
+# 	if not_inc > 0: score -= 1
+# 	if excluded > 0: score -= 1
+	
+# 	return score 
+
+# def get_clinical_trials(patient_details):
+#     # Check if environment variables are set
+#     if not os.getenv("OPENAI_ENDPOINT") or not os.getenv("OPENAI_API_KEY"):
+#         return "**Error:** Please set OPENAI_ENDPOINT and OPENAI_API_KEY environment variables before using this app."
+    
+#     if not patient_details or patient_details.strip() == "":
+#         return "**Error:** Please enter patient details."
+    
+#     try:
+#         summary, conditions = generate_keywords(patient_details)
+#         if isinstance(summary, str) and summary.startswith("Error"):
+#             return f"**Error:** {summary}"
+        
+#         if not conditions:
+#             return "## No Conditions Identified\n\nUnable to identify medical conditions from the patient description. Please provide more detailed medical information."
+
+#         retrieved_nctids = retrieve_trials(conditions)
+#         if not retrieved_nctids:
+#             return "## No Matching Clinical Trials Found\n\nNo clinical trials were found matching the patient's conditions. Please try with different search terms or broader criteria."
+        
+#         top_n_trials = retrieved_nctids[:5]
+        
+#         sents = sent_tokenize(patient_details)
+#         sents.append("The patient will provide informed consent, and will comply with the trial protocol without any practical issues.")
+#         patient_processed = "\\n".join([f"{idx}. {sent}" for idx, sent in enumerate(sents)])
+
+#         results = []
+#         for nctid in top_n_trials:
+#             if nctid in corpus_data:
+#                 try:
+#                     trial_info = corpus_data[nctid]
+#                     matching_result = trialgpt_matching(trial_info, patient_processed, "gpt-4")
+#                     score = get_matching_score(matching_result)
+#                     title = trial_info.get('title', 'Unknown Title')
+#                     results.append({"title": title, "nctid": nctid, "score": score})
+#                 except Exception as e:
+#                     print(f"Warning: Error processing trial {nctid}: {e}")
+#                     continue
+
+#         # Sort results by score
+#         results.sort(key=lambda x: x["score"], reverse=True)
+
+#         # Format output as a table
+#         output_str = "## Matching Clinical Trials\n\n"
+#         output_str += "| Rank | Trial | NCT ID | Summary | Score | Link |\n"
+#         output_str += "|------|-------|--------|---------|-------|------|\n"
+        
+#         for rank, res in enumerate(results):
+#             # Get trial info for summary
+#             trial_info = corpus_data.get(res['nctid'], {})
+#             metadata = trial_info.get('metadata', {})
+#             brief_summary = metadata.get('brief_summary', 'No summary available')
+            
+#             # Truncate summary to ~100 characters
+#             if len(brief_summary) > 100:
+#                 brief_summary = brief_summary[:97] + "..."
+            
+#             # Color coding based on rank
+#             if rank == 0:
+#                 color = "🟢"  # Green for top rank
+#             elif rank == 1:
+#                 color = "🟡"  # Yellow for second rank
+#             elif rank == 2:
+#                 color = "🟠"  # Orange for third rank
+#             elif rank == 3:
+#                 color = "🔴"  # Red for fourth rank
+#             else:
+#                 color = "⚫"  # Black for fifth rank
+            
+#             # Create ClinicalTrials.gov link
+#             ct_link = f"https://clinicaltrials.gov/study/{res['nctid']}"
+            
+#             output_str += f"| {color} {rank+1} | {res['title']} | {res['nctid']} | {brief_summary} | {res['score']:.2f} | [View Trial]({ct_link}) |\n"
+            
+#         return output_str
+        
+#     except Exception as e:
+#         return f"**Error:** An unexpected error occurred: {str(e)}"
+
+# if __name__ == "__main__":
+#     # Wrapper function for ChatInterface
+#     def chat_function(message, history):
+#         return get_clinical_trials(message)
+    
+#     # Create chatbot using the new ChatInterface
+#     demo = gr.ChatInterface(
+#         fn=chat_function,
+#         title="Clinical Trial Matching Chatbot",
+#         description="AI-Powered Clinical Trial Matching Assistant. Describe a patient's condition and find matching clinical trials.",
+#         examples=[
+#             ["58-year-old African-American woman with hypertension and obesity presenting with new-onset episodic chest pain, radiating to the back, accompanied by nausea, diaphoresis, and mild dyspnea. EKG shows nonspecific changes."],
+#             ["45-year-old male with type 2 diabetes mellitus, HbA1c 8.2%, BMI 32, on metformin 1000mg twice daily. Recent diagnosis of diabetic neuropathy in both feet."],
+#             ["67-year-old female with stage III non-small cell lung cancer, ECOG performance status 1, no prior chemotherapy, EGFR mutation positive."]
+#         ],
+#         theme=gr.themes.Soft(
+#             primary_hue="blue",
+#             secondary_hue="gray",
+#             neutral_hue="slate"
+#         )
+#     )
+    
+#     demo.launch(share=True) 
+
+
+import gradio as gr
+import os
+import json
+import asyncio
+import concurrent.futures
+from openai import AzureOpenAI
+from trialgpt_retrieval.retriever import retrieve_trials_gpu, create_retriever_session
+from nltk.tokenize import sent_tokenize
+import nltk
+import hashlib
+import time
+from functools import lru_cache
+from threading import Lock
+
+# Download required NLTK data
+try:
+    nltk.download('punkt')
+    nltk.download('punkt_tab')
+except Exception as e:
+    print(f"Warning: Could not download NLTK data: {e}")
+
+# Check environment variables
+if not os.getenv("OPENAI_ENDPOINT") or not os.getenv("OPENAI_API_KEY"):
+    print("Warning: OPENAI_ENDPOINT and OPENAI_API_KEY environment variables are required.")
+
+# Initialize the Azure OpenAI client
+client = AzureOpenAI(
+    api_version="2023-09-01-preview",
+    azure_endpoint=os.getenv("OPENAI_ENDPOINT"),
+    api_key=os.getenv("OPENAI_API_KEY"),
+)
+
+# Global cache and locks for thread safety
+cache = {}
+cache_lock = Lock()
+MAX_CACHE_SIZE = 1000
+
+# Initialize persistent retriever session (major speedup)
+print("Initializing GPU retriever session...")
+retriever_session = create_retriever_session(use_gpu=True)
+print("Retriever session ready!")
+
+def get_cache_key(text):
+    """Generate cache key from text."""
+    return hashlib.md5(text.encode()).hexdigest()
+
+def get_from_cache(key):
+    """Thread-safe cache retrieval."""
+    with cache_lock:
+        return cache.get(key)
+
+def set_cache(key, value):
+    """Thread-safe cache storage with size limit."""
+    with cache_lock:
+        if len(cache) >= MAX_CACHE_SIZE:
+            # Remove oldest entries (simple FIFO)
+            oldest_keys = list(cache.keys())[:len(cache)//2]
+            for old_key in oldest_keys:
+                del cache[old_key]
+        cache[key] = value
+
+@lru_cache(maxsize=100)
+def generate_keywords_cached(note_hash, note):
+    """Cached keyword generation."""
+    system = 'You are a helpful assistant and your task is to help search relevant clinical trials for a given patient description. Please first summarize the main medical problems of the patient. Then generate up to 32 key conditions for searching relevant clinical trials for this patient. The key condition list should be ranked by priority. Please output only a JSON dict formatted as Dict{{"summary": Str(summary), "conditions": List[Str(condition)]}}.'
+    prompt = f"Here is the patient description: \\n{note}\\n\\nJSON output:"
+    messages = [
+        {"role": "system", "content": system},
+        {"role": "user", "content": prompt}
+    ]
+    
+    try:
+        response = client.chat.completions.create(
+            model="gpt-35-turbo", 
+            messages=messages,
+            temperature=0,
+            max_tokens=2048,
+        )
+
+        output = response.choices[0].message.content
+        if output is None:
+            return "Error: Received empty response from the model.", []
+
+        try:
+            start = output.find('{')
+            end = output.rfind('}')
+            if start != -1 and end > start:
+                json_output = output[start:end+1]
+                data = json.loads(json_output)
+                return data["summary"], data["conditions"]
+            else:
+                return "Error: No JSON found in model response.", []
+        except (json.JSONDecodeError, KeyError) as e:
+            return f"Error parsing model output: {e}", []
+            
+    except Exception as e:
+        return f"Error calling OpenAI API: {e}", []
+
+def generate_keywords(note):
+    """Wrapper for cached keyword generation."""
+    note_hash = get_cache_key(note)
+    return generate_keywords_cached(note_hash, note)
+
+def parse_criteria(criteria):
+    output = ""
+    criteria = criteria.split("\\n\\n")
+    
+    idx = 0
+    for criterion in criteria:
+        criterion = criterion.strip()
+        if "inclusion criteria" in criterion.lower() or "exclusion criteria" in criterion.lower():
+            continue
+        if len(criterion) < 5:
+            continue
+        output += f"{idx}. {criterion}\\n" 
+        idx += 1
+    return output
+
+def print_trial(trial_info: dict, inc_exc: str) -> str:
+    trial = f"Title: {trial_info.get('title', 'N/A')}\\n"
+    
+    metadata = trial_info.get('metadata', {})
+    diseases_list = metadata.get('diseases_list', [])
+    drugs_list = metadata.get('drugs_list', [])
+    
+    trial += f"Target diseases: {', '.join(diseases_list) if diseases_list else 'N/A'}\\n"
+    trial += f"Interventions: {', '.join(drugs_list) if drugs_list else 'N/A'}\\n"
+    trial += f"Summary: {trial_info.get('text', 'N/A')}\\n"
+    
+    if inc_exc == "inclusion":
+        inclusion_criteria = metadata.get('inclusion_criteria', 'No inclusion criteria available')
+        trial += "Inclusion criteria:\\n %s\\n" % parse_criteria(inclusion_criteria)
+    elif inc_exc == "exclusion":
+        exclusion_criteria = metadata.get('exclusion_criteria', 'No exclusion criteria available')
+        trial += "Exclusion criteria:\\n %s\\n" % parse_criteria(exclusion_criteria) 
+    return trial
+
+def get_matching_prompt(trial_info: dict, inc_exc: str, patient: str) -> tuple[str, str]:
+    """Output the prompt."""
+    prompt = f"You are a helpful assistant for clinical trial recruitment. Your task is to compare a given patient note and the {inc_exc} criteria of a clinical trial to determine the patient's eligibility at the criterion level.\\n"
+
+    if inc_exc == "inclusion":
+        prompt += "The factors that allow someone to participate in a clinical study are called inclusion criteria. They are based on characteristics such as age, gender, the type and stage of a disease, previous treatment history, and other medical conditions.\\n"
+    elif inc_exc == "exclusion":
+        prompt += "The factors that disqualify someone from participating are called exclusion criteria. They are based on characteristics such as age, gender, the type and stage of a disease, previous treatment history, and other medical conditions.\\n"
+
+    prompt += f"You should check the {inc_exc} criteria one-by-one, and output the following three elements for each criterion:\\n"
+    prompt += f"\\tElement 1. For each {inc_exc} criterion, briefly generate your reasoning process: First, judge whether the criterion is not applicable (not very common), where the patient does not meet the premise of the criterion. Then, check if the patient note contains direct evidence. If so, judge whether the patient meets or does not meet the criterion. If there is no direct evidence, try to infer from existing evidence, and answer one question: If the criterion is true, is it possible that a good patient note will miss such information? If impossible, then you can assume that the criterion is not true. Otherwise, there is not enough information.\\n"
+    prompt += f"\\tElement 2. If there is relevant information, you must generate a list of relevant sentence IDs in the patient note. If there is no relevant information, you must annotate an empty list.\\n" 
+    prompt += f"\\tElement 3. Classify the patient eligibility for this specific {inc_exc} criterion: "
+    
+    if inc_exc == "inclusion":
+        prompt += 'the label must be chosen from {"not applicable", "not enough information", "included", "not included"}. "not applicable" should only be used for criteria that are not applicable to the patient. "not enough information" should be used where the patient note does not contain sufficient information for making the classification. Try to use as less "not enough information" as possible because if the note does not mention a medically important fact, you can assume that the fact is not true for the patient. "included" denotes that the patient meets the inclusion criterion, while "not included" means the reverse.\\n'
+    elif inc_exc == "exclusion":
+        prompt += 'the label must be chosen from {"not applicable", "not enough information", "excluded", "not excluded"}. "not applicable" should only be used for criteria that are not applicable to the patient. "not enough information" should be used where the patient note does not contain sufficient information for making the classification. Try to use as less "not enough information" as possible because if the note does not mention a medically important fact, you can assume that the fact is not true for the patient. "excluded" denotes that the patient meets the exclusion criterion and should be excluded in the trial, while "not excluded" means the reverse.\\n'
+    
+    prompt += "You should output only a JSON dict exactly formatted as: dict{str(criterion_number): list[str(element_1_brief_reasoning), list[int(element_2_sentence_id)], str(element_3_eligibility_label)]}."
+    
+    user_prompt = f"Here is the patient note, each sentence is led by a sentence_id:\\n{patient}\\n\\n" 
+    user_prompt += f"Here is the clinical trial:\\n{print_trial(trial_info, inc_exc)}\\n\\n"
+    user_prompt += f"Plain JSON output:"
+
+    return prompt, user_prompt
+
+def trialgpt_matching_single(trial: dict, patient: str, inc_exc: str, model: str):
+    """Single matching call for parallel processing."""
+    cache_key = get_cache_key(f"{trial.get('_id', '')}-{inc_exc}-{get_cache_key(patient)}")
+    cached_result = get_from_cache(cache_key)
+    if cached_result:
+        return inc_exc, cached_result
+    
+    system_prompt, user_prompt = get_matching_prompt(trial, inc_exc, patient)
+    messages = [
+        {"role": "system", "content": system_prompt},
+        {"role": "user", "content": user_prompt},
+    ]
+    
+    try:
+        response = client.chat.completions.create(
+            model=model,
+            messages=messages,
+            temperature=0,
+        )
+        message = response.choices[0].message.content.strip()
+        start = message.find('{')
+        end = message.rfind('}')
+        if start != -1 and end > start:
+            message = message[start:end+1]
+        
+        result = json.loads(message)
+        set_cache(cache_key, result)
+        return inc_exc, result
+    except (json.JSONDecodeError, TypeError, Exception) as e:
+        result = {"error": f"Failed to parse model output: {e}", "raw": message if 'message' in locals() else "No response"}
+        return inc_exc, result
+
+def trialgpt_matching_parallel(trial: dict, patient: str, model: str):
+    """Parallel processing of inclusion and exclusion criteria."""
+    with concurrent.futures.ThreadPoolExecutor(max_workers=2) as executor:
+        futures = [
+            executor.submit(trialgpt_matching_single, trial, patient, "inclusion", model),
+            executor.submit(trialgpt_matching_single, trial, patient, "exclusion", model)
+        ]
+        
+        results = {}
+        for future in concurrent.futures.as_completed(futures):
+            try:
+                inc_exc, result = future.result(timeout=30)  # 30 second timeout
+                results[inc_exc] = result
+            except concurrent.futures.TimeoutError:
+                print(f"Timeout for trial matching")
+                results[inc_exc] = {"error": "Timeout"}
+            except Exception as e:
+                print(f"Error in parallel matching: {e}")
+                results[inc_exc] = {"error": str(e)}
+    
+    return results
+
+# Load corpus once at startup
+print("Loading corpus data...")
+def load_corpus(corpus="sigir"):
+    corpus_path = f"dataset/{corpus}/corpus.jsonl"
+    corpus_map = {}
+    with open(corpus_path, "r") as f:
+        for line in f:
+            entry = json.loads(line)
+            corpus_map[entry["_id"]] = entry
+    return corpus_map
+
+corpus_data = load_corpus()
+print(f"Corpus loaded with {len(corpus_data)} trials")
+
+# --- Optimized Trial Ranking ---
+eps = 1e-9
+
+def get_matching_score(matching):
+    included, not_inc, na_inc, no_info_inc = 0, 0, 0, 0
+    excluded, not_exc, na_exc, no_info_exc = 0, 0, 0, 0
+    
+    if "inclusion" in matching:
+        for criteria, info in matching["inclusion"].items():
+            if len(info) == 3:
+                if info[2] == "included": included += 1	
+                elif info[2] == "not included": not_inc += 1
+                elif info[2] == "not applicable": na_inc += 1
+                elif info[2] == "not enough information": no_info_inc += 1
+    
+    if "exclusion" in matching:
+        for criteria, info in matching["exclusion"].items():
+            if len(info) == 3:
+                if info[2] == "excluded": excluded += 1	
+                elif info[2] == "not excluded": not_exc += 1
+                elif info[2] == "not applicable": na_exc += 1
+                elif info[2] == "not enough information": no_info_exc += 1
+
+    score = 0
+    if (included + not_inc + no_info_inc) > 0:
+        score += included / (included + not_inc + no_info_inc + eps)
+    if not_inc > 0: score -= 1
+    if excluded > 0: score -= 1
+    
+    return score 
+
+def get_clinical_trials(patient_details):
+    start_time = time.time()
+    
+    # Check environment variables
+    if not os.getenv("OPENAI_ENDPOINT") or not os.getenv("OPENAI_API_KEY"):
+        return "**Error:** Please set OPENAI_ENDPOINT and OPENAI_API_KEY environment variables before using this app."
+    
+    if not patient_details or patient_details.strip() == "":
+        return "**Error:** Please enter patient details."
+    
+    try:
+        # Step 1: Generate keywords (cached)
+        keyword_start = time.time()
+        summary, conditions = generate_keywords(patient_details)
+        keyword_time = time.time() - keyword_start
+        
+        if isinstance(summary, str) and summary.startswith("Error"):
+            return f"**Error:** {summary}"
+        
+        if not conditions:
+            return "## No Conditions Identified\n\nUnable to identify medical conditions from the patient description."
+
+        # Step 2: Fast GPU-accelerated trial retrieval
+        retrieval_start = time.time()
+        retrieved_nctids = retrieve_trials_gpu(
+            conditions, 
+            corpus="sigir", 
+            top_k=5, 
+            use_gpu=True, 
+            retriever=retriever_session  # Use persistent session
+        )
+        retrieval_time = time.time() - retrieval_start
+        
+        if not retrieved_nctids:
+            return "## No Matching Clinical Trials Found\n\nNo clinical trials were found matching the patient's conditions."
+        
+        # Step 3: Preprocess patient data once
+        processing_start = time.time()
+        sents = sent_tokenize(patient_details)
+        sents.append("The patient will provide informed consent, and will comply with the trial protocol without any practical issues.")
+        patient_processed = "\\n".join([f"{idx}. {sent}" for idx, sent in enumerate(sents)])
+        processing_time = time.time() - processing_start
+
+        # Step 4: Parallel trial matching
+        matching_start = time.time()
+        results = []
+        
+        # Process trials in parallel batches
+        with concurrent.futures.ThreadPoolExecutor(max_workers=3) as executor:
+            future_to_nctid = {
+                executor.submit(
+                    trialgpt_matching_parallel, 
+                    corpus_data[nctid], 
+                    patient_processed, 
+                    "gpt-4"
+                ): nctid 
+                for nctid in retrieved_nctids[:5] 
+                if nctid in corpus_data
+            }
+            
+            for future in concurrent.futures.as_completed(future_to_nctid):
+                nctid = future_to_nctid[future]
+                try:
+                    matching_result = future.result(timeout=60)  # 60 second timeout per trial
+                    score = get_matching_score(matching_result)
+                    title = corpus_data[nctid].get('title', 'Unknown Title')
+                    results.append({"title": title, "nctid": nctid, "score": score})
+                except Exception as e:
+                    print(f"Warning: Error processing trial {nctid}: {e}")
+                    continue
+        
+        matching_time = time.time() - matching_start
+
+        # Step 5: Sort and format results
+        results.sort(key=lambda x: x["score"], reverse=True)
+        
+        total_time = time.time() - start_time
+        
+        # Format output
+        output_str = f"## Matching Clinical Trials\n\n"
+        output_str += f"*Processing time: {total_time:.1f}s (Keywords: {keyword_time:.1f}s, Retrieval: {retrieval_time:.1f}s, Matching: {matching_time:.1f}s)*\n\n"
+        output_str += "| Rank | Trial | NCT ID | Summary | Score | Link |\n"
+        output_str += "|------|-------|--------|---------|-------|------|\n"
+        
+        for rank, res in enumerate(results):
+            trial_info = corpus_data.get(res['nctid'], {})
+            metadata = trial_info.get('metadata', {})
+            brief_summary = metadata.get('brief_summary', 'No summary available')
+            
+            if len(brief_summary) > 100:
+                brief_summary = brief_summary[:97] + "..."
+            
+            colors = ["🟢", "🟡", "🟠", "🔴", "⚫"]
+            color = colors[min(rank, len(colors)-1)]
+            
+            ct_link = f"https://clinicaltrials.gov/study/{res['nctid']}"
+            output_str += f"| {color} {rank+1} | {res['title']} | {res['nctid']} | {brief_summary} | {res['score']:.2f} | [View Trial]({ct_link}) |\n"
+            
+        return output_str
+        
+    except Exception as e:
+        return f"**Error:** An unexpected error occurred: {str(e)}"
+
+if __name__ == "__main__":
+    def chat_function(message, history):
+        return get_clinical_trials(message)
+    
+    # Create optimized chatbot
+    demo = gr.ChatInterface(
+        fn=chat_function,
+        title="⚡ Fast Clinical Trial Matching Chatbot",
+        description="AI-Powered Clinical Trial Matching with GPU acceleration and parallel processing. Describe a patient's condition to find matching trials quickly.",
+        examples=[
+            ["A 39-year-old Thai male presents for study screening. He received his second dose of an approved mRNA COVID-19 vaccine (Pfizer) over 4 months ago and has remained in good health since. He has no history of COVID-19 infection, confirmed by a negative SARS-CoV-2 rapid antigen test today. He denies any chronic illness, does not take any medications, and has never had an adverse reaction to vaccines. He is sexually active and reports consistent condom use with his partner. He is interested in participating in a study assessing immune response to a new booster dose."],
+            ["A 19-year-old pregnant patient, receiving prenatal care at Johns Hopkins Hospital, is 28 weeks gestation and preparing to receive her first mRNA COVID-19 vaccine dose. She has no previous history of COVID-19 and no significant medical history. She meets eligibility criteria and is interested in contributing to research on pregnancy and vaccine-related immune responses."],
+            ["A 67-year-old male with relapsed/refractory multiple myeloma presents for enrollment into a post-trial access study. He previously participated in a Pfizer-sponsored parent study evaluating elranatamab, during which he achieved a partial response and remained clinically stable. At the time the parent trial ended, he was continuing on elranatamab with no evidence of disease progression or significant toxicity. He reports no history of psychiatric illness or lab abnormalities, and he wishes to continue treatment through this access program."]
+        ],
+        theme=gr.themes.Soft(
+            primary_hue="blue",
+            secondary_hue="gray",
+            neutral_hue="slate"
+        )
+    )
+    
+    demo.launch(share=True)

dataset/sigir/corpus.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eac4586f7bbba85701d947a3e5f29982bab5c13d0872152dbe88f7c734c0541a
+size 16439575

dataset/sigir/id2queries.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7ca512ed2544fc598426e244ee5fe5d1095bea771c482102d6e84dda71de878a
+size 77704

dataset/sigir/id2queries_old.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:787f959fa2d9121249e232d4aa675b95bac9614c6bcdf92548c1a78f1a9979c6
+size 83410

dataset/sigir/qrels/test.tsv

@@ -0,0 +1,2144 @@
+query-id	corpus-id	score
+sigir-20141	NCT00000408	0
+sigir-20141	NCT00000492	1
+sigir-20141	NCT00000501	0
+sigir-20141	NCT00001853	0
+sigir-20141	NCT00004727	0
+sigir-20141	NCT00005127	2
+sigir-20141	NCT00005485	2
+sigir-20141	NCT00005757	2
+sigir-20141	NCT00077948	0
+sigir-20141	NCT00108381	0
+sigir-20141	NCT00124969	0
+sigir-20141	NCT00129233	1
+sigir-20141	NCT00134160	0
+sigir-20141	NCT00141583	1
+sigir-20141	NCT00143195	2
+sigir-20141	NCT00149227	1
+sigir-20141	NCT00162344	1
+sigir-20141	NCT00166231	0
+sigir-20141	NCT00175279	2
+sigir-20141	NCT00185120	0
+sigir-20141	NCT00190489	0
+sigir-20141	NCT00202566	0
+sigir-20141	NCT00225355	1
+sigir-20141	NCT00252421	0
+sigir-20141	NCT00285649	0
+sigir-20141	NCT00337116	0
+sigir-20141	NCT00348803	0
+sigir-20141	NCT00356707	0
+sigir-20141	NCT00373828	2
+sigir-20141	NCT00375089	0
+sigir-20141	NCT00413712	0
+sigir-20141	NCT00418392	0
+sigir-20141	NCT00424021	0
+sigir-20141	NCT00454662	0
+sigir-20141	NCT00462241	0
+sigir-20141	NCT00479908	1
+sigir-20141	NCT00486954	0
+sigir-20141	NCT00536224	0
+sigir-20141	NCT00598728	0
+sigir-20141	NCT00623454	0
+sigir-20141	NCT00624273	0
+sigir-20141	NCT00644319	0
+sigir-20141	NCT00683813	2
+sigir-20141	NCT00705952	0
+sigir-20141	NCT00736333	0
+sigir-20141	NCT00745043	0
+sigir-20141	NCT00752674	0
+sigir-20141	NCT00780260	0
+sigir-20141	NCT00797953	1
+sigir-20141	NCT00802230	1
+sigir-20141	NCT00808652	1
+sigir-20141	NCT00809029	2
+sigir-20141	NCT00832442	2
+sigir-20141	NCT00840593	0
+sigir-20141	NCT00848250	0
+sigir-20141	NCT00883987	0
+sigir-20141	NCT00896285	0
+sigir-20141	NCT00925119	0
+sigir-20141	NCT00932867	0
+sigir-20141	NCT00937365	0
+sigir-20141	NCT00951808	0
+sigir-20141	NCT00952744	2
+sigir-20141	NCT00974116	0
+sigir-20141	NCT00979199	1
+sigir-20141	NCT00986180	0
+sigir-20141	NCT00992927	0
+sigir-20141	NCT01028534	0
+sigir-20141	NCT01056367	0
+sigir-20141	NCT01080274	0
+sigir-20141	NCT01087723	0
+sigir-20141	NCT01134328	0
+sigir-20141	NCT01162902	1
+sigir-20141	NCT01167205	0
+sigir-20141	NCT01171911	0
+sigir-20141	NCT01175889	0
+sigir-20141	NCT01203696	1
+sigir-20141	NCT01211171	0
+sigir-20141	NCT01216020	0
+sigir-20141	NCT01228123	0
+sigir-20141	NCT01231165	1
+sigir-20141	NCT01250262	0
+sigir-20141	NCT01253486	2
+sigir-20141	NCT01276548	0
+sigir-20141	NCT01284179	0
+sigir-20141	NCT01290770	0
+sigir-20141	NCT01293019	0
+sigir-20141	NCT01332188	0
+sigir-20141	NCT01334203	0
+sigir-20141	NCT01365286	0
+sigir-20141	NCT01397994	1
+sigir-20141	NCT01407146	1
+sigir-20141	NCT01429194	1
+sigir-20141	NCT01444833	0
+sigir-20141	NCT01450020	0
+sigir-20141	NCT01484912	0
+sigir-20141	NCT01494870	0
+sigir-20141	NCT01506999	0
+sigir-20141	NCT01559948	0
+sigir-20141	NCT01561001	0
+sigir-20141	NCT01576978	0
+sigir-20141	NCT01657136	0
+sigir-20141	NCT01658033	0
+sigir-20141	NCT01660594	1
+sigir-20141	NCT01675362	0
+sigir-20141	NCT01682057	1
+sigir-20141	NCT01719952	0
+sigir-20141	NCT01724567	2
+sigir-20141	NCT01724996	0
+sigir-20141	NCT01736306	0
+sigir-20141	NCT01770080	0
+sigir-20141	NCT01829659	1
+sigir-20141	NCT01838616	0
+sigir-20141	NCT01847014	0
+sigir-20141	NCT01879293	0
+sigir-20141	NCT01904695	0
+sigir-20141	NCT02001545	1
+sigir-20141	NCT02011646	0
+sigir-20141	NCT02015858	0
+sigir-20141	NCT02062424	1
+sigir-20141	NCT02072694	0
+sigir-20141	NCT02078921	0
+sigir-20141	NCT02105220	0
+sigir-20141	NCT02144636	1
+sigir-20141	NCT02148289	0
+sigir-20141	NCT02158962	0
+sigir-20141	NCT02159625	0
+sigir-20141	NCT02190123	0
+sigir-20141	NCT02226510	0
+sigir-20141	NCT02232607	0
+sigir-20141	NCT02271568	1
+sigir-20141	NCT02272920	1
+sigir-20141	NCT02319161	0
+sigir-20141	NCT02347111	0
+sigir-20141	NCT02357212	1
+sigir-20141	NCT02381340	2
+sigir-20141	NCT02390323	0
+sigir-20141	NCT02408263	0
+sigir-20141	NCT02472938	0
+sigir-20141	NCT02507050	0
+sigir-20141	NCT02516839	0
+sigir-20141	NCT02532699	2
+sigir-20141	NCT02608255	1
+sigir-20141	NCT02626741	0
+sigir-201410	NCT00043277	0
+sigir-201410	NCT00102037	0
+sigir-201410	NCT00154674	0
+sigir-201410	NCT00171054	0
+sigir-201410	NCT00314002	0
+sigir-201410	NCT00324233	0
+sigir-201410	NCT00392639	0
+sigir-201410	NCT00471289	0
+sigir-201410	NCT00497094	0
+sigir-201410	NCT00533156	1
+sigir-201410	NCT00633503	0
+sigir-201410	NCT00667381	1
+sigir-201410	NCT00674778	1
+sigir-201410	NCT00736086	1
+sigir-201410	NCT00742014	0
+sigir-201410	NCT00827957	0
+sigir-201410	NCT00838175	0
+sigir-201410	NCT00843297	0
+sigir-201410	NCT00846846	0
+sigir-201410	NCT00970476	0
+sigir-201410	NCT01001663	0
+sigir-201410	NCT01108120	0
+sigir-201410	NCT01121510	1
+sigir-201410	NCT01133119	1
+sigir-201410	NCT01297387	0
+sigir-201410	NCT01397175	0
+sigir-201410	NCT01443104	0
+sigir-201410	NCT01479153	0
+sigir-201410	NCT01556373	0
+sigir-201410	NCT01599195	2
+sigir-201410	NCT01600482	1
+sigir-201410	NCT01631448	0
+sigir-201410	NCT01683448	0
+sigir-201410	NCT01691911	2
+sigir-201410	NCT01745692	0
+sigir-201410	NCT01776151	0
+sigir-201410	NCT01855698	0
+sigir-201410	NCT01868698	0
+sigir-201410	NCT01870310	0
+sigir-201410	NCT01943344	1
+sigir-201410	NCT02033720	0
+sigir-201410	NCT02069275	0
+sigir-201410	NCT02074371	0
+sigir-201410	NCT02097186	2
+sigir-201410	NCT02163031	0
+sigir-201410	NCT02200783	1
+sigir-201410	NCT02216305	0
+sigir-201410	NCT02237430	0
+sigir-201410	NCT02241642	1
+sigir-201410	NCT02264964	1
+sigir-201410	NCT02341196	0
+sigir-201410	NCT02389023	1
+sigir-201410	NCT02419053	0
+sigir-201410	NCT02426710	0
+sigir-201410	NCT02453321	0
+sigir-201410	NCT02519933	0
+sigir-201410	NCT02525926	0
+sigir-201410	NCT02583906	0
+sigir-201410	NCT02626156	0
+sigir-201410	NCT02634138	2
+sigir-201411	NCT00243152	0
+sigir-201411	NCT00322621	0
+sigir-201411	NCT00382421	1
+sigir-201411	NCT00413855	0
+sigir-201411	NCT00760903	0
+sigir-201411	NCT00769275	0
+sigir-201411	NCT00929773	0
+sigir-201411	NCT00944333	1
+sigir-201411	NCT01184846	0
+sigir-201411	NCT01188993	0
+sigir-201411	NCT01200628	0
+sigir-201411	NCT01219023	0
+sigir-201411	NCT01296789	1
+sigir-201411	NCT01464671	1
+sigir-201411	NCT01526382	1
+sigir-201411	NCT01533519	0
+sigir-201411	NCT01539161	0
+sigir-201411	NCT01558206	2
+sigir-201411	NCT01665521	0
+sigir-201411	NCT01672697	0
+sigir-201411	NCT01723137	2
+sigir-201411	NCT01792336	0
+sigir-201411	NCT01827007	0
+sigir-201411	NCT01839786	0
+sigir-201411	NCT01890044	0
+sigir-201411	NCT01973907	0
+sigir-201411	NCT01997047	0
+sigir-201411	NCT02011477	0
+sigir-201411	NCT02072538	0
+sigir-201411	NCT02099071	0
+sigir-201411	NCT02104050	0
+sigir-201411	NCT02168998	0
+sigir-201411	NCT02177188	0
+sigir-201411	NCT02214030	1
+sigir-201411	NCT02258243	0
+sigir-201411	NCT02302534	0
+sigir-201411	NCT02336542	0
+sigir-201411	NCT02353793	0
+sigir-201411	NCT02481076	0
+sigir-201411	NCT02502513	0
+sigir-201411	NCT02517944	0
+sigir-201411	NCT02623556	0
+sigir-201412	NCT00011102	0
+sigir-201412	NCT00111735	0
+sigir-201412	NCT00124384	0
+sigir-201412	NCT00194259	0
+sigir-201412	NCT00261898	0
+sigir-201412	NCT00264901	0
+sigir-201412	NCT00271427	2
+sigir-201412	NCT00360074	1
+sigir-201412	NCT00497575	2
+sigir-201412	NCT00565890	0
+sigir-201412	NCT00593736	0
+sigir-201412	NCT00606840	2
+sigir-201412	NCT00646451	0
+sigir-201412	NCT00686426	2
+sigir-201412	NCT00715572	2
+sigir-201412	NCT00759993	0
+sigir-201412	NCT00812162	0
+sigir-201412	NCT00819624	0
+sigir-201412	NCT00915642	0
+sigir-201412	NCT00948610	0
+sigir-201412	NCT01041261	0
+sigir-201412	NCT01131429	0
+sigir-201412	NCT01197183	2
+sigir-201412	NCT01221935	0
+sigir-201412	NCT01260532	0
+sigir-201412	NCT01280292	0
+sigir-201412	NCT01307618	0
+sigir-201412	NCT01319370	0
+sigir-201412	NCT01551498	1
+sigir-201412	NCT01574859	0
+sigir-201412	NCT01576705	0
+sigir-201412	NCT01739972	0
+sigir-201412	NCT01835236	0
+sigir-201412	NCT01840332	1
+sigir-201412	NCT01848171	0
+sigir-201412	NCT01874704	0
+sigir-201412	NCT01986140	0
+sigir-201412	NCT02015936	0
+sigir-201412	NCT02027077	0
+sigir-201412	NCT02086799	0
+sigir-201412	NCT02094079	0
+sigir-201412	NCT02126683	0
+sigir-201412	NCT02190214	0
+sigir-201412	NCT02201069	0
+sigir-201412	NCT02290405	0
+sigir-201412	NCT02316418	0
+sigir-201412	NCT02379728	0
+sigir-201412	NCT02399423	0
+sigir-201412	NCT02491567	1
+sigir-201412	NCT02548715	0
+sigir-201412	NCT02575001	0
+sigir-201412	NCT02597582	0
+sigir-201412	NCT02627937	0
+sigir-201413	NCT00067691	0
+sigir-201413	NCT00163709	2
+sigir-201413	NCT00302601	1
+sigir-201413	NCT00314548	0
+sigir-201413	NCT00318071	0
+sigir-201413	NCT00341848	0
+sigir-201413	NCT00356551	0
+sigir-201413	NCT00522977	0
+sigir-201413	NCT00573053	0
+sigir-201413	NCT00692198	0
+sigir-201413	NCT00712998	0
+sigir-201413	NCT00782912	1
+sigir-201413	NCT00839124	0
+sigir-201413	NCT00848731	1
+sigir-201413	NCT00974337	0
+sigir-201413	NCT01004939	0
+sigir-201413	NCT01095406	1
+sigir-201413	NCT01097928	1
+sigir-201413	NCT01098851	0
+sigir-201413	NCT01232387	0
+sigir-201413	NCT01262742	0
+sigir-201413	NCT01274637	0
+sigir-201413	NCT01321788	0
+sigir-201413	NCT01428817	0
+sigir-201413	NCT01463202	0
+sigir-201413	NCT01468194	0
+sigir-201413	NCT01498094	0
+sigir-201413	NCT01535248	0
+sigir-201413	NCT01566058	1
+sigir-201413	NCT01620723	0
+sigir-201413	NCT01621685	0
+sigir-201413	NCT01630655	0
+sigir-201413	NCT01684592	0
+sigir-201413	NCT01689961	0
+sigir-201413	NCT01692821	1
+sigir-201413	NCT01787773	1
+sigir-201413	NCT01828697	0
+sigir-201413	NCT01858129	1
+sigir-201413	NCT01870830	0
+sigir-201413	NCT01935141	2
+sigir-201413	NCT01959009	0
+sigir-201413	NCT01991886	0
+sigir-201413	NCT02010840	0
+sigir-201413	NCT02043223	0
+sigir-201413	NCT02094547	0
+sigir-201413	NCT02150616	0
+sigir-201413	NCT02210702	0
+sigir-201413	NCT02221895	1
+sigir-201413	NCT02231398	0
+sigir-201413	NCT02258958	0
+sigir-201413	NCT02264769	1
+sigir-201413	NCT02307201	0
+sigir-201413	NCT02309489	0
+sigir-201413	NCT02372461	0
+sigir-201413	NCT02399774	0
+sigir-201413	NCT02447731	0
+sigir-201413	NCT02449928	0
+sigir-201413	NCT02458625	1
+sigir-201413	NCT02523586	1
+sigir-201413	NCT02526433	2
+sigir-201413	NCT02600260	0
+sigir-201413	NCT02615834	0
+sigir-201413	NCT02616991	1
+sigir-201414	NCT00006196	0
+sigir-201414	NCT00094952	0
+sigir-201414	NCT00182884	0
+sigir-201414	NCT00189046	0
+sigir-201414	NCT00302692	1
+sigir-201414	NCT00419874	2
+sigir-201414	NCT00424736	0
+sigir-201414	NCT00450047	0
+sigir-201414	NCT00482001	0
+sigir-201414	NCT00493714	0
+sigir-201414	NCT00512655	0
+sigir-201414	NCT00613886	0
+sigir-201414	NCT00760734	0
+sigir-201414	NCT00906711	0
+sigir-201414	NCT00928915	0
+sigir-201414	NCT00928941	0
+sigir-201414	NCT01075035	2
+sigir-201414	NCT01162343	0
+sigir-201414	NCT01182961	0
+sigir-201414	NCT01196299	2
+sigir-201414	NCT01199276	0
+sigir-201414	NCT01263067	0
+sigir-201414	NCT01316965	0
+sigir-201414	NCT01331512	0
+sigir-201414	NCT01392820	0
+sigir-201414	NCT01397643	0
+sigir-201414	NCT01470040	0
+sigir-201414	NCT01564303	0
+sigir-201414	NCT01574950	0
+sigir-201414	NCT01619293	2
+sigir-201414	NCT01624545	2
+sigir-201414	NCT01633593	0
+sigir-201414	NCT01674140	0
+sigir-201414	NCT01774240	0
+sigir-201414	NCT01804426	0
+sigir-201414	NCT01836848	0
+sigir-201414	NCT01862952	0
+sigir-201414	NCT01869855	1
+sigir-201414	NCT01922115	0
+sigir-201414	NCT01947959	0
+sigir-201414	NCT01991405	0
+sigir-201414	NCT02028260	0
+sigir-201414	NCT02035436	0
+sigir-201414	NCT02039999	0
+sigir-201414	NCT02051270	0
+sigir-201414	NCT02086981	2
+sigir-201414	NCT02120664	0
+sigir-201414	NCT02131181	0
+sigir-201414	NCT02172625	0
+sigir-201414	NCT02199093	0
+sigir-201414	NCT02213380	0
+sigir-201414	NCT02227225	0
+sigir-201414	NCT02237703	0
+sigir-201414	NCT02266563	0
+sigir-201414	NCT02267538	0
+sigir-201414	NCT02323984	0
+sigir-201414	NCT02355483	0
+sigir-201414	NCT02366299	0
+sigir-201414	NCT02394418	0
+sigir-201414	NCT02404181	0
+sigir-201414	NCT02465307	1
+sigir-201414	NCT02477007	0
+sigir-201414	NCT02505061	0
+sigir-201414	NCT02513316	0
+sigir-201414	NCT02518646	1
+sigir-201414	NCT02527499	0
+sigir-201414	NCT02531542	1
+sigir-201414	NCT02558777	0
+sigir-201414	NCT02564939	0
+sigir-201414	NCT02577731	0
+sigir-201414	NCT02591043	0
+sigir-201414	NCT02603731	0
+sigir-201414	NCT02631304	0
+sigir-201415	NCT00003702	1
+sigir-201415	NCT00044876	0
+sigir-201415	NCT00150644	1
+sigir-201415	NCT00156208	0
+sigir-201415	NCT00166790	2
+sigir-201415	NCT00180739	1
+sigir-201415	NCT00190918	2
+sigir-201415	NCT00237965	0
+sigir-201415	NCT00248144	0
+sigir-201415	NCT00277680	2
+sigir-201415	NCT00288938	0
+sigir-201415	NCT00340288	1
+sigir-201415	NCT00361036	0
+sigir-201416	NCT00001252	2
+sigir-201416	NCT00001552	1
+sigir-201416	NCT00001986	1
+sigir-201416	NCT00061139	0
+sigir-201416	NCT00080366	0
+sigir-201416	NCT00102596	0
+sigir-201416	NCT00393926	0
+sigir-201416	NCT00432666	0
+sigir-201416	NCT00462449	0
+sigir-201416	NCT00565201	0
+sigir-201416	NCT00597701	0
+sigir-201416	NCT00656097	0
+sigir-201416	NCT00703521	0
+sigir-201416	NCT00740090	0
+sigir-201416	NCT00906971	0
+sigir-201416	NCT00948272	0
+sigir-201416	NCT00955838	0
+sigir-201416	NCT01025726	1
+sigir-201416	NCT01044199	0
+sigir-201416	NCT01047839	0
+sigir-201416	NCT01060748	0
+sigir-201416	NCT01075269	0
+sigir-201416	NCT01081275	0
+sigir-201416	NCT01228383	0
+sigir-201416	NCT01281475	0
+sigir-201416	NCT01286493	1
+sigir-201416	NCT01373879	0
+sigir-201416	NCT01387074	0
+sigir-201416	NCT01388400	0
+sigir-201416	NCT01466387	0
+sigir-201416	NCT01603875	0
+sigir-201416	NCT01635647	0
+sigir-201416	NCT01683838	0
+sigir-201416	NCT01767480	0
+sigir-201416	NCT01827917	0
+sigir-201416	NCT01930357	0
+sigir-201416	NCT01932463	0
+sigir-201416	NCT01936298	0
+sigir-201416	NCT01949909	0
+sigir-201416	NCT02043145	0
+sigir-201416	NCT02074150	0
+sigir-201416	NCT02122159	0
+sigir-201416	NCT02177032	0
+sigir-201416	NCT02199015	0
+sigir-201416	NCT02238756	1
+sigir-201416	NCT02241135	0
+sigir-201416	NCT02339896	0
+sigir-201416	NCT02359253	0
+sigir-201416	NCT02359812	0
+sigir-201416	NCT02374814	1
+sigir-201416	NCT02418858	0
+sigir-201416	NCT02420106	0
+sigir-201416	NCT02427854	0
+sigir-201416	NCT02491541	0
+sigir-201416	NCT02547727	0
+sigir-201416	NCT02552290	0
+sigir-201416	NCT02564471	0
+sigir-201416	NCT02596659	0
+sigir-201418	NCT00001676	0
+sigir-201418	NCT00004496	0
+sigir-201418	NCT00004696	1
+sigir-201418	NCT00006137	0
+sigir-201418	NCT00027417	0
+sigir-201418	NCT00226369	0
+sigir-201418	NCT00231322	0
+sigir-201418	NCT00232401	0
+sigir-201418	NCT00298454	1
+sigir-201418	NCT00313807	0
+sigir-201418	NCT00362869	0
+sigir-201418	NCT00539292	0
+sigir-201418	NCT00549107	1
+sigir-201418	NCT00555321	0
+sigir-201418	NCT00556491	0
+sigir-201418	NCT00557219	2
+sigir-201418	NCT00572715	1
+sigir-201418	NCT00577525	1
+sigir-201418	NCT00624637	0
+sigir-201418	NCT00738283	0
+sigir-201418	NCT00747864	1
+sigir-201418	NCT00770705	0
+sigir-201418	NCT00774137	1
+sigir-201418	NCT00776646	0
+sigir-201418	NCT00778349	0
+sigir-201418	NCT00821522	1
+sigir-201418	NCT00908037	0
+sigir-201418	NCT00934843	0
+sigir-201418	NCT00952263	0
+sigir-201418	NCT00974337	1
+sigir-201418	NCT01062828	0
+sigir-201418	NCT01084980	0
+sigir-201418	NCT01142115	0
+sigir-201418	NCT01143246	0
+sigir-201418	NCT01156428	0
+sigir-201418	NCT01219998	1
+sigir-201418	NCT01233869	0
+sigir-201418	NCT01243996	0
+sigir-201418	NCT01260259	0
+sigir-201418	NCT01260883	2
+sigir-201418	NCT01309477	0
+sigir-201418	NCT01569698	1
+sigir-201418	NCT01641289	0
+sigir-201418	NCT01700231	0
+sigir-201418	NCT01795638	0
+sigir-201418	NCT01869933	0
+sigir-201418	NCT01870011	0
+sigir-201418	NCT01918618	0
+sigir-201418	NCT01973088	0
+sigir-201418	NCT02016053	0
+sigir-201418	NCT02046135	0
+sigir-201418	NCT02081235	0
+sigir-201418	NCT02081261	0
+sigir-201418	NCT02081274	0
+sigir-201418	NCT02147470	0
+sigir-201418	NCT02195713	0
+sigir-201418	NCT02213757	0
+sigir-201418	NCT02233010	0
+sigir-201418	NCT02276170	0
+sigir-201418	NCT02282930	1
+sigir-201418	NCT02306642	0
+sigir-201418	NCT02311530	0
+sigir-201418	NCT02312115	0
+sigir-201418	NCT02313636	0
+sigir-201418	NCT02338895	0
+sigir-201418	NCT02361736	0
+sigir-201418	NCT02415049	0
+sigir-201418	NCT02443389	0
+sigir-201418	NCT02489669	0
+sigir-201418	NCT02495220	0
+sigir-201418	NCT02495909	0
+sigir-201418	NCT02531412	0
+sigir-201418	NCT02560584	0
+sigir-201418	NCT02599155	1
+sigir-201419	NCT00002711	1
+sigir-201419	NCT00002897	1
+sigir-201419	NCT00002930	1
+sigir-201419	NCT00021320	1
+sigir-201419	NCT00040287	0
+sigir-201419	NCT00193882	1
+sigir-201419	NCT00256529	2
+sigir-201419	NCT00306501	0
+sigir-201419	NCT00308399	0
+sigir-201419	NCT00336609	1
+sigir-201419	NCT00352391	0
+sigir-201419	NCT00375180	1
+sigir-201419	NCT00402987	0
+sigir-201419	NCT00408590	0
+sigir-201419	NCT00412490	0
+sigir-201419	NCT00465907	1
+sigir-201419	NCT00506324	1
+sigir-201419	NCT00525044	0
+sigir-201419	NCT00591175	0
+sigir-201419	NCT00608205	1
+sigir-201419	NCT00628368	1
+sigir-201419	NCT00647829	0
+sigir-201419	NCT00662662	1
+sigir-201419	NCT00727376	1
+sigir-201419	NCT00773227	0
+sigir-201419	NCT00816634	1
+sigir-201419	NCT00826813	1
+sigir-201419	NCT00827671	1
+sigir-201419	NCT00861094	1
+sigir-201419	NCT00873535	0
+sigir-201419	NCT00880906	0
+sigir-201419	NCT00897364	0
+sigir-201419	NCT00898495	1
+sigir-201419	NCT00905385	0
+sigir-201419	NCT00908817	0
+sigir-201419	NCT00918307	0
+sigir-201419	NCT00934453	0
+sigir-201419	NCT01023828	0
+sigir-201419	NCT01052038	0
+sigir-201419	NCT01054274	1
+sigir-201419	NCT01125384	0
+sigir-201419	NCT01131429	0
+sigir-201419	NCT01137123	1
+sigir-201419	NCT01199185	0
+sigir-201419	NCT01243398	1
+sigir-201419	NCT01252225	0
+sigir-201419	NCT01279824	0
+sigir-201419	NCT01291823	0
+sigir-201419	NCT01335269	0
+sigir-201419	NCT01359397	1
+sigir-201419	NCT01366833	1
+sigir-201419	NCT01396707	1
+sigir-201419	NCT01450748	0
+sigir-201419	NCT01453400	0
+sigir-201419	NCT01471249	1
+sigir-201419	NCT01514552	0
+sigir-201419	NCT01578018	0
+sigir-201419	NCT01586962	0
+sigir-201419	NCT01651975	2
+sigir-201419	NCT01688700	1
+sigir-201419	NCT01704690	1
+sigir-201419	NCT01723358	0
+sigir-201419	NCT01724411	0
+sigir-201419	NCT01731847	0
+sigir-201419	NCT01742312	1
+sigir-201419	NCT01745107	1
+sigir-201419	NCT01809756	1
+sigir-201419	NCT01862809	0
+sigir-201419	NCT01863264	0
+sigir-201419	NCT01867983	0
+sigir-201419	NCT01900392	0
+sigir-201419	NCT01923831	0
+sigir-201419	NCT01954407	0
+sigir-201419	NCT01986361	0
+sigir-201419	NCT01988350	0
+sigir-201419	NCT02011035	0
+sigir-201419	NCT02017288	0
+sigir-201419	NCT02045342	0
+sigir-201419	NCT02130011	1
+sigir-201419	NCT02144545	0
+sigir-201419	NCT02206971	2
+sigir-201419	NCT02296528	0
+sigir-201419	NCT02304250	0
+sigir-201419	NCT02322411	0
+sigir-201419	NCT02333201	0
+sigir-201419	NCT02375581	0
+sigir-201419	NCT02405832	0
+sigir-201419	NCT02445209	1
+sigir-201419	NCT02483663	0
+sigir-201419	NCT02492646	0
+sigir-201419	NCT02494960	0
+sigir-201419	NCT02509286	1
+sigir-201419	NCT02591784	1
+sigir-20142	NCT00000932	0
+sigir-20142	NCT00086879	0
+sigir-20142	NCT00094887	0
+sigir-20142	NCT00095316	0
+sigir-20142	NCT00096772	0
+sigir-20142	NCT00132951	0
+sigir-20142	NCT00161772	0
+sigir-20142	NCT00161798	0
+sigir-20142	NCT00161824	0
+sigir-20142	NCT00161863	0
+sigir-20142	NCT00161967	0
+sigir-20142	NCT00163540	0
+sigir-20142	NCT00163618	0
+sigir-20142	NCT00237016	1
+sigir-20142	NCT00311441	0
+sigir-20142	NCT00385632	0
+sigir-20142	NCT00386035	0
+sigir-20142	NCT00432003	0
+sigir-20142	NCT00455468	0
+sigir-20142	NCT00460486	0
+sigir-20142	NCT00498654	0
+sigir-20142	NCT00503529	0
+sigir-20142	NCT00507871	0
+sigir-20142	NCT00513500	0
+sigir-20142	NCT00617344	0
+sigir-20142	NCT00680836	0
+sigir-20142	NCT00694655	0
+sigir-20142	NCT00711399	2
+sigir-20142	NCT00740155	0
+sigir-20142	NCT00840801	0
+sigir-20142	NCT00851487	0
+sigir-20142	NCT00894686	0
+sigir-20142	NCT01017081	2
+sigir-20142	NCT01070732	0
+sigir-20142	NCT01075204	0
+sigir-20142	NCT01087892	0
+sigir-20142	NCT01099943	1
+sigir-20142	NCT01147445	0
+sigir-20142	NCT01177657	0
+sigir-20142	NCT01201252	0
+sigir-20142	NCT01202201	0
+sigir-20142	NCT01241201	0
+sigir-20142	NCT01253967	0
+sigir-20142	NCT01253980	1
+sigir-20142	NCT01269554	1
+sigir-20142	NCT01359020	0
+sigir-20142	NCT01521403	0
+sigir-20142	NCT01589926	0
+sigir-20142	NCT01627327	0
+sigir-20142	NCT01663155	0
+sigir-20142	NCT01670149	0
+sigir-20142	NCT01707485	2
+sigir-20142	NCT01739673	0
+sigir-20142	NCT02052934	0
+sigir-20142	NCT02139163	0
+sigir-20142	NCT02269761	2
+sigir-20142	NCT02380352	2
+sigir-20142	NCT02383680	0
+sigir-20142	NCT02391909	0
+sigir-20142	NCT02441699	0
+sigir-20142	NCT02618655	1
+sigir-20142	NCT02625129	0
+sigir-201421	NCT00001789	1
+sigir-201421	NCT00005436	1
+sigir-201421	NCT00006055	2
+sigir-201421	NCT00008749	2
+sigir-201421	NCT00036491	1
+sigir-201421	NCT00065806	2
+sigir-201421	NCT00071175	2
+sigir-201421	NCT00071487	1
+sigir-201421	NCT00074373	0
+sigir-201421	NCT00119678	1
+sigir-201421	NCT00125307	2
+sigir-201421	NCT00172666	2
+sigir-201421	NCT00181298	1
+sigir-201421	NCT00215800	1
+sigir-201421	NCT00222183	1
+sigir-201421	NCT00230035	1
+sigir-201421	NCT00275899	2
+sigir-201421	NCT00282347	1
+sigir-201421	NCT00297284	1
+sigir-201421	NCT00342329	1
+sigir-201421	NCT00343187	0
+sigir-201421	NCT00364728	2
+sigir-201421	NCT00371501	2
+sigir-201421	NCT00392093	0
+sigir-201421	NCT00413361	1
+sigir-201421	NCT00430677	1
+sigir-201421	NCT00470522	1
+sigir-201421	NCT00504244	1
+sigir-201421	NCT00512694	1
+sigir-201421	NCT00539799	1
+sigir-201421	NCT00581763	2
+sigir-201421	NCT00582465	2
+sigir-201421	NCT00611663	1
+sigir-201421	NCT00637819	2
+sigir-201421	NCT00731302	1
+sigir-201421	NCT00760669	0
+sigir-201421	NCT00779194	1
+sigir-201421	NCT00866229	1
+sigir-201421	NCT00908037	0
+sigir-201421	NCT00965809	0
+sigir-201421	NCT00982670	2
+sigir-201421	NCT00997100	1
+sigir-201421	NCT01018238	1
+sigir-201421	NCT01031797	1
+sigir-201421	NCT01076101	1
+sigir-201421	NCT01101802	1
+sigir-201421	NCT01176643	2
+sigir-201421	NCT01187927	2
+sigir-201421	NCT01230827	0
+sigir-201421	NCT01266915	2
+sigir-201421	NCT01276782	0
+sigir-201421	NCT01330368	2
+sigir-201421	NCT01381536	1
+sigir-201421	NCT01413100	1
+sigir-201421	NCT01413230	2
+sigir-201421	NCT01421316	0
+sigir-201421	NCT01440231	1
+sigir-201421	NCT01512277	0
+sigir-201421	NCT01520155	2
+sigir-201421	NCT01549184	2
+sigir-201421	NCT01551069	0
+sigir-201421	NCT01597622	0
+sigir-201421	NCT01632241	1
+sigir-201421	NCT01649765	0
+sigir-201421	NCT01689025	2
+sigir-201421	NCT01689350	2
+sigir-201421	NCT01705977	1
+sigir-201421	NCT01729455	1
+sigir-201421	NCT01784523	1
+sigir-201421	NCT01923415	2
+sigir-201421	NCT01924000	0
+sigir-201421	NCT01931150	0
+sigir-201421	NCT01946880	1
+sigir-201421	NCT01950780	1
+sigir-201421	NCT01992666	1
+sigir-201421	NCT02006784	0
+sigir-201421	NCT02034344	2
+sigir-201421	NCT02074020	0
+sigir-201421	NCT02080195	1
+sigir-201421	NCT02119156	1
+sigir-201421	NCT02176148	0
+sigir-201421	NCT02239731	0
+sigir-201421	NCT02260934	0
+sigir-201421	NCT02284984	1
+sigir-201421	NCT02444728	0
+sigir-201421	NCT02455089	0
+sigir-201421	NCT02477254	1
+sigir-201421	NCT02530944	2
+sigir-201421	NCT02554019	1
+sigir-201421	NCT02558517	0
+sigir-201421	NCT02591355	0
+sigir-201421	NCT02633163	1
+sigir-201422	NCT00172822	1
+sigir-201422	NCT00236912	2
+sigir-201422	NCT00674726	2
+sigir-201422	NCT00677989	0
+sigir-201422	NCT00723788	2
+sigir-201422	NCT00908804	2
+sigir-201422	NCT00913445	0
+sigir-201422	NCT01022567	0
+sigir-201422	NCT01024439	0
+sigir-201422	NCT01027455	0
+sigir-201422	NCT01068288	0
+sigir-201422	NCT01096927	2
+sigir-201422	NCT01115153	1
+sigir-201422	NCT01192620	2
+sigir-201422	NCT01610193	0
+sigir-201422	NCT01678365	0
+sigir-201422	NCT01698099	1
+sigir-201422	NCT01718171	2
+sigir-201422	NCT01825863	0
+sigir-201422	NCT01925014	1
+sigir-201422	NCT01953289	2
+sigir-201422	NCT01979029	0
+sigir-201422	NCT02004470	0
+sigir-201422	NCT02017951	1
+sigir-201422	NCT02018016	1
+sigir-201422	NCT02029781	0
+sigir-201422	NCT02047786	0
+sigir-201422	NCT02134652	0
+sigir-201422	NCT02137603	1
+sigir-201422	NCT02194140	0
+sigir-201422	NCT02271932	1
+sigir-201422	NCT02375529	0
+sigir-201422	NCT02439593	0
+sigir-201422	NCT02447224	2
+sigir-201422	NCT02469935	0
+sigir-201422	NCT02507674	2
+sigir-201422	NCT02535962	0
+sigir-201422	NCT02574364	2
+sigir-201422	NCT02624089	1
+sigir-201422	NCT02633735	2
+sigir-201423	NCT00018408	1
+sigir-201423	NCT00084266	0
+sigir-201423	NCT00108823	1
+sigir-201423	NCT00116584	0
+sigir-201423	NCT00170222	1
+sigir-201423	NCT00181272	1
+sigir-201423	NCT00181285	1
+sigir-201423	NCT00190437	2
+sigir-201423	NCT00263380	2
+sigir-201423	NCT00274222	2
+sigir-201423	NCT00440687	1
+sigir-201423	NCT00441649	0
+sigir-201423	NCT00455468	0
+sigir-201423	NCT00476736	1
+sigir-201423	NCT00512954	1
+sigir-201423	NCT00595114	1
+sigir-201423	NCT00605007	2
+sigir-201423	NCT00628225	1
+sigir-201423	NCT00774176	1
+sigir-201423	NCT00788645	1
+sigir-201423	NCT00791505	1
+sigir-201423	NCT00807469	1
+sigir-201423	NCT00839189	1
+sigir-201423	NCT00860938	1
+sigir-201423	NCT00883701	1
+sigir-201423	NCT00893009	1
+sigir-201423	NCT00974064	0
+sigir-201423	NCT00981851	1
+sigir-201423	NCT00985244	1
+sigir-201423	NCT01091493	1
+sigir-201423	NCT01213693	1
+sigir-201423	NCT01232140	1
+sigir-201423	NCT01248507	1
+sigir-201423	NCT01252225	0
+sigir-201423	NCT01254032	1
+sigir-201423	NCT01296477	0
+sigir-201423	NCT01297790	1
+sigir-201423	NCT01314807	1
+sigir-201423	NCT01337336	1
+sigir-201423	NCT01344005	0
+sigir-201423	NCT01381367	1
+sigir-201423	NCT01395875	1
+sigir-201423	NCT01396798	0
+sigir-201423	NCT01431911	1
+sigir-201423	NCT01460082	1
+sigir-201423	NCT01589926	0
+sigir-201423	NCT01621685	0
+sigir-201423	NCT01694069	0
+sigir-201423	NCT01701869	1
+sigir-201423	NCT01710488	1
+sigir-201423	NCT01742338	1
+sigir-201423	NCT01798420	1
+sigir-201423	NCT01801657	0
+sigir-201423	NCT01877707	0
+sigir-201423	NCT01892488	1
+sigir-201423	NCT01950936	1
+sigir-201423	NCT01997047	0
+sigir-201423	NCT02092675	0
+sigir-201423	NCT02136875	1
+sigir-201423	NCT02181023	1
+sigir-201423	NCT02209974	1
+sigir-201423	NCT02219360	2
+sigir-201423	NCT02220387	1
+sigir-201423	NCT02264483	1
+sigir-201423	NCT02269644	1
+sigir-201423	NCT02282436	1
+sigir-201423	NCT02284295	1
+sigir-201423	NCT02299375	1
+sigir-201423	NCT02333812	2
+sigir-201423	NCT02386735	1
+sigir-201423	NCT02490358	1
+sigir-201423	NCT02534766	2
+sigir-201423	NCT02605122	0
+sigir-201423	NCT02619903	1
+sigir-201423	NCT02633033	0
+sigir-201424	NCT00115128	0
+sigir-201424	NCT00222651	0
+sigir-201424	NCT00228111	0
+sigir-201424	NCT00344149	0
+sigir-201424	NCT00424736	0
+sigir-201424	NCT00440830	0
+sigir-201424	NCT00587132	0
+sigir-201424	NCT00625053	0
+sigir-201424	NCT00625092	0
+sigir-201424	NCT00630513	0
+sigir-201424	NCT00726544	0
+sigir-201424	NCT00754806	0
+sigir-201424	NCT00769873	0
+sigir-201424	NCT00775528	0
+sigir-201424	NCT00778362	0
+sigir-201424	NCT00870766	0
+sigir-201424	NCT00950625	0
+sigir-201424	NCT00965744	0
+sigir-201424	NCT00970515	0
+sigir-201424	NCT00997633	0
+sigir-201424	NCT01013428	0
+sigir-201424	NCT01044173	0
+sigir-201424	NCT01074112	0
+sigir-201424	NCT01103999	0
+sigir-201424	NCT01113645	0
+sigir-201424	NCT01196715	0
+sigir-201424	NCT01265004	0
+sigir-201424	NCT01276561	2
+sigir-201424	NCT01279369	0
+sigir-201424	NCT01297725	0
+sigir-201424	NCT01322828	0
+sigir-201424	NCT01376466	0
+sigir-201424	NCT01387828	0
+sigir-201424	NCT01412684	0
+sigir-201424	NCT01441492	0
+sigir-201424	NCT01451931	0
+sigir-201424	NCT01518816	0
+sigir-201424	NCT01594385	0
+sigir-201424	NCT01632774	0
+sigir-201424	NCT01762761	0
+sigir-201424	NCT01771861	1
+sigir-201424	NCT01816256	0
+sigir-201424	NCT01828749	0
+sigir-201424	NCT01852773	0
+sigir-201424	NCT01869049	0
+sigir-201424	NCT01887028	0
+sigir-201424	NCT01895127	0
+sigir-201424	NCT01911702	2
+sigir-201424	NCT01944397	0
+sigir-201424	NCT02029339	0
+sigir-201424	NCT02073890	0
+sigir-201424	NCT02140710	0
+sigir-201424	NCT02149472	0
+sigir-201424	NCT02198885	0
+sigir-201424	NCT02217956	0
+sigir-201424	NCT02229695	2
+sigir-201424	NCT02232191	2
+sigir-201424	NCT02233569	0
+sigir-201424	NCT02269878	0
+sigir-201424	NCT02302664	0
+sigir-201424	NCT02310854	0
+sigir-201424	NCT02331134	0
+sigir-201424	NCT02376192	0
+sigir-201424	NCT02378311	0
+sigir-201424	NCT02399150	2
+sigir-201424	NCT02494232	0
+sigir-201424	NCT02505347	0
+sigir-201424	NCT02513316	0
+sigir-201424	NCT02527031	0
+sigir-201424	NCT02584777	0
+sigir-201424	NCT02624089	0
+sigir-201425	NCT00035139	2
+sigir-201425	NCT00040339	0
+sigir-201425	NCT00061399	0
+sigir-201425	NCT00163774	0
+sigir-201425	NCT00174980	0
+sigir-201425	NCT00178711	0
+sigir-201425	NCT00222742	2
+sigir-201425	NCT00282269	2
+sigir-201425	NCT00451789	0
+sigir-201425	NCT00490997	0
+sigir-201425	NCT00754481	0
+sigir-201425	NCT00810940	0
+sigir-201425	NCT00908063	0
+sigir-201425	NCT00915928	0
+sigir-201425	NCT01056146	2
+sigir-201425	NCT01279928	0
+sigir-201425	NCT01315379	0
+sigir-201425	NCT01463878	0
+sigir-201425	NCT01512277	0
+sigir-201425	NCT01595906	0
+sigir-201425	NCT01613235	0
+sigir-201425	NCT01619943	0
+sigir-201425	NCT01632813	0
+sigir-201425	NCT01692769	2
+sigir-201425	NCT01712477	0
+sigir-201425	NCT01753921	0
+sigir-201425	NCT01763892	0
+sigir-201425	NCT01785797	0
+sigir-201425	NCT01796886	0
+sigir-201425	NCT01819597	0
+sigir-201425	NCT01869855	0
+sigir-201425	NCT01933217	0
+sigir-201425	NCT01942564	0
+sigir-201425	NCT01990768	0
+sigir-201425	NCT02024373	0
+sigir-201425	NCT02054117	0
+sigir-201425	NCT02064959	0
+sigir-201425	NCT02119533	0
+sigir-201425	NCT02192320	0
+sigir-201425	NCT02203968	0
+sigir-201425	NCT02282228	0
+sigir-201425	NCT02304328	0
+sigir-201425	NCT02309242	0
+sigir-201425	NCT02318290	0
+sigir-201425	NCT02362321	0
+sigir-201425	NCT02378311	2
+sigir-201425	NCT02392975	0
+sigir-201425	NCT02415257	0
+sigir-201425	NCT02416752	0
+sigir-201425	NCT02475226	1
+sigir-201425	NCT02477007	0
+sigir-201425	NCT02485873	0
+sigir-201425	NCT02568124	0
+sigir-201427	NCT00026468	1
+sigir-201427	NCT00055848	0
+sigir-201427	NCT00140894	2
+sigir-201427	NCT00224679	1
+sigir-201427	NCT00253812	0
+sigir-201427	NCT00319007	0
+sigir-201427	NCT00485069	0
+sigir-201427	NCT00525655	2
+sigir-201427	NCT00586053	0
+sigir-201427	NCT00641147	0
+sigir-201427	NCT00663819	0
+sigir-201427	NCT00685568	1
+sigir-201427	NCT00728026	2
+sigir-201427	NCT00748449	1
+sigir-201427	NCT00770991	1
+sigir-201427	NCT00808743	1
+sigir-201427	NCT00823836	0
+sigir-201427	NCT00843375	0
+sigir-201427	NCT00900016	0
+sigir-201427	NCT00905710	0
+sigir-201427	NCT01173250	0
+sigir-201427	NCT01187901	2
+sigir-201427	NCT01245816	0
+sigir-201427	NCT01422577	0
+sigir-201427	NCT01483144	2
+sigir-201427	NCT01495806	0
+sigir-201427	NCT01647776	0
+sigir-201427	NCT01713881	2
+sigir-201427	NCT01722149	0
+sigir-201427	NCT01725490	2
+sigir-201427	NCT01729416	0
+sigir-201427	NCT01737567	0
+sigir-201427	NCT01761279	2
+sigir-201427	NCT01767870	0
+sigir-201427	NCT01960348	1
+sigir-201427	NCT02016326	0
+sigir-201427	NCT02081742	0
+sigir-201427	NCT02113202	1
+sigir-201427	NCT02470416	0
+sigir-201427	NCT02521727	0
+sigir-201427	NCT02538406	2
+sigir-201429	NCT00000402	0
+sigir-201429	NCT00000430	0
+sigir-201429	NCT00001720	0
+sigir-201429	NCT00001951	0
+sigir-201429	NCT00010686	0
+sigir-201429	NCT00010712	1
+sigir-201429	NCT00011323	0
+sigir-201429	NCT00021827	1
+sigir-201429	NCT00073190	1
+sigir-201429	NCT00092053	0
+sigir-201429	NCT00145080	1
+sigir-201429	NCT00170339	0
+sigir-201429	NCT00224991	0
+sigir-201429	NCT00226031	0
+sigir-201429	NCT00273260	0
+sigir-201429	NCT00280943	0
+sigir-201429	NCT00301353	1
+sigir-201429	NCT00303485	0
+sigir-201429	NCT00327847	1
+sigir-201429	NCT00347737	0
+sigir-201429	NCT00376662	0
+sigir-201429	NCT00386022	0
+sigir-201429	NCT00432692	0
+sigir-201429	NCT00540267	0
+sigir-201429	NCT00545779	0
+sigir-201429	NCT00575250	0
+sigir-201429	NCT00578981	1
+sigir-201429	NCT00609830	0
+sigir-201429	NCT00698984	0
+sigir-201429	NCT00703417	2
+sigir-201429	NCT00762775	2
+sigir-201429	NCT00787904	0
+sigir-201429	NCT00846638	1
+sigir-201429	NCT00860964	0
+sigir-201429	NCT00982371	0
+sigir-201429	NCT01035177	0
+sigir-201429	NCT01128257	0
+sigir-201429	NCT01141972	0
+sigir-201429	NCT01152580	1
+sigir-201429	NCT01206491	2
+sigir-201429	NCT01227369	0
+sigir-201429	NCT01232647	0
+sigir-201429	NCT01314807	0
+sigir-201429	NCT01341067	1
+sigir-201429	NCT01360424	0
+sigir-201429	NCT01440803	0
+sigir-201429	NCT01452243	0
+sigir-201429	NCT01512446	0
+sigir-201429	NCT01607775	0
+sigir-201429	NCT01677637	2
+sigir-201429	NCT01698463	0
+sigir-201429	NCT01821885	1
+sigir-201429	NCT01846533	1
+sigir-201429	NCT01854086	0
+sigir-201429	NCT01870557	1
+sigir-201429	NCT01928134	0
+sigir-201429	NCT01934400	0
+sigir-201429	NCT01935232	1
+sigir-201429	NCT01978834	2
+sigir-201429	NCT01998737	2
+sigir-201429	NCT02066480	0
+sigir-201429	NCT02115711	1
+sigir-201429	NCT02164552	0
+sigir-201429	NCT02184962	1
+sigir-201429	NCT02273128	0
+sigir-201429	NCT02347865	0
+sigir-201429	NCT02422069	2
+sigir-201429	NCT02472782	0
+sigir-201429	NCT02534714	0
+sigir-201429	NCT02551315	2
+sigir-201429	NCT02594592	2
+sigir-20143	NCT00001776	1
+sigir-20143	NCT00001777	0
+sigir-20143	NCT00003562	1
+sigir-20143	NCT00003859	0
+sigir-20143	NCT00005666	0
+sigir-20143	NCT00006087	0
+sigir-20143	NCT00024908	1
+sigir-20143	NCT00047801	1
+sigir-20143	NCT00049543	0
+sigir-20143	NCT00088569	0
+sigir-20143	NCT00124761	2
+sigir-20143	NCT00128986	0
+sigir-20143	NCT00172575	1
+sigir-20143	NCT00181272	0
+sigir-20143	NCT00188279	0
+sigir-20143	NCT00188435	0
+sigir-20143	NCT00193765	0
+sigir-20143	NCT00202176	0
+sigir-20143	NCT00303901	1
+sigir-20143	NCT00339066	0
+sigir-20143	NCT00382707	0
+sigir-20143	NCT00423332	0
+sigir-20143	NCT00466947	0
+sigir-20143	NCT00517959	0
+sigir-20143	NCT00578084	2
+sigir-20143	NCT00579852	1
+sigir-20143	NCT00591838	0
+sigir-20143	NCT00613041	0
+sigir-20143	NCT00629460	0
+sigir-20143	NCT00633035	2
+sigir-20143	NCT00643318	0
+sigir-20143	NCT00757120	0
+sigir-20143	NCT00839488	0
+sigir-20143	NCT00852644	0
+sigir-20143	NCT00894569	0
+sigir-20143	NCT00897650	2
+sigir-20143	NCT00907543	0
+sigir-20143	NCT00918320	2
+sigir-20143	NCT00963651	2
+sigir-20143	NCT00992966	0
+sigir-20143	NCT01028417	0
+sigir-20143	NCT01085864	1
+sigir-20143	NCT01109147	0
+sigir-20143	NCT01141842	1
+sigir-20143	NCT01149187	1
+sigir-20143	NCT01152463	1
+sigir-20143	NCT01152788	0
+sigir-20143	NCT01201824	1
+sigir-20143	NCT01221493	1
+sigir-20143	NCT01272622	0
+sigir-20143	NCT01349400	0
+sigir-20143	NCT01351870	0
+sigir-20143	NCT01354912	0
+sigir-20143	NCT01452971	2
+sigir-20143	NCT01465425	2
+sigir-20143	NCT01561729	0
+sigir-20143	NCT01566682	1
+sigir-20143	NCT01572571	0
+sigir-20143	NCT01572584	0
+sigir-20143	NCT01586611	0
+sigir-20143	NCT01605214	0
+sigir-20143	NCT01649752	0
+sigir-20143	NCT01654887	0
+sigir-20143	NCT01663155	0
+sigir-20143	NCT01668459	0
+sigir-20143	NCT01669135	0
+sigir-20143	NCT01696292	0
+sigir-20143	NCT01696968	0
+sigir-20143	NCT01700257	0
+sigir-20143	NCT01739881	2
+sigir-20143	NCT01802125	0
+sigir-20143	NCT01803542	0
+sigir-20143	NCT01847209	1
+sigir-20143	NCT01863069	0
+sigir-20143	NCT01864538	0
+sigir-20143	NCT01908933	0
+sigir-20143	NCT01951846	1
+sigir-20143	NCT02013063	0
+sigir-20143	NCT02022371	0
+sigir-20143	NCT02050724	1
+sigir-20143	NCT02051868	0
+sigir-20143	NCT02081105	1
+sigir-20143	NCT02095808	2
+sigir-20143	NCT02096445	0
+sigir-20143	NCT02137291	0
+sigir-20143	NCT02225392	0
+sigir-20143	NCT02289144	0
+sigir-20143	NCT02374242	0
+sigir-20143	NCT02434107	0
+sigir-20143	NCT02439086	0
+sigir-20143	NCT02441478	0
+sigir-20143	NCT02445183	0
+sigir-20143	NCT02448225	0
+sigir-20143	NCT02490059	1
+sigir-20143	NCT02501668	2
+sigir-20143	NCT02505750	0
+sigir-20143	NCT02519972	2
+sigir-20143	NCT02563691	0
+sigir-20143	NCT02623712	1
+sigir-201430	NCT00005134	0
+sigir-201430	NCT00005254	0
+sigir-201430	NCT00005338	0
+sigir-201430	NCT00118560	0
+sigir-201430	NCT00171054	0
+sigir-201430	NCT00173602	0
+sigir-201430	NCT00185068	1
+sigir-201430	NCT00262470	1
+sigir-201430	NCT00310752	0
+sigir-201430	NCT00311805	0
+sigir-201430	NCT00368095	1
+sigir-201430	NCT00471289	0
+sigir-201430	NCT00481741	0
+sigir-201430	NCT00485329	0
+sigir-201430	NCT00603590	1
+sigir-201430	NCT00721006	2
+sigir-201430	NCT00725881	0
+sigir-201430	NCT00739596	0
+sigir-201430	NCT00753025	0
+sigir-201430	NCT00762086	0
+sigir-201430	NCT00768495	0
+sigir-201430	NCT00781950	0
+sigir-201430	NCT00883246	1
+sigir-201430	NCT00926770	0
+sigir-201430	NCT00931112	0
+sigir-201430	NCT00996281	0
+sigir-201430	NCT01002235	0
+sigir-201430	NCT01019681	1
+sigir-201430	NCT01041729	1
+sigir-201430	NCT01076738	0
+sigir-201430	NCT01118000	0
+sigir-201430	NCT01250275	1
+sigir-201430	NCT01289548	0
+sigir-201430	NCT01335022	0
+sigir-201430	NCT01362283	0
+sigir-201430	NCT01382277	1
+sigir-201430	NCT01397474	0
+sigir-201430	NCT01422343	1
+sigir-201430	NCT01475097	1
+sigir-201430	NCT01475786	0
+sigir-201430	NCT01499134	0
+sigir-201430	NCT01505738	1
+sigir-201430	NCT01624519	0
+sigir-201430	NCT01625832	1
+sigir-201430	NCT01691911	0
+sigir-201430	NCT01698853	0
+sigir-201430	NCT01741896	0
+sigir-201430	NCT01801592	1
+sigir-201430	NCT01902654	0
+sigir-201430	NCT01970332	1
+sigir-201430	NCT02003781	0
+sigir-201430	NCT02051088	1
+sigir-201430	NCT02057328	0
+sigir-201430	NCT02097186	0
+sigir-201430	NCT02110251	1
+sigir-201430	NCT02123004	0
+sigir-201430	NCT02133911	0
+sigir-201430	NCT02217501	1
+sigir-201430	NCT02254356	2
+sigir-201430	NCT02273232	2
+sigir-201430	NCT02406131	1
+sigir-201430	NCT02427464	0
+sigir-201430	NCT02427867	0
+sigir-201430	NCT02431819	1
+sigir-201430	NCT02433990	0
+sigir-201430	NCT02436200	0
+sigir-201430	NCT02516072	1
+sigir-20144	NCT00044239	0
+sigir-20144	NCT00053456	0
+sigir-20144	NCT00100126	0
+sigir-20144	NCT00162032	0
+sigir-20144	NCT00254930	0
+sigir-20144	NCT00271570	2
+sigir-20144	NCT00305201	0
+sigir-20144	NCT00591916	0
+sigir-20144	NCT00604786	0
+sigir-20144	NCT00643149	0
+sigir-20144	NCT00793403	0
+sigir-20144	NCT00841789	2
+sigir-20144	NCT01050062	0
+sigir-20144	NCT01173939	0
+sigir-20144	NCT01212770	0
+sigir-20144	NCT01244217	0
+sigir-20144	NCT01281969	0
+sigir-20144	NCT01307423	0
+sigir-20144	NCT01433861	0
+sigir-20144	NCT01440075	0
+sigir-20144	NCT01491776	0
+sigir-20144	NCT01494792	0
+sigir-20144	NCT01596335	2
+sigir-20144	NCT01917721	1
+sigir-20144	NCT02031120	0
+sigir-20144	NCT02063789	0
+sigir-20144	NCT02247765	0
+sigir-20144	NCT02317913	2
+sigir-20144	NCT02390596	1
+sigir-20144	NCT02439996	2
+sigir-20144	NCT02457403	0
+sigir-20144	NCT02557334	0
+sigir-20144	NCT02624882	0
+sigir-20145	NCT00000566	1
+sigir-20145	NCT00003674	0
+sigir-20145	NCT00005351	0
+sigir-20145	NCT00041509	0
+sigir-20145	NCT00043927	0
+sigir-20145	NCT00075179	0
+sigir-20145	NCT00083772	0
+sigir-20145	NCT00123734	2
+sigir-20145	NCT00138099	0
+sigir-20145	NCT00163709	1
+sigir-20145	NCT00178789	2
+sigir-20145	NCT00182364	0
+sigir-20145	NCT00196417	0
+sigir-20145	NCT00203970	2
+sigir-20145	NCT00264901	0
+sigir-20145	NCT00277394	1
+sigir-20145	NCT00314548	0
+sigir-20145	NCT00321048	1
+sigir-20145	NCT00329433	0
+sigir-20145	NCT00361959	0
+sigir-20145	NCT00365950	2
+sigir-20145	NCT00368836	0
+sigir-20145	NCT00455468	0
+sigir-20145	NCT00456196	1
+sigir-20145	NCT00618579	0
+sigir-20145	NCT00714597	0
+sigir-20145	NCT00719784	2
+sigir-20145	NCT00726570	0
+sigir-20145	NCT00726947	1
+sigir-20145	NCT00743106	0
+sigir-20145	NCT00756145	0
+sigir-20145	NCT00771563	0
+sigir-20145	NCT00789399	0
+sigir-20145	NCT00816920	0
+sigir-20145	NCT00997737	0
+sigir-20145	NCT01137578	0
+sigir-20145	NCT01164046	1
+sigir-20145	NCT01183026	2
+sigir-20145	NCT01212757	0
+sigir-20145	NCT01258725	0
+sigir-20145	NCT01298986	0
+sigir-20145	NCT01300832	0
+sigir-20145	NCT01307423	0
+sigir-20145	NCT01317160	0
+sigir-20145	NCT01318174	0
+sigir-20145	NCT01344954	0
+sigir-20145	NCT01394107	0
+sigir-20145	NCT01444612	0
+sigir-20145	NCT01535248	0
+sigir-20145	NCT01542723	0
+sigir-20145	NCT01542762	0
+sigir-20145	NCT01557725	0
+sigir-20145	NCT01559532	0
+sigir-20145	NCT01604538	2
+sigir-20145	NCT01662843	2
+sigir-20145	NCT01708239	0
+sigir-20145	NCT01724996	0
+sigir-20145	NCT01729559	2
+sigir-20145	NCT01752673	0
+sigir-20145	NCT01770912	0
+sigir-20145	NCT01793194	0
+sigir-20145	NCT01849406	0
+sigir-20145	NCT01856842	0
+sigir-20145	NCT01899339	1
+sigir-20145	NCT01935141	2
+sigir-20145	NCT01940328	0
+sigir-20145	NCT01956955	1
+sigir-20145	NCT01959529	0
+sigir-20145	NCT01970202	0
+sigir-20145	NCT02070354	0
+sigir-20145	NCT02102828	0
+sigir-20145	NCT02134652	0
+sigir-20145	NCT02356120	2
+sigir-20145	NCT02369263	2
+sigir-20145	NCT02393963	0
+sigir-20145	NCT02406885	0
+sigir-20145	NCT02447731	0
+sigir-20145	NCT02507180	0
+sigir-20145	NCT02553200	0
+sigir-20145	NCT02584725	0
+sigir-20146	NCT00004572	0
+sigir-20146	NCT00006426	1
+sigir-20146	NCT00013286	1
+sigir-20146	NCT00015626	1
+sigir-20146	NCT00034788	2
+sigir-20146	NCT00095446	2
+sigir-20146	NCT00117364	0
+sigir-20146	NCT00134550	0
+sigir-20146	NCT00235196	0
+sigir-20146	NCT00269100	0
+sigir-20146	NCT00289497	0
+sigir-20146	NCT00304733	1
+sigir-20146	NCT00311922	2
+sigir-20146	NCT00338702	0
+sigir-20146	NCT00355602	0
+sigir-20146	NCT00358059	0
+sigir-20146	NCT00399425	2
+sigir-20146	NCT00410449	0
+sigir-20146	NCT00446823	1
+sigir-20146	NCT00469105	2
+sigir-20146	NCT00477022	0
+sigir-20146	NCT00512538	0
+sigir-20146	NCT00519493	0
+sigir-20146	NCT00519935	1
+sigir-20146	NCT00521027	0
+sigir-20146	NCT00621608	2
+sigir-20146	NCT00655044	1
+sigir-20146	NCT00754741	1
+sigir-20146	NCT00789646	0
+sigir-20146	NCT00807664	0
+sigir-20146	NCT00830453	0
+sigir-20146	NCT00839865	0
+sigir-20146	NCT00866424	0
+sigir-20146	NCT00916500	0
+sigir-20146	NCT00950040	1
+sigir-20146	NCT00955669	0
+sigir-20146	NCT00970697	1
+sigir-20146	NCT00990522	1
+sigir-20146	NCT00999635	1
+sigir-20146	NCT01005264	0
+sigir-20146	NCT01013792	0
+sigir-20146	NCT01028404	0
+sigir-20146	NCT01043354	1
+sigir-20146	NCT01094899	1
+sigir-20146	NCT01181453	0
+sigir-20146	NCT01211236	1
+sigir-20146	NCT01217463	0
+sigir-20146	NCT01217567	0
+sigir-20146	NCT01237392	0
+sigir-20146	NCT01259206	2
+sigir-20146	NCT01263288	0
+sigir-20146	NCT01285973	0
+sigir-20146	NCT01319123	0
+sigir-20146	NCT01339637	0
+sigir-20146	NCT01356459	0
+sigir-20146	NCT01370837	0
+sigir-20146	NCT01396304	1
+sigir-20146	NCT01396837	0
+sigir-20146	NCT01417208	1
+sigir-20146	NCT01432925	0
+sigir-20146	NCT01449422	0
+sigir-20146	NCT01477047	0
+sigir-20146	NCT01480414	0
+sigir-20146	NCT01531517	0
+sigir-20146	NCT01542463	2
+sigir-20146	NCT01580917	1
+sigir-20146	NCT01583465	0
+sigir-20146	NCT01584817	0
+sigir-20146	NCT01586481	0
+sigir-20146	NCT01657318	0
+sigir-20146	NCT01666093	1
+sigir-20146	NCT01716624	0
+sigir-20146	NCT01733524	0
+sigir-20146	NCT01796548	0
+sigir-20146	NCT01827683	0
+sigir-20146	NCT01865292	0
+sigir-20146	NCT01912976	0
+sigir-20146	NCT01924806	0
+sigir-20146	NCT01951768	1
+sigir-20146	NCT01966380	2
+sigir-20146	NCT01986205	0
+sigir-20146	NCT01998932	0
+sigir-20146	NCT02007824	0
+sigir-20146	NCT02052817	0
+sigir-20146	NCT02073188	0
+sigir-20146	NCT02087215	0
+sigir-20146	NCT02136654	0
+sigir-20146	NCT02144142	0
+sigir-20146	NCT02169167	0
+sigir-20146	NCT02217722	0
+sigir-20146	NCT02222376	2
+sigir-20146	NCT02235935	1
+sigir-20146	NCT02251418	1
+sigir-20146	NCT02294175	0
+sigir-20146	NCT02329366	1
+sigir-20146	NCT02379507	1
+sigir-20146	NCT02405741	0
+sigir-20146	NCT02447172	0
+sigir-20146	NCT02458495	0
+sigir-20146	NCT02482948	1
+sigir-20146	NCT02512159	2
+sigir-20146	NCT02557698	0
+sigir-20146	NCT02563678	2
+sigir-20146	NCT02583958	0
+sigir-20146	NCT02586519	0
+sigir-20146	NCT02598180	0
+sigir-20146	NCT02619877	1
+sigir-20146	NCT02621476	0
+sigir-20147	NCT00000369	2
+sigir-20147	NCT00005015	1
+sigir-20147	NCT00006170	0
+sigir-20147	NCT00012558	2
+sigir-20147	NCT00043654	1
+sigir-20147	NCT00044187	0
+sigir-20147	NCT00063362	1
+sigir-20147	NCT00074789	0
+sigir-20147	NCT00088387	0
+sigir-20147	NCT00114439	0
+sigir-20147	NCT00116259	0
+sigir-20147	NCT00156325	0
+sigir-20147	NCT00156533	0
+sigir-20147	NCT00167479	0
+sigir-20147	NCT00174876	0
+sigir-20147	NCT00177216	0
+sigir-20147	NCT00194038	0
+sigir-20147	NCT00202293	0
+sigir-20147	NCT00203450	0
+sigir-20147	NCT00221442	0
+sigir-20147	NCT00223509	2
+sigir-20147	NCT00226135	2
+sigir-20147	NCT00275834	0
+sigir-20147	NCT00277641	0
+sigir-20147	NCT00277654	0
+sigir-20147	NCT00296179	0
+sigir-20147	NCT00296790	0
+sigir-20147	NCT00318448	0
+sigir-20147	NCT00332098	0
+sigir-20147	NCT00348036	0
+sigir-20147	NCT00374543	1
+sigir-20147	NCT00380081	1
+sigir-20147	NCT00383643	0
+sigir-20147	NCT00384332	0
+sigir-20147	NCT00393978	0
+sigir-20147	NCT00394095	0
+sigir-20147	NCT00422331	0
+sigir-20147	NCT00431171	0
+sigir-20147	NCT00432198	0
+sigir-20147	NCT00464191	2
+sigir-20147	NCT00466193	0
+sigir-20147	NCT00469937	0
+sigir-20147	NCT00485680	1
+sigir-20147	NCT00492232	0
+sigir-20147	NCT00494468	0
+sigir-20147	NCT00501540	0
+sigir-20147	NCT00545233	0
+sigir-20147	NCT00552760	1
+sigir-20147	NCT00563992	2
+sigir-20147	NCT00573196	2
+sigir-20147	NCT00579280	0
+sigir-20147	NCT00582712	0
+sigir-20147	NCT00584961	2
+sigir-20147	NCT00592696	0
+sigir-20147	NCT00601536	1
+sigir-20147	NCT00601575	0
+sigir-20147	NCT00602394	0
+sigir-20147	NCT00602537	0
+sigir-20147	NCT00606840	0
+sigir-20147	NCT00608296	1
+sigir-20147	NCT00630175	1
+sigir-20147	NCT00641108	0
+sigir-20147	NCT00658541	0
+sigir-20147	NCT00662155	0
+sigir-20147	NCT00665366	1
+sigir-20147	NCT00672490	2
+sigir-20147	NCT00684814	0
+sigir-20147	NCT00709202	0
+sigir-20147	NCT00716521	0
+sigir-20147	NCT00720395	0
+sigir-20147	NCT00721955	1
+sigir-20147	NCT00741299	2
+sigir-20147	NCT00750061	0
+sigir-20147	NCT00781482	0
+sigir-20147	NCT00806234	0
+sigir-20147	NCT00814502	0
+sigir-20147	NCT00820794	0
+sigir-20147	NCT00838032	0
+sigir-20147	NCT00845507	0
+sigir-20147	NCT00845988	2
+sigir-20147	NCT00883493	1
+sigir-20147	NCT00934908	0
+sigir-20147	NCT00936598	0
+sigir-20147	NCT00956319	1
+sigir-20147	NCT00957359	0
+sigir-20147	NCT00986128	2
+sigir-20147	NCT00993850	2
+sigir-20147	NCT00997672	0
+sigir-20147	NCT00998634	0
+sigir-20147	NCT00999219	0
+sigir-20147	NCT01006525	0
+sigir-20147	NCT01012180	1
+sigir-20147	NCT01072929	0
+sigir-20147	NCT01074073	0
+sigir-20147	NCT01075295	0
+sigir-20147	NCT01093937	0
+sigir-20147	NCT01117961	0
+sigir-20147	NCT01134731	0
+sigir-20147	NCT01159652	0
+sigir-20147	NCT01166425	0
+sigir-20147	NCT01172652	0
+sigir-20147	NCT01181232	1
+sigir-20147	NCT01181583	0
+sigir-20147	NCT01184443	0
+sigir-20147	NCT01259388	0
+sigir-20147	NCT01293825	1
+sigir-20147	NCT01363310	0
+sigir-20147	NCT01385709	2
+sigir-20147	NCT01395238	0
+sigir-20147	NCT01416220	0
+sigir-20147	NCT01428518	0
+sigir-20147	NCT01471613	0
+sigir-20147	NCT01491490	0
+sigir-20147	NCT01526148	2
+sigir-20147	NCT01551485	0
+sigir-20147	NCT01689909	1
+sigir-20147	NCT01704352	1
+sigir-20147	NCT01736709	0
+sigir-20147	NCT01764035	0
+sigir-20147	NCT01783652	0
+sigir-20147	NCT01784666	2
+sigir-20147	NCT01804036	0
+sigir-20147	NCT01855594	0
+sigir-20147	NCT01863628	1
+sigir-20147	NCT01892306	0
+sigir-20147	NCT01893229	2
+sigir-20147	NCT01896336	0
+sigir-20147	NCT01928446	0
+sigir-20147	NCT01944293	2
+sigir-20147	NCT01954680	0
+sigir-20147	NCT02003690	0
+sigir-20147	NCT02033915	0
+sigir-20147	NCT02039479	0
+sigir-20147	NCT02090595	0
+sigir-20147	NCT02096900	0
+sigir-20147	NCT02116400	0
+sigir-20147	NCT02129790	0
+sigir-20147	NCT02139098	0
+sigir-20147	NCT02163564	0
+sigir-20147	NCT02198859	0
+sigir-20147	NCT02281162	0
+sigir-20147	NCT02287259	0
+sigir-20147	NCT02323763	2
+sigir-20147	NCT02353299	1
+sigir-20147	NCT02399423	0
+sigir-20147	NCT02484963	0
+sigir-20147	NCT02490241	0
+sigir-20147	NCT02491307	2
+sigir-20147	NCT02525991	0
+sigir-20147	NCT02527564	0
+sigir-20147	NCT02593643	0
+sigir-20147	NCT02604277	2
+sigir-20147	NCT02607696	0
+sigir-20148	NCT00002414	1
+sigir-20148	NCT00004771	0
+sigir-20148	NCT00031369	1
+sigir-20148	NCT00040196	0
+sigir-20148	NCT00077896	1
+sigir-20148	NCT00104663	0
+sigir-20148	NCT00108849	0
+sigir-20148	NCT00169468	0
+sigir-20148	NCT00297271	0
+sigir-20148	NCT00306124	0
+sigir-20148	NCT00340301	1
+sigir-20148	NCT00393588	0
+sigir-20148	NCT00439127	0
+sigir-20148	NCT00474292	0
+sigir-20148	NCT00476372	0
+sigir-20148	NCT00526084	0
+sigir-20148	NCT00631332	0
+sigir-20148	NCT00718107	1
+sigir-20148	NCT00718458	0
+sigir-20148	NCT00781833	0
+sigir-20148	NCT00786032	0
+sigir-20148	NCT00812474	0
+sigir-20148	NCT00816556	0
+sigir-20148	NCT00832403	0
+sigir-20148	NCT00897299	0
+sigir-20148	NCT00921648	0
+sigir-20148	NCT01022307	0
+sigir-20148	NCT01050881	0
+sigir-20148	NCT01122342	0
+sigir-20148	NCT01231672	0
+sigir-20148	NCT01238926	1
+sigir-20148	NCT01315704	1
+sigir-20148	NCT01388400	0
+sigir-20148	NCT01390922	2
+sigir-20148	NCT01438060	0
+sigir-20148	NCT01463475	0
+sigir-20148	NCT01507454	0
+sigir-20148	NCT01519271	2
+sigir-20148	NCT01555827	0
+sigir-20148	NCT01586520	0
+sigir-20148	NCT01592825	0
+sigir-20148	NCT01628315	2
+sigir-20148	NCT01637051	0
+sigir-20148	NCT01641627	0
+sigir-20148	NCT01677793	0
+sigir-20148	NCT01725178	0
+sigir-20148	NCT01862172	0
+sigir-20148	NCT01910363	1
+sigir-20148	NCT01942759	0
+sigir-20148	NCT01944397	0
+sigir-20148	NCT01960985	1
+sigir-20148	NCT01962779	0
+sigir-20148	NCT02002884	0
+sigir-20148	NCT02020980	0
+sigir-20148	NCT02045004	0
+sigir-20148	NCT02056613	0
+sigir-20148	NCT02067247	0
+sigir-20148	NCT02152449	0
+sigir-20148	NCT02208219	0
+sigir-20148	NCT02212119	1
+sigir-20148	NCT02275312	0
+sigir-20148	NCT02440815	0
+sigir-20148	NCT02480803	0
+sigir-20148	NCT02544178	0
+sigir-20148	NCT02571504	0
+sigir-20148	NCT02579278	0
+sigir-20148	NCT02588053	0
+sigir-20148	NCT02589925	0
+sigir-20148	NCT02604186	0
+sigir-20149	NCT00124163	0
+sigir-20149	NCT00237276	0
+sigir-20149	NCT00291486	0
+sigir-20149	NCT00476905	2
+sigir-20149	NCT00610818	0
+sigir-20149	NCT00617825	0
+sigir-20149	NCT00667225	0
+sigir-20149	NCT00684294	0
+sigir-20149	NCT00911352	0
+sigir-20149	NCT00937690	1
+sigir-20149	NCT01010321	1
+sigir-20149	NCT01014819	2
+sigir-20149	NCT01023984	0
+sigir-20149	NCT01072045	0
+sigir-20149	NCT01125150	0
+sigir-20149	NCT01259037	2
+sigir-20149	NCT01384357	0
+sigir-20149	NCT01405755	1
+sigir-20149	NCT01406379	0
+sigir-20149	NCT01444989	2
+sigir-20149	NCT01492647	0
+sigir-20149	NCT01509599	0
+sigir-20149	NCT01541553	1
+sigir-20149	NCT01565993	0
+sigir-20149	NCT01574664	0
+sigir-20149	NCT01633515	0
+sigir-20149	NCT01636453	0
+sigir-20149	NCT01680562	2
+sigir-20149	NCT01700101	1
+sigir-20149	NCT01800656	0
+sigir-20149	NCT01950026	1
+sigir-20149	NCT01994616	0
+sigir-20149	NCT02005796	0
+sigir-20149	NCT02009566	2
+sigir-20149	NCT02063243	0
+sigir-20149	NCT02063971	0
+sigir-20149	NCT02147353	0
+sigir-20149	NCT02156193	0
+sigir-20149	NCT02160626	0
+sigir-20149	NCT02196246	1
+sigir-20149	NCT02220127	0
+sigir-20149	NCT02260180	0
+sigir-20149	NCT02366559	0
+sigir-20149	NCT02432118	0
+sigir-20149	NCT02476747	0
+sigir-20149	NCT02525822	0
+sigir-20149	NCT02537483	0
+sigir-20149	NCT02615912	1
+sigir-20151	NCT00038272	0
+sigir-20151	NCT00040495	2
+sigir-20151	NCT00147030	0
+sigir-20151	NCT00152399	2
+sigir-20151	NCT00153712	0
+sigir-20151	NCT00164905	1
+sigir-20151	NCT00164931	1
+sigir-20151	NCT00165009	2
+sigir-20151	NCT00282373	0
+sigir-20151	NCT00314704	2
+sigir-20151	NCT00316017	0
+sigir-20151	NCT00374101	1
+sigir-20151	NCT00392639	0
+sigir-20151	NCT00432991	0
+sigir-20151	NCT00441727	1
+sigir-20151	NCT00612196	0
+sigir-20151	NCT00624169	2
+sigir-20151	NCT00687336	1
+sigir-20151	NCT00727974	0
+sigir-20151	NCT00731601	2
+sigir-20151	NCT00751634	0
+sigir-20151	NCT00753064	0
+sigir-20151	NCT00771563	0
+sigir-20151	NCT00797641	1
+sigir-20151	NCT00810121	0
+sigir-20151	NCT00838682	2
+sigir-20151	NCT00843063	1
+sigir-20151	NCT00854776	0
+sigir-20151	NCT00881413	2
+sigir-20151	NCT00950417	0
+sigir-20151	NCT01040416	0
+sigir-20151	NCT01041196	0
+sigir-20151	NCT01080326	2
+sigir-20151	NCT01123837	0
+sigir-20151	NCT01138969	0
+sigir-20151	NCT01142180	1
+sigir-20151	NCT01142245	1
+sigir-20151	NCT01155401	2
+sigir-20151	NCT01165866	0
+sigir-20151	NCT01180179	1
+sigir-20151	NCT01182597	2
+sigir-20151	NCT01292915	2
+sigir-20151	NCT01353144	0
+sigir-20151	NCT01371591	2
+sigir-20151	NCT01408186	1
+sigir-20151	NCT01411852	0
+sigir-20151	NCT01474915	0
+sigir-20151	NCT01551641	1
+sigir-20151	NCT01560702	2
+sigir-20151	NCT01822600	2
+sigir-20151	NCT01838863	0
+sigir-20151	NCT01845168	0
+sigir-20151	NCT01888588	0
+sigir-20151	NCT01911702	0
+sigir-20151	NCT01923064	0
+sigir-20151	NCT01926600	2
+sigir-20151	NCT01956955	0
+sigir-20151	NCT01993381	0
+sigir-20151	NCT02017379	2
+sigir-20151	NCT02049619	0
+sigir-20151	NCT02088385	2
+sigir-20151	NCT02105532	1
+sigir-20151	NCT02150941	2
+sigir-20151	NCT02166008	0
+sigir-20151	NCT02197039	1
+sigir-20151	NCT02241044	2
+sigir-20151	NCT02245802	2
+sigir-20151	NCT02316990	0
+sigir-20151	NCT02342470	0
+sigir-20151	NCT02352155	2
+sigir-20151	NCT02358850	0
+sigir-20151	NCT02417519	0
+sigir-20151	NCT02446678	0
+sigir-20151	NCT02456012	1
+sigir-20151	NCT02461329	1
+sigir-20151	NCT02477501	0
+sigir-20151	NCT02535624	0
+sigir-20152	NCT00000665	0
+sigir-20152	NCT00000673	0
+sigir-20152	NCT00000726	0
+sigir-20152	NCT00000970	0
+sigir-20152	NCT00000981	0
+sigir-20152	NCT00001471	1
+sigir-20152	NCT00001890	0
+sigir-20152	NCT00002025	0
+sigir-20152	NCT00002275	0
+sigir-20152	NCT00005547	0
+sigir-20152	NCT00005572	2
+sigir-20152	NCT00034437	1
+sigir-20152	NCT00077909	2
+sigir-20152	NCT00159055	1
+sigir-20152	NCT00225394	1
+sigir-20152	NCT00241345	0
+sigir-20152	NCT00411645	0
+sigir-20152	NCT00442975	1
+sigir-20152	NCT00471250	2
+sigir-20152	NCT00495963	0
+sigir-20152	NCT00497796	0
+sigir-20152	NCT00547235	0
+sigir-20152	NCT00662012	0
+sigir-20152	NCT00674648	0
+sigir-20152	NCT00699868	1
+sigir-20152	NCT00766415	0
+sigir-20152	NCT00817908	2
+sigir-20152	NCT00838552	0
+sigir-20152	NCT00907686	0
+sigir-20152	NCT00954200	0
+sigir-20152	NCT01088243	0
+sigir-20152	NCT01128907	0
+sigir-20152	NCT01220895	0
+sigir-20152	NCT01251744	0
+sigir-20152	NCT01259141	0
+sigir-20152	NCT01292213	0
+sigir-20152	NCT01301131	0
+sigir-20152	NCT01376778	0
+sigir-20152	NCT01395498	2
+sigir-20152	NCT01587807	0
+sigir-20152	NCT01646645	1
+sigir-20152	NCT01810731	0
+sigir-20152	NCT01818726	0
+sigir-20152	NCT01833416	0
+sigir-20152	NCT01862159	0
+sigir-20152	NCT01903928	0
+sigir-20152	NCT01945619	0
+sigir-20152	NCT02025335	1
+sigir-20152	NCT02081716	1
+sigir-20152	NCT02083042	0
+sigir-20152	NCT02136797	1
+sigir-20152	NCT02152358	0
+sigir-20152	NCT02154880	0
+sigir-20152	NCT02156479	0
+sigir-20152	NCT02328963	0
+sigir-20152	NCT02370758	0
+sigir-20152	NCT02406729	0
+sigir-20152	NCT02474901	0
+sigir-20152	NCT02496494	0
+sigir-20152	NCT02509091	0
+sigir-20152	NCT02527291	0
+sigir-20152	NCT02532452	1
+sigir-20152	NCT02622009	0
+sigir-20152	NCT02630537	0
+sigir-20153	NCT00035971	0
+sigir-20153	NCT00043927	0
+sigir-20153	NCT00045760	0
+sigir-20153	NCT00154895	0
+sigir-20153	NCT00182143	1
+sigir-20153	NCT00182364	0
+sigir-20153	NCT00187408	0
+sigir-20153	NCT00188032	2
+sigir-20153	NCT00219921	0
+sigir-20153	NCT00225108	0
+sigir-20153	NCT00264901	0
+sigir-20153	NCT00302601	0
+sigir-20153	NCT00311753	0
+sigir-20153	NCT00314548	0
+sigir-20153	NCT00358735	0
+sigir-20153	NCT00361959	1
+sigir-20153	NCT00378027	1
+sigir-20153	NCT00413374	1
+sigir-20153	NCT00413504	0
+sigir-20153	NCT00421538	0
+sigir-20153	NCT00425542	1
+sigir-20153	NCT00451412	0
+sigir-20153	NCT00462241	0
+sigir-20153	NCT00539487	0
+sigir-20153	NCT00643370	0
+sigir-20153	NCT00700739	1
+sigir-20153	NCT00707811	1
+sigir-20153	NCT00720915	2
+sigir-20153	NCT00740883	0
+sigir-20153	NCT00823563	0
+sigir-20153	NCT00873587	0
+sigir-20153	NCT00878475	0
+sigir-20153	NCT00954395	0
+sigir-20153	NCT00957021	0
+sigir-20153	NCT00967304	2
+sigir-20153	NCT01014156	0
+sigir-20153	NCT01084239	0
+sigir-20153	NCT01096927	0
+sigir-20153	NCT01097928	2
+sigir-20153	NCT01112436	0
+sigir-20153	NCT01117610	0
+sigir-20153	NCT01133002	2
+sigir-20153	NCT01139632	1
+sigir-20153	NCT01164046	0
+sigir-20153	NCT01169389	0
+sigir-20153	NCT01214954	1
+sigir-20153	NCT01230931	0
+sigir-20153	NCT01240044	0
+sigir-20153	NCT01242150	0
+sigir-20153	NCT01245283	1
+sigir-20153	NCT01274637	0
+sigir-20153	NCT01274871	0
+sigir-20153	NCT01285648	0
+sigir-20153	NCT01291979	0
+sigir-20153	NCT01298531	0
+sigir-20153	NCT01304108	0
+sigir-20153	NCT01308385	0
+sigir-20153	NCT01312077	0
+sigir-20153	NCT01312376	1
+sigir-20153	NCT01313676	0
+sigir-20153	NCT01319682	0
+sigir-20153	NCT01326507	2
+sigir-20153	NCT01355965	0
+sigir-20153	NCT01499784	0
+sigir-20153	NCT01531829	1
+sigir-20153	NCT01535248	0
+sigir-20153	NCT01546857	0
+sigir-20153	NCT01599403	0
+sigir-20153	NCT01604538	1
+sigir-20153	NCT01608360	0
+sigir-20153	NCT01608373	0
+sigir-20153	NCT01633671	0
+sigir-20153	NCT01635257	2
+sigir-20153	NCT01660594	0
+sigir-20153	NCT01721837	0
+sigir-20153	NCT01724996	0
+sigir-20153	NCT01743313	1
+sigir-20153	NCT01787773	2
+sigir-20153	NCT01803022	0
+sigir-20153	NCT01817257	0
+sigir-20153	NCT01849614	0
+sigir-20153	NCT01871194	0
+sigir-20153	NCT01898637	1
+sigir-20153	NCT01935141	2
+sigir-20153	NCT01959529	0
+sigir-20153	NCT01992224	0
+sigir-20153	NCT02008370	0
+sigir-20153	NCT02015858	1
+sigir-20153	NCT02134652	0
+sigir-20153	NCT02204514	0
+sigir-20153	NCT02230865	0
+sigir-20153	NCT02235558	0
+sigir-20153	NCT02248610	2
+sigir-20153	NCT02258958	0
+sigir-20153	NCT02271399	0
+sigir-20153	NCT02289937	0
+sigir-20153	NCT02334007	0
+sigir-20153	NCT02355548	1
+sigir-20153	NCT02356120	2
+sigir-20153	NCT02375919	2
+sigir-20153	NCT02386696	0
+sigir-20153	NCT02402881	1
+sigir-20153	NCT02408263	0
+sigir-20153	NCT02427516	1
+sigir-20153	NCT02437942	0
+sigir-20153	NCT02440529	0
+sigir-20153	NCT02446587	0
+sigir-20153	NCT02458729	1
+sigir-20153	NCT02476721	0
+sigir-20153	NCT02507180	0
+sigir-20153	NCT02531542	1
+sigir-20153	NCT02532075	0
+sigir-20153	NCT02546817	2
+sigir-20153	NCT02553733	0
+sigir-20153	NCT02558465	2
+sigir-20153	NCT02596555	2
+sigir-20153	NCT02601846	1
+sigir-20153	NCT02604550	0
+pfizer-covidbooster-001	NCT05930730	2
+pfizer-covidbooster-001	NCT05197621	1
+pfizer-covidbooster-001	NCT05170399	1
+pfizer-covidbooster-002	NCT05930730	2
+pfizer-covidbooster-002	NCT05197621	1
+pfizer-covidbooster-003	NCT05930730	2
+elranatamab-pta-001	NCT06057402	2
+elranatamab-pta-002	NCT06057402	2
+covid-maternal-001	NCT05197621	2
+covid-maternal-002	NCT05197621	2
+covid-maternal-003	NCT05197621	2
+hemophilia-gene-001	NCT05568719	2
+hemophilia-gene-002	NCT05568719	2
+hemophilia-gene-003	NCT05568719	2
+mevpro1-mcrpc-001	NCT06551324	2
+mevpro1-mcrpc-002	NCT06551324	2
+mevpro1-mcrpc-003	NCT06551324	2
+bcell-vax-001	NCT05170399	2
+bcell-vax-002	NCT05170399	2
+bcell-vax-003	NCT05170399	2
+txa-orbit-001	NCT00736086	1
+txa-orbit-002	NCT00736086	1
+ama-001	NCT00001951	1
+ama-002	NCT00001951	1
+ama-003	NCT00001951	1
+mythic-3001	NCT01577771	1
+mythic-3002	NCT01577771	1
+mpa-lngius-0001	NCT00001951	1
+mpa-lngius-0002	NCT00001951	1
+mpa-lngius-0003	NCT00001951	1
+mpa-lngius-0004	NCT00001951	1
+axspa-001	NCT02592434	1
+axspa-002	NCT02592434	1
+axspa-003	NCT02592434	1
+ama-001	NCT04669717	2
+ama-002	NCT04669717	2
+ama-003	NCT04669717	2
+mythic-3001	NCT06325293	2
+mythic-3002	NCT06325293	2
+mythic-3001	NCT01280461	1
+mythic-3002	NCT01280461	1
+mpa-lngius-0001	NCT05565573	2
+mpa-lngius-0002	NCT05565573	2
+mpa-lngius-0003	NCT05565573	2
+mpa-lngius-0004	NCT05565573	2
+txa-orbit-001	NCT00313365	1
+txa-orbit-002	NCT00313365	1
+axspa-001	NCT00672490	1
+axspa-002	NCT00672490	1
+axspa-003	NCT00672490	1

dataset/sigir/queries.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9feeb8a450d544b5e121ac9c591c9ee48da3ef90542ec0be97a72304d07efe82
+size 46261

dataset/sigir/retrieved_trials.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:529823a52827e0623fb2eb6b450ec2cdfc56cf890b6b095ce75a0291f2d78595
+size 284678

dataset/sigir/s.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a422252f465ddc173d111231b6427051503f478d98c327a6b6d53c5a5320c251
+size 281830

dataset/sigir/wrap_gpt35turbo.py

@@ -0,0 +1,16 @@
+import json
+
+# Load the original file
+with open('/l/users/salma.hassan/TrialGPT_scratch/dataset/sigir/id2queries.json', 'r') as f:
+    data = json.load(f)
+
+# Wrap each entry with 'gpt-35-turbo'
+wrapped = {}
+for k, v in data.items():
+    wrapped[k] = {"gpt-35-turbo": v}
+
+# Write to a new file (to avoid overwriting the original)
+with open('/l/users/salma.hassan/TrialGPT_scratch/dataset/sigir/id2queries_gpt35turbo.json', 'w') as f:
+    json.dump(wrapped, f, indent=2)
+
+print('Done. Output written to id2queries_gpt35turbo.json') 

dataset/trial_info.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1ce098c5b511353533bc409a5b66ea2cf0263e6564d0c6431b3034e441f7e2e1
+size 68526681

requirements.txt

@@ -0,0 +1,13 @@
+beir==2.0.0
+faiss==1.8.0
+faiss_cpu==1.8.0
+nltk==3.8.1
+numpy==1.26.4
+openai==1.30.5
+pandas==2.2.2
+rank_bm25==0.2.2
+scikit_learn==1.2.2
+sentence_transformers==2.6.1
+torch==2.2.2
+tqdm==4.65.0
+transformers==4.39.3

trialgpt_matching/TrialGPT.py

@@ -0,0 +1,118 @@
+__author__ = "qiao"
+
+"""
+TrialGPT-Matching main functions.
+"""
+
+import json
+from nltk.tokenize import sent_tokenize
+import time
+import os
+
+from openai import AzureOpenAI
+
+client = AzureOpenAI(
+	api_version="2023-09-01-preview",
+	azure_endpoint=os.getenv("OPENAI_ENDPOINT"),
+	api_key=os.getenv("OPENAI_API_KEY"),
+)
+
+def parse_criteria(criteria):
+	output = ""
+	criteria = criteria.split("\n\n")
+	
+	idx = 0
+	for criterion in criteria:
+		criterion = criterion.strip()
+
+		if "inclusion criteria" in criterion.lower() or "exclusion criteria" in criterion.lower():
+			continue
+
+		if len(criterion) < 5:
+			continue
+	
+		output += f"{idx}. {criterion}\n" 
+		idx += 1
+	
+	return output
+
+
+def print_trial(
+	trial_info: dict,
+	inc_exc: str,
+) -> str:
+	"""Given a dict of trial information, returns a string of trial."""
+	
+	trial = f"Title: {trial_info['brief_title']}\n"
+	trial += f"Target diseases: {', '.join(trial_info['diseases_list'])}\n"
+	trial += f"Interventions: {', '.join(trial_info['drugs_list'])}\n"
+	trial += f"Summary: {trial_info['brief_summary']}\n"
+	
+	if inc_exc == "inclusion":
+		trial += "Inclusion criteria:\n %s\n" % parse_criteria(trial_info['inclusion_criteria'])
+	elif inc_exc == "exclusion":
+		trial += "Exclusion criteria:\n %s\n" % parse_criteria(trial_info['exclusion_criteria']) 
+
+	return trial
+
+
+def get_matching_prompt(
+	trial_info: dict,
+	inc_exc: str,
+	patient: str,
+) -> str:
+	"""Output the prompt."""
+	prompt = f"You are a helpful assistant for clinical trial recruitment. Your task is to compare a given patient note and the {inc_exc} criteria of a clinical trial to determine the patient's eligibility at the criterion level.\n"
+
+	if inc_exc == "inclusion":
+		prompt += "The factors that allow someone to participate in a clinical study are called inclusion criteria. They are based on characteristics such as age, gender, the type and stage of a disease, previous treatment history, and other medical conditions.\n"
+	
+	elif inc_exc == "exclusion":
+		prompt += "The factors that disqualify someone from participating are called exclusion criteria. They are based on characteristics such as age, gender, the type and stage of a disease, previous treatment history, and other medical conditions.\n"
+
+	prompt += f"You should check the {inc_exc} criteria one-by-one, and output the following three elements for each criterion:\n"
+	prompt += f"\tElement 1. For each {inc_exc} criterion, briefly generate your reasoning process: First, judge whether the criterion is not applicable (not very common), where the patient does not meet the premise of the criterion. Then, check if the patient note contains direct evidence. If so, judge whether the patient meets or does not meet the criterion. If there is no direct evidence, try to infer from existing evidence, and answer one question: If the criterion is true, is it possible that a good patient note will miss such information? If impossible, then you can assume that the criterion is not true. Otherwise, there is not enough information.\n"
+	prompt += f"\tElement 2. If there is relevant information, you must generate a list of relevant sentence IDs in the patient note. If there is no relevant information, you must annotate an empty list.\n" 
+	prompt += f"\tElement 3. Classify the patient eligibility for this specific {inc_exc} criterion: "
+	
+	if inc_exc == "inclusion":
+		prompt += 'the label must be chosen from {"not applicable", "not enough information", "included", "not included"}. "not applicable" should only be used for criteria that are not applicable to the patient. "not enough information" should be used where the patient note does not contain sufficient information for making the classification. Try to use as less "not enough information" as possible because if the note does not mention a medically important fact, you can assume that the fact is not true for the patient. "included" denotes that the patient meets the inclusion criterion, while "not included" means the reverse.\n'
+	elif inc_exc == "exclusion":
+		prompt += 'the label must be chosen from {"not applicable", "not enough information", "excluded", "not excluded"}. "not applicable" should only be used for criteria that are not applicable to the patient. "not enough information" should be used where the patient note does not contain sufficient information for making the classification. Try to use as less "not enough information" as possible because if the note does not mention a medically important fact, you can assume that the fact is not true for the patient. "excluded" denotes that the patient meets the exclusion criterion and should be excluded in the trial, while "not excluded" means the reverse.\n'
+	
+	prompt += "You should output only a JSON dict exactly formatted as: dict{str(criterion_number): list[str(element_1_brief_reasoning), list[int(element_2_sentence_id)], str(element_3_eligibility_label)]}."
+	
+	user_prompt = f"Here is the patient note, each sentence is led by a sentence_id:\n{patient}\n\n" 
+	user_prompt += f"Here is the clinical trial:\n{print_trial(trial_info, inc_exc)}\n\n"
+	user_prompt += f"Plain JSON output:"
+
+	return prompt, user_prompt
+
+
+def trialgpt_matching(trial: dict, patient: str, model: str):
+	results = {}
+
+	# doing inclusions and exclusions in separate prompts
+	for inc_exc in ["inclusion", "exclusion"]:
+		system_prompt, user_prompt = get_matching_prompt(trial, inc_exc, patient)
+	
+		messages = [
+			{"role": "system", "content": system_prompt},
+			{"role": "user", "content": user_prompt},
+		]
+
+		response = client.chat.completions.create(
+			model=model,
+			messages=messages,
+			temperature=0,
+		)
+
+		message = response.choices[0].message.content.strip()
+		message = message.strip("`").strip("json")
+
+		try:
+			results[inc_exc] = json.loads(message)
+		except:
+			results[inc_exc] = message
+
+	return results

trialgpt_matching/run_matching.py

@@ -0,0 +1,61 @@
+__author__ = "qiao"
+
+"""
+Running the TrialGPT matching for three cohorts (sigir, TREC 2021, TREC 2022).
+"""
+
+import json
+from nltk.tokenize import sent_tokenize
+import os
+import sys
+from tqdm import tqdm
+from TrialGPT import trialgpt_matching 
+
+if __name__ == "__main__":
+	corpus = sys.argv[1]
+	model = sys.argv[2] 
+	
+	dataset = json.load(open(f"dataset/{corpus}/retrieved_trials.json"))
+
+	output_path = f"results/matching_results_{corpus}_{model}.json" 
+
+	# Dict{Str(patient_id): Dict{Str(label): Dict{Str(trial_id): Str(output)}}}
+	if os.path.exists(output_path):
+		output = json.load(open(output_path))
+	else:
+		output = {}
+
+	for instance in tqdm(dataset):
+		# Dict{'patient': Str(patient), '0': Str(NCTID), ...}
+		patient_id = instance["patient_id"]
+		patient = instance["patient"]
+		sents = sent_tokenize(patient)
+		sents.append("The patient will provide informed consent, and will comply with the trial protocol without any practical issues.")
+		sents = [f"{idx}. {sent}" for idx, sent in enumerate(sents)]
+		patient = "\n".join(sents)
+
+		# initialize the patient id in the output 
+		if patient_id not in output:
+			output[patient_id] = {"0": {}, "1": {}, "2": {}}
+		
+		for label in ["2", "1", "0"]:
+			if label not in instance: continue
+
+			for trial in instance[label]: 
+				trial_id = trial["NCTID"]
+
+				# already calculated and cached
+				if trial_id in output[patient_id][label]:
+					continue
+				
+				# in case anything goes wrong (e.g., API calling errors)
+				try:
+					results = trialgpt_matching(trial, patient, model)
+					output[patient_id][label][trial_id] = results
+
+					with open(output_path, "w") as f:
+						json.dump(output, f, indent=4)
+
+				except Exception as e:
+					print(e)
+					continue

trialgpt_ranking/TrialGPT.py

@@ -0,0 +1,120 @@
+__author__ = "qiao"
+
+"""
+TrialGPT-Ranking main functions.
+"""
+
+import json
+from nltk.tokenize import sent_tokenize
+import time
+import os
+
+from openai import AzureOpenAI
+
+client = AzureOpenAI(
+	api_version="2023-09-01-preview",
+	azure_endpoint=os.getenv("OPENAI_ENDPOINT"),
+	api_key=os.getenv("OPENAI_API_KEY"),
+)
+
+def convert_criteria_pred_to_string(
+		prediction: dict,
+		trial_info: dict,
+) -> str:
+	"""Given the TrialGPT prediction, output the linear string of the criteria."""
+	output = ""
+
+	for inc_exc in ["inclusion", "exclusion"]:
+
+		# first get the idx2criterion dict
+		idx2criterion = {}
+		criteria = trial_info[inc_exc + "_criteria"].split("\n\n")
+		
+		idx = 0
+		for criterion in criteria:
+			criterion = criterion.strip()
+
+			if "inclusion criteria" in criterion.lower() or "exclusion criteria" in criterion.lower():
+				continue
+
+			if len(criterion) < 5:
+				continue
+		
+			idx2criterion[str(idx)] = criterion
+			idx += 1
+
+		for idx, info in enumerate(prediction[inc_exc].items()):
+			criterion_idx, preds = info
+
+			if criterion_idx not in idx2criterion:
+				continue
+
+			criterion = idx2criterion[criterion_idx]
+
+			if len(preds) != 3:
+				continue
+
+			output += f"{inc_exc} criterion {idx}: {criterion}\n"
+			output += f"\tPatient relevance: {preds[0]}\n"
+			if len(preds[1]) > 0:
+				output += f"\tEvident sentences: {preds[1]}\n"
+			output += f"\tPatient eligibility: {preds[2]}\n"
+	
+	return output
+
+
+def convert_pred_to_prompt(
+		patient: str,
+		pred: dict,
+		trial_info: dict,
+) -> str:
+	"""Convert the prediction to a prompt string."""
+	# get the trial string
+	trial = f"Title: {trial_info['brief_title']}\n"
+	trial += f"Target conditions: {', '.join(trial_info['diseases_list'])}\n"
+	trial += f"Summary: {trial_info['brief_summary']}"
+
+	# then get the prediction strings
+	pred = convert_criteria_pred_to_string(pred, trial_info)
+
+	# construct the prompt
+	prompt = "You are a helpful assistant for clinical trial recruitment. You will be given a patient note, a clinical trial, and the patient eligibility predictions for each criterion.\n"
+	prompt += "Your task is to output two scores, a relevance score (R) and an eligibility score (E), between the patient and the clinical trial.\n"
+	prompt += "First explain the consideration for determining patient-trial relevance. Predict the relevance score R (0~100), which represents the overall relevance between the patient and the clinical trial. R=0 denotes the patient is totally irrelevant to the clinical trial, and R=100 denotes the patient is exactly relevant to the clinical trial.\n"
+	prompt += "Then explain the consideration for determining patient-trial eligibility. Predict the eligibility score E (-R~R), which represents the patient's eligibility to the clinical trial. Note that -R <= E <= R (the absolute value of eligibility cannot be higher than the relevance), where E=-R denotes that the patient is ineligible (not included by any inclusion criteria, or excluded by all exclusion criteria), E=R denotes that the patient is eligible (included by all inclusion criteria, and not excluded by any exclusion criteria), E=0 denotes the patient is neutral (i.e., no relevant information for all inclusion and exclusion criteria).\n"
+	prompt += 'Please output a JSON dict formatted as Dict{"relevance_explanation": Str, "relevance_score_R": Float, "eligibility_explanation": Str, "eligibility_score_E": Float}.'
+
+
+	user_prompt = "Here is the patient note:\n"
+	user_prompt += patient + "\n\n"
+	user_prompt += "Here is the clinical trial description:\n"
+	user_prompt += trial + "\n\n"
+	user_prompt += "Here are the criterion-level eligibility prediction:\n"
+	user_prompt += pred + "\n\n"
+	user_prompt += "Plain JSON output:"
+
+	return prompt, user_prompt
+
+
+def trialgpt_aggregation(patient: str, trial_results: dict, trial_info: dict, model: str):
+	system_prompt, user_prompt = convert_pred_to_prompt(
+			patient,
+			trial_results,
+			trial_info
+	)   
+
+	messages = [
+		{"role": "system", "content": system_prompt},
+		{"role": "user", "content": user_prompt}
+	]
+
+	response = client.chat.completions.create(
+		model=model,
+		messages=messages,
+		temperature=0,
+	)
+	result = response.choices[0].message.content.strip()
+	result = result.strip("`").strip("json")
+	result = json.loads(result)
+
+	return result

trialgpt_ranking/rank_results.py

@@ -0,0 +1,120 @@
+__author__ = "qiao"
+
+"""
+Rank the trials given the matching and aggregation results
+"""
+
+import json
+import sys
+
+eps = 1e-9
+
+def get_matching_score(matching):
+	# count only the valid ones
+	included = 0
+	not_inc = 0
+	na_inc = 0
+	no_info_inc = 0
+
+	excluded = 0
+	not_exc = 0
+	na_exc = 0
+	no_info_exc = 0
+	
+	# first count inclusions
+	for criteria, info in matching["inclusion"].items():
+		
+		if len(info) != 3:
+			continue
+
+		if info[2] == "included":
+			included += 1	
+		elif info[2] == "not included":
+			not_inc += 1
+		elif info[2] == "not applicable":
+			na_inc += 1
+		elif info[2] == "not enough information":
+			no_info_inc += 1
+	
+	# then count exclusions
+	for criteria, info in matching["exclusion"].items():
+
+		if len(info) != 3:
+			continue
+
+		if info[2] == "excluded":
+			excluded += 1	
+		elif info[2] == "not excluded":
+			not_exc += 1
+		elif info[2] == "not applicable":
+			na_exc += 1
+		elif info[2] == "not enough information":
+			no_info_exc += 1
+
+	# get the matching score
+	score = 0
+	
+	score += included / (included + not_inc + no_info_inc + eps)
+	
+	if not_inc > 0:
+		score -= 1
+	
+	if excluded > 0:
+		score -= 1
+	
+	return score 
+
+
+def get_agg_score(assessment):
+	try:
+		rel_score = float(assessment["relevance_score_R"])
+		eli_score = float(assessment["eligibility_score_E"])
+	except:
+		rel_score = 0
+		eli_score = 0
+	
+	score = (rel_score + eli_score) / 100
+
+	return score 
+
+
+if __name__ == "__main__":
+	# args are the results paths
+	matching_results_path = sys.argv[1]
+	agg_results_path = sys.argv[2]
+
+	# loading the results
+	matching_results = json.load(open(matching_results_path))
+	agg_results = json.load(open(agg_results_path))
+	
+	# loop over the patients
+	for patient_id, label2trial2results in matching_results.items():
+
+		trial2score = {}
+
+		for _, trial2results in label2trial2results.items():
+			for trial_id, results in trial2results.items():
+
+				matching_score = get_matching_score(results)
+				
+				if patient_id not in agg_results or trial_id not in agg_results[patient_id]:
+					print(f"Patient {patient_id} Trial {trial_id} not in the aggregation results.")
+					agg_score = 0
+				else:
+					agg_score = get_agg_score(agg_results[patient_id][trial_id])
+
+				trial_score = matching_score + agg_score
+				
+				trial2score[trial_id] = trial_score
+
+		sorted_trial2score = sorted(trial2score.items(), key=lambda x: -x[1])
+		
+		print()
+		print(f"Patient ID: {patient_id}")
+		print("Clinical trial ranking:")
+		
+		for trial, score in sorted_trial2score:
+			print(trial, score)
+
+		print("===")
+		print()

trialgpt_ranking/run_aggregation.py

@@ -0,0 +1,78 @@
+__author__ = "qiao"
+
+"""
+Using GPT to aggregate the scores by itself.
+"""
+
+from beir.datasets.data_loader import GenericDataLoader
+import json
+from nltk.tokenize import sent_tokenize
+import os
+import sys
+import time
+
+from TrialGPT import trialgpt_aggregation
+
+if __name__ == "__main__":
+	corpus = sys.argv[1] 
+	model = sys.argv[2]
+
+	# the path of the matching results
+	matching_results_path = sys.argv[3]
+	results = json.load(open(matching_results_path))
+
+	# loading the trial2info dict
+	trial2info = json.load(open("dataset/trial_info.json"))
+	
+	# loading the patient info
+	_, queries, _ = GenericDataLoader(data_folder=f"dataset/{corpus}/").load(split="test")
+	
+	# output file path
+	output_path = f"results/aggregation_results_{corpus}_{model}.json"
+
+	if os.path.exists(output_path):
+		output = json.load(open(output_path))
+	else:
+		output = {}
+
+	# patient-level
+	for patient_id, info in results.items():
+		# get the patient note
+		patient = queries[patient_id]
+		sents = sent_tokenize(patient)
+		sents.append("The patient will provide informed consent, and will comply with the trial protocol without any practical issues.")
+		sents = [f"{idx}. {sent}" for idx, sent in enumerate(sents)]
+		patient = "\n".join(sents)
+
+		if patient_id not in output:
+			output[patient_id] = {}
+		
+		# label-level, 3 label / patient
+		for label, trials in info.items():
+				
+			# trial-level
+			for trial_id, trial_results in trials.items():
+				# already cached results
+				if trial_id in output[patient_id]:
+					continue
+
+				if type(trial_results) is not dict:
+					output[patient_id][trial_id] = "matching result error"
+
+					with open(output_path, "w") as f:
+						json.dump(output, f, indent=4)
+
+					continue
+
+				# specific trial information
+				trial_info = trial2info[trial_id]	
+
+				try:
+					result = trialgpt_aggregation(patient, trial_results, trial_info, model)
+					output[patient_id][trial_id] = result 
+
+					with open(output_path, "w") as f:
+						json.dump(output, f, indent=4)
+
+				except:
+					continue

trialgpt_retrieval/bm25_corpus_sigir.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e801cf14babb39ac15836939d6826a4ad077847dcb8a4cf2218e549f8061d253
+size 28346027

trialgpt_retrieval/hybrid_fusion_retrieval.py

@@ -0,0 +1,230 @@
+__author__ = "qiao"
+
+"""
+Conduct the first stage retrieval by the hybrid retriever 
+"""
+
+from beir.datasets.data_loader import GenericDataLoader
+import faiss
+import json
+from nltk import word_tokenize
+import numpy as np
+import os
+from rank_bm25 import BM25Okapi
+import sys
+import tqdm
+import torch
+from transformers import AutoTokenizer, AutoModel
+from beir import util, LoggingHandler
+
+# Device detection - use CUDA if available, otherwise CPU
+device = "cuda" if torch.cuda.is_available() else "cpu"
+print(f"Using device: {device}")
+
+def get_bm25_corpus_index(corpus):
+	corpus_path = os.path.join(f"trialgpt_retrieval/bm25_corpus_{corpus}.json")
+
+	# if already cached then load, otherwise build
+	if os.path.exists(corpus_path):
+		corpus_data = json.load(open(corpus_path))
+		tokenized_corpus = corpus_data["tokenized_corpus"]
+		corpus_nctids = corpus_data["corpus_nctids"]
+
+	else:
+		tokenized_corpus = []
+		corpus_nctids = []
+
+		with open(f"dataset/{corpus}/corpus.jsonl", "r") as f:
+			for line in f.readlines():
+				entry = json.loads(line)
+				corpus_nctids.append(entry["_id"])
+				
+				# weighting: 3 * title, 2 * condition, 1 * text
+				tokens = word_tokenize(entry["title"].lower()) * 3
+				for disease in entry["metadata"]["diseases_list"]:
+					tokens += word_tokenize(disease.lower()) * 2
+				tokens += word_tokenize(entry["text"].lower())
+
+				tokenized_corpus.append(tokens)
+
+		corpus_data = {
+			"tokenized_corpus": tokenized_corpus,
+			"corpus_nctids": corpus_nctids,
+		}
+
+		with open(corpus_path, "w") as f:
+			json.dump(corpus_data, f, indent=4)
+	
+	bm25 = BM25Okapi(tokenized_corpus)
+
+	return bm25, corpus_nctids
+
+			
+def get_medcpt_corpus_index(corpus):
+	corpus_path = f"trialgpt_retrieval/{corpus}_embeds.npy" 
+	nctids_path = f"trialgpt_retrieval/{corpus}_nctids.json"
+
+	# if already cached then load, otherwise build
+	if os.path.exists(corpus_path):
+		embeds = np.load(corpus_path)
+		corpus_nctids = json.load(open(nctids_path)) 
+
+	else:
+		embeds = []
+		corpus_nctids = []
+
+		model = AutoModel.from_pretrained("ncbi/MedCPT-Article-Encoder").to(device)
+		tokenizer = AutoTokenizer.from_pretrained("ncbi/MedCPT-Article-Encoder")
+
+		with open(f"dataset/{corpus}/corpus.jsonl", "r") as f:
+			print("Encoding the corpus")
+			for line in tqdm.tqdm(f.readlines()):
+				entry = json.loads(line)
+				corpus_nctids.append(entry["_id"])
+
+				title = entry["title"]
+				text = entry["text"]
+
+				with torch.no_grad():
+					# tokenize the articles
+					encoded = tokenizer(
+						[[title, text]], 
+						truncation=True, 
+						padding=True, 
+						return_tensors='pt', 
+						max_length=512,
+					).to(device)
+					
+					embed = model(**encoded).last_hidden_state[:, 0, :]
+
+					embeds.append(embed[0].cpu().numpy())
+
+		embeds = np.array(embeds)
+
+		np.save(corpus_path, embeds)
+		with open(nctids_path, "w") as f:
+			json.dump(corpus_nctids, f, indent=4)
+
+	index = faiss.IndexFlatIP(768)
+	index.add(embeds)
+	
+	return index, corpus_nctids
+	
+
+if __name__ == "__main__":
+	# different corpora, "trec_2021", "trec_2022", "sigir"
+	corpus = sys.argv[1]
+
+	# query type
+	q_type = sys.argv[2]
+
+	# different k for fusion
+	k = int(sys.argv[3])
+
+	# bm25 weight 
+	bm25_wt = int(sys.argv[4])
+
+	# medcpt weight
+	medcpt_wt = int(sys.argv[5])
+
+	# how many to rank
+	N = 2000 
+
+	# loading the qrels
+	_, _, qrels = GenericDataLoader(data_folder=f"dataset/{corpus}/").load(split="test")
+
+	# loading all types of queries
+	id2queries = json.load(open(f"dataset/{corpus}/id2queries.json"))
+
+	# loading the indices
+	bm25, bm25_nctids = get_bm25_corpus_index(corpus)
+	medcpt, medcpt_nctids = get_medcpt_corpus_index(corpus)
+
+	# loading the query encoder for MedCPT
+	model = AutoModel.from_pretrained("ncbi/MedCPT-Query-Encoder").to(device)
+	tokenizer = AutoTokenizer.from_pretrained("ncbi/MedCPT-Query-Encoder")
+	
+	# then conduct the searches, saving top 1k
+	output_path = f"results/qid2nctids_results_{q_type}_{corpus}_k{k}_bm25wt{bm25_wt}_medcptwt{medcpt_wt}_N{N}.json"
+	
+	qid2nctids = {}
+	recalls = []
+
+	with open(f"dataset/{corpus}/queries.jsonl", "r") as f:
+		for line in tqdm.tqdm(f.readlines()):
+			entry = json.loads(line)
+			query = entry["text"]
+			qid = entry["_id"]
+
+			if qid not in qrels:
+				continue
+
+			truth_sum = sum(qrels[qid].values())
+			
+			# get the keyword list
+			if q_type in ["raw", "human_summary"]:
+				conditions = [id2queries[qid][q_type]]
+			elif "turbo" in q_type:
+				conditions = id2queries[qid][q_type]["conditions"]
+			elif "Clinician" in q_type:
+				conditions = id2queries[qid].get(q_type, [])
+
+			if len(conditions) == 0:
+				nctid2score = {}
+			else:
+				# a list of nctid lists for the bm25 retriever
+				bm25_condition_top_nctids = []
+
+				for condition in conditions:
+					tokens = word_tokenize(condition.lower())
+					top_nctids = bm25.get_top_n(tokens, bm25_nctids, n=N)
+					bm25_condition_top_nctids.append(top_nctids)
+				
+				# doing MedCPT retrieval
+				with torch.no_grad():
+					encoded = tokenizer(
+						conditions, 
+						truncation=True, 
+						padding=True, 
+						return_tensors='pt', 
+						max_length=256,
+					).to(device)
+
+					# encode the queries (use the [CLS] last hidden states as the representations)
+					embeds = model(**encoded).last_hidden_state[:, 0, :].cpu().numpy()
+
+					# search the Faiss index
+					scores, inds = medcpt.search(embeds, k=N)				
+
+				medcpt_condition_top_nctids = []
+				for ind_list in inds:
+					top_nctids = [medcpt_nctids[ind] for ind in ind_list]
+					medcpt_condition_top_nctids.append(top_nctids)
+
+				nctid2score = {}
+
+				for condition_idx, (bm25_top_nctids, medcpt_top_nctids) in enumerate(zip(bm25_condition_top_nctids, medcpt_condition_top_nctids)):
+
+					if bm25_wt > 0:
+						for rank, nctid in enumerate(bm25_top_nctids):
+							if nctid not in nctid2score:
+								nctid2score[nctid] = 0
+							
+							nctid2score[nctid] += (1 / (rank + k)) * (1 / (condition_idx + 1))
+					
+					if medcpt_wt > 0:
+						for rank, nctid in enumerate(medcpt_top_nctids):
+							if nctid not in nctid2score:
+								nctid2score[nctid] = 0
+							
+							nctid2score[nctid] += (1 / (rank + k)) * (1 / (condition_idx + 1))
+
+			nctid2score = sorted(nctid2score.items(), key=lambda x: -x[1])
+			top_nctids = [nctid for nctid, _ in nctid2score[:N]]
+			qid2nctids[qid] = top_nctids
+
+			actual_sum = sum([qrels[qid].get(nctid, 0) for nctid in top_nctids])
+			recalls.append(actual_sum / truth_sum)
+	
+	with open(output_path, "w") as f:
+		json.dump(qid2nctids, f, indent=4)

trialgpt_retrieval/keyword_generation.py

@@ -0,0 +1,166 @@
+__author__ = "qiao"
+
+"""
+generate the search keywords for each patient
+"""
+
+import json
+import os
+import time
+from openai import AzureOpenAI
+
+import sys
+
+client = AzureOpenAI(
+	api_version="2023-09-01-preview",
+	azure_endpoint=os.getenv("OPENAI_ENDPOINT"),
+	api_key=os.getenv("OPENAI_API_KEY"),
+)
+
+
+def get_keyword_generation_messages(note):
+	system = 'You are a helpful assistant and your task is to help search relevant clinical trials for a given patient description. Please first summarize the main medical problems of the patient. Then generate up to 32 key conditions for searching relevant clinical trials for this patient. The key condition list should be ranked by priority. Please output only a JSON dict formatted as Dict{{"summary": Str(summary), "conditions": List[Str(condition)]}}.'
+
+	prompt =  f"Here is the patient description: \n{note}\n\nJSON output:"
+
+	messages = [
+		{"role": "system", "content": system},
+		{"role": "user", "content": prompt}
+	]
+	
+	return messages
+
+
+def is_valid_json(json_str):
+	"""Check if a string is valid JSON"""
+	try:
+		json.loads(json_str)
+		return True
+	except (json.JSONDecodeError, TypeError):
+		return False
+
+
+def fix_truncated_json(json_str):
+	"""Attempt to fix common JSON truncation issues"""
+	if not json_str or not isinstance(json_str, str):
+		return None
+	
+	# Remove any trailing commas before closing brackets/braces
+	json_str = json_str.rstrip()
+	
+	# If it ends with a comma, remove it
+	if json_str.endswith(','):
+		json_str = json_str[:-1]
+	
+	# If it ends with an incomplete string, try to close it
+	if json_str.count('"') % 2 != 0:
+		# Find the last quote and see if we can close the string
+		last_quote_pos = json_str.rfind('"')
+		if last_quote_pos != -1:
+			# Check if this is an opening quote (no closing quote after it)
+			remaining = json_str[last_quote_pos+1:]
+			if '"' not in remaining:
+				# Try to close the string and the JSON
+				json_str += '"]}'
+	
+	# If it ends with an incomplete array, try to close it
+	if json_str.endswith('['):
+		json_str += ']}'
+	elif json_str.endswith('["'):
+		json_str += ']}'
+	
+	# If it ends with an incomplete object, try to close it
+	if json_str.endswith('{'):
+		json_str += '}'
+	
+	return json_str
+
+
+if __name__ == "__main__":
+	# the corpus: trec_2021, trec_2022, or sigir
+	corpus = sys.argv[1]
+
+	# the model index to use
+	model = sys.argv[2]
+
+	outputs = {}
+	
+	with open(f"dataset/{corpus}/queries.jsonl", "r") as f:
+		for line in f.readlines():
+			entry = json.loads(line)
+			messages = get_keyword_generation_messages(entry["text"])
+
+			max_retries = 3
+			retry_count = 0
+			
+			while retry_count < max_retries:
+				try:
+					response = client.chat.completions.create(
+						model=model,
+						messages=messages,
+						temperature=0,
+						max_tokens=2000,  # Increase max tokens to avoid truncation
+					)
+
+					output = response.choices[0].message.content
+					
+					# Check if output is None or empty
+					if output is None or output.strip() == "":
+						print(f"Warning: Empty response for entry {entry['_id']}, retry {retry_count + 1}/{max_retries}")
+						retry_count += 1
+						if retry_count < max_retries:
+							time.sleep(2)  # Wait before retrying
+							continue
+						else:
+							print(f"Error: Failed to get response for entry {entry['_id']} after {max_retries} retries, skipping...")
+							break
+					
+					output = output.strip("`").strip("json")
+					
+					# Try to fix truncated JSON
+					if not is_valid_json(output):
+						fixed_output = fix_truncated_json(output)
+						if fixed_output and is_valid_json(fixed_output):
+							output = fixed_output
+							print(f"Info: Fixed truncated JSON for entry {entry['_id']}")
+						else:
+							print(f"Warning: Invalid JSON for entry {entry['_id']}, retry {retry_count + 1}/{max_retries}")
+							print(f"Raw output: {output[:200]}...")  # Show first 200 chars
+							retry_count += 1
+							if retry_count < max_retries:
+								time.sleep(2)  # Wait before retrying
+								continue
+							else:
+								print(f"Error: Failed to parse JSON for entry {entry['_id']} after {max_retries} retries, skipping...")
+								break
+					
+					try:
+						parsed_output = json.loads(output)
+						outputs[entry["_id"]] = parsed_output
+						print(f"Success: Processed entry {entry['_id']}")
+						break  # Success, exit retry loop
+						
+					except json.JSONDecodeError as e:
+						print(f"Warning: Failed to parse JSON for entry {entry['_id']}: {e}")
+						print(f"Raw output: {output[:200]}...")  # Show first 200 chars
+						retry_count += 1
+						if retry_count < max_retries:
+							time.sleep(2)  # Wait before retrying
+							continue
+						else:
+							print(f"Error: Failed to parse JSON for entry {entry['_id']} after {max_retries} retries, skipping...")
+							break
+						
+				except Exception as e:
+					print(f"Error processing entry {entry['_id']}: {e}")
+					retry_count += 1
+					if retry_count < max_retries:
+						time.sleep(2)  # Wait before retrying
+						continue
+					else:
+						print(f"Error: Failed to process entry {entry['_id']} after {max_retries} retries, skipping...")
+						break
+
+			# Save progress after each entry
+			with open(f"results/retrieval_keywords_{model}_{corpus}.json", "w") as f:
+				json.dump(outputs, f, indent=4)

trialgpt_retrieval/retriever.py

@@ -0,0 +1,348 @@
+# """
+# Retriever module for TrialGPT
+# """
+
+# import json
+# import os
+# import numpy as np
+# import torch
+# from nltk import word_tokenize
+# from rank_bm25 import BM25Okapi
+# import faiss
+# from transformers import AutoTokenizer, AutoModel
+
+# def get_bm25_corpus_index(corpus="sigir"):
+#     """Get BM25 corpus index for the specified corpus."""
+#     corpus_path = os.path.join(f"trialgpt_retrieval/bm25_corpus_{corpus}.json")
+
+#     if os.path.exists(corpus_path):
+#         corpus_data = json.load(open(corpus_path))
+#         tokenized_corpus = corpus_data["tokenized_corpus"]
+#         corpus_nctids = corpus_data["corpus_nctids"]
+#     else:
+#         # If the pre-built index doesn't exist, we'll need to build it
+#         # For now, return None to indicate the index needs to be built
+#         return None, None
+    
+#     bm25 = BM25Okapi(tokenized_corpus)
+#     return bm25, corpus_nctids
+
+# def get_medcpt_corpus_index(corpus="sigir"):
+#     """Get MedCPT corpus index for the specified corpus."""
+#     corpus_path = f"trialgpt_retrieval/{corpus}_embeds.npy" 
+#     nctids_path = f"trialgpt_retrieval/{corpus}_nctids.json"
+
+#     if os.path.exists(corpus_path):
+#         embeds = np.load(corpus_path)
+#         corpus_nctids = json.load(open(nctids_path))
+#     else:
+#         # If the pre-built index doesn't exist, return None
+#         return None, None
+
+#     index = faiss.IndexFlatIP(768)
+#     index.add(embeds)
+    
+#     return index, corpus_nctids
+
+# def retrieve_trials(conditions, corpus="sigir", top_k=5, bm25_weight=1, medcpt_weight=1):
+#     """
+#     Retrieve clinical trials based on conditions using hybrid BM25 + MedCPT retrieval.
+    
+#     Args:
+#         conditions (list): List of condition strings to search for
+#         corpus (str): Corpus to search in ("trec_2021", "trec_2022", "sigir")
+#         top_k (int): Number of top trials to return
+#         bm25_weight (int): Weight for BM25 scores
+#         medcpt_weight (int): Weight for MedCPT scores
+    
+#     Returns:
+#         list: List of NCT IDs for the top matching trials
+#     """
+#     # Get the retrieval indices
+#     bm25, bm25_nctids = get_bm25_corpus_index(corpus)
+#     medcpt, medcpt_nctids = get_medcpt_corpus_index(corpus)
+    
+#     if bm25 is None or medcpt is None:
+#         print(f"Warning: Pre-built indices for corpus '{corpus}' not found.")
+#         print("Please run the hybrid_fusion_retrieval.py script first to build the indices.")
+#         return []
+    
+#     if len(conditions) == 0:
+#         return []
+    
+#     # BM25 retrieval
+#     bm25_condition_top_nctids = []
+#     if bm25_weight > 0:
+#         for condition in conditions:
+#             tokens = word_tokenize(condition.lower())
+#             top_nctids = bm25.get_top_n(tokens, bm25_nctids, n=top_k*2)  # Get more for fusion
+#             bm25_condition_top_nctids.append(top_nctids)
+    
+#     # MedCPT retrieval
+#     medcpt_condition_top_nctids = []
+#     if medcpt_weight > 0:
+#         try:
+#             model = AutoModel.from_pretrained("ncbi/MedCPT-Query-Encoder")
+#             tokenizer = AutoTokenizer.from_pretrained("ncbi/MedCPT-Query-Encoder")
+            
+#             with torch.no_grad():
+#                 encoded = tokenizer(
+#                     conditions, 
+#                     truncation=True, 
+#                     padding=True, 
+#                     return_tensors='pt', 
+#                     max_length=256,
+#                 )
+
+#                 # encode the queries
+#                 embeds = model(**encoded).last_hidden_state[:, 0, :].numpy()
+
+#                 # search the Faiss index
+#                 scores, inds = medcpt.search(embeds, k=top_k*2)  # Get more for fusion
+
+#             for ind_list in inds:
+#                 top_nctids = [medcpt_nctids[ind] for ind in ind_list]
+#                 medcpt_condition_top_nctids.append(top_nctids)
+                
+#         except Exception as e:
+#             print(f"Warning: MedCPT retrieval failed: {e}")
+#             medcpt_weight = 0
+    
+#     # Fusion of results
+#     nctid2score = {}
+    
+#     for condition_idx, condition in enumerate(conditions):
+#         # BM25 scoring
+#         if bm25_weight > 0 and condition_idx < len(bm25_condition_top_nctids):
+#             for rank, nctid in enumerate(bm25_condition_top_nctids[condition_idx]):
+#                 if nctid not in nctid2score:
+#                     nctid2score[nctid] = 0
+#                 nctid2score[nctid] += bm25_weight * (1 / (rank + 1)) * (1 / (condition_idx + 1))
+        
+#         # MedCPT scoring
+#         if medcpt_weight > 0 and condition_idx < len(medcpt_condition_top_nctids):
+#             for rank, nctid in enumerate(medcpt_condition_top_nctids[condition_idx]):
+#                 if nctid not in nctid2score:
+#                     nctid2score[nctid] = 0
+#                 nctid2score[nctid] += medcpt_weight * (1 / (rank + 1)) * (1 / (condition_idx + 1))
+    
+#     # Sort by score and return top_k
+#     nctid2score = sorted(nctid2score.items(), key=lambda x: -x[1])
+#     top_nctids = [nctid for nctid, _ in nctid2score[:top_k]]
+    
+#     return top_nctids 
+
+"""
+GPU-accelerated Retriever module for TrialGPT
+"""
+
+import json
+import os
+import numpy as np
+import torch
+from nltk import word_tokenize
+from rank_bm25 import BM25Okapi
+import faiss
+from transformers import AutoTokenizer, AutoModel
+
+class GPUTrialRetriever:
+    """GPU-accelerated trial retriever with model caching."""
+    
+    def __init__(self, device=None):
+        self.device = device or ('cuda' if torch.cuda.is_available() else 'cpu')
+        self.model = None
+        self.tokenizer = None
+        self._model_loaded = False
+        
+    def _load_medcpt_model(self):
+        """Load MedCPT model once and cache it."""
+        if not self._model_loaded:
+            self.tokenizer = AutoTokenizer.from_pretrained("ncbi/MedCPT-Query-Encoder")
+            self.model = AutoModel.from_pretrained("ncbi/MedCPT-Query-Encoder")
+            
+            # Move model to GPU if available
+            if self.device != 'cpu':
+                self.model = self.model.to(self.device)
+            
+            # Set to evaluation mode for inference
+            self.model.eval()
+            self._model_loaded = True
+            print(f"MedCPT model loaded on {self.device}")
+
+def get_bm25_corpus_index(corpus="sigir"):
+    """Get BM25 corpus index for the specified corpus."""
+    corpus_path = os.path.join(f"trialgpt_retrieval/bm25_corpus_{corpus}.json")
+
+    if os.path.exists(corpus_path):
+        corpus_data = json.load(open(corpus_path))
+        tokenized_corpus = corpus_data["tokenized_corpus"]
+        corpus_nctids = corpus_data["corpus_nctids"]
+    else:
+        return None, None
+    
+    bm25 = BM25Okapi(tokenized_corpus)
+    return bm25, corpus_nctids
+
+def get_medcpt_corpus_index_gpu(corpus="sigir", use_gpu=True):
+    """Get GPU-accelerated MedCPT corpus index."""
+    corpus_path = f"trialgpt_retrieval/{corpus}_embeds.npy" 
+    nctids_path = f"trialgpt_retrieval/{corpus}_nctids.json"
+
+    if os.path.exists(corpus_path):
+        embeds = np.load(corpus_path).astype(np.float32)  # Ensure float32 for GPU
+        corpus_nctids = json.load(open(nctids_path))
+    else:
+        return None, None
+
+    # Use GPU FAISS index if available and requested
+    if use_gpu and torch.cuda.is_available():
+        try:
+            # Create GPU resources
+            res = faiss.StandardGpuResources()
+            
+            # Create CPU index first
+            cpu_index = faiss.IndexFlatIP(768)
+            cpu_index.add(embeds)
+            
+            # Move to GPU
+            gpu_index = faiss.index_cpu_to_gpu(res, 0, cpu_index)
+            print(f"FAISS index moved to GPU with {len(corpus_nctids)} embeddings")
+            return gpu_index, corpus_nctids
+            
+        except Exception as e:
+            print(f"GPU FAISS failed, falling back to CPU: {e}")
+            # Fall back to CPU index
+            index = faiss.IndexFlatIP(768)
+            index.add(embeds)
+            return index, corpus_nctids
+    else:
+        # CPU index
+        index = faiss.IndexFlatIP(768)
+        index.add(embeds)
+        return index, corpus_nctids
+
+def retrieve_trials_gpu(conditions, corpus="sigir", top_k=5, bm25_weight=1, medcpt_weight=1, 
+                       use_gpu=True, batch_size=32, retriever=None):
+    """
+    GPU-accelerated clinical trial retrieval with optimized batching.
+    
+    Args:
+        conditions (list): List of condition strings to search for
+        corpus (str): Corpus to search in ("trec_2021", "trec_2022", "sigir")
+        top_k (int): Number of top trials to return
+        bm25_weight (int): Weight for BM25 scores
+        medcpt_weight (int): Weight for MedCPT scores
+        use_gpu (bool): Whether to use GPU acceleration
+        batch_size (int): Batch size for MedCPT encoding
+        retriever (GPUTrialRetriever): Cached retriever instance
+    
+    Returns:
+        list: List of NCT IDs for the top matching trials
+    """
+    if len(conditions) == 0:
+        return []
+    
+    # Get the retrieval indices
+    bm25, bm25_nctids = get_bm25_corpus_index(corpus)
+    medcpt, medcpt_nctids = get_medcpt_corpus_index_gpu(corpus, use_gpu)
+    
+    if bm25 is None or medcpt is None:
+        print(f"Warning: Pre-built indices for corpus '{corpus}' not found.")
+        print("Please run the hybrid_fusion_retrieval.py script first to build the indices.")
+        return []
+    
+    # Initialize retriever if not provided
+    if retriever is None:
+        device = 'cuda' if use_gpu and torch.cuda.is_available() else 'cpu'
+        retriever = GPUTrialRetriever(device)
+    
+    # BM25 retrieval (CPU-bound, hard to optimize further)
+    bm25_condition_top_nctids = []
+    if bm25_weight > 0:
+        for condition in conditions:
+            tokens = word_tokenize(condition.lower())
+            top_nctids = bm25.get_top_n(tokens, bm25_nctids, n=top_k*2)
+            bm25_condition_top_nctids.append(top_nctids)
+    
+    # MedCPT retrieval with GPU acceleration
+    medcpt_condition_top_nctids = []
+    if medcpt_weight > 0:
+        try:
+            retriever._load_medcpt_model()
+            
+            # Process conditions in batches for better GPU utilization
+            all_embeds = []
+            
+            for i in range(0, len(conditions), batch_size):
+                batch_conditions = conditions[i:i + batch_size]
+                
+                with torch.no_grad():
+                    # Tokenize batch
+                    encoded = retriever.tokenizer(
+                        batch_conditions,
+                        truncation=True,
+                        padding=True,
+                        return_tensors='pt',
+                        max_length=256,
+                    )
+                    
+                    # Move tensors to GPU
+                    if retriever.device != 'cpu':
+                        encoded = {k: v.to(retriever.device) for k, v in encoded.items()}
+                    
+                    # Get embeddings
+                    batch_embeds = retriever.model(**encoded).last_hidden_state[:, 0, :]
+                    
+                    # Always move to CPU before appending (fixes the CUDA tensor issue)
+                    batch_embeds = batch_embeds.cpu()
+                    all_embeds.append(batch_embeds)
+            
+            # Concatenate all embeddings and convert to numpy
+            embeds = torch.cat(all_embeds, dim=0).numpy()
+            
+            # Search the FAISS index
+            scores, inds = medcpt.search(embeds, k=top_k*2)
+            
+            # Convert indices to NCT IDs
+            for ind_list in inds:
+                top_nctids = [medcpt_nctids[ind] for ind in ind_list]
+                medcpt_condition_top_nctids.append(top_nctids)
+                
+        except Exception as e:
+            print(f"Warning: MedCPT retrieval failed: {e}")
+            medcpt_weight = 0
+    
+    # Fusion of results (same as original)
+    nctid2score = {}
+    
+    for condition_idx, condition in enumerate(conditions):
+        # BM25 scoring
+        if bm25_weight > 0 and condition_idx < len(bm25_condition_top_nctids):
+            for rank, nctid in enumerate(bm25_condition_top_nctids[condition_idx]):
+                if nctid not in nctid2score:
+                    nctid2score[nctid] = 0
+                nctid2score[nctid] += bm25_weight * (1 / (rank + 1)) * (1 / (condition_idx + 1))
+        
+        # MedCPT scoring
+        if medcpt_weight > 0 and condition_idx < len(medcpt_condition_top_nctids):
+            for rank, nctid in enumerate(medcpt_condition_top_nctids[condition_idx]):
+                if nctid not in nctid2score:
+                    nctid2score[nctid] = 0
+                nctid2score[nctid] += medcpt_weight * (1 / (rank + 1)) * (1 / (condition_idx + 1))
+    
+    # Sort by score and return top_k
+    nctid2score = sorted(nctid2score.items(), key=lambda x: -x[1])
+    top_nctids = [nctid for nctid, _ in nctid2score[:top_k]]
+    
+    return top_nctids
+
+# Convenience function for backward compatibility
+def retrieve_trials(conditions, corpus="sigir", top_k=5, bm25_weight=1, medcpt_weight=1):
+    """Original interface with GPU acceleration."""
+    return retrieve_trials_gpu(conditions, corpus, top_k, bm25_weight, medcpt_weight)
+
+# Example usage with persistent retriever for multiple calls
+def create_retriever_session(use_gpu=True):
+    """Create a retriever session for multiple queries to avoid model reloading."""
+    device = 'cuda' if use_gpu and torch.cuda.is_available() else 'cpu'
+    return GPUTrialRetriever(device)

trialgpt_retrieval/sigir_embeds.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6d0d11b601ae441f3f5545445844890d194c27a3ce1482c6da766c4dde8ca555
+size 11157632

trialgpt_retrieval/sigir_nctids.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c3d425afa157e3bc7265d288902836b0516484fd0f3297b2a3216f7728c30605
+size 69010