my latest updates

agent.py

@@ -1,21 +1,18 @@
-import pandas as pd
-import requests
 from pydantic import Field, BaseModel
 
-from omegaconf import OmegaConf
-
 from vectara_agentic.agent import Agent
-from vectara_agentic.tools import ToolsFactory, VectaraToolFactory
+from vectara_agentic.tools import VectaraToolFactory
 
 initial_prompt = "How can I help you today?"
 
-prompt = """
+prompt_old = """
 [
-  {"role": "system", "content": "You are an AI assistant that forms a coherent answer to a user query based on search results that are provided to you." },
+  {"role": "system", "content": "You are an AI assistant that forms a detailed and comprehensive answer to a user query based on search results that are provided to you." },
   {"role": "user", "content": " 
     [INSTRUCTIONS]
     If the search results are irrelevant to the question respond with *** I do not have enough information to answer this question.***
-    Search results may include tables in a markdown format. When answering a question using a table be careful about which rows and columns contain the answer and include all relevant information from the relevant rows and columns that the query is asking about.
+    Search results may include tables in a markdown format. 
+    When answering a question using a table be careful about which rows and columns contain the answer and include all relevant information from the relevant rows and columns that the query is asking about.
     Do not base your response on information or knowledge that is not in the search results.
     Make sure your response is answering the query asked. If the query is related to an entity (such as a person or place), make sure you use search results related to that entity.
     Consider that each search result is a partial segment from a bigger text, and may be incomplete.
@@ -30,23 +27,62 @@ prompt = """
         $qResult.getText()
       #end
     #end
-    Generate a coherent response to the query *** $vectaraQuery *** using information and facts in the search results provided. 
+    Generate a comprehensive response to the query *** $vectaraQuery *** using information and facts in the search results provided. 
     Give a slight preference to search results that appear earlier in the list.
     Include statistical and numerical evidence to support and contextualize your response.
-    Your response should be comprehensive and include all relevant information from the search results. Do not omit any relevant information.
-    If multiple trials are mentioned in the search results, include all relevant trials in your response, and highlight the various trials included.
-    Only cite relevant search results in your answer following these specific instructions: $vectaraCitationInstructions
-    If the search results are irrelevant to the query, respond with ***I do not have enough information to answer this question.***. 
+    Your response should include all relevant information and values from the search results. Do not omit anything relevant.
+    Prioritize a long, detailed, thorough and comprehensive response over a short one.
+    Cite relevant search results in your answer following these specific instructions: $vectaraCitationInstructions
     Respond always in the $vectaraLangName language, and only in that language."}
 ]
 """
 
-def create_assistant_tools(cfg):
 
+prompt = """
+[
+  {"role": "system", "content": "
+      You are an AI assistant that forms a detailed and comprehensive answer to a user question based on search results that are provided to you.
+      You are an expert in clinical trial and statistical data analysis with extensive experience in designing, analyzing, and interpreting clinical research data. 
+      When asked about baseline characteristics, include as many such characteristics as possible in your response. Be detailed and comprehensive.
+      For example, always include in baseline characteristics the sample size (number of patients), population demographics (male/female), age, race, BMI, and any other relevant characteristics.
+      Include statistical and numerical evidence to support and contextualize your response.
+      Your response should include all relevant information and values from the search results. Do not omit anything relevant.
+      Prioritize a long, detailed, thorough and comprehensive response over a short one."
+  },
+  {"role": "user", "content": " 
+    [INSTRUCTIONS]
+    - Generate a comprehensive response to the question *** $vectaraQuery *** using information and facts in the search results provided. 
+    - If the search results are irrelevant to the question respond with *** I do not have enough information to answer this question.***
+    - Do not base your response on information or knowledge that is not in the search results.
+    - Make sure your response is answering the question asked. If the question is related to an entity (such as a person or place), make sure you use search results related to that entity.
+    - Consider that each search result is a partial segment from a bigger text, and may be incomplete.
+    - Your output should always be in a single language - the $vectaraLangName language. Check spelling and grammar for the $vectaraLangName language.
+    - Give a slight preference to search results that appear earlier in the list.
+    - Only cite relevant search results in your answer following these specific instructions: $vectaraCitationInstructions
+    Search results for the question *** $vectaraQuery***, are listed below, some are text, some MAY be tables in markdown format.
+    #foreach ($qResult in $vectaraQueryResultsDeduped)
+      [$esc.java($foreach.index + 1)]
+      #if($qResult.hasTable())
+        Table Title: $qResult.getTable().title() || Table Description: $qResult.getTable().description() || Table Data:
+        $qResult.getTable().markdown()
+      #else
+        $qResult.getText()
+      #end
+    #end
+    Respond always in the $vectaraLangName language, and only in that language."}
+]
+"""
+
+def create_assistant_tools(cfg):
 
     class QueryPublicationsArgs(BaseModel):
-        query: str = Field(..., description="The user query, always in the form of a question", 
-                           examples=["what are the risks reported?", "which drug was use on the and how big was the population?"])        
+        query: str = Field(..., description="The user query, always in the form of a question?", 
+                                examples=[
+                                    "what are the risks reported?", 
+                                    "which drug was tested?", 
+                                    "what is the baseline population in the trial?"
+                                ]),
+        name: str = Field(..., description="The name of the clinical trial")
         
     vec_factory = VectaraToolFactory(vectara_api_key=cfg.api_key,
                                      vectara_corpus_key=cfg.corpus_key)
@@ -54,57 +90,69 @@ def create_assistant_tools(cfg):
     ask_publications = vec_factory.create_rag_tool(
         tool_name = "ask_publications",
         tool_description = """
-        Responds to an user question about a particular result, based on the publications.
+        Responds to an user question about clinical trials, focusing on a specific information and data.
         """,
         tool_args_schema = QueryPublicationsArgs,
-        reranker = "chain", rerank_k = 100, 
-        rerank_chain = [
-            {
-                "type": "multilingual_reranker_v1",
-                "cutoff": 0.1
-            },
-            {
-                "type": "mmr",
-                "diversity_bias": 0.1,
-                "limit": 100
-            }
-        ],
-        n_sentences_before = 3, n_sentences_after = 3, lambda_val = 0.005,
-        summary_num_results = 25,
+        reranker = "slingshot", rerank_k = 100, rerank_cutoff = 0.1,
+        n_sentences_before = 1, n_sentences_after = 1, lambda_val = 0.1,
+        summary_num_results = 15,
+        max_response_chars = 8192, max_tokens = 4096,
         vectara_summarizer = summarizer,
         include_citations = True,
         vectara_prompt_text = prompt,
         save_history = True,
-        verbose = True
+        verbose = False
     )
 
-    tools_factory = ToolsFactory()
+    class SearchPublicationsArgs(BaseModel):
+        query: str = Field(..., description="The user query, always in the form of a question?", 
+                                examples=[
+                                    "what are the risks reported?", 
+                                    "which drug was tested?", 
+                                    "what is the baseline population in the trial?"
+                                ]),
+    search_publications = vec_factory.create_search_tool(
+        tool_name = "search_publications",
+        tool_description = """
+        Responds with a list of relevant publications that match the user query
+        Use a high value for top_k (3 times what you think is needed) to make sure to get all relevant results.
+        """,
+        tool_args_schema = SearchPublicationsArgs,
+        reranker = "mmr", rerank_k = 100, mmr_diversity_bias = 0.5,
+        n_sentences_before = 2, n_sentences_after = 2, lambda_val = 0.3,
+        save_history = True,
+        verbose = False
+    )
+
+
     return (
-            tools_factory.standard_tools() +
-            [ask_publications]
+        [ask_publications, search_publications]
     )
 
 def initialize_agent(_cfg, agent_progress_callback=None):
     menarini_bot_instructions = """
     - You are an expert in clinical trial and statistical data analysis with extensive experience in designing, analyzing, and interpreting clinical research data.
-    - Your responses should be technically rigorous, data-driven, and written for an audience familiar with advanced statistical methodologies, regulatory standards, 
-      and the nuances of clinical trial design.
-    - If asked about clinical trial, cover all relevant trials in your response (avoid aggregating results across trials, unless specifically asked to do so).
-    - Call the ask_publications tool to retreive information to answer the user query.
-      If the initial query lacks comprehensive data, try to call ask_publications with a different or refined query until you retrieve all necessary numerical details.
-      You can try multiple times. You can include additional information about the context of the query (like clinical trial name) to get more relevant results.
-    - If you are missing information about a specific trial, or some data item, call the ask_publication tool with a specific query to retrieve the missing information.
-    - If ask_publications returns citations or references, include them in your response.
-    - When responding to queries:
+    - Your task is to answer user question, using the tools you have available.
+    - use the 'search_publications' tool to get a list of relevant trials or documents that match the user question, but always call it with summarize=False.
+    - Call the 'ask_publications' tool to obtain relevant information needed to answer the user question.
+      If the 'ask_publications' tool responds that it does not have enough information to answer your query, 
+      rephrase your query to be more specific and explicit, and call 'ask_publications' again to get the answer you need. 
+      Retry in this manner up to 10 times.
+    - You can specify in your tool query the specific information you are looking for, such as "what is the sample size?" or "what is the percentage of patients with Diabetes".
+    - Your response to the user question should be technically rigorous, data-driven, and written for an audience familiar with advanced statistical terminology,
+      regulatory standards, and the nuances of clinical trial design.
+    - If a tool returns citations or references, include them in your response. Avoid including citations inside table cells.
+    - Form queries to tool as questions. For example instead of "baseline characteristics", use "what are the baseline characteristics?"
+    - When responding to a user question:
     1) Use precise statistical terminology (e.g., randomization, blinding, intention-to-treat, type I/II error, p-values, confidence intervals, Bayesian methods, etc.) 
        and reference common methodologies or guidelines where applicable (e.g., CONSORT, FDA, EMA).
-    2) Your responses must include contextual information like sample size (n) and population characteristics. This nuance is crucial in clinical trial analysis.
-       When considering or reporting sample sizes, consider participants who were eligible for the study, those who were randomized, and those who completed the study.
-       If it's unclear which one is being referred to, clarify this in your response or ask the user for clarification.
+    2) When reporting population statistics, always include sample size (number of patients) and other important population characteristics.
+       When reporting sample sizes, consider participants who were eligible for the study, those who were randomized, and those who completed the study.
+       Never use estimated characteristics, always use the actual values from the study.
     3) Provide clear explanations of statistical concepts, including assumptions, potential biases, and limitations in the context of clinical trial data.
     4) Ensure that your analysis is evidence-based and reflects current best practices in the field of clinical research and data analysis.
-    5) Before finalizing your answer, review the analysis to ensure that all relevant data has been incorporated and that your conclusions are well-supported by the evidence.
-    6) Provide sources and citations for all data and statistical information included in your responses, as provided in the response from the tools.
+    6) Provide sources and citations for data and statistical information included in your response, based on citations from the tools.
+    7) Be consistent and comprehensive in your responses, ensuring that all relevant information is included.    
     """
 
     agent = Agent(