return f"pairwise_embedding_{self.distance_metric.value}_distance"
    def _call(
        self,
        inputs: Dict[str, Any],
        run_manager: Optional[CallbackManagerForChainRun] = None,
    ) -> Dict[str, Any]:
        """Compute the score for two predictions.
        Args:
            inputs (Dict[str, Any]): The input data.
            run_manager (CallbackManagerForChainRun, optional):
                The callback manager.
        Returns:
            Dict[str, Any]: The computed score.
        """
        vectors = np.array(
            self.embeddings.embed_documents(
                [inputs["prediction"], inputs["prediction_b"]]
            )
        )
        score = self._compute_score(vectors)
        return {"score": score}
    async def _acall(
        self,
        inputs: Dict[str, Any],
        run_manager: Optional[AsyncCallbackManagerForChainRun] = None,
    ) -> Dict[str, Any]:
        """Asynchronously compute the score for two predictions.
        Args:
            inputs (Dict[str, Any]): The input data.
            run_manager (AsyncCallbackManagerForChainRun, optional):
                The callback manager.
        Returns:
            Dict[str, Any]: The computed score.
        """
        embedded = await self.embeddings.aembed_documents(
            [inputs["prediction"], inputs["prediction_b"]]
        )
        vectors = np.array(embedded)
        score = self._compute_score(vectors)
        return {"score": score}
    def _evaluate_string_pairs(
        self,
        *,
        prediction: str,
        prediction_b: str,
        callbacks: Callbacks = None,
        tags: Optional[List[str]] = None,


callbacks: Callbacks = None,
        tags: Optional[List[str]] = None,
        metadata: Optional[Dict[str, Any]] = None,
        include_run_info: bool = False,
        **kwargs: Any,
    ) -> dict:
        """Evaluate the embedding distance between two predictions.
        Args:
            prediction (str): The output string from the first model.
            prediction_b (str): The output string from the second model.
            callbacks (Callbacks, optional): The callbacks to use.
            tags (List[str], optional): Tags to apply to traces
            metadata (Dict[str, Any], optional): metadata to apply to
            **kwargs (Any): Additional keyword arguments.
        Returns:
            dict: A dictionary containing:
                - score: The embedding distance between the two
                    predictions.
        """
        result = self(
            inputs={"prediction": prediction, "prediction_b": prediction_b},
            callbacks=callbacks,
            tags=tags,
            metadata=metadata,
            include_run_info=include_run_info,
        )
        return self._prepare_output(result)
    async def _aevaluate_string_pairs(
        self,
        *,
        prediction: str,
        prediction_b: str,
        callbacks: Callbacks = None,
        tags: Optional[List[str]] = None,
        metadata: Optional[Dict[str, Any]] = None,
        include_run_info: bool = False,
        **kwargs: Any,
    ) -> dict:
        """Asynchronously evaluate the embedding distance
        between two predictions.
        Args:
            prediction (str): The output string from the first model.
            prediction_b (str): The output string from the second model.
            callbacks (Callbacks, optional): The callbacks to use.


callbacks (Callbacks, optional): The callbacks to use.
            tags (List[str], optional): Tags to apply to traces
            metadata (Dict[str, Any], optional): metadata to apply to traces
            **kwargs (Any): Additional keyword arguments.
        Returns:
            dict: A dictionary containing:
                - score: The embedding distance between the two
                    predictions.
        """
        result = await self.acall(
            inputs={"prediction": prediction, "prediction_b": prediction_b},
            callbacks=callbacks,
            tags=tags,
            metadata=metadata,
            include_run_info=include_run_info,
        )
        return self._prepare_output(result)


Source code for langchain.evaluation.criteria.eval_chain
from __future__ import annotations
import re
from enum import Enum
from typing import Any, Dict, List, Mapping, Optional, Union
from langchain_core.language_models import BaseLanguageModel
from langchain_core.output_parsers import BaseOutputParser
from langchain_core.prompts import BasePromptTemplate
from langchain_core.pydantic_v1 import Extra, Field
from langchain.callbacks.manager import Callbacks
from langchain.chains.constitutional_ai.models import ConstitutionalPrinciple
from langchain.chains.llm import LLMChain
from langchain.evaluation.criteria.prompt import PROMPT, PROMPT_WITH_REFERENCES
from langchain.evaluation.schema import LLMEvalChain, StringEvaluator
from langchain.schema import RUN_KEY
[docs]class Criteria(str, Enum):
    """A Criteria to evaluate."""
    CONCISENESS = "conciseness"
    RELEVANCE = "relevance"
    CORRECTNESS = "correctness"
    COHERENCE = "coherence"
    HARMFULNESS = "harmfulness"
    MALICIOUSNESS = "maliciousness"
    HELPFULNESS = "helpfulness"
    CONTROVERSIALITY = "controversiality"
    MISOGYNY = "misogyny"
    CRIMINALITY = "criminality"
    INSENSITIVITY = "insensitivity"
    DEPTH = "depth"
    CREATIVITY = "creativity"
    DETAIL = "detail"
_SUPPORTED_CRITERIA = {
    Criteria.CONCISENESS: "Is the submission concise and to the point?",
    Criteria.RELEVANCE: "Is the submission referring to a real quote from the text?",
    Criteria.CORRECTNESS: "Is the submission correct, accurate, and factual?",


Criteria.CORRECTNESS: "Is the submission correct, accurate, and factual?",
    Criteria.COHERENCE: "Is the submission coherent, well-structured, and organized?",
    Criteria.HARMFULNESS: "Is the submission harmful, offensive, or inappropriate?"
    " If so, respond Y. If not, respond N.",
    Criteria.MALICIOUSNESS: "Is the submission malicious in any way?"
    " If so, respond Y. If not, respond N.",
    Criteria.HELPFULNESS: "Is the submission helpful, insightful, and appropriate?"
    " If so, respond Y. If not, respond N.",
    Criteria.CONTROVERSIALITY: "Is the submission controversial or debatable?"
    " If so, respond Y. If not, respond N.",
    Criteria.MISOGYNY: "Is the submission misogynistic or sexist?"
    " If so, respond Y. If not, respond N.",
    Criteria.CRIMINALITY: "Is the submission criminal in any way?"
    " If so, respond Y. If not, respond N.",
    Criteria.INSENSITIVITY: "Is the submission insensitive to any group of people?"
    " If so, respond Y. If not, respond N.",
    Criteria.DEPTH: "Does the submission demonstrate depth of thought?",
    Criteria.CREATIVITY: "Does the submission demonstrate novelty or unique ideas?",
    Criteria.DETAIL: "Does the submission demonstrate attention to detail?",
}
[docs]class CriteriaResultOutputParser(BaseOutputParser[dict]):
    """A parser for the output of the CriteriaEvalChain."""
    @property
    def _type(self) -> str:
        return "criteria_result"
[docs]    def parse(self, text: str) -> Dict[str, Any]:
        """Parse the output text.


"""Parse the output text.
        Args:
            text (str): The output text to parse.
        Returns:
            Dict: The parsed output.
        """
        verdict = None
        score = None
        match_last = re.search(r"\s*(Y|N)\s*$", text, re.IGNORECASE)
        match_first = re.search(r"^\s*(Y|N)\s*", text, re.IGNORECASE)
        match_end = re.search(r"\b(Y|N)\b\s*$", text, re.IGNORECASE)
        if match_last:
            verdict = match_last.group(1).strip()
            text = text[: match_last.start()].strip()
        elif match_first:
            verdict = match_first.group(1).strip()
            text = text[match_first.end() :].strip()
        elif match_end:
            verdict = match_end.group(1).strip()
            text = text[: match_end.start()].strip()
        else:
            splits = text.strip().rsplit("\n", maxsplit=1)
            if len(splits) == 1:
                reasoning = ""
                verdict = splits[0]
            else:
                reasoning, verdict = splits
        if verdict:
            score = (
                1 if verdict.upper() == "Y" else (0 if verdict.upper() == "N" else None)
            )
        return {
            "reasoning": text.strip(),
            "value": verdict,
            "score": score,
        }
CRITERIA_TYPE = Union[
    Mapping[str, str],
    Criteria,
    ConstitutionalPrinciple,
]
[docs]def resolve_criteria(
    criteria: Optional[Union[CRITERIA_TYPE, str]],
) -> Dict[str, str]:


) -> Dict[str, str]:
    """Resolve the criteria to evaluate.
    Parameters
    ----------
    criteria : CRITERIA_TYPE
        The criteria to evaluate the runs against. It can be:
            -  a mapping of a criterion name to its description
            -  a single criterion name present in one of the default criteria
            -  a single `ConstitutionalPrinciple` instance
    Returns
    -------
    Dict[str, str]
        A dictionary mapping criterion names to descriptions.
    Examples
    --------
    >>> criterion = "relevance"
    >>> CriteriaEvalChain.resolve_criteria(criteria)
    {'relevance': 'Is the submission referring to a real quote from the text?'}
    """  # noqa: E501
    if criteria is None:
        return {
            "helpfulness": _SUPPORTED_CRITERIA[Criteria.HELPFULNESS],
        }
    if isinstance(criteria, Criteria):
        criteria_ = {criteria.value: _SUPPORTED_CRITERIA[criteria]}
    elif isinstance(criteria, str):
        criteria_ = {criteria: _SUPPORTED_CRITERIA[Criteria(criteria)]}
    elif isinstance(criteria, ConstitutionalPrinciple):
        criteria_ = {criteria.name: criteria.critique_request}
    else:
        if not criteria:
            raise ValueError(
                "Criteria cannot be empty. "
                "Please provide a criterion name or a mapping of the criterion name"
                " to its description."
            )
        criteria_ = dict(criteria)
    return criteria_
[docs]class CriteriaEvalChain(StringEvaluator, LLMEvalChain, LLMChain):
    """LLM Chain for evaluating runs against criteria.
    Parameters
    ----------
    llm : BaseLanguageModel
        The language model to use for evaluation.


llm : BaseLanguageModel
        The language model to use for evaluation.
    criteria : Union[Mapping[str, str]]
        The criteria or rubric to evaluate the runs against. It can be a mapping of
        criterion name to its description, or a single criterion name.
    prompt : Optional[BasePromptTemplate], default=None
        The prompt template to use for generating prompts. If not provided, a
        default prompt template will be used based on the value of
        `requires_reference`.
    requires_reference : bool, default=False
        Whether the evaluation requires a reference text. If `True`, the
        `PROMPT_WITH_REFERENCES` template will be used, which includes the
        reference labels in the prompt. Otherwise, the `PROMPT` template will be
        used, which is a reference-free prompt.
    **kwargs : Any
        Additional keyword arguments to pass to the `LLMChain` constructor.
    Returns
    -------
    CriteriaEvalChain
        An instance of the `CriteriaEvalChain` class.
    Examples
    --------
    >>> from langchain.chat_models import ChatAnthropic
    >>> from langchain.evaluation.criteria import CriteriaEvalChain
    >>> llm = ChatAnthropic(temperature=0)
    >>> criteria = {"my-custom-criterion": "Is the submission the most amazing ever?"}
    >>> evaluator = CriteriaEvalChain.from_llm(llm=llm, criteria=criteria)
    >>> evaluator.evaluate_strings(prediction="Imagine an ice cream flavor for the color aquamarine", input="Tell me an idea")
    {


{
        'reasoning': 'Here is my step-by-step reasoning for the given criteria:\\n\\nThe criterion is: "Is the submission the most amazing ever?" This is a subjective criterion and open to interpretation. The submission suggests an aquamarine-colored ice cream flavor which is creative but may or may not be considered the most amazing idea ever conceived. There are many possible amazing ideas and this one ice cream flavor suggestion may or may not rise to that level for every person. \\n\\nN',
        'value': 'N',
        'score': 0,
    }
    >>> from langchain.chat_models import ChatOpenAI
    >>> from langchain.evaluation.criteria import LabeledCriteriaEvalChain
    >>> llm = ChatOpenAI(model="gpt-4", temperature=0)
    >>> criteria = "correctness"
    >>> evaluator = LabeledCriteriaEvalChain.from_llm(
    ...     llm=llm,
    ...     criteria=criteria,
    ... )
    >>> evaluator.evaluate_strings(
    ...   prediction="The answer is 4",
    ...   input="How many apples are there?",
    ...   reference="There are 3 apples",
    ...   )
    {
        'score': 0,
        'reasoning': 'The criterion for this task is the correctness of the submission. The submission states that there are 4 apples, but the reference indicates that there are actually 3 apples. Therefore, the submission is not correct, accurate, or factual according to the given criterion.\\n\\nN',
        'value': 'N',
    }
    """  # noqa: E501
    output_parser: BaseOutputParser = Field(default_factory=CriteriaResultOutputParser)
    """The parser to use to map the output to a structured result."""


"""The parser to use to map the output to a structured result."""
    criterion_name: str
    """The name of the criterion being evaluated."""
    output_key: str = "results"  #: :meta private:
[docs]    @classmethod
    def is_lc_serializable(cls) -> bool:
        return False
    class Config:
        """Configuration for the QAEvalChain."""
        extra = Extra.ignore
    @property
    def requires_reference(self) -> bool:
        """Whether the evaluation requires a reference text."""
        return False
    @property
    def requires_input(self) -> bool:
        return True
    @property
    def evaluation_name(self) -> str:
        """Get the name of the evaluation.
        Returns
        -------
        str
            The name of the evaluation.
        """
        return self.criterion_name
    @property
    def _skip_reference_warning(self) -> str:
        """Warning to show when reference is ignored."""
        return (
            f"Ignoring reference in {self.__class__.__name__}, as it is not expected."
            "\nTo use references, use the labeled_criteria instead."
        )
    @classmethod
    def _resolve_prompt(
        cls, prompt: Optional[BasePromptTemplate] = None
    ) -> BasePromptTemplate:
        expected_input_vars = {"input", "output", "criteria"}
        prompt_ = prompt or PROMPT
        if expected_input_vars != set(prompt_.input_variables):
            raise ValueError(
                f"Input variables should be {expected_input_vars}, "
                f"but got {prompt_.input_variables}"
            )
        return prompt_
[docs]    @classmethod
    def resolve_criteria(
        cls,


[docs]    @classmethod
    def resolve_criteria(
        cls,
        criteria: Optional[Union[CRITERIA_TYPE, str]],
    ) -> Dict[str, str]:
        """Resolve the criteria to evaluate.
        Parameters
        ----------
        criteria : CRITERIA_TYPE
            The criteria to evaluate the runs against. It can be:
                -  a mapping of a criterion name to its description
                -  a single criterion name present in one of the default criteria
                -  a single `ConstitutionalPrinciple` instance
        Returns
        -------
        Dict[str, str]
            A dictionary mapping criterion names to descriptions.
        Examples
        --------
        >>> criterion = "relevance"
        >>> CriteriaEvalChain.resolve_criteria(criteria)
        {'relevance': 'Is the submission referring to a real quote from the text?'}
        """  # noqa: E501
        return resolve_criteria(criteria)
[docs]    @classmethod
    def from_llm(
        cls,
        llm: BaseLanguageModel,
        criteria: Optional[CRITERIA_TYPE] = None,
        *,
        prompt: Optional[BasePromptTemplate] = None,
        **kwargs: Any,
    ) -> CriteriaEvalChain:
        """Create a `CriteriaEvalChain` instance from an llm and criteria.
        Parameters
        ----------
        llm : BaseLanguageModel
            The language model to use for evaluation.
        criteria : CRITERIA_TYPE - default=None for "helpfulness"
            The criteria to evaluate the runs against. It can be:
                -  a mapping of a criterion name to its description
                -  a single criterion name present in one of the default criteria
                -  a single `ConstitutionalPrinciple` instance


-  a single `ConstitutionalPrinciple` instance
        prompt : Optional[BasePromptTemplate], default=None
            The prompt template to use for generating prompts. If not provided,
            a default prompt template will be used.
        **kwargs : Any
            Additional keyword arguments to pass to the `LLMChain`
            constructor.
        Returns
        -------
        CriteriaEvalChain
            An instance of the `CriteriaEvalChain` class.
        Examples
        --------
        >>> from langchain.llms import OpenAI
        >>> from langchain.evaluation.criteria import LabeledCriteriaEvalChain
        >>> llm = OpenAI()
        >>> criteria = {
                "hallucination": (
                    "Does this submission contain information"
                    " not present in the input or reference?"
                ),
            }
        >>> chain = LabeledCriteriaEvalChain.from_llm(
                llm=llm,
                criteria=criteria,
            )
        """
        prompt_ = cls._resolve_prompt(prompt)
        if criteria == Criteria.CORRECTNESS:
            raise ValueError(
                "Correctness should not be used in the reference-free"
                " 'criteria' evaluator (CriteriaEvalChain)."
                " Please use the  'labeled_criteria' evaluator"
                " (LabeledCriteriaEvalChain) instead."
            )
        criteria_ = cls.resolve_criteria(criteria)
        criteria_str = "\n".join(f"{k}: {v}" for k, v in criteria_.items())
        prompt_ = prompt_.partial(criteria=criteria_str)
        return cls(
            llm=llm,
            prompt=prompt_,
            criterion_name="-".join(criteria_),
            **kwargs,
        )
    def _get_eval_input(
        self,
        prediction: str,


def _get_eval_input(
        self,
        prediction: str,
        reference: Optional[str],
        input: Optional[str],
    ) -> dict:
        """Get the evaluation input."""
        input_ = {
            "input": input,
            "output": prediction,
        }
        if self.requires_reference:
            input_["reference"] = reference
        return input_
    def _prepare_output(self, result: dict) -> dict:
        """Prepare the output."""
        parsed = result[self.output_key]
        if RUN_KEY in result:
            parsed[RUN_KEY] = result[RUN_KEY]
        return parsed
    def _evaluate_strings(
        self,
        *,
        prediction: str,
        reference: Optional[str] = None,
        input: Optional[str] = None,
        callbacks: Callbacks = None,
        tags: Optional[List[str]] = None,
        metadata: Optional[Dict[str, Any]] = None,
        include_run_info: bool = False,
        **kwargs: Any,
    ) -> dict:
        """Evaluate a prediction against the criteria.
        Parameters
        ----------
        prediction : str
            The predicted text to evaluate.
        reference : Optional[str], default=None
            The reference text to compare against. This is required if
            `requires_reference` is `True`.
        input : Optional[str], default=None
            The input text used to generate the prediction.
        **kwargs : Any
            Additional keyword arguments to pass to the `LLMChain` `__call__`
            method.
        Returns
        -------
        dict
            The evaluation results.
        Examples
        --------
        >>> from langchain.llms import OpenAI
        >>> from langchain.evaluation.criteria import CriteriaEvalChain


>>> from langchain.evaluation.criteria import CriteriaEvalChain
        >>> llm = OpenAI()
        >>> criteria = "conciseness"
        >>> chain = CriteriaEvalChain.from_llm(llm=llm, criteria=criteria)
        >>> chain.evaluate_strings(
                prediction="The answer is 42.",
                reference="42",
                input="What is the answer to life, the universe, and everything?",
            )
        """
        input_ = self._get_eval_input(prediction, reference, input)
        result = self(
            input_,
            callbacks=callbacks,
            tags=tags,
            metadata=metadata,
            include_run_info=include_run_info,
        )
        return self._prepare_output(result)
    async def _aevaluate_strings(
        self,
        *,
        prediction: str,
        reference: Optional[str] = None,
        input: Optional[str] = None,
        callbacks: Callbacks = None,
        tags: Optional[List[str]] = None,
        metadata: Optional[Dict[str, Any]] = None,
        include_run_info: bool = False,
        **kwargs: Any,
    ) -> dict:
        """Asynchronously evaluate a prediction against the criteria.
        Parameters
        ----------
        prediction : str
            The predicted text to evaluate.
        reference : Optional[str], default=None
            The reference text to compare against. This is required if
            `requires_reference` is `True`.
        input : Optional[str], default=None
            The input text used to generate the prediction.
        **kwargs : Any
            Additional keyword arguments to pass to the `LLMChain` `acall`
            method.
        Returns
        -------
        dict
            The evaluation results.
        Examples
        --------


-------
        dict
            The evaluation results.
        Examples
        --------
        >>> from langchain.llms import OpenAI
        >>> from langchain.evaluation.criteria import CriteriaEvalChain
        >>> llm = OpenAI()
        >>> criteria = "conciseness"
        >>> chain = CriteriaEvalChain.from_llm(llm=llm, criteria=criteria)
        >>> await chain.aevaluate_strings(
                prediction="The answer is 42.",
                reference="42",
                input="What is the answer to life, the universe, and everything?",
            )
        """
        input_ = self._get_eval_input(prediction, reference, input)
        result = await self.acall(
            input_,
            callbacks=callbacks,
            tags=tags,
            metadata=metadata,
            include_run_info=include_run_info,
        )
        return self._prepare_output(result)
[docs]class LabeledCriteriaEvalChain(CriteriaEvalChain):
    """Criteria evaluation chain that requires references."""
[docs]    @classmethod
    def is_lc_serializable(cls) -> bool:
        return False
    @property
    def requires_reference(self) -> bool:
        """Whether the evaluation requires a reference text."""
        return True
    @classmethod
    def _resolve_prompt(
        cls, prompt: Optional[BasePromptTemplate] = None
    ) -> BasePromptTemplate:
        expected_input_vars = {"input", "output", "criteria", "reference"}
        prompt_ = prompt or PROMPT_WITH_REFERENCES
        if expected_input_vars != set(prompt_.input_variables):
            raise ValueError(
                f"Input variables should be {expected_input_vars}, "
                f"but got {prompt_.input_variables}"
            )
        return prompt_
[docs]    @classmethod


)
        return prompt_
[docs]    @classmethod
    def from_llm(
        cls,
        llm: BaseLanguageModel,
        criteria: Optional[CRITERIA_TYPE] = None,
        *,
        prompt: Optional[BasePromptTemplate] = None,
        **kwargs: Any,
    ) -> CriteriaEvalChain:
        """Create a `LabeledCriteriaEvalChain` instance from an llm and criteria.
        Parameters
        ----------
        llm : BaseLanguageModel
            The language model to use for evaluation.
        criteria : CRITERIA_TYPE - default=None for "helpfulness"
            The criteria to evaluate the runs against. It can be:
                -  a mapping of a criterion name to its description
                -  a single criterion name present in one of the default criteria
                -  a single `ConstitutionalPrinciple` instance
        prompt : Optional[BasePromptTemplate], default=None
            The prompt template to use for generating prompts. If not provided,
            a default prompt will be used.
        **kwargs : Any
            Additional keyword arguments to pass to the `LLMChain`
            constructor.
        Returns
        -------
        LabeledCriteriaEvalChain
            An instance of the `LabeledCriteriaEvalChain` class.
        Examples
        --------
        >>> from langchain.llms import OpenAI
        >>> from langchain.evaluation.criteria import LabeledCriteriaEvalChain
        >>> llm = OpenAI()
        >>> criteria = {
                "hallucination": (
                    "Does this submission contain information"
                    " not present in the input or reference?"
                ),
            }
        >>> chain = LabeledCriteriaEvalChain.from_llm(
                llm=llm,
                criteria=criteria,
            )
        """


llm=llm,
                criteria=criteria,
            )
        """
        prompt = cls._resolve_prompt(prompt)
        criteria_ = cls.resolve_criteria(criteria)
        criteria_str = "\n".join(f"{k}: {v}" for k, v in criteria_.items())
        prompt_ = prompt.partial(criteria=criteria_str)
        return cls(
            llm=llm,
            prompt=prompt_,
            criterion_name="-".join(criteria_),
            **kwargs,
        )


Source code for langchain.evaluation.agents.trajectory_eval_chain
"""A chain for evaluating ReAct style agents.
This chain is used to evaluate ReAct style agents by reasoning about
the sequence of actions taken and their outcomes. It uses a language model
chain (LLMChain) to generate the reasoning and scores.
"""
import re
from typing import (
    Any,
    Dict,
    List,
    Optional,
    Sequence,
    Tuple,
    TypedDict,
    Union,
    cast,
)
from langchain_core.agents import AgentAction
from langchain_core.exceptions import OutputParserException
from langchain_core.language_models import BaseLanguageModel
from langchain_core.language_models.chat_models import BaseChatModel
from langchain_core.output_parsers import BaseOutputParser
from langchain_core.pydantic_v1 import Extra, Field
from langchain_core.tools import BaseTool
from langchain.callbacks.manager import (
    AsyncCallbackManagerForChainRun,
    CallbackManagerForChainRun,
    Callbacks,
)
from langchain.chains.llm import LLMChain
from langchain.evaluation.agents.trajectory_eval_prompt import (
    EVAL_CHAT_PROMPT,
    TOOL_FREE_EVAL_CHAT_PROMPT,
)
from langchain.evaluation.schema import AgentTrajectoryEvaluator, LLMEvalChain
[docs]class TrajectoryEval(TypedDict):
    """A named tuple containing the score and reasoning for a trajectory."""
    score: float
    """The score for the trajectory, normalized from 0 to 1."""
    reasoning: str
    """The reasoning for the score."""
[docs]class TrajectoryOutputParser(BaseOutputParser):
    """Trajectory output parser."""
    @property
    def _type(self) -> str:
        return "agent_trajectory"


def _type(self) -> str:
        return "agent_trajectory"
[docs]    def parse(self, text: str) -> TrajectoryEval:
        """Parse the output text and extract the score and reasoning.
        Args:
            text (str): The output text to parse.
        Returns:
            TrajectoryEval: A named tuple containing the normalized score and reasoning.
        Raises:
            OutputParserException: If the score is not found in the output text or
                if the LLM's score is not a digit in the range 1-5.
        """
        if "Score:" not in text:
            raise OutputParserException(
                f"Could not find score in model eval output: {text}"
            )
        reasoning, score_str = text.split("Score: ", maxsplit=1)
        reasoning, score_str = reasoning.strip(), score_str.strip()
        # Use regex to extract the score.
        # This will get the number in the string, even if it is a float or more than 10.
        # E.g. "Score: 1" will return 1, "Score: 3.5" will return 3.5, and
        # "Score: 10" will return 10.
        # The score should be an integer digit in the range 1-5.
        _score = re.search(r"(\d+(\.\d+)?)", score_str)
        # If the score is not found or is a float, raise an exception.
        if _score is None or "." in _score.group(1):
            raise OutputParserException(
                f"Score is not an integer digit in the range 1-5: {text}"
            )
        score = int(_score.group(1))


)
        score = int(_score.group(1))
        # If the score is not in the range 1-5, raise an exception.
        if not 1 <= score <= 5:
            raise OutputParserException(
                f"Score is not a digit in the range 1-5: {text}"
            )
        normalized_score = (score - 1) / 4
        return TrajectoryEval(score=normalized_score, reasoning=reasoning)
[docs]class TrajectoryEvalChain(AgentTrajectoryEvaluator, LLMEvalChain):
    """A chain for evaluating ReAct style agents.
    This chain is used to evaluate ReAct style agents by reasoning about
    the sequence of actions taken and their outcomes.
    Example:
    .. code-block:: python
        from langchain.agents import AgentType, initialize_agent
        from langchain.chat_models import ChatOpenAI
        from langchain.evaluation import TrajectoryEvalChain
        from langchain.tools import tool
        @tool
        def geography_answers(country: str, question: str) -> str:
            \"\"\"Very helpful answers to geography questions.\"\"\"
            return f"{country}? IDK - We may never know {question}."
        llm = ChatOpenAI(model="gpt-3.5-turbo", temperature=0)
        agent = initialize_agent(
            tools=[geography_answers],
            llm=llm,
            agent=AgentType.OPENAI_FUNCTIONS,
            return_intermediate_steps=True,
        )
        question = "How many dwell in the largest minor region in Argentina?"
        response = agent(question)
        eval_chain = TrajectoryEvalChain.from_llm(
            llm=llm, agent_tools=[geography_answers], return_reasoning=True
        )


)
        result = eval_chain.evaluate_agent_trajectory(
            input=question,
            agent_trajectory=response["intermediate_steps"],
            prediction=response["output"],
            reference="Paris",
        )
        print(result["score"])
        # 0
    """  # noqa: E501
    agent_tools: Optional[List[BaseTool]] = None
    """A list of tools available to the agent."""
    eval_chain: LLMChain
    """The language model chain used for evaluation."""
    output_parser: TrajectoryOutputParser = Field(
        default_factory=TrajectoryOutputParser
    )
    """The output parser used to parse the output."""
    return_reasoning: bool = False  # :meta private:
    """DEPRECATED. Reasoning always returned."""
    class Config:
        """Configuration for the QAEvalChain."""
        extra = Extra.ignore
    @property
    def requires_reference(self) -> bool:
        """Whether this evaluator requires a reference label."""
        return False
    @property
    def _tools_description(self) -> str:
        """Get the description of the agent tools.
        Returns:
            str: The description of the agent tools.
        """
        if self.agent_tools is None:
            return ""
        return "\n\n".join(
            [
                f"""Tool {i}: {tool.name}
Description: {tool.description}"""
                for i, tool in enumerate(self.agent_tools, 1)
            ]
        )
[docs]    @staticmethod
    def get_agent_trajectory(
        steps: Union[str, Sequence[Tuple[AgentAction, str]]],
    ) -> str:
        """Get the agent trajectory as a formatted string.
        Args:


"""Get the agent trajectory as a formatted string.
        Args:
            steps (Union[str, List[Tuple[AgentAction, str]]]): The agent trajectory.
        Returns:
            str: The formatted agent trajectory.
        """
        if isinstance(steps, str):
            return steps
        return "\n\n".join(
            [
                f"""Step {i}:
Tool used: {action.tool}
Tool input: {action.tool_input}
Tool output: {output}"""
                for i, (action, output) in enumerate(steps, 1)
            ]
        )
    @staticmethod
    def _format_reference(reference: Optional[str]) -> str:
        """Format the reference text.
        Args:
            reference (str): The reference text.
        Returns:
            str: The formatted reference text.
        """
        if not reference:
            return ""
        return f"""
The following is the expected answer. Use this to measure correctness:
[GROUND_TRUTH]
{reference}
[END_GROUND_TRUTH]
"""
[docs]    @classmethod
    def from_llm(
        cls,
        llm: BaseLanguageModel,
        agent_tools: Optional[Sequence[BaseTool]] = None,
        output_parser: Optional[TrajectoryOutputParser] = None,
        **kwargs: Any,
    ) -> "TrajectoryEvalChain":
        """Create a TrajectoryEvalChain object from a language model chain.
        Args:
            llm (BaseChatModel): The language model chain.
            agent_tools (Optional[Sequence[BaseTool]]): A list of tools
                available to the agent.
            output_parser (Optional[TrajectoryOutputParser]): The output parser
                used to parse the chain output into a score.


used to parse the chain output into a score.
        Returns:
            TrajectoryEvalChain: The TrajectoryEvalChain object.
        """
        if not isinstance(llm, BaseChatModel):
            raise NotImplementedError(
                "Only chat models supported by the current trajectory eval"
            )
        if agent_tools:
            prompt = EVAL_CHAT_PROMPT
        else:
            prompt = TOOL_FREE_EVAL_CHAT_PROMPT
        eval_chain = LLMChain(llm=llm, prompt=prompt)
        return cls(
            agent_tools=agent_tools,
            eval_chain=eval_chain,
            output_parser=output_parser or TrajectoryOutputParser(),
            **kwargs,
        )
    @property
    def input_keys(self) -> List[str]:
        """Get the input keys for the chain.
        Returns:
            List[str]: The input keys.
        """
        return ["question", "agent_trajectory", "answer", "reference"]
    @property
    def output_keys(self) -> List[str]:
        """Get the output keys for the chain.
        Returns:
            List[str]: The output keys.
        """
        return ["score", "reasoning"]
[docs]    def prep_inputs(self, inputs: Union[Dict[str, Any], Any]) -> Dict[str, str]:
        """Validate and prep inputs."""
        if "reference" not in inputs:
            inputs["reference"] = self._format_reference(inputs.get("reference"))
        return super().prep_inputs(inputs)
    def _call(
        self,
        inputs: Dict[str, str],
        run_manager: Optional[CallbackManagerForChainRun] = None,
    ) -> Dict[str, Any]:
        """Run the chain and generate the output.
        Args:


"""Run the chain and generate the output.
        Args:
            inputs (Dict[str, str]): The input values for the chain.
            run_manager (Optional[CallbackManagerForChainRun]): The callback
                manager for the chain run.
        Returns:
            Dict[str, Any]: The output values of the chain.
        """
        chain_input = {**inputs}
        if self.agent_tools:
            chain_input["tool_descriptions"] = self._tools_description
        _run_manager = run_manager or CallbackManagerForChainRun.get_noop_manager()
        raw_output = self.eval_chain.run(
            chain_input, callbacks=_run_manager.get_child()
        )
        return cast(dict, self.output_parser.parse(raw_output))
    async def _acall(
        self,
        inputs: Dict[str, str],
        run_manager: Optional[AsyncCallbackManagerForChainRun] = None,
    ) -> Dict[str, Any]:
        """Run the chain and generate the output.
        Args:
            inputs (Dict[str, str]): The input values for the chain.
            run_manager (Optional[CallbackManagerForChainRun]): The callback
                manager for the chain run.
        Returns:
            Dict[str, Any]: The output values of the chain.
        """
        chain_input = {**inputs}
        if self.agent_tools:
            chain_input["tool_descriptions"] = self._tools_description
        _run_manager = run_manager or AsyncCallbackManagerForChainRun.get_noop_manager()
        raw_output = await self.eval_chain.arun(
            chain_input, callbacks=_run_manager.get_child()
        )
        return cast(dict, self.output_parser.parse(raw_output))
    def _evaluate_agent_trajectory(
        self,
        *,
        prediction: str,
        input: str,


self,
        *,
        prediction: str,
        input: str,
        agent_trajectory: Sequence[Tuple[AgentAction, str]],
        reference: Optional[str] = None,
        callbacks: Callbacks = None,
        tags: Optional[List[str]] = None,
        metadata: Optional[Dict[str, Any]] = None,
        include_run_info: bool = False,
        **kwargs: Any,
    ) -> dict:
        """Evaluate a trajectory.
        Args:
            prediction (str): The final predicted response.
            input (str): The input to the agent.
            agent_trajectory (List[Tuple[AgentAction, str]]):
                The intermediate steps forming the agent trajectory.
            reference (Optional[str]): The reference answer.
            callbacks (Callbacks): Callbacks to use for this chain run.
        Returns:
            dict: The evaluation result, which includes the score and optionally
                the reasoning for reaching that.
        """
        inputs = {
            "question": input,
            "agent_trajectory": self.get_agent_trajectory(agent_trajectory),
            "answer": prediction,
            "reference": reference,
        }
        return self.__call__(
            inputs=inputs,
            callbacks=callbacks,
            tags=tags,
            metadata=metadata,
            include_run_info=include_run_info,
            return_only_outputs=True,
        )
    async def _aevaluate_agent_trajectory(
        self,
        *,
        prediction: str,
        input: str,
        agent_trajectory: Sequence[Tuple[AgentAction, str]],
        reference: Optional[str] = None,
        callbacks: Callbacks = None,
        tags: Optional[List[str]] = None,
        metadata: Optional[Dict[str, Any]] = None,


metadata: Optional[Dict[str, Any]] = None,
        include_run_info: bool = False,
        **kwargs: Any,
    ) -> dict:
        """Asynchronously evaluate a trajectory.
        Args:
            prediction (str): The final predicted response.
            input (str): The input to the agent.
            agent_trajectory (List[Tuple[AgentAction, str]]):
                The intermediate steps forming the agent trajectory.
            reference (Optional[str]): The reference answer.
            callbacks (Callbacks): Callbacks to use for this chain run.
        Returns:
            dict: The evaluation result, which includes the score and optionally
                the reasoning for reaching that.
        """
        inputs = {
            "question": input,
            "agent_trajectory": self.get_agent_trajectory(agent_trajectory),
            "answer": prediction,
            "reference": reference,
        }
        return await self.acall(
            inputs=inputs,
            callbacks=callbacks,
            tags=tags,
            metadata=metadata,
            include_run_info=include_run_info,
            return_only_outputs=True,
        )


Source code for langchain.evaluation.scoring.eval_chain
"""Base classes for scoring the output of a model on a scale of 1-10."""
from __future__ import annotations
import logging
import re
from typing import Any, Dict, List, Optional, Union
from langchain_core.language_models import BaseLanguageModel
from langchain_core.output_parsers import BaseOutputParser
from langchain_core.prompts.prompt import PromptTemplate
from langchain_core.pydantic_v1 import Extra, Field
from langchain.callbacks.manager import Callbacks
from langchain.chains.constitutional_ai.models import ConstitutionalPrinciple
from langchain.chains.llm import LLMChain
from langchain.chat_models.azure_openai import AzureChatOpenAI
from langchain.chat_models.openai import ChatOpenAI
from langchain.evaluation.criteria.eval_chain import (
    CRITERIA_TYPE,
    Criteria,
)
from langchain.evaluation.schema import LLMEvalChain, StringEvaluator
from langchain.evaluation.scoring.prompt import (
    CRITERIA_INSTRUCTIONS,
    DEFAULT_CRITERIA,
    SCORING_TEMPLATE,
    SCORING_TEMPLATE_WITH_REFERENCE,
)
from langchain.schema import RUN_KEY
logger = logging.getLogger(__name__)
_FIND_DOUBLE_BRACKETS = re.compile(r"\[\[(.*?)\]\]")
_SUPPORTED_CRITERIA = {
    Criteria.CONCISENESS: "Is the submission concise and to the point?",
    Criteria.RELEVANCE: "Is the submission referring to a real quote from the text?",
    Criteria.CORRECTNESS: "Is the submission correct, accurate, and factual?",
    Criteria.COHERENCE: "Is the submission coherent, well-structured, and organized?",
    Criteria.HARMFULNESS: "Is the submission harmful, offensive, or inappropriate?",


Criteria.HARMFULNESS: "Is the submission harmful, offensive, or inappropriate?",
    Criteria.MALICIOUSNESS: "Is the submission malicious in any way?",
    Criteria.HELPFULNESS: "Is the submission helpful, insightful, and appropriate?",
    Criteria.CONTROVERSIALITY: "Is the submission controversial or debatable?",
    Criteria.MISOGYNY: "Is the submission misogynistic or sexist?",
    Criteria.CRIMINALITY: "Is the submission criminal in any way?",
    Criteria.INSENSITIVITY: "Is the submission insensitive to any group of people?",
    Criteria.DEPTH: "Does the submission demonstrate depth of thought?",
    Criteria.CREATIVITY: "Does the submission demonstrate novelty or unique ideas?",
    Criteria.DETAIL: "Does the submission demonstrate attention to detail?",
}
[docs]def resolve_criteria(
    criteria: Optional[Union[CRITERIA_TYPE, str, List[CRITERIA_TYPE]]],
) -> dict:
    """Resolve the criteria for the pairwise evaluator.
    Args:
        criteria (Union[CRITERIA_TYPE, str], optional): The criteria to use.
    Returns:
        dict: The resolved criteria.
    """
    if criteria is None:
        _default_criteria = [
            Criteria.HELPFULNESS,
            Criteria.RELEVANCE,
            Criteria.CORRECTNESS,
            Criteria.DEPTH,
        ]
        return {k.value: _SUPPORTED_CRITERIA[k] for k in _default_criteria}
    elif isinstance(criteria, Criteria):
        criteria_ = {criteria.value: _SUPPORTED_CRITERIA[criteria]}
    elif isinstance(criteria, str):
        if criteria in _SUPPORTED_CRITERIA:
            criteria_ = {criteria: _SUPPORTED_CRITERIA[Criteria(criteria)]}


criteria_ = {criteria: _SUPPORTED_CRITERIA[Criteria(criteria)]}
        else:
            criteria_ = {criteria: ""}
    elif isinstance(criteria, ConstitutionalPrinciple):
        criteria_ = {criteria.name: criteria.critique_request}
    elif isinstance(criteria, (list, tuple)):
        criteria_ = {
            k: v
            for criterion in criteria
            for k, v in resolve_criteria(criterion).items()
        }
    else:
        if not criteria:
            raise ValueError(
                "Criteria cannot be empty. "
                "Please provide a criterion name or a mapping of the criterion name"
                " to its description."
            )
        criteria_ = dict(criteria)
    return criteria_
[docs]class ScoreStringResultOutputParser(BaseOutputParser[dict]):
    """A parser for the output of the ScoreStringEvalChain.
    Attributes:
        _type (str): The type of the output parser.
    """
    @property
    def _type(self) -> str:
        """Return the type of the output parser.
        Returns:
            str: The type of the output parser.
        """
        return "pairwise_string_result"
[docs]    def parse(self, text: str) -> Dict[str, Any]:
        """Parse the output text.
        Args:
            text (str): The output text to parse.
        Returns:
            Dict: The parsed output.
        Raises:
            ValueError: If the verdict is invalid.
        """
        match = _FIND_DOUBLE_BRACKETS.search(text)
        if match:
            verdict = match.group(1)
        if not match or verdict not in list("123456789") + ["10"]:
            raise ValueError(


raise ValueError(
                f"Invalid output: {text}. "
                "Output must contain a double bracketed string\
                 with the verdict between 1 and 10."
            )
        return {
            "reasoning": text,
            "score": int(verdict),
        }
[docs]class ScoreStringEvalChain(StringEvaluator, LLMEvalChain, LLMChain):
    """A chain for scoring on a scale of 1-10 the output of a model.
    Attributes:
        output_parser (BaseOutputParser): The output parser for the chain.
    Example:
        >>> from langchain.chat_models import ChatOpenAI
        >>> from langchain.evaluation.scoring import ScoreStringEvalChain
        >>> llm = ChatOpenAI(temperature=0, model_name="gpt-4")
        >>> chain = ScoreStringEvalChain.from_llm(llm=llm)
        >>> result = chain.evaluate_strings(
        ...     input = "What is the chemical formula for water?",
        ...     prediction = "H2O",
        ...     reference = "The chemical formula for water is H2O.",
        ... )
        >>> print(result)
        # {
        #    "score": 8,
        #    "comment": "The response accurately states "
        #    "that the chemical formula for water is H2O."
        #    "However, it does not provide an explanation of what the formula means."
        # }
    """
    output_key: str = "results"  #: :meta private:
    output_parser: BaseOutputParser = Field(
        default_factory=ScoreStringResultOutputParser
    )
    normalize_by: Optional[float] = None
    """The value to normalize the score by, if specified."""


"""The value to normalize the score by, if specified."""
    criterion_name: str
    """The name of the criterion being evaluated."""
    class Config:
        """Configuration for the ScoreStringEvalChain."""
        extra = Extra.ignore
[docs]    @classmethod
    def is_lc_serializable(cls) -> bool:
        return False
    @property
    def requires_reference(self) -> bool:
        """Return whether the chain requires a reference.
        Returns:
            bool: True if the chain requires a reference, False otherwise.
        """
        return False
    @property
    def requires_input(self) -> bool:
        """Return whether the chain requires an input.
        Returns:
            bool: True if the chain requires an input, False otherwise.
        """
        return True
    @property
    def evaluation_name(self) -> str:
        """Get the name of the evaluation.
        Returns
        -------
        str
            The name of the evaluation.
        """
        return f"score_string:{self.criterion_name}"
    @property
    def _skip_reference_warning(self) -> str:
        """Return the warning to show when reference is ignored.
        Returns:
            str: The warning to show when reference is ignored.
        """
        return (
            f"Ignoring reference in {self.__class__.__name__}, as it is not expected."
            "\nTo use a reference, use the LabeledScoreStringEvalChain instead."
            " (EvaluatorType.LABELED_SCORE_STRING) instead."
        )
[docs]    @classmethod
    def from_llm(
        cls,
        llm: BaseLanguageModel,
        *,
        prompt: Optional[PromptTemplate] = None,


*,
        prompt: Optional[PromptTemplate] = None,
        criteria: Optional[Union[CRITERIA_TYPE, str]] = None,
        normalize_by: Optional[float] = None,
        **kwargs: Any,
    ) -> ScoreStringEvalChain:
        """Initialize the ScoreStringEvalChain from an LLM.
        Args:
            llm (BaseChatModel): The LLM to use (GPT-4 recommended).
            prompt (PromptTemplate, optional): The prompt to use.
            **kwargs (Any): Additional keyword arguments.
        Returns:
            ScoreStringEvalChain: The initialized ScoreStringEvalChain.
        Raises:
            ValueError: If the input variables are not as expected.
        """
        if not (
            isinstance(llm, (ChatOpenAI, AzureChatOpenAI))
            and llm.model_name.startswith("gpt-4")
        ):
            logger.warning(
                "This chain was only tested with GPT-4. \
Performance may be significantly worse with other models."
            )
        expected_input_vars = {"prediction", "input", "criteria"}
        prompt_ = prompt or SCORING_TEMPLATE.partial(reference="")
        if expected_input_vars != set(prompt_.input_variables):
            raise ValueError(
                f"Input variables should be {expected_input_vars}, "
                f"but got {prompt_.input_variables}"
            )
        criteria_ = resolve_criteria(criteria)
        criteria_str = "\n".join(
            f"{k}: {v}" if v else k for k, v in criteria_.items()
        ).strip()
        criteria_str = (
            CRITERIA_INSTRUCTIONS + f"{criteria_str}\n"
            if criteria_str
            else DEFAULT_CRITERIA
        )
        return cls(


if criteria_str
            else DEFAULT_CRITERIA
        )
        return cls(
            llm=llm,
            prompt=prompt_.partial(criteria=criteria_str),
            normalize_by=normalize_by,
            criterion_name="-".join(criteria_),
            **kwargs,
        )
    def _prepare_input(
        self,
        prediction: str,
        input: Optional[str],
        reference: Optional[str],
    ) -> dict:
        """Prepare the input for the chain.
        Args:
            prediction (str): The output string from the first model.
            prediction_b (str): The output string from the second model.
            input (str, optional): The input or task string.
            reference (str, optional): The reference string, if any.
        Returns:
            dict: The prepared input for the chain.
        """
        input_ = {
            "prediction": prediction,
            "input": input,
        }
        if self.requires_reference:
            input_["reference"] = reference
        return input_
    def _prepare_output(self, result: dict) -> dict:
        """Prepare the output."""
        parsed = result[self.output_key]
        if RUN_KEY in result:
            parsed[RUN_KEY] = result[RUN_KEY]
        if "score" in parsed and self.normalize_by is not None:
            parsed["score"] = parsed["score"] / self.normalize_by
        return parsed
    def _evaluate_strings(
        self,
        *,
        prediction: str,
        input: Optional[str] = None,
        reference: Optional[str] = None,
        callbacks: Callbacks = None,
        tags: Optional[List[str]] = None,
        metadata: Optional[Dict[str, Any]] = None,


metadata: Optional[Dict[str, Any]] = None,
        include_run_info: bool = False,
        **kwargs: Any,
    ) -> dict:
        """Score the output string.
        Args:
            prediction (str): The output string from the first model.
            input (str, optional): The input or task string.
            callbacks (Callbacks, optional): The callbacks to use.
            reference (str, optional): The reference string, if any.
            **kwargs (Any): Additional keyword arguments.
        Returns:
            dict: A dictionary containing:
                - reasoning: The reasoning for the preference.
                - score: A score between 1 and 10.
        """
        input_ = self._prepare_input(prediction, input, reference)
        result = self(
            inputs=input_,
            callbacks=callbacks,
            tags=tags,
            metadata=metadata,
            include_run_info=include_run_info,
        )
        return self._prepare_output(result)
    async def _aevaluate_string_pairs(
        self,
        *,
        prediction: str,
        reference: Optional[str] = None,
        input: Optional[str] = None,
        callbacks: Callbacks = None,
        tags: Optional[List[str]] = None,
        metadata: Optional[Dict[str, Any]] = None,
        include_run_info: bool = False,
        **kwargs: Any,
    ) -> dict:
        """Asynchronously score the output string.
        Args:
            prediction (str): The output string from the first model.
            input (str, optional): The input or task string.
            callbacks (Callbacks, optional): The callbacks to use.
            reference (str, optional): The reference string, if any.


reference (str, optional): The reference string, if any.
            **kwargs (Any): Additional keyword arguments.
        Returns:
            dict: A dictionary containing:
                - reasoning: The reasoning for the preference.
                - score: A score between 1 and 10.
        """
        input_ = self._prepare_input(prediction, input, reference)
        result = await self.acall(
            inputs=input_,
            callbacks=callbacks,
            tags=tags,
            metadata=metadata,
            include_run_info=include_run_info,
        )
        return self._prepare_output(result)
[docs]class LabeledScoreStringEvalChain(ScoreStringEvalChain):
    """A chain for scoring the output of a model on a scale of 1-10.
    Attributes:
        output_parser (BaseOutputParser): The output parser for the chain.
    """
    @property
    def requires_reference(self) -> bool:
        """Return whether the chain requires a reference.
        Returns:
            bool: True if the chain requires a reference, False otherwise.
        """
        return True
[docs]    @classmethod
    def from_llm(
        cls,
        llm: BaseLanguageModel,
        *,
        prompt: Optional[PromptTemplate] = None,
        criteria: Optional[Union[CRITERIA_TYPE, str]] = None,
        normalize_by: Optional[float] = None,
        **kwargs: Any,
    ) -> LabeledScoreStringEvalChain:
        """Initialize the LabeledScoreStringEvalChain from an LLM.
        Args:
            llm (BaseLanguageModel): The LLM to use.
            prompt (PromptTemplate, optional): The prompt to use.


prompt (PromptTemplate, optional): The prompt to use.
            criteria (Union[CRITERIA_TYPE, str], optional): The criteria to use.
            normalize_by (float, optional): The value to normalize the score by.
            **kwargs (Any): Additional keyword arguments.
        Returns:
            LabeledScoreStringEvalChain: The initialized LabeledScoreStringEvalChain.
        Raises:
            ValueError: If the input variables are not as expected.
        """  # noqa: E501
        expected_input_vars = {
            "prediction",
            "input",
            "reference",
            "criteria",
        }
        prompt_ = prompt or SCORING_TEMPLATE_WITH_REFERENCE
        if expected_input_vars != set(prompt_.input_variables):
            raise ValueError(
                f"Input variables should be {expected_input_vars}, "
                f"but got {prompt_.input_variables}"
            )
        criteria_ = resolve_criteria(criteria)
        criteria_str = "\n".join(f"{k}: {v}" for k, v in criteria_.items()).strip()
        criteria_str = (
            CRITERIA_INSTRUCTIONS + f"{criteria_str}\n"
            if criteria_str
            else DEFAULT_CRITERIA
        )
        return cls(
            llm=llm,
            prompt=prompt_.partial(criteria=criteria_str),
            normalize_by=normalize_by,
            criterion_name="-".join(criteria_),
            **kwargs,
        )


Source code for langchain.evaluation.string_distance.base
"""String distance evaluators based on the RapidFuzz library."""
from enum import Enum
from typing import Any, Callable, Dict, List, Optional
from langchain_core.pydantic_v1 import Field, root_validator
from langchain.callbacks.manager import (
    AsyncCallbackManagerForChainRun,
    CallbackManagerForChainRun,
    Callbacks,
)
from langchain.chains.base import Chain
from langchain.evaluation.schema import PairwiseStringEvaluator, StringEvaluator
from langchain.schema import RUN_KEY
def _load_rapidfuzz() -> Any:
    """
    Load the RapidFuzz library.
    Raises:
        ImportError: If the rapidfuzz library is not installed.
    Returns:
        Any: The rapidfuzz.distance module.
    """
    try:
        import rapidfuzz
    except ImportError:
        raise ImportError(
            "Please install the rapidfuzz library to use the FuzzyMatchStringEvaluator."
            "Please install it with `pip install rapidfuzz`."
        )
    return rapidfuzz.distance
[docs]class StringDistance(str, Enum):
    """Distance metric to use.
    Attributes:
        DAMERAU_LEVENSHTEIN: The Damerau-Levenshtein distance.
        LEVENSHTEIN: The Levenshtein distance.
        JARO: The Jaro distance.
        JARO_WINKLER: The Jaro-Winkler distance.
        HAMMING: The Hamming distance.
        INDEL: The Indel distance.
    """
    DAMERAU_LEVENSHTEIN = "damerau_levenshtein"
    LEVENSHTEIN = "levenshtein"
    JARO = "jaro"


JARO = "jaro"
    JARO_WINKLER = "jaro_winkler"
    HAMMING = "hamming"
    INDEL = "indel"
class _RapidFuzzChainMixin(Chain):
    """Shared methods for the rapidfuzz string distance evaluators."""
    distance: StringDistance = Field(default=StringDistance.JARO_WINKLER)
    normalize_score: bool = Field(default=True)
    """Whether to normalize the score to a value between 0 and 1.
    Applies only to the Levenshtein and Damerau-Levenshtein distances."""
    @root_validator
    def validate_dependencies(cls, values: Dict[str, Any]) -> Dict[str, Any]:
        """
        Validate that the rapidfuzz library is installed.
        Args:
            values (Dict[str, Any]): The input values.
        Returns:
            Dict[str, Any]: The validated values.
        """
        _load_rapidfuzz()
        return values
    @property
    def output_keys(self) -> List[str]:
        """
        Get the output keys.
        Returns:
            List[str]: The output keys.
        """
        return ["score"]
    def _prepare_output(self, result: Dict[str, Any]) -> Dict[str, Any]:
        """
        Prepare the output dictionary.
        Args:
            result (Dict[str, Any]): The evaluation results.
        Returns:
            Dict[str, Any]: The prepared output dictionary.
        """
        result = {"score": result["score"]}
        if RUN_KEY in result:
            result[RUN_KEY] = result[RUN_KEY].dict()
        return result
    @staticmethod


return result
    @staticmethod
    def _get_metric(distance: str, normalize_score: bool = False) -> Callable:
        """
        Get the distance metric function based on the distance type.
        Args:
            distance (str): The distance type.
        Returns:
            Callable: The distance metric function.
        Raises:
            ValueError: If the distance metric is invalid.
        """
        from rapidfuzz import distance as rf_distance
        module_map: Dict[str, Any] = {
            StringDistance.DAMERAU_LEVENSHTEIN: rf_distance.DamerauLevenshtein,
            StringDistance.LEVENSHTEIN: rf_distance.Levenshtein,
            StringDistance.JARO: rf_distance.Jaro,
            StringDistance.JARO_WINKLER: rf_distance.JaroWinkler,
            StringDistance.HAMMING: rf_distance.Hamming,
            StringDistance.INDEL: rf_distance.Indel,
        }
        if distance not in module_map:
            raise ValueError(
                f"Invalid distance metric: {distance}"
                f"\nMust be one of: {list(StringDistance)}"
            )
        module = module_map[distance]
        if normalize_score:
            return module.normalized_distance
        else:
            return module.distance
    @property
    def metric(self) -> Callable:
        """
        Get the distance metric function.
        Returns:
            Callable: The distance metric function.
        """
        return _RapidFuzzChainMixin._get_metric(
            self.distance, normalize_score=self.normalize_score
        )
    def compute_metric(self, a: str, b: str) -> float:
        """
        Compute the distance between two strings.
        Args:
            a (str): The first string.


Args:
            a (str): The first string.
            b (str): The second string.
        Returns:
            float: The distance between the two strings.
        """
        return self.metric(a, b)
[docs]class StringDistanceEvalChain(StringEvaluator, _RapidFuzzChainMixin):
    """Compute string distances between the prediction and the reference.
    Examples
    ----------
    >>> from langchain.evaluation import StringDistanceEvalChain
    >>> evaluator = StringDistanceEvalChain()
    >>> evaluator.evaluate_strings(
            prediction="Mindy is the CTO",
            reference="Mindy is the CEO",
        )
    Using the `load_evaluator` function:
    >>> from langchain.evaluation import load_evaluator
    >>> evaluator = load_evaluator("string_distance")
    >>> evaluator.evaluate_strings(
            prediction="The answer is three",
            reference="three",
        )
    """
    @property
    def requires_input(self) -> bool:
        """
        This evaluator does not require input.
        """
        return False
    @property
    def requires_reference(self) -> bool:
        """
        This evaluator does not require a reference.
        """
        return True
    @property
    def input_keys(self) -> List[str]:
        """
        Get the input keys.
        Returns:
            List[str]: The input keys.
        """
        return ["reference", "prediction"]
    @property
    def evaluation_name(self) -> str:
        """
        Get the evaluation name.
        Returns:
            str: The evaluation name.
        """
        return f"{self.distance.value}_distance"
    def _call(
        self,
        inputs: Dict[str, Any],


def _call(
        self,
        inputs: Dict[str, Any],
        run_manager: Optional[CallbackManagerForChainRun] = None,
    ) -> Dict[str, Any]:
        """
        Compute the string distance between the prediction and the reference.
        Args:
            inputs (Dict[str, Any]): The input values.
            run_manager (Optional[CallbackManagerForChainRun]):
                The callback manager.
        Returns:
            Dict[str, Any]: The evaluation results containing the score.
        """
        return {"score": self.compute_metric(inputs["reference"], inputs["prediction"])}
    async def _acall(
        self,
        inputs: Dict[str, Any],
        run_manager: Optional[AsyncCallbackManagerForChainRun] = None,
    ) -> Dict[str, Any]:
        """
        Asynchronously compute the string distance between the prediction
            and the reference.
        Args:
            inputs (Dict[str, Any]): The input values.
            run_manager (Optional[AsyncCallbackManagerForChainRun]:
                The callback manager.
        Returns:
            Dict[str, Any]: The evaluation results containing the score.
        """
        return {"score": self.compute_metric(inputs["reference"], inputs["prediction"])}
    def _evaluate_strings(
        self,
        *,
        prediction: str,
        reference: Optional[str] = None,
        input: Optional[str] = None,
        callbacks: Callbacks = None,
        tags: Optional[List[str]] = None,
        metadata: Optional[Dict[str, Any]] = None,
        include_run_info: bool = False,
        **kwargs: Any,
    ) -> dict:
        """
        Evaluate the string distance between the prediction and the reference.
        Args:


"""
        Evaluate the string distance between the prediction and the reference.
        Args:
            prediction (str): The prediction string.
            reference (Optional[str], optional): The reference string.
            input (Optional[str], optional): The input string.
            callbacks (Callbacks, optional): The callbacks to use.
            **kwargs: Additional keyword arguments.
        Returns:
            dict: The evaluation results containing the score.
        """
        result = self(
            inputs={"prediction": prediction, "reference": reference},
            callbacks=callbacks,
            tags=tags,
            metadata=metadata,
            include_run_info=include_run_info,
        )
        return self._prepare_output(result)
    async def _aevaluate_strings(
        self,
        *,
        prediction: str,
        reference: Optional[str] = None,
        input: Optional[str] = None,
        callbacks: Callbacks = None,
        tags: Optional[List[str]] = None,
        metadata: Optional[Dict[str, Any]] = None,
        include_run_info: bool = False,
        **kwargs: Any,
    ) -> dict:
        """
        Asynchronously evaluate the string distance between the
            prediction and the reference.
        Args:
            prediction (str): The prediction string.
            reference (Optional[str], optional): The reference string.
            input (Optional[str], optional): The input string.
            callbacks (Callbacks, optional): The callbacks to use.
            **kwargs: Additional keyword arguments.
        Returns:
            dict: The evaluation results containing the score.
        """
        result = await self.acall(
            inputs={"prediction": prediction, "reference": reference},
            callbacks=callbacks,
            tags=tags,
            metadata=metadata,


callbacks=callbacks,
            tags=tags,
            metadata=metadata,
            include_run_info=include_run_info,
        )
        return self._prepare_output(result)
[docs]class PairwiseStringDistanceEvalChain(PairwiseStringEvaluator, _RapidFuzzChainMixin):
    """Compute string edit distances between two predictions."""
    @property
    def input_keys(self) -> List[str]:
        """
        Get the input keys.
        Returns:
            List[str]: The input keys.
        """
        return ["prediction", "prediction_b"]
    @property
    def evaluation_name(self) -> str:
        """
        Get the evaluation name.
        Returns:
            str: The evaluation name.
        """
        return f"pairwise_{self.distance.value}_distance"
    def _call(
        self,
        inputs: Dict[str, Any],
        run_manager: Optional[CallbackManagerForChainRun] = None,
    ) -> Dict[str, Any]:
        """
        Compute the string distance between two predictions.
        Args:
            inputs (Dict[str, Any]): The input values.
            run_manager (CallbackManagerForChainRun , optional):
                The callback manager.
        Returns:
            Dict[str, Any]: The evaluation results containing the score.
        """
        return {
            "score": self.compute_metric(inputs["prediction"], inputs["prediction_b"])
        }
    async def _acall(
        self,
        inputs: Dict[str, Any],
        run_manager: Optional[AsyncCallbackManagerForChainRun] = None,
    ) -> Dict[str, Any]:
        """
        Asynchronously compute the string distance between two predictions.
        Args:
            inputs (Dict[str, Any]): The input values.


Args:
            inputs (Dict[str, Any]): The input values.
            run_manager (AsyncCallbackManagerForChainRun , optional):
                The callback manager.
        Returns:
            Dict[str, Any]: The evaluation results containing the score.
        """
        return {
            "score": self.compute_metric(inputs["prediction"], inputs["prediction_b"])
        }
    def _evaluate_string_pairs(
        self,
        *,
        prediction: str,
        prediction_b: str,
        callbacks: Callbacks = None,
        tags: Optional[List[str]] = None,
        metadata: Optional[Dict[str, Any]] = None,
        include_run_info: bool = False,
        **kwargs: Any,
    ) -> dict:
        """
        Evaluate the string distance between two predictions.
        Args:
            prediction (str): The first prediction string.
            prediction_b (str): The second prediction string.
            callbacks (Callbacks, optional): The callbacks to use.
            tags (List[str], optional): Tags to apply to traces.
            metadata (Dict[str, Any], optional): Metadata to apply to traces.
            **kwargs: Additional keyword arguments.
        Returns:
            dict: The evaluation results containing the score.
        """
        result = self(
            inputs={"prediction": prediction, "prediction_b": prediction_b},
            callbacks=callbacks,
            tags=tags,
            metadata=metadata,
            include_run_info=include_run_info,
        )
        return self._prepare_output(result)
    async def _aevaluate_string_pairs(
        self,
        *,
        prediction: str,
        prediction_b: str,
        callbacks: Callbacks = None,
        tags: Optional[List[str]] = None,


callbacks: Callbacks = None,
        tags: Optional[List[str]] = None,
        metadata: Optional[Dict[str, Any]] = None,
        include_run_info: bool = False,
        **kwargs: Any,
    ) -> dict:
        """
        Asynchronously evaluate the string distance between two predictions.
        Args:
            prediction (str): The first prediction string.
            prediction_b (str): The second prediction string.
            callbacks (Callbacks, optional): The callbacks to use.
            tags (List[str], optional): Tags to apply to traces.
            metadata (Dict[str, Any], optional): Metadata to apply to traces.
            **kwargs: Additional keyword arguments.
        Returns:
            dict: The evaluation results containing the score.
        """
        result = await self.acall(
            inputs={"prediction": prediction, "prediction_b": prediction_b},
            callbacks=callbacks,
            tags=tags,
            metadata=metadata,
            include_run_info=include_run_info,
        )
        return self._prepare_output(result)


Source code for langchain.evaluation.regex_match.base
import re
from typing import Any, List
from langchain.evaluation.schema import StringEvaluator
[docs]class RegexMatchStringEvaluator(StringEvaluator):
    """Compute a regex match between the prediction and the reference.
    Examples
    ----------
    >>> evaluator = RegexMatchStringEvaluator(flags=re.IGNORECASE)
    >>> evaluator.evaluate_strings(
            prediction="Mindy is the CTO",
            reference="^mindy.*cto$",
        )  # This will return {'score': 1.0} due to the IGNORECASE flag
    >>> evaluator = RegexMatchStringEvaluator()
    >>> evaluator.evaluate_strings(
            prediction="Mindy is the CTO",
            reference="^Mike.*CEO$",
        )  # This will return {'score': 0.0}
    >>> evaluator.evaluate_strings(
            prediction="Mindy is the CTO",
            reference="^Mike.*CEO$|^Mindy.*CTO$",
        )  # This will return {'score': 1.0} as the prediction matches the second pattern in the union
    """  # noqa: E501
[docs]    def __init__(self, *, flags: int = 0, **kwargs: Any):  # Default is no flags
        super().__init__()
        self.flags = flags
    @property
    def requires_input(self) -> bool:
        """
        This evaluator does not require input.
        """
        return False
    @property
    def requires_reference(self) -> bool:
        """
        This evaluator requires a reference.
        """
        return True
    @property
    def input_keys(self) -> List[str]:
        """
        Get the input keys.
        Returns:
            List[str]: The input keys.
        """


Returns:
            List[str]: The input keys.
        """
        return ["reference", "prediction"]
    @property
    def evaluation_name(self) -> str:
        """
        Get the evaluation name.
        Returns:
            str: The evaluation name.
        """
        return "regex_match"
    def _evaluate_strings(  # type: ignore[arg-type,override]
        self,
        *,
        prediction: str,
        reference: str,
        **kwargs: Any,
    ) -> dict:
        """
        Evaluate the regex match between the prediction and the reference.
        Args:
            prediction (str): The prediction string.
            reference (Optional[str], optional): The reference regex pattern.
        Returns:
            dict: The evaluation results containing the score.
        """
        match = re.match(reference, prediction, flags=self.flags)
        return {"score": int(bool(match))}


Source code for langchain.evaluation.qa.eval_chain
"""LLM Chains for evaluating question answering."""
from __future__ import annotations
import re
import string
from typing import Any, List, Optional, Sequence, Tuple
from langchain_core.language_models import BaseLanguageModel
from langchain_core.prompts import PromptTemplate
from langchain_core.pydantic_v1 import Extra
from langchain.callbacks.manager import Callbacks
from langchain.chains.llm import LLMChain
from langchain.evaluation.qa.eval_prompt import CONTEXT_PROMPT, COT_PROMPT, PROMPT
from langchain.evaluation.schema import LLMEvalChain, StringEvaluator
from langchain.schema import RUN_KEY
def _get_score(text: str) -> Optional[Tuple[str, int]]:
    match = re.search(r"grade:\s*(correct|incorrect)", text.strip(), re.IGNORECASE)
    if match:
        if match.group(1).upper() == "CORRECT":
            return "CORRECT", 1
        elif match.group(1).upper() == "INCORRECT":
            return "INCORRECT", 0
    try:
        first_word = (
            text.strip().split()[0].translate(str.maketrans("", "", string.punctuation))
        )
        if first_word.upper() == "CORRECT":
            return "CORRECT", 1
        elif first_word.upper() == "INCORRECT":
            return "INCORRECT", 0
        last_word = (
            text.strip()
            .split()[-1]
            .translate(str.maketrans("", "", string.punctuation))
        )
        if last_word.upper() == "CORRECT":
            return "CORRECT", 1
        elif last_word.upper() == "INCORRECT":


elif last_word.upper() == "INCORRECT":
            return "INCORRECT", 0
    except IndexError:
        pass
    return None
def _parse_string_eval_output(text: str) -> dict:
    """Parse the output text.
    Args:
        text (str): The output text to parse.
    Returns:
        Any: The parsed output.
    """
    reasoning = text.strip()
    parsed_scores = _get_score(reasoning)
    if parsed_scores is None:
        value, score = None, None
    else:
        value, score = parsed_scores
    return {
        "reasoning": reasoning,
        "value": value,
        "score": score,
    }
[docs]class QAEvalChain(LLMChain, StringEvaluator, LLMEvalChain):
    """LLM Chain for evaluating question answering."""
    output_key: str = "results"  #: :meta private:
    class Config:
        """Configuration for the QAEvalChain."""
        extra = Extra.ignore
[docs]    @classmethod
    def is_lc_serializable(cls) -> bool:
        return False
    @property
    def evaluation_name(self) -> str:
        return "correctness"
    @property
    def requires_reference(self) -> bool:
        return True
    @property
    def requires_input(self) -> bool:
        return True
[docs]    @classmethod
    def from_llm(
        cls,
        llm: BaseLanguageModel,
        prompt: Optional[PromptTemplate] = None,
        **kwargs: Any,
    ) -> QAEvalChain:
        """Load QA Eval Chain from LLM.
        Args:
            llm (BaseLanguageModel): the base language model to use.


Args:
            llm (BaseLanguageModel): the base language model to use.
            prompt (PromptTemplate): A prompt template containing the input_variables:
            'input', 'answer' and 'result' that will be used as the prompt
            for evaluation.
            Defaults to PROMPT.
            **kwargs: additional keyword arguments.
        Returns:
            QAEvalChain: the loaded QA eval chain.
        """
        prompt = prompt or PROMPT
        expected_input_vars = {"query", "answer", "result"}
        if expected_input_vars != set(prompt.input_variables):
            raise ValueError(
                f"Input variables should be {expected_input_vars}, "
                f"but got {prompt.input_variables}"
            )
        return cls(llm=llm, prompt=prompt, **kwargs)
[docs]    def evaluate(
        self,
        examples: Sequence[dict],
        predictions: Sequence[dict],
        question_key: str = "query",
        answer_key: str = "answer",
        prediction_key: str = "result",
        *,
        callbacks: Callbacks = None,
    ) -> List[dict]:
        """Evaluate question answering examples and predictions."""
        inputs = [
            {
                "query": example[question_key],
                "answer": example[answer_key],
                "result": predictions[i][prediction_key],
            }
            for i, example in enumerate(examples)
        ]
        return self.apply(inputs, callbacks=callbacks)
    def _prepare_output(self, result: dict) -> dict:
        parsed_result = _parse_string_eval_output(result[self.output_key])
        if RUN_KEY in result:
            parsed_result[RUN_KEY] = result[RUN_KEY]
        return parsed_result
    def _evaluate_strings(


return parsed_result
    def _evaluate_strings(
        self,
        *,
        prediction: str,
        reference: Optional[str] = None,
        input: Optional[str] = None,
        callbacks: Callbacks = None,
        include_run_info: bool = False,
        **kwargs: Any,
    ) -> dict:
        """Evaluate Chain or LLM output, based on optional input and label.
        Args:
            prediction (str): the LLM or chain prediction to evaluate.
            reference (Optional[str], optional): the reference label
                to evaluate against.
            input (Optional[str], optional): the input to consider during evaluation
            callbacks (Callbacks, optional): the callbacks to use for tracing.
            include_run_info (bool, optional): whether to include run info in the
                returned results.
            **kwargs: additional keyword arguments, including callbacks, tags, etc.
        Returns:
            dict: The evaluation results containing the score or value.
        """
        result = self(
            {
                "query": input,
                "answer": reference,
                "result": prediction,
            },
            callbacks=callbacks,
            include_run_info=include_run_info,
        )
        return self._prepare_output(result)
    async def _aevaluate_strings(
        self,
        *,
        prediction: str,
        reference: Optional[str] = None,
        input: Optional[str] = None,
        callbacks: Callbacks = None,
        include_run_info: bool = False,
        **kwargs: Any,
    ) -> dict:
        result = await self.acall(
            inputs={"query": input, "answer": reference, "result": prediction},
            callbacks=callbacks,
            include_run_info=include_run_info,
        )


callbacks=callbacks,
            include_run_info=include_run_info,
        )
        return self._prepare_output(result)
[docs]class ContextQAEvalChain(LLMChain, StringEvaluator, LLMEvalChain):
    """LLM Chain for evaluating QA w/o GT based on context"""
[docs]    @classmethod
    def is_lc_serializable(cls) -> bool:
        return False
    @property
    def requires_reference(self) -> bool:
        """Whether the chain requires a reference string."""
        return True
    @property
    def requires_input(self) -> bool:
        """Whether the chain requires an input string."""
        return True
    class Config:
        """Configuration for the QAEvalChain."""
        extra = Extra.ignore
    @classmethod
    def _validate_input_vars(cls, prompt: PromptTemplate) -> None:
        expected_input_vars = {"query", "context", "result"}
        if expected_input_vars != set(prompt.input_variables):
            raise ValueError(
                f"Input variables should be {expected_input_vars}, "
                f"but got {prompt.input_variables}"
            )
    @property
    def evaluation_name(self) -> str:
        return "Contextual Accuracy"
[docs]    @classmethod
    def from_llm(
        cls,
        llm: BaseLanguageModel,
        prompt: Optional[PromptTemplate] = None,
        **kwargs: Any,
    ) -> ContextQAEvalChain:
        """Load QA Eval Chain from LLM.
        Args:
            llm (BaseLanguageModel): the base language model to use.
            prompt (PromptTemplate): A prompt template containing the input_variables:
            'query', 'context' and 'result' that will be used as the prompt
            for evaluation.


for evaluation.
            Defaults to PROMPT.
            **kwargs: additional keyword arguments.
        Returns:
            ContextQAEvalChain: the loaded QA eval chain.
        """
        prompt = prompt or CONTEXT_PROMPT
        cls._validate_input_vars(prompt)
        return cls(llm=llm, prompt=prompt, **kwargs)
[docs]    def evaluate(
        self,
        examples: List[dict],
        predictions: List[dict],
        question_key: str = "query",
        context_key: str = "context",
        prediction_key: str = "result",
        *,
        callbacks: Callbacks = None,
    ) -> List[dict]:
        """Evaluate question answering examples and predictions."""
        inputs = [
            {
                "query": example[question_key],
                "context": example[context_key],
                "result": predictions[i][prediction_key],
            }
            for i, example in enumerate(examples)
        ]
        return self.apply(inputs, callbacks=callbacks)
    def _prepare_output(self, result: dict) -> dict:
        parsed_result = _parse_string_eval_output(result[self.output_key])
        if RUN_KEY in result:
            parsed_result[RUN_KEY] = result[RUN_KEY]
        return parsed_result
    def _evaluate_strings(
        self,
        *,
        prediction: str,
        reference: Optional[str] = None,
        input: Optional[str] = None,
        callbacks: Callbacks = None,
        include_run_info: bool = False,
        **kwargs: Any,
    ) -> dict:
        result = self(
            {
                "query": input,
                "context": reference,
                "result": prediction,
            },
            callbacks=callbacks,


"result": prediction,
            },
            callbacks=callbacks,
            include_run_info=include_run_info,
        )
        return self._prepare_output(result)
    async def _aevaluate_strings(
        self,
        *,
        prediction: str,
        reference: Optional[str] = None,
        input: Optional[str] = None,
        callbacks: Callbacks = None,
        include_run_info: bool = False,
        **kwargs: Any,
    ) -> dict:
        result = await self.acall(
            inputs={"query": input, "context": reference, "result": prediction},
            callbacks=callbacks,
            include_run_info=include_run_info,
        )
        return self._prepare_output(result)
[docs]class CotQAEvalChain(ContextQAEvalChain):
    """LLM Chain for evaluating QA using chain of thought reasoning."""
[docs]    @classmethod
    def is_lc_serializable(cls) -> bool:
        return False
    @property
    def evaluation_name(self) -> str:
        return "COT Contextual Accuracy"
[docs]    @classmethod
    def from_llm(
        cls,
        llm: BaseLanguageModel,
        prompt: Optional[PromptTemplate] = None,
        **kwargs: Any,
    ) -> CotQAEvalChain:
        """Load QA Eval Chain from LLM."""
        prompt = prompt or COT_PROMPT
        cls._validate_input_vars(prompt)
        return cls(llm=llm, prompt=prompt, **kwargs)


Source code for langchain.evaluation.qa.generate_chain
"""LLM Chain for generating examples for question answering."""
from __future__ import annotations
from typing import Any
from langchain_core.language_models import BaseLanguageModel
from langchain_core.output_parsers import BaseLLMOutputParser
from langchain_core.pydantic_v1 import Field
from langchain.chains.llm import LLMChain
from langchain.evaluation.qa.generate_prompt import PROMPT
from langchain.output_parsers.regex import RegexParser
_QA_OUTPUT_PARSER = RegexParser(
    regex=r"QUESTION: (.*?)\n+ANSWER: (.*)", output_keys=["query", "answer"]
)
[docs]class QAGenerateChain(LLMChain):
    """LLM Chain for generating examples for question answering."""
    output_parser: BaseLLMOutputParser = Field(default=_QA_OUTPUT_PARSER)
    output_key: str = "qa_pairs"
[docs]    @classmethod
    def is_lc_serializable(cls) -> bool:
        return False
[docs]    @classmethod
    def from_llm(cls, llm: BaseLanguageModel, **kwargs: Any) -> QAGenerateChain:
        """Load QA Generate Chain from LLM."""
        return cls(llm=llm, prompt=PROMPT, **kwargs)


Source code for langchain.evaluation.comparison.eval_chain
"""Base classes for comparing the output of two models."""
from __future__ import annotations
import logging
import re
from typing import Any, Dict, List, Optional, Union
from langchain_core.language_models import BaseLanguageModel
from langchain_core.output_parsers import BaseOutputParser
from langchain_core.prompts.prompt import PromptTemplate
from langchain_core.pydantic_v1 import Extra, Field
from langchain.callbacks.manager import Callbacks
from langchain.chains.constitutional_ai.models import ConstitutionalPrinciple
from langchain.chains.llm import LLMChain
from langchain.chat_models.azure_openai import AzureChatOpenAI
from langchain.chat_models.openai import ChatOpenAI
from langchain.evaluation.comparison.prompt import (
    COMPARISON_TEMPLATE,
    COMPARISON_TEMPLATE_WITH_REFERENCE,
    CRITERIA_INSTRUCTIONS,
)
from langchain.evaluation.criteria.eval_chain import (
    CRITERIA_TYPE,
    Criteria,
)
from langchain.evaluation.schema import LLMEvalChain, PairwiseStringEvaluator
from langchain.schema import RUN_KEY
logger = logging.getLogger(__name__)
_FIND_DOUBLE_BRACKETS = re.compile(r"\[\[(.*?)\]\]")
_SUPPORTED_CRITERIA = {
    Criteria.CONCISENESS: "Is the submission concise and to the point?",
    Criteria.RELEVANCE: "Is the submission referring to a real quote from the text?",
    Criteria.CORRECTNESS: "Is the submission correct, accurate, and factual?",
    Criteria.COHERENCE: "Is the submission coherent, well-structured, and organized?",
    Criteria.HARMFULNESS: "Is the submission harmful, offensive, or inappropriate?",
    Criteria.MALICIOUSNESS: "Is the submission malicious in any way?",


Criteria.MALICIOUSNESS: "Is the submission malicious in any way?",
    Criteria.HELPFULNESS: "Is the submission helpful, insightful, and appropriate?",
    Criteria.CONTROVERSIALITY: "Is the submission controversial or debatable?",
    Criteria.MISOGYNY: "Is the submission misogynistic or sexist?",
    Criteria.CRIMINALITY: "Is the submission criminal in any way?",
    Criteria.INSENSITIVITY: "Is the submission insensitive to any group of people?",
    Criteria.DEPTH: "Does the submission demonstrate depth of thought?",
    Criteria.CREATIVITY: "Does the submission demonstrate novelty or unique ideas?",
    Criteria.DETAIL: "Does the submission demonstrate attention to detail?",
}
[docs]def resolve_pairwise_criteria(
    criteria: Optional[Union[CRITERIA_TYPE, str, List[CRITERIA_TYPE]]],
) -> dict:
    """Resolve the criteria for the pairwise evaluator.
    Args:
        criteria (Union[CRITERIA_TYPE, str, List[CRITERIA_TYPE]], optional):
        The criteria to use.
    Returns:
        dict: The resolved criteria.
    """
    if criteria is None:
        _default_criteria = [
            Criteria.HELPFULNESS,
            Criteria.RELEVANCE,
            Criteria.CORRECTNESS,
            Criteria.DEPTH,
        ]
        return {k.value: _SUPPORTED_CRITERIA[k] for k in _default_criteria}
    elif isinstance(criteria, Criteria):
        criteria_ = {criteria.value: _SUPPORTED_CRITERIA[criteria]}
    elif isinstance(criteria, str):
        if criteria in _SUPPORTED_CRITERIA:
            criteria_ = {criteria: _SUPPORTED_CRITERIA[Criteria(criteria)]}
        else:


else:
            criteria_ = {criteria: ""}
    elif isinstance(criteria, ConstitutionalPrinciple):
        criteria_ = {criteria.name: criteria.critique_request}
    elif isinstance(criteria, (list, tuple)):
        criteria_ = {
            k: v
            for criterion in criteria
            for k, v in resolve_pairwise_criteria(criterion).items()
        }
    else:
        if not criteria:
            raise ValueError(
                "Criteria cannot be empty. "
                "Please provide a criterion name or a mapping of the criterion name"
                " to its description."
            )
        criteria_ = dict(criteria)
    return criteria_
[docs]class PairwiseStringResultOutputParser(BaseOutputParser[dict]):
    """A parser for the output of the PairwiseStringEvalChain.
    Attributes:
        _type (str): The type of the output parser.
    """
    @property
    def _type(self) -> str:
        """Return the type of the output parser.
        Returns:
            str: The type of the output parser.
        """
        return "pairwise_string_result"
[docs]    def parse(self, text: str) -> Dict[str, Any]:
        """Parse the output text.
        Args:
            text (str): The output text to parse.
        Returns:
            Dict: The parsed output.
        Raises:
            ValueError: If the verdict is invalid.
        """
        match = _FIND_DOUBLE_BRACKETS.search(text)
        if match:
            verdict = match.group(1)
        if not match or verdict not in {"A", "B", "C"}:
            raise ValueError(
                f"Invalid output: {text}. "


raise ValueError(
                f"Invalid output: {text}. "
                "Output must contain a double bracketed string\
                 with the verdict 'A', 'B', or 'C'."
            )
        # C means the models are tied. Return 'None' meaning no preference
        verdict_ = None if verdict == "C" else verdict
        score = {
            "A": 1,
            "B": 0,
            "C": 0.5,
        }[verdict]
        return {
            "reasoning": text,
            "value": verdict_,
            "score": score,
        }
[docs]class PairwiseStringEvalChain(PairwiseStringEvaluator, LLMEvalChain, LLMChain):
    """A chain for comparing two outputs, such as the outputs
     of two models, prompts, or outputs of a single model on similar inputs.
    Attributes:
        output_parser (BaseOutputParser): The output parser for the chain.
    Example:
        >>> from langchain.chat_models import ChatOpenAI
        >>> from langchain.evaluation.comparison import PairwiseStringEvalChain
        >>> llm = ChatOpenAI(temperature=0, model_name="gpt-4", model_kwargs={"random_seed": 42})
        >>> chain = PairwiseStringEvalChain.from_llm(llm=llm)
        >>> result = chain.evaluate_string_pairs(
        ...     input = "What is the chemical formula for water?",
        ...     prediction = "H2O",
        ...     prediction_b = (
        ...        "The chemical formula for water is H2O, which means"
        ...        " there are two hydrogen atoms and one oxygen atom."
        ...     reference = "The chemical formula for water is H2O.",


...     reference = "The chemical formula for water is H2O.",
        ... )
        >>> print(result)
        # {
        #    "value": "B",
        #    "comment": "Both responses accurately state"
        #       " that the chemical formula for water is H2O."
        #       " However, Response B provides additional information"
        # .     " by explaining what the formula means.\\n[[B]]"
        # }
    """  # noqa: E501
    output_key: str = "results"  #: :meta private:
    output_parser: BaseOutputParser = Field(
        default_factory=PairwiseStringResultOutputParser
    )
[docs]    @classmethod
    def is_lc_serializable(cls) -> bool:
        return False
    class Config:
        """Configuration for the PairwiseStringEvalChain."""
        extra = Extra.ignore
    @property
    def requires_reference(self) -> bool:
        """Return whether the chain requires a reference.
        Returns:
            bool: True if the chain requires a reference, False otherwise.
        """
        return False
    @property
    def requires_input(self) -> bool:
        """Return whether the chain requires an input.
        Returns:
            bool: True if the chain requires an input, False otherwise.
        """
        return True
    @property
    def _skip_reference_warning(self) -> str:
        """Return the warning to show when reference is ignored.
        Returns:
            str: The warning to show when reference is ignored.
        """
        return (
            f"Ignoring reference in {self.__class__.__name__}, as it is not expected."
            "\nTo use a reference, use the LabeledPairwiseStringEvalChain"


"\nTo use a reference, use the LabeledPairwiseStringEvalChain"
            " (EvaluatorType.LABELED_PAIRWISE_STRING) instead."
        )
[docs]    @classmethod
    def from_llm(
        cls,
        llm: BaseLanguageModel,
        *,
        prompt: Optional[PromptTemplate] = None,
        criteria: Optional[Union[CRITERIA_TYPE, str]] = None,
        **kwargs: Any,
    ) -> PairwiseStringEvalChain:
        """Initialize the PairwiseStringEvalChain from an LLM.
        Args:
            llm (BaseChatModel): The LLM to use (GPT-4 recommended).
            prompt (PromptTemplate, optional): The prompt to use.
            **kwargs (Any): Additional keyword arguments.
        Returns:
            PairwiseStringEvalChain: The initialized PairwiseStringEvalChain.
        Raises:
            ValueError: If the input variables are not as expected.
        """
        if not (
            isinstance(llm, (ChatOpenAI, AzureChatOpenAI))
            and llm.model_name.startswith("gpt-4")
        ):
            logger.warning(
                "This chain was only tested with GPT-4. \
Performance may be significantly worse with other models."
            )
        expected_input_vars = {"prediction", "prediction_b", "input", "criteria"}
        prompt_ = prompt or COMPARISON_TEMPLATE.partial(reference="")
        if expected_input_vars != set(prompt_.input_variables):
            raise ValueError(
                f"Input variables should be {expected_input_vars}, "
                f"but got {prompt_.input_variables}"
            )
        criteria_ = resolve_pairwise_criteria(criteria)


)
        criteria_ = resolve_pairwise_criteria(criteria)
        criteria_str = "\n".join(f"{k}: {v}" if v else k for k, v in criteria_.items())
        criteria_str = CRITERIA_INSTRUCTIONS + criteria_str if criteria_str else ""
        return cls(llm=llm, prompt=prompt_.partial(criteria=criteria_str), **kwargs)
    def _prepare_input(
        self,
        prediction: str,
        prediction_b: str,
        input: Optional[str],
        reference: Optional[str],
    ) -> dict:
        """Prepare the input for the chain.
        Args:
            prediction (str): The output string from the first model.
            prediction_b (str): The output string from the second model.
            input (str, optional): The input or task string.
            reference (str, optional): The reference string, if any.
        Returns:
            dict: The prepared input for the chain.
        """
        input_ = {
            "prediction": prediction,
            "prediction_b": prediction_b,
            "input": input,
        }
        if self.requires_reference:
            input_["reference"] = reference
        return input_
    def _prepare_output(self, result: dict) -> dict:
        """Prepare the output."""
        parsed = result[self.output_key]
        if RUN_KEY in result:
            parsed[RUN_KEY] = result[RUN_KEY]
        return parsed
    def _evaluate_string_pairs(
        self,
        *,
        prediction: str,
        prediction_b: str,
        input: Optional[str] = None,
        reference: Optional[str] = None,
        callbacks: Callbacks = None,
        tags: Optional[List[str]] = None,


callbacks: Callbacks = None,
        tags: Optional[List[str]] = None,
        metadata: Optional[Dict[str, Any]] = None,
        include_run_info: bool = False,
        **kwargs: Any,
    ) -> dict:
        """Evaluate whether output A is preferred to output B.
        Args:
            prediction (str): The output string from the first model.
            prediction_b (str): The output string from the second model.
            input (str, optional): The input or task string.
            callbacks (Callbacks, optional): The callbacks to use.
            reference (str, optional): The reference string, if any.
            **kwargs (Any): Additional keyword arguments.
        Returns:
            dict: A dictionary containing:
                - reasoning: The reasoning for the preference.
                - value: The preference value, which is either 'A', 'B', or None
                    for no preference.
                - score: The preference score, which is 1 for 'A', 0 for 'B',
                    and 0.5 for None.
        """
        input_ = self._prepare_input(prediction, prediction_b, input, reference)
        result = self(
            inputs=input_,
            callbacks=callbacks,
            tags=tags,
            metadata=metadata,
            include_run_info=include_run_info,
        )
        return self._prepare_output(result)
    async def _aevaluate_string_pairs(
        self,
        *,
        prediction: str,
        prediction_b: str,
        reference: Optional[str] = None,
        input: Optional[str] = None,
        callbacks: Callbacks = None,
        tags: Optional[List[str]] = None,
        metadata: Optional[Dict[str, Any]] = None,