Source code for langchain.evaluation.qa.eval_chain
"""LLM Chains for evaluating question answering."""
from __future__ import annotations
import re
from typing import Any, List, Optional, Sequence
from pydantic import Extra
from langchain import PromptTemplate
from langchain.callbacks.manager import Callbacks
from langchain.chains.llm import LLMChain
from langchain.evaluation.qa.eval_prompt import CONTEXT_PROMPT, COT_PROMPT, PROMPT
from langchain.evaluation.schema import LLMEvalChain, StringEvaluator
from langchain.schema import RUN_KEY
from langchain.schema.language_model import BaseLanguageModel
def _get_score(verdict: str) -> Optional[int]:
    match = re.search(r"(?i)(?:grade:\s*)?(correct|incorrect)", verdict)
    if match:
        if match.group(1).upper() == "CORRECT":
            return 1
        elif match.group(1).upper() == "INCORRECT":
            return 0
    return None
def _parse_string_eval_output(text: str) -> dict:
    """Parse the output text.
    Args:
        text (str): The output text to parse.
    Returns:
        Any: The parsed output.
    """
    splits = text.strip().rsplit("\n", maxsplit=1)
    if len(splits) == 1:
        verdict = splits[0]
        reasoning = None
    else:
        reasoning, verdict = splits
        reasoning = reasoning.strip()
    score = _get_score(verdict)
    return {
        "reasoning": reasoning,
        "value": verdict,
        "score": score,
    }
[docs]class QAEvalChain(LLMChain, StringEvaluator, LLMEvalChain):