Source code for langchain.smith.evaluation.config
"""Configuration for run evaluators."""
from typing import Any, Dict, List, Optional, Union
from langsmith import RunEvaluator
from pydantic import BaseModel, Field
from langchain.embeddings.base import Embeddings
from langchain.evaluation.criteria.eval_chain import CRITERIA_TYPE
from langchain.evaluation.embedding_distance.base import (
    EmbeddingDistance as EmbeddingDistanceEnum,
)
from langchain.evaluation.schema import EvaluatorType, StringEvaluator
from langchain.evaluation.string_distance.base import (
    StringDistance as StringDistanceEnum,
)
from langchain.schema.language_model import BaseLanguageModel
from langchain.schema.prompt_template import BasePromptTemplate
[docs]class EvalConfig(BaseModel):
    """Configuration for a given run evaluator.
    Parameters
    ----------
    evaluator_type : EvaluatorType
        The type of evaluator to use.
    Methods
    -------
    get_kwargs()
        Get the keyword arguments for the evaluator configuration.
    """
    evaluator_type: EvaluatorType
[docs]    def get_kwargs(self) -> Dict[str, Any]:
        """Get the keyword arguments for the load_evaluator call.
        Returns
        -------
        Dict[str, Any]
            The keyword arguments for the load_evaluator call.
        """
        kwargs = {}
        for field, val in self:
            if field == "evaluator_type":
                continue
            kwargs[field] = val
        return kwargs
[docs]class RunEvalConfig(BaseModel):
    """Configuration for a run evaluation.
    Parameters
    ----------
    evaluators : List[Union[EvaluatorType, EvalConfig]]
        Configurations for which evaluators to apply to the dataset run.