def requires_input(self) -> bool:
        return False
    @property
    def requires_reference(self) -> bool:
        return True
    @property
    def evaluation_name(self) -> str:
        return "exact_match"
    def _evaluate_strings(self, prediction, reference=None, input=None, **kwargs) -> dict:
        return {"score": prediction == reference}
evaluation_config = RunEvalConfig(
    custom_evaluators = [MyStringEvaluator()],
)
run_on_dataset(
    client,
    "<my_dataset_name>",
    construct_chain,
    evaluation=evaluation_config,
)
Examples using run_on_dataset¶
LangSmith Walkthrough