Remove evaluation ceremony from gepa judge
This commit is contained in:
parent
8bac8b4104
commit
7c399d1ff1
1 changed files with 7 additions and 26 deletions
|
|
@ -4,8 +4,8 @@ from dataclasses import dataclass
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
from pydantic_evals import Case, Dataset as EvalDataset
|
from pydantic_evals import Case
|
||||||
from pydantic_evals.evaluators import LLMJudge
|
from pydantic_evals.evaluators.llm_as_a_judge import judge_input_output_expected
|
||||||
|
|
||||||
from gepa.core.adapter import EvaluationBatch
|
from gepa.core.adapter import EvaluationBatch
|
||||||
|
|
||||||
|
|
@ -126,33 +126,14 @@ class QAPromptAdapter:
|
||||||
self, question: str, answer: str, expected: str
|
self, question: str, answer: str, expected: str
|
||||||
) -> tuple[float, str | None]:
|
) -> tuple[float, str | None]:
|
||||||
"""Score an answer using pydantic-evals LLMJudge with float scoring."""
|
"""Score an answer using pydantic-evals LLMJudge with float scoring."""
|
||||||
judge = LLMJudge(
|
result = await judge_input_output_expected(
|
||||||
|
inputs=question,
|
||||||
|
output=answer,
|
||||||
|
expected_output=expected,
|
||||||
rubric=OPTIMIZATION_SCORING_RUBRIC,
|
rubric=OPTIMIZATION_SCORING_RUBRIC,
|
||||||
include_input=True,
|
|
||||||
include_expected_output=True,
|
|
||||||
model=self.judge_model,
|
model=self.judge_model,
|
||||||
score={"evaluation_name": "accuracy", "include_reason": True},
|
|
||||||
assertion=False,
|
|
||||||
)
|
)
|
||||||
|
return result.score, result.reason
|
||||||
dataset = EvalDataset(
|
|
||||||
cases=[Case(inputs=question, expected_output=expected)],
|
|
||||||
evaluators=[judge],
|
|
||||||
)
|
|
||||||
|
|
||||||
async def identity(q: str) -> str:
|
|
||||||
return answer
|
|
||||||
|
|
||||||
report = await dataset.evaluate(identity, max_concurrency=1, progress=False)
|
|
||||||
case_report = report.cases[0]
|
|
||||||
|
|
||||||
score_result = case_report.scores.get("accuracy")
|
|
||||||
if score_result is not None:
|
|
||||||
score = float(score_result.value)
|
|
||||||
reason = getattr(score_result, "reason", None)
|
|
||||||
return score, reason
|
|
||||||
|
|
||||||
return 0.0, None
|
|
||||||
|
|
||||||
def make_reflective_dataset(
|
def make_reflective_dataset(
|
||||||
self,
|
self,
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue