danvics/haiku.rag
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Use non-thinking judge in evals

Browse source
This commit is contained in:
Yiorgis Gozadinos 2025-11-25 13:00:25 +02:00
parent e83978f88b
commit f843fb8140
No known key found for this signature in database
1 changed files with 12 additions and 13 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

25
evaluations/evaluations/benchmark.py
View file

@ -6,8 +6,6 @@ from typing import Any, cast
import logfire import logfire
import typer import typer
from dotenv import load_dotenv from dotenv import load_dotenv
from pydantic_ai.models.openai import OpenAIChatModel
from pydantic_ai.providers.ollama import OllamaProvider
from pydantic_evals import Case, Dataset as EvalDataset from pydantic_evals import Case, Dataset as EvalDataset
from pydantic_evals.evaluators import LLMJudge from pydantic_evals.evaluators import LLMJudge
from pydantic_evals.reporting import ReportCaseFailure from pydantic_evals.reporting import ReportCaseFailure
@ -20,13 +18,13 @@ from evaluations.evaluators import ANSWER_EQUIVALENCE_RUBRIC
from evaluations.prompts import WIX_SUPPORT_PROMPT from evaluations.prompts import WIX_SUPPORT_PROMPT
from haiku.rag.client import HaikuRAG from haiku.rag.client import HaikuRAG
from haiku.rag.config import AppConfig, find_config_file, load_yaml_config from haiku.rag.config import AppConfig, find_config_file, load_yaml_config
from haiku.rag.config.models import ModelConfig
from haiku.rag.logging import configure_cli_logging from haiku.rag.logging import configure_cli_logging
from haiku.rag.qa import get_qa_agent from haiku.rag.qa import get_qa_agent
from haiku.rag.utils import get_model
load_dotenv() load_dotenv()
QA_JUDGE_MODEL = "qwen3"
logfire.configure(send_to_logfire="if-token-present", service_name="evals") logfire.configure(send_to_logfire="if-token-present", service_name="evals")
logfire.instrument_pydantic_ai() logfire.instrument_pydantic_ai()
configure_cli_logging() configure_cli_logging()
@ -37,7 +35,7 @@ def build_experiment_metadata(
dataset_key: str, dataset_key: str,
test_cases: int, test_cases: int,
config: AppConfig, config: AppConfig,
judge_model: str, judge_config: ModelConfig,
) -> dict[str, Any]: ) -> dict[str, Any]:
"""Build experiment metadata for Logfire tracking.""" """Build experiment metadata for Logfire tracking."""
return { return {
@ -54,8 +52,8 @@ def build_experiment_metadata(
"rerank_model": config.reranking.model.name if config.reranking.model else None, "rerank_model": config.reranking.model.name if config.reranking.model else None,
"qa_provider": config.qa.model.provider, "qa_provider": config.qa.model.provider,
"qa_model": config.qa.model.name, "qa_model": config.qa.model.name,
"judge_provider": "ollama", "judge_provider": judge_config.provider,
"judge_model": judge_model, "judge_model": judge_config.name,
} }
@ -163,11 +161,14 @@ async def run_retrieval_benchmark(
eval_name = name if name is not None else f"{spec.key}_retrieval_evaluation" eval_name = name if name is not None else f"{spec.key}_retrieval_evaluation"
judge_config = ModelConfig(
provider="ollama", name="gpt-oss", enable_thinking=False
)
experiment_metadata = build_experiment_metadata( experiment_metadata = build_experiment_metadata(
dataset_key=spec.key, dataset_key=spec.key,
test_cases=len(cases), test_cases=len(cases),
config=config, config=config,
judge_model=QA_JUDGE_MODEL, judge_config=judge_config,
) )
report = await dataset.evaluate( report = await dataset.evaluate(
@ -215,10 +216,8 @@ async def run_qa_benchmark(
for index, doc in enumerate(corpus, start=1) for index, doc in enumerate(corpus, start=1)
] ]
judge_model = OpenAIChatModel( judge_config = ModelConfig(provider="ollama", name="gpt-oss", enable_thinking=False)
model_name=QA_JUDGE_MODEL, judge_model = get_model(judge_config, config)
provider=OllamaProvider(base_url=f"{config.providers.ollama.base_url}/v1"),
)
evaluation_dataset = EvalDataset[str, str, dict[str, str]]( evaluation_dataset = EvalDataset[str, str, dict[str, str]](
name=spec.key, name=spec.key,
@ -251,7 +250,7 @@ async def run_qa_benchmark(
dataset_key=spec.key, dataset_key=spec.key,
test_cases=len(cases), test_cases=len(cases),
config=config, config=config,
judge_model=QA_JUDGE_MODEL, judge_config=judge_config,
) )
report = await evaluation_dataset.evaluate( report = await evaluation_dataset.evaluate(