danvics/haiku.rag
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Fixes

Browse source
This commit is contained in:
Yiorgis Gozadinos 2026-03-12 12:19:21 +02:00
parent c785a54f5f
commit 24275d2ef9
No known key found for this signature in database
2 changed files with 11 additions and 10 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
evaluations/evaluations/benchmark.py
View file

@ -303,7 +303,7 @@ async def run_qa_benchmark(
db = spec.db_path(db_path) db = spec.db_path(db_path)
async with HaikuRAG(db, config=config) as rag: async with HaikuRAG(db, config=config) as rag:
qa = get_qa_agent(rag, system_prompt=spec.resolve_system_prompt(config)) qa = get_qa_agent(rag, config, system_prompt=spec.resolve_system_prompt(config))
async def answer_question(question: str) -> str: async def answer_question(question: str) -> str:
answer, _ = await qa.answer(question) answer, _ = await qa.answer(question)
@ -424,13 +424,7 @@ def _resolve_datasets(dataset: str) -> list[DatasetSpec]:
"""Resolve 'all' or a single dataset key to a list of DatasetSpecs.""" """Resolve 'all' or a single dataset key to a list of DatasetSpecs."""
if dataset.lower() == "all": if dataset.lower() == "all":
return list(DATASETS.values()) return list(DATASETS.values())
spec = DATASETS.get(dataset.lower()) return [_resolve_dataset(dataset)]
if spec is None:
valid_datasets = ", ".join(sorted(DATASETS))
raise typer.BadParameter(
f"Unknown dataset '{dataset}'. Choose from: {valid_datasets}, all"
)
return [spec]
@app.command() @app.command()
@ -441,7 +435,7 @@ def run(
), ),
db: Path | None = typer.Option(None, "--db", help="Override the database path."), db: Path | None = typer.Option(None, "--db", help="Override the database path."),
skip_db: bool = typer.Option( skip_db: bool = typer.Option(
False, "--skip-db", help="Skip updateing the evaluation db." False, "--skip-db", help="Skip updating the evaluation db."
), ),
skip_retrieval: bool = typer.Option( skip_retrieval: bool = typer.Option(
False, "--skip-retrieval", help="Skip retrieval benchmark." False, "--skip-retrieval", help="Skip retrieval benchmark."

9
evaluations/evaluations/optimization.py
View file

@ -1,9 +1,11 @@
import asyncio import asyncio
import logging
from collections.abc import Mapping, Sequence from collections.abc import Mapping, Sequence
from dataclasses import dataclass from dataclasses import dataclass
from pathlib import Path from pathlib import Path
from typing import Any from typing import Any
from pydantic_ai.models import Model
from pydantic_evals import Case from pydantic_evals import Case
from pydantic_evals.evaluators.llm_as_a_judge import judge_input_output_expected from pydantic_evals.evaluators.llm_as_a_judge import judge_input_output_expected
@ -17,6 +19,8 @@ from haiku.rag.client import HaikuRAG
from haiku.rag.config.models import AppConfig, ModelConfig from haiku.rag.config.models import AppConfig, ModelConfig
from haiku.rag.utils import get_model from haiku.rag.utils import get_model
logger = logging.getLogger(__name__)
OPTIMIZATION_SCORING_RUBRIC = """You are evaluating the quality of an answer to a question, OPTIMIZATION_SCORING_RUBRIC = """You are evaluating the quality of an answer to a question,
comparing it against a reference answer. comparing it against a reference answer.
@ -65,7 +69,7 @@ class QAPromptAdapter:
config: AppConfig config: AppConfig
db_path: Path db_path: Path
judge_model: Any judge_model: Model
def evaluate( def evaluate(
self, self,
@ -96,6 +100,9 @@ class QAPromptAdapter:
try: try:
answer, _ = await qa.answer(question) answer, _ = await qa.answer(question)
except Exception: except Exception:
logger.warning(
"QA agent failed for question: %s", question, exc_info=True
)
answer = None answer = None
if answer is not None: if answer is not None: