From 2a99cb09bf107df452dad0b6b1a5a56461dda218 Mon Sep 17 00:00:00 2001 From: Yiorgis Gozadinos Date: Thu, 29 Jan 2026 14:22:18 +0200 Subject: [PATCH] Evaluate using ask --deep --- CHANGELOG.md | 5 +++ docs/benchmarks.md | 18 +++++++++++ evaluations/evaluations/benchmark.py | 46 ++++++++++++++++++++++++---- 3 files changed, 63 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e41b53eb..39846f51 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,11 @@ ### Added +- **Deep Ask Evaluations**: QA benchmarks can now use the research graph for multi-step reasoning + - New `--deep` flag on `evaluations run` enables deep ask mode + - Uses research graph with `max_iterations=2` and `confidence_threshold=0.0` + - Evaluation name automatically suffixed with `_deep` when enabled + - Experiment metadata includes `deep_ask` field for tracking - **Chat Agent Document Awareness Tools**: Two new tools for browsing and understanding the knowledge base - `list_documents` — Returns `DocumentListResponse` with paginated documents (50 per page), page number, total pages, and total count; respects session document filter - `summarize_document` — Generate LLM-powered summaries of specific documents diff --git a/docs/benchmarks.md b/docs/benchmarks.md index 8d7b002c..52240ced 100644 --- a/docs/benchmarks.md +++ b/docs/benchmarks.md @@ -60,9 +60,27 @@ evaluations run repliqa --config /path/to/haiku.rag.yaml --db /path/to/custom.la - `--skip-qa` - Skip QA benchmark - `--limit N` - Limit number of test cases - `--name NAME` - Override the evaluation name +- `--deep` - Use deep QA mode (multi-step reasoning with research graph) If no config file is specified, the script searches standard locations: `./haiku.rag.yaml`, user config directory, then falls back to defaults. +### Deep QA Mode + +The `--deep` flag enables multi-step reasoning using the research graph instead of the simple QA agent: + +```bash +evaluations run repliqa --skip-db --deep +``` + +In deep mode: + +- Questions are decomposed into sub-questions by a planning agent +- Each sub-question is answered by searching the knowledge base +- A synthesis agent combines findings into a comprehensive answer +- The graph runs for up to 2 iterations with no early exit (confidence threshold disabled) + +This matches the behavior of `haiku-rag ask --deep` in the CLI. Deep mode typically produces more thorough answers but requires more LLM calls per question. + ## Methodology ### Retrieval Metrics diff --git a/evaluations/evaluations/benchmark.py b/evaluations/evaluations/benchmark.py index 2dcae976..c6e98a96 100644 --- a/evaluations/evaluations/benchmark.py +++ b/evaluations/evaluations/benchmark.py @@ -17,6 +17,9 @@ from rich.progress import Progress from evaluations.config import DatasetSpec from evaluations.datasets import DATASETS from evaluations.evaluators import ANSWER_EQUIVALENCE_RUBRIC +from haiku.rag.agents.research.dependencies import ResearchContext +from haiku.rag.agents.research.graph import build_research_graph +from haiku.rag.agents.research.state import ResearchDeps, ResearchState from haiku.rag.client import HaikuRAG from haiku.rag.config import AppConfig, find_config_file, load_yaml_config from haiku.rag.config.models import ModelConfig @@ -39,11 +42,13 @@ def build_experiment_metadata( test_cases: int, config: AppConfig, judge_config: ModelConfig, + deep: bool = False, ) -> dict[str, Any]: """Build experiment metadata for Logfire tracking.""" return { "dataset": dataset_key, "test_cases": test_cases, + "deep_ask": deep, "embedder_provider": config.embeddings.model.provider, "embedder_model": config.embeddings.model.name, "embedder_dim": config.embeddings.model.vector_dim, @@ -265,6 +270,7 @@ async def run_qa_benchmark( limit: int | None = None, name: str | None = None, db_path: Path | None = None, + deep: bool = False, ) -> ReportCaseFailure[str, str, dict[str, str]] | None: corpus = spec.qa_loader() if limit is not None: @@ -297,19 +303,37 @@ async def run_qa_benchmark( db = spec.db_path(db_path) async with HaikuRAG(db, config=config) as rag: - qa = get_qa_agent(rag, system_prompt=spec.system_prompt) + if deep: + graph = build_research_graph(config=config) - async def answer_question(question: str) -> str: - answer, _ = await qa.answer(question) - return answer + async def answer_question(question: str) -> str: + context = ResearchContext(original_question=question) + state = ResearchState.from_config( + context=context, + config=config, + max_iterations=2, + confidence_threshold=0.0, + ) + deps = ResearchDeps(client=rag) + report = await graph.run(state=state, deps=deps) + return report.executive_summary if report else "" + else: + qa = get_qa_agent(rag, system_prompt=spec.system_prompt) + + async def answer_question(question: str) -> str: + answer, _ = await qa.answer(question) + return answer eval_name = name if name is not None else f"{spec.key}_qa_evaluation" + if deep: + eval_name = f"{eval_name}_deep" experiment_metadata = build_experiment_metadata( dataset_key=spec.key, test_cases=len(cases), config=config, judge_config=judge_config, + deep=deep, ) report = await evaluation_dataset.evaluate( @@ -359,6 +383,7 @@ async def evaluate_dataset( db_path: Path | None, vacuum_interval: int = 100, multimodal_only: bool = False, + deep: bool = False, ) -> None: if not skip_db: console.print(f"Using dataset: {spec.key}", style="bold magenta") @@ -378,8 +403,11 @@ async def evaluate_dataset( ) if not skip_qa: - console.print("\nRunning QA benchmarks...", style="bold yellow") - await run_qa_benchmark(spec, config, limit=limit, name=name, db_path=db_path) + mode_label = "deep QA" if deep else "QA" + console.print(f"\nRunning {mode_label} benchmarks...", style="bold yellow") + await run_qa_benchmark( + spec, config, limit=limit, name=name, db_path=db_path, deep=deep + ) app = typer.Typer(help="Run retrieval and QA benchmarks for configured datasets.") @@ -411,6 +439,11 @@ def run( "--multimodal-only", help="Only evaluate queries requiring image understanding.", ), + deep: bool = typer.Option( + False, + "--deep", + help="Use deep QA mode (multi-step reasoning with research graph).", + ), ) -> None: spec = DATASETS.get(dataset.lower()) if spec is None: @@ -449,6 +482,7 @@ def run( db_path=db, vacuum_interval=vacuum_interval, multimodal_only=multimodal_only, + deep=deep, ) )