From c785a54f5fca657ebee10aa359e5dcf158242359 Mon Sep 17 00:00:00 2001
From: Yiorgis Gozadinos <ggozadinos@gmail.com>
Date: Tue, 10 Mar 2026 09:20:24 +0200
Subject: [PATCH] Respect config.prompts.qa in evaluations benchmark and
 optimization

---
 evaluations/evaluations/benchmark.py    |  2 +-
 evaluations/evaluations/config.py       | 10 ++++++++++
 evaluations/evaluations/optimization.py |  2 +-
 evaluations/tests/test_config.py        | 19 +++++++++++++++++++
 4 files changed, 31 insertions(+), 2 deletions(-)

diff --git a/evaluations/evaluations/benchmark.py b/evaluations/evaluations/benchmark.py
index bdbf985c..cf693612 100644
--- a/evaluations/evaluations/benchmark.py
+++ b/evaluations/evaluations/benchmark.py
@@ -303,7 +303,7 @@ async def run_qa_benchmark(
 
     db = spec.db_path(db_path)
     async with HaikuRAG(db, config=config) as rag:
-        qa = get_qa_agent(rag, system_prompt=spec.system_prompt)
+        qa = get_qa_agent(rag, system_prompt=spec.resolve_system_prompt(config))
 
         async def answer_question(question: str) -> str:
             answer, _ = await qa.answer(question)
diff --git a/evaluations/evaluations/config.py b/evaluations/evaluations/config.py
index 5e1f7a6e..580235fc 100644
--- a/evaluations/evaluations/config.py
+++ b/evaluations/evaluations/config.py
@@ -7,6 +7,8 @@ from datasets import Dataset
 from pydantic_evals import Case
 from pydantic_evals.evaluators import Evaluator
 
+from haiku.rag.config.models import AppConfig
+
 
 @dataclass
 class DocumentPayload:
@@ -63,3 +65,11 @@ class DatasetSpec:
 
         data_dir = get_default_data_dir()
         return data_dir / "evaluations" / "dbs" / self.db_filename
+
+    def resolve_system_prompt(self, config: AppConfig) -> str | None:
+        """Resolve the QA system prompt.
+
+        Precedence: config.prompts.qa > spec.system_prompt > None
+        (get_qa_agent handles the final fallback to QA_SYSTEM_PROMPT)
+        """
+        return config.prompts.qa or self.system_prompt
diff --git a/evaluations/evaluations/optimization.py b/evaluations/evaluations/optimization.py
index bbd1984d..6cc28e9b 100644
--- a/evaluations/evaluations/optimization.py
+++ b/evaluations/evaluations/optimization.py
@@ -216,7 +216,7 @@ def run_optimization(
 
     reflection_lm = ReflectionLM(config.qa.model, config)
 
-    seed_prompt = spec.system_prompt or QA_SYSTEM_PROMPT
+    seed_prompt = spec.resolve_system_prompt(config) or QA_SYSTEM_PROMPT
     seed_candidate = {"instructions": seed_prompt}
 
     mid = len(cases) // 2
diff --git a/evaluations/tests/test_config.py b/evaluations/tests/test_config.py
index cbb7c0d0..8ad52669 100644
--- a/evaluations/tests/test_config.py
+++ b/evaluations/tests/test_config.py
@@ -2,6 +2,7 @@ from pathlib import Path
 from unittest.mock import patch
 
 from evaluations.config import DatasetSpec, DocumentPayload, RetrievalSample
+from haiku.rag.config.models import AppConfig
 
 
 def _make_spec(**kwargs: object) -> DatasetSpec:
@@ -54,6 +55,24 @@ class TestDatasetSpecDefaults:
         assert spec.system_prompt is None
 
 
+class TestResolveSystemPrompt:
+    def test_config_prompt_overrides_spec_prompt(self) -> None:
+        spec = _make_spec(system_prompt="spec prompt")
+        config = AppConfig()
+        config.prompts.qa = "config prompt"
+        assert spec.resolve_system_prompt(config) == "config prompt"
+
+    def test_spec_prompt_used_when_config_unset(self) -> None:
+        spec = _make_spec(system_prompt="spec prompt")
+        config = AppConfig()
+        assert spec.resolve_system_prompt(config) == "spec prompt"
+
+    def test_returns_none_when_both_unset(self) -> None:
+        spec = _make_spec()
+        config = AppConfig()
+        assert spec.resolve_system_prompt(config) is None
+
+
 class TestDocumentPayload:
     def test_defaults(self) -> None:
         payload = DocumentPayload(uri="test://doc")