remove dataset-specific system prompts

evaluations/evaluations/benchmark.py

@@ -406,9 +406,7 @@ async def run_qa_benchmark(
 
     if target == "qa":
         async with HaikuRAG(db, config=config) as rag:
-            qa = get_qa_agent(
-                rag, config, system_prompt=spec.resolve_system_prompt(config)
-            )
+            qa = get_qa_agent(rag, config)
 
             async def answer_question(question: str) -> str:
                 answer, _ = await qa.answer(question)

evaluations/evaluations/config.py

@@ -7,8 +7,6 @@ from datasets import Dataset
 from pydantic_evals import Case
 from pydantic_evals.evaluators import Evaluator
 
-from haiku.rag.config.models import AppConfig
-
 
 @dataclass
 class DocumentPayload:
@@ -47,7 +45,6 @@ class DatasetSpec:
     retrieval_mapper: RetrievalMapper | None = None
     retrieval_evaluator: Evaluator | None = None
     document_limit: int | None = None
-    system_prompt: str | None = None
 
     def db_path(self, override_path: Path | None = None) -> Path:
         """Get the database path.
@@ -65,11 +62,3 @@ class DatasetSpec:
 
         data_dir = get_default_data_dir()
         return data_dir / "evaluations" / "dbs" / self.db_filename
-
-    def resolve_system_prompt(self, config: AppConfig) -> str | None:
-        """Resolve the QA system prompt.
-
-        Precedence: config.prompts.qa > spec.system_prompt > None
-        (get_qa_agent handles the final fallback to QA_SYSTEM_PROMPT)
-        """
-        return config.prompts.qa or self.system_prompt

evaluations/evaluations/datasets/open_rag_bench.py

@@ -14,46 +14,6 @@ from evaluations.evaluators import MAPEvaluator
 
 logger = logging.getLogger(__name__)
 
-ORB_SYSTEM_PROMPT = """You are a knowledgeable assistant that answers questions using a document knowledge base.
-
-Process:
-1. Call search_documents with relevant keywords from the question
-2. Review the results ordered by relevance
-3. If needed, perform follow-up searches with different keywords (max 3 total)
-4. Provide a concise answer based strictly on the retrieved content
-
-The search tool returns results like:
-[chunk_abc123] [rank 1 of 5]
-Source: "Document Title" > Section > Subsection
-Type: paragraph
-Content:
-The actual text content here...
-
-[chunk_def456] [rank 2 of 5]
-Source: "Another Document"
-Type: table
-Content:
-| Column 1 | Column 2 |
-...
-
-Each result includes:
-- chunk_id in brackets and rank position (rank 1 = most relevant)
-- Source: document title and section hierarchy (when available)
-- Type: content type like paragraph, table, code, list_item (when available)
-- Content: the actual text
-
-In your response, include the chunk IDs you used in cited_chunks.
-
-Guidelines:
-- Base answers strictly on retrieved content - do not use external knowledge
-- Use the Source and Type metadata to understand context
-- If multiple results are relevant, synthesize them coherently
-- If information is insufficient, say: "I cannot find enough information in the knowledge base to answer this question."
-- Be concise and direct - avoid elaboration unless asked
-- Results are ordered by relevance, with rank 1 being most relevant
-- IMPORTANT: Do not use LaTeX notation (like \\(...\\) or $...$) in your answers. Use plain text or Unicode math symbols instead.
-"""
-
 REPO_ID = "vectara/open_ragbench"
 PDF_SUBDIR = "pdf/arxiv"
 
@@ -266,5 +226,4 @@ OPEN_RAG_BENCH_SPEC = DatasetSpec(
     retrieval_loader=load_orb_retrieval,
     retrieval_mapper=map_orb_retrieval,
     retrieval_evaluator=MAPEvaluator(),
-    system_prompt=ORB_SYSTEM_PROMPT,
 )

evaluations/evaluations/datasets/wix.py

@@ -8,27 +8,6 @@ from pydantic_evals import Case
 from evaluations.config import DatasetSpec, DocumentPayload, RetrievalSample
 from evaluations.evaluators import MAPEvaluator
 
-WIX_SUPPORT_PROMPT = """You are a WIX technical support expert helping users with questions about the WIX platform.
-
-Your process:
-1. When a user asks a question, use the search_documents tool to find relevant information
-2. Search with specific keywords and phrases from the user's question
-3. Review the search results ordered by relevance (rank 1 = most relevant)
-4. If you need additional context, perform follow-up searches with different keywords
-5. Provide a short and to the point comprehensive answer based only on the retrieved documents
-
-Guidelines:
-- Base your answers strictly on the provided document content
-- Quote or reference specific information when possible
-- If multiple documents contain relevant information, synthesize them coherently
-- Indicate when information is incomplete or when you need to search for additional context
-- If the retrieved documents don't contain sufficient information, clearly state: "I cannot find enough information in the knowledge base to answer this question."
-- For complex questions, consider breaking them down and performing multiple searches
-- Stick to the answer, do not ellaborate or provide context unless explicitly asked for it.
-
-Be concise, and always maintain accuracy over completeness. Prefer short, direct answers that are well-supported by the documents.
-"""
-
 
 def load_wix_corpus() -> Dataset:
     dataset_dict = load_dataset("Wix/WixQA", "wix_kb_corpus")
@@ -102,5 +81,4 @@ WIX_SPEC = DatasetSpec(
     retrieval_loader=load_wix_qa,
     retrieval_mapper=map_wix_retrieval,
     retrieval_evaluator=MAPEvaluator(),
-    system_prompt=WIX_SUPPORT_PROMPT,
 )

evaluations/evaluations/optimization.py

@@ -236,7 +236,7 @@ def run_optimization(
     reflect_config = reflect_model or config.qa.model
     reflection_lm = ReflectionLM(reflect_config, config)
 
-    seed_prompt = spec.resolve_system_prompt(config) or QA_SYSTEM_PROMPT
+    seed_prompt = config.prompts.qa or QA_SYSTEM_PROMPT
     seed_candidate = {"instructions": seed_prompt}
 
     mid = len(cases) // 2

evaluations/tests/test_config.py

@@ -2,7 +2,6 @@ from pathlib import Path
 from unittest.mock import patch
 
 from evaluations.config import DatasetSpec, DocumentPayload, RetrievalSample
-from haiku.rag.config.models import AppConfig
 
 
 def _make_spec(**kwargs: object) -> DatasetSpec:
@@ -52,25 +51,6 @@ class TestDatasetSpecDefaults:
         assert spec.retrieval_mapper is None
         assert spec.retrieval_evaluator is None
         assert spec.document_limit is None
-        assert spec.system_prompt is None
-
-
-class TestResolveSystemPrompt:
-    def test_config_prompt_overrides_spec_prompt(self) -> None:
-        spec = _make_spec(system_prompt="spec prompt")
-        config = AppConfig()
-        config.prompts.qa = "config prompt"
-        assert spec.resolve_system_prompt(config) == "config prompt"
-
-    def test_spec_prompt_used_when_config_unset(self) -> None:
-        spec = _make_spec(system_prompt="spec prompt")
-        config = AppConfig()
-        assert spec.resolve_system_prompt(config) == "spec prompt"
-
-    def test_returns_none_when_both_unset(self) -> None:
-        spec = _make_spec()
-        config = AppConfig()
-        assert spec.resolve_system_prompt(config) is None
 
 
 class TestDocumentPayload:

evaluations/tests/test_optimization.py

@@ -291,7 +291,6 @@ class TestRunOptimization:
             document_mapper=lambda doc: None,
             qa_loader=lambda: None,  # type: ignore[arg-type]  # ty: ignore[invalid-argument-type]
             qa_case_builder=lambda idx, doc: None,  # type: ignore[arg-type]  # ty: ignore[invalid-argument-type]
-            system_prompt="You are a test assistant.",
         )
 
     def test_returns_results(self, tmp_path: Path, gepa_mock_result: MagicMock) -> None: