diff --git a/evaluations/evaluations/benchmark.py b/evaluations/evaluations/benchmark.py index d5979c7a..d963da4a 100644 --- a/evaluations/evaluations/benchmark.py +++ b/evaluations/evaluations/benchmark.py @@ -406,9 +406,7 @@ async def run_qa_benchmark( if target == "qa": async with HaikuRAG(db, config=config) as rag: - qa = get_qa_agent( - rag, config, system_prompt=spec.resolve_system_prompt(config) - ) + qa = get_qa_agent(rag, config) async def answer_question(question: str) -> str: answer, _ = await qa.answer(question) diff --git a/evaluations/evaluations/config.py b/evaluations/evaluations/config.py index 580235fc..063fb104 100644 --- a/evaluations/evaluations/config.py +++ b/evaluations/evaluations/config.py @@ -7,8 +7,6 @@ from datasets import Dataset from pydantic_evals import Case from pydantic_evals.evaluators import Evaluator -from haiku.rag.config.models import AppConfig - @dataclass class DocumentPayload: @@ -47,7 +45,6 @@ class DatasetSpec: retrieval_mapper: RetrievalMapper | None = None retrieval_evaluator: Evaluator | None = None document_limit: int | None = None - system_prompt: str | None = None def db_path(self, override_path: Path | None = None) -> Path: """Get the database path. @@ -65,11 +62,3 @@ class DatasetSpec: data_dir = get_default_data_dir() return data_dir / "evaluations" / "dbs" / self.db_filename - - def resolve_system_prompt(self, config: AppConfig) -> str | None: - """Resolve the QA system prompt. - - Precedence: config.prompts.qa > spec.system_prompt > None - (get_qa_agent handles the final fallback to QA_SYSTEM_PROMPT) - """ - return config.prompts.qa or self.system_prompt diff --git a/evaluations/evaluations/datasets/open_rag_bench.py b/evaluations/evaluations/datasets/open_rag_bench.py index 79b869e2..f11be906 100644 --- a/evaluations/evaluations/datasets/open_rag_bench.py +++ b/evaluations/evaluations/datasets/open_rag_bench.py @@ -14,46 +14,6 @@ from evaluations.evaluators import MAPEvaluator logger = logging.getLogger(__name__) -ORB_SYSTEM_PROMPT = """You are a knowledgeable assistant that answers questions using a document knowledge base. - -Process: -1. Call search_documents with relevant keywords from the question -2. Review the results ordered by relevance -3. If needed, perform follow-up searches with different keywords (max 3 total) -4. Provide a concise answer based strictly on the retrieved content - -The search tool returns results like: -[chunk_abc123] [rank 1 of 5] -Source: "Document Title" > Section > Subsection -Type: paragraph -Content: -The actual text content here... - -[chunk_def456] [rank 2 of 5] -Source: "Another Document" -Type: table -Content: -| Column 1 | Column 2 | -... - -Each result includes: -- chunk_id in brackets and rank position (rank 1 = most relevant) -- Source: document title and section hierarchy (when available) -- Type: content type like paragraph, table, code, list_item (when available) -- Content: the actual text - -In your response, include the chunk IDs you used in cited_chunks. - -Guidelines: -- Base answers strictly on retrieved content - do not use external knowledge -- Use the Source and Type metadata to understand context -- If multiple results are relevant, synthesize them coherently -- If information is insufficient, say: "I cannot find enough information in the knowledge base to answer this question." -- Be concise and direct - avoid elaboration unless asked -- Results are ordered by relevance, with rank 1 being most relevant -- IMPORTANT: Do not use LaTeX notation (like \\(...\\) or $...$) in your answers. Use plain text or Unicode math symbols instead. -""" - REPO_ID = "vectara/open_ragbench" PDF_SUBDIR = "pdf/arxiv" @@ -266,5 +226,4 @@ OPEN_RAG_BENCH_SPEC = DatasetSpec( retrieval_loader=load_orb_retrieval, retrieval_mapper=map_orb_retrieval, retrieval_evaluator=MAPEvaluator(), - system_prompt=ORB_SYSTEM_PROMPT, ) diff --git a/evaluations/evaluations/datasets/wix.py b/evaluations/evaluations/datasets/wix.py index 0fdcbda6..7e81fb8f 100644 --- a/evaluations/evaluations/datasets/wix.py +++ b/evaluations/evaluations/datasets/wix.py @@ -8,27 +8,6 @@ from pydantic_evals import Case from evaluations.config import DatasetSpec, DocumentPayload, RetrievalSample from evaluations.evaluators import MAPEvaluator -WIX_SUPPORT_PROMPT = """You are a WIX technical support expert helping users with questions about the WIX platform. - -Your process: -1. When a user asks a question, use the search_documents tool to find relevant information -2. Search with specific keywords and phrases from the user's question -3. Review the search results ordered by relevance (rank 1 = most relevant) -4. If you need additional context, perform follow-up searches with different keywords -5. Provide a short and to the point comprehensive answer based only on the retrieved documents - -Guidelines: -- Base your answers strictly on the provided document content -- Quote or reference specific information when possible -- If multiple documents contain relevant information, synthesize them coherently -- Indicate when information is incomplete or when you need to search for additional context -- If the retrieved documents don't contain sufficient information, clearly state: "I cannot find enough information in the knowledge base to answer this question." -- For complex questions, consider breaking them down and performing multiple searches -- Stick to the answer, do not ellaborate or provide context unless explicitly asked for it. - -Be concise, and always maintain accuracy over completeness. Prefer short, direct answers that are well-supported by the documents. -""" - def load_wix_corpus() -> Dataset: dataset_dict = load_dataset("Wix/WixQA", "wix_kb_corpus") @@ -102,5 +81,4 @@ WIX_SPEC = DatasetSpec( retrieval_loader=load_wix_qa, retrieval_mapper=map_wix_retrieval, retrieval_evaluator=MAPEvaluator(), - system_prompt=WIX_SUPPORT_PROMPT, ) diff --git a/evaluations/evaluations/optimization.py b/evaluations/evaluations/optimization.py index 6a768406..ae68af96 100644 --- a/evaluations/evaluations/optimization.py +++ b/evaluations/evaluations/optimization.py @@ -236,7 +236,7 @@ def run_optimization( reflect_config = reflect_model or config.qa.model reflection_lm = ReflectionLM(reflect_config, config) - seed_prompt = spec.resolve_system_prompt(config) or QA_SYSTEM_PROMPT + seed_prompt = config.prompts.qa or QA_SYSTEM_PROMPT seed_candidate = {"instructions": seed_prompt} mid = len(cases) // 2 diff --git a/evaluations/tests/test_config.py b/evaluations/tests/test_config.py index 60f96aee..37342701 100644 --- a/evaluations/tests/test_config.py +++ b/evaluations/tests/test_config.py @@ -2,7 +2,6 @@ from pathlib import Path from unittest.mock import patch from evaluations.config import DatasetSpec, DocumentPayload, RetrievalSample -from haiku.rag.config.models import AppConfig def _make_spec(**kwargs: object) -> DatasetSpec: @@ -52,25 +51,6 @@ class TestDatasetSpecDefaults: assert spec.retrieval_mapper is None assert spec.retrieval_evaluator is None assert spec.document_limit is None - assert spec.system_prompt is None - - -class TestResolveSystemPrompt: - def test_config_prompt_overrides_spec_prompt(self) -> None: - spec = _make_spec(system_prompt="spec prompt") - config = AppConfig() - config.prompts.qa = "config prompt" - assert spec.resolve_system_prompt(config) == "config prompt" - - def test_spec_prompt_used_when_config_unset(self) -> None: - spec = _make_spec(system_prompt="spec prompt") - config = AppConfig() - assert spec.resolve_system_prompt(config) == "spec prompt" - - def test_returns_none_when_both_unset(self) -> None: - spec = _make_spec() - config = AppConfig() - assert spec.resolve_system_prompt(config) is None class TestDocumentPayload: diff --git a/evaluations/tests/test_optimization.py b/evaluations/tests/test_optimization.py index 7f108072..0e832b19 100644 --- a/evaluations/tests/test_optimization.py +++ b/evaluations/tests/test_optimization.py @@ -291,7 +291,6 @@ class TestRunOptimization: document_mapper=lambda doc: None, qa_loader=lambda: None, # type: ignore[arg-type] # ty: ignore[invalid-argument-type] qa_case_builder=lambda idx, doc: None, # type: ignore[arg-type] # ty: ignore[invalid-argument-type] - system_prompt="You are a test assistant.", ) def test_returns_results(self, tmp_path: Path, gepa_mock_result: MagicMock) -> None: