From c7a9ad8583dc2d6c1f161842a50c3460dbd89af1 Mon Sep 17 00:00:00 2001 From: Yiorgis Gozadinos Date: Sat, 24 Jan 2026 13:48:09 +0200 Subject: [PATCH] Customize orb QA prompt to not use LaTeX as gpt-oss Ollama implementation fails to parse it properly --- evaluations/evaluations/benchmark.py | 4 +- evaluations/evaluations/config.py | 1 + .../evaluations/datasets/open_rag_bench.py | 41 +++++++++++++++++++ evaluations/evaluations/datasets/wix.py | 22 ++++++++++ evaluations/evaluations/prompts.py | 21 ---------- 5 files changed, 65 insertions(+), 24 deletions(-) delete mode 100644 evaluations/evaluations/prompts.py diff --git a/evaluations/evaluations/benchmark.py b/evaluations/evaluations/benchmark.py index a2dcbacd..b17756cd 100644 --- a/evaluations/evaluations/benchmark.py +++ b/evaluations/evaluations/benchmark.py @@ -15,7 +15,6 @@ from rich.progress import Progress from evaluations.config import DatasetSpec from evaluations.datasets import DATASETS from evaluations.evaluators import ANSWER_EQUIVALENCE_RUBRIC -from evaluations.prompts import WIX_SUPPORT_PROMPT from haiku.rag.client import HaikuRAG from haiku.rag.config import AppConfig, find_config_file, load_yaml_config from haiku.rag.config.models import ModelConfig @@ -294,8 +293,7 @@ async def run_qa_benchmark( db = spec.db_path(db_path) async with HaikuRAG(db, config=config) as rag: - system_prompt = WIX_SUPPORT_PROMPT if spec.key == "wix" else None - qa = get_qa_agent(rag, system_prompt=system_prompt) + qa = get_qa_agent(rag, system_prompt=spec.system_prompt) async def answer_question(question: str) -> str: answer, _ = await qa.answer(question) diff --git a/evaluations/evaluations/config.py b/evaluations/evaluations/config.py index 063fb104..5e1f7a6e 100644 --- a/evaluations/evaluations/config.py +++ b/evaluations/evaluations/config.py @@ -45,6 +45,7 @@ class DatasetSpec: retrieval_mapper: RetrievalMapper | None = None retrieval_evaluator: Evaluator | None = None document_limit: int | None = None + system_prompt: str | None = None def db_path(self, override_path: Path | None = None) -> Path: """Get the database path. diff --git a/evaluations/evaluations/datasets/open_rag_bench.py b/evaluations/evaluations/datasets/open_rag_bench.py index f11be906..79b869e2 100644 --- a/evaluations/evaluations/datasets/open_rag_bench.py +++ b/evaluations/evaluations/datasets/open_rag_bench.py @@ -14,6 +14,46 @@ from evaluations.evaluators import MAPEvaluator logger = logging.getLogger(__name__) +ORB_SYSTEM_PROMPT = """You are a knowledgeable assistant that answers questions using a document knowledge base. + +Process: +1. Call search_documents with relevant keywords from the question +2. Review the results ordered by relevance +3. If needed, perform follow-up searches with different keywords (max 3 total) +4. Provide a concise answer based strictly on the retrieved content + +The search tool returns results like: +[chunk_abc123] [rank 1 of 5] +Source: "Document Title" > Section > Subsection +Type: paragraph +Content: +The actual text content here... + +[chunk_def456] [rank 2 of 5] +Source: "Another Document" +Type: table +Content: +| Column 1 | Column 2 | +... + +Each result includes: +- chunk_id in brackets and rank position (rank 1 = most relevant) +- Source: document title and section hierarchy (when available) +- Type: content type like paragraph, table, code, list_item (when available) +- Content: the actual text + +In your response, include the chunk IDs you used in cited_chunks. + +Guidelines: +- Base answers strictly on retrieved content - do not use external knowledge +- Use the Source and Type metadata to understand context +- If multiple results are relevant, synthesize them coherently +- If information is insufficient, say: "I cannot find enough information in the knowledge base to answer this question." +- Be concise and direct - avoid elaboration unless asked +- Results are ordered by relevance, with rank 1 being most relevant +- IMPORTANT: Do not use LaTeX notation (like \\(...\\) or $...$) in your answers. Use plain text or Unicode math symbols instead. +""" + REPO_ID = "vectara/open_ragbench" PDF_SUBDIR = "pdf/arxiv" @@ -226,4 +266,5 @@ OPEN_RAG_BENCH_SPEC = DatasetSpec( retrieval_loader=load_orb_retrieval, retrieval_mapper=map_orb_retrieval, retrieval_evaluator=MAPEvaluator(), + system_prompt=ORB_SYSTEM_PROMPT, ) diff --git a/evaluations/evaluations/datasets/wix.py b/evaluations/evaluations/datasets/wix.py index 7e81fb8f..0fdcbda6 100644 --- a/evaluations/evaluations/datasets/wix.py +++ b/evaluations/evaluations/datasets/wix.py @@ -8,6 +8,27 @@ from pydantic_evals import Case from evaluations.config import DatasetSpec, DocumentPayload, RetrievalSample from evaluations.evaluators import MAPEvaluator +WIX_SUPPORT_PROMPT = """You are a WIX technical support expert helping users with questions about the WIX platform. + +Your process: +1. When a user asks a question, use the search_documents tool to find relevant information +2. Search with specific keywords and phrases from the user's question +3. Review the search results ordered by relevance (rank 1 = most relevant) +4. If you need additional context, perform follow-up searches with different keywords +5. Provide a short and to the point comprehensive answer based only on the retrieved documents + +Guidelines: +- Base your answers strictly on the provided document content +- Quote or reference specific information when possible +- If multiple documents contain relevant information, synthesize them coherently +- Indicate when information is incomplete or when you need to search for additional context +- If the retrieved documents don't contain sufficient information, clearly state: "I cannot find enough information in the knowledge base to answer this question." +- For complex questions, consider breaking them down and performing multiple searches +- Stick to the answer, do not ellaborate or provide context unless explicitly asked for it. + +Be concise, and always maintain accuracy over completeness. Prefer short, direct answers that are well-supported by the documents. +""" + def load_wix_corpus() -> Dataset: dataset_dict = load_dataset("Wix/WixQA", "wix_kb_corpus") @@ -81,4 +102,5 @@ WIX_SPEC = DatasetSpec( retrieval_loader=load_wix_qa, retrieval_mapper=map_wix_retrieval, retrieval_evaluator=MAPEvaluator(), + system_prompt=WIX_SUPPORT_PROMPT, ) diff --git a/evaluations/evaluations/prompts.py b/evaluations/evaluations/prompts.py deleted file mode 100644 index 0a0e1b83..00000000 --- a/evaluations/evaluations/prompts.py +++ /dev/null @@ -1,21 +0,0 @@ -WIX_SUPPORT_PROMPT = """ -You are a WIX technical support expert helping users with questions about the WIX platform. - -Your process: -1. When a user asks a question, use the search_documents tool to find relevant information -2. Search with specific keywords and phrases from the user's question -3. Review the search results ordered by relevance (rank 1 = most relevant) -4. If you need additional context, perform follow-up searches with different keywords -5. Provide a short and to the point comprehensive answer based only on the retrieved documents - -Guidelines: -- Base your answers strictly on the provided document content -- Quote or reference specific information when possible -- If multiple documents contain relevant information, synthesize them coherently -- Indicate when information is incomplete or when you need to search for additional context -- If the retrieved documents don't contain sufficient information, clearly state: "I cannot find enough information in the knowledge base to answer this question." -- For complex questions, consider breaking them down and performing multiple searches -- Stick to the answer, do not ellaborate or provide context unless explicitly asked for it. - -Be concise, and always maintain accuracy over completeness. Prefer short, direct answers that are well-supported by the documents. -"""