From c1d53348041f22c7c6ad6264e0cb89e4856dd2c7 Mon Sep 17 00:00:00 2001 From: Yiorgis Gozadinos Date: Sat, 19 Jul 2025 19:02:47 +0300 Subject: [PATCH] Update benchmarks with reranking --- docs/benchmarks.md | 20 +++++++++++--------- src/haiku/rag/qa/prompts.py | 3 ++- tests/llm_judge.py | 34 +++++++++++++++++++++++----------- 3 files changed, 36 insertions(+), 21 deletions(-) diff --git a/docs/benchmarks.md b/docs/benchmarks.md index d5a85a04..6b043fcf 100644 --- a/docs/benchmarks.md +++ b/docs/benchmarks.md @@ -12,17 +12,19 @@ In order to calculate recall, we load the `News Stories` from `repliqa_3` which The recall obtained is ~0.73 for matching in the top result, raising to ~0.75 for the top 3 results. -| Model | Document in top 1 | Document in top 3 | -|---------------------------------------|-------------------|-------------------| -| Ollama / `mxbai-embed-large` | 0.77 | 0.89 | -| Ollama / `nomic-embed-text` | 0.74 | 0.88 | -| OpenAI / `text-embeddings-3-small` | 0.75 | 0.88 | +| Model | Document in top 1 | Document in top 3 | Reranker | +|---------------------------------------|-------------------|-------------------|----------------------| +| Ollama / `mxbai-embed-large` | 0.77 | 0.89 | None | +| Ollama / `mxbai-embed-large` | 0.81 | 0.91 | mxbai-rerank-base-v2 | +| Ollama / `nomic-embed-text` | 0.74 | 0.88 | None | +| OpenAI / `text-embeddings-3-small` | 0.75 | 0.88 | None | ## Question/Answer evaluation Again using the same dataset, we use a QA agent to answer the question. In addition we use an LLM judge (using the Ollama `qwen3`) to evaluate whether the answer is correct or not. The obtained accuracy is as follows: -| Embedding Model | QA Model | Accuracy | -|------------------------------|-----------------------------------|-----------| -| Ollama / `mxbai-embed-large` | Ollama / `qwen3` | 0.64 | -| Ollama / `mxbai-embed-large` | Anthropic / `Claude Sonnet 3.7` | 0.79 | +| Embedding Model | QA Model | Accuracy | Reranker | +|------------------------------|-----------------------------------|-----------|----------------------| +| Ollama / `mxbai-embed-large` | Ollama / `qwen3` | 0.64 | None | +| Ollama / `mxbai-embed-large` | Ollama / `qwen3` | 0.72 | mxbai-rerank-base-v2 | +| Ollama / `mxbai-embed-large` | Anthropic / `Claude Sonnet 3.7` | 0.79 | None | diff --git a/src/haiku/rag/qa/prompts.py b/src/haiku/rag/qa/prompts.py index 283c40e2..68d42cc2 100644 --- a/src/haiku/rag/qa/prompts.py +++ b/src/haiku/rag/qa/prompts.py @@ -6,7 +6,7 @@ Your process: 2. Search with specific keywords and phrases from the user's question 3. Review the search results and their relevance scores 4. If you need additional context, perform follow-up searches with different keywords -5. Provide a comprehensive answer based only on the retrieved documents +5. Provide a short and to the point comprehensive answer based only on the retrieved documents Guidelines: - Base your answers strictly on the provided document content @@ -15,6 +15,7 @@ Guidelines: - Indicate when information is incomplete or when you need to search for additional context - If the retrieved documents don't contain sufficient information, clearly state: "I cannot find enough information in the knowledge base to answer this question." - For complex questions, consider breaking them down and performing multiple searches +- Stick to the answer, do not ellaborate or provde context unless asked for it. Be concise, and always maintain accuracy over completeness. Prefer short, direct answers that are well-supported by the documents. """ diff --git a/tests/llm_judge.py b/tests/llm_judge.py index 5af4cf0e..f7e3f6e2 100644 --- a/tests/llm_judge.py +++ b/tests/llm_judge.py @@ -35,23 +35,35 @@ class LLMJudge: - score: str rating from 1-5 """ - prompt = f""" - You are an expert judge evaluating the equivalence of two answers to the same question. + prompt = f"""You are an expert evaluator determining whether two answers to the same question are semantically equivalent. - Question: {question} +QUESTION: {question} - Generated Answer: {answer} +GENERATED ANSWER: {answer} - Expected Answer: {expected_answer} +EXPECTED ANSWER: {expected_answer} - Your task is to determine if these two answers are equivalent in meaning and both correctly answer the question. Consider: +EVALUATION CRITERIA: +Rate as EQUIVALENT (true) if: +✓ Both answers contain the same core factual information +✓ Both directly address the question asked +✓ The key claims and conclusions are consistent +✓ Any additional detail in one answer doesn't contradict the other - 1. Do both answers provide the same answer? - 2. Do both answers directly address the question asked? - 3. Minor differences in wording or style are acceptable if the meaning of the answer is the same. - 4. If one answer is more detailed but the other is correct, they can still be considered equivalent. +Rate as NOT EQUIVALENT (false) if: +✗ Factual contradictions exist between the answers +✗ One answer fails to address the core question +✗ Key information is missing from one answer that changes the meaning +✗ The answers lead to different conclusions or implications - Be strict but fair in your evaluation. Focus on factual correctness and whether both answers would satisfy someone asking the question.""" +GUIDELINES: +- Ignore minor differences in phrasing, style, or formatting +- Focus on semantic meaning rather than exact wording +- Consider both answers correct if they convey the same essential information +- Be tolerant of different levels of detail if the core answer is preserved +- Evaluate based on what a person asking this question would need to know + +Respond with JSON containing only: {{"equivalent": true}} or {{"equivalent": false}}""" response = await self.client.chat( model=self.model,