From 27b16bdf8dc7b4e4eb12ece35d1d7f77e131e5e1 Mon Sep 17 00:00:00 2001 From: Yiorgis Gozadinos Date: Mon, 1 Sep 2025 21:53:36 +0300 Subject: [PATCH] Update basic benchmarks --- docs/benchmarks.md | 8 ++++---- tests/generate_benchmark_db.py | 6 +++++- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/docs/benchmarks.md b/docs/benchmarks.md index 4096931c..4650db4d 100644 --- a/docs/benchmarks.md +++ b/docs/benchmarks.md @@ -7,14 +7,14 @@ You can perform your own evaluations using as example the script found at ## Recall -In order to calculate recall, we load the `News Stories` from `repliqa_3` which is 1035 documents and index them. Subsequently, we run a search over the `question` field for each row of the dataset and check whether we match the document that answers the question. +In order to calculate recall, we load the `News Stories` from `repliqa_3` (1035 documents) and index them. Subsequently, we run a search over the `question` field for each row of the dataset and check whether we match the document that answers the question. Questions for which the answer cannot be found in the documents are ignored. -The recall obtained is ~0.73 for matching in the top result, raising to ~0.75 for the top 3 results. +The recall obtained is ~0.79 for matching in the top result, raising to ~0.91 for the top 3 results with the "bare" default settings (Ollama `qwen3`, `mxbai-embed-large` embeddings, no reranking). | Embedding Model | Document in top 1 | Document in top 3 | Reranker | |---------------------------------------|-------------------|-------------------|------------------------| -| Ollama / `mxbai-embed-large` | 0.77 | 0.89 | None | +| Ollama / `mxbai-embed-large` | 0.79 | 0.91 | None | | Ollama / `mxbai-embed-large` | 0.81 | 0.91 | `mxbai-rerank-base-v2` | | Ollama / `nomic-embed-text` | 0.74 | 0.88 | None | | OpenAI / `text-embeddings-3-small` | 0.75 | 0.88 | None | @@ -27,7 +27,7 @@ Again using the same dataset, we use a QA agent to answer the question. In addit | Embedding Model | QA Model | Accuracy | Reranker | |------------------------------------|-----------------------------------|-----------|------------------------| -| Ollama / `mxbai-embed-large` | Ollama / `qwen3` | 0.64 | None | +| Ollama / `mxbai-embed-large` | Ollama / `qwen3` | 0.85 | None | | Ollama / `mxbai-embed-large` | Ollama / `qwen3` | 0.72 | `mxbai-rerank-base-v2` | | Ollama / `mxbai-embed-large` | Anthropic / `Claude Sonnet 3.7` | 0.79 | None | | OpenAI / `text-embeddings-3-small` | OpenAI / `gpt-4-turbo` | 0.62 | None | diff --git a/tests/generate_benchmark_db.py b/tests/generate_benchmark_db.py index 28dfe271..90abd8c7 100644 --- a/tests/generate_benchmark_db.py +++ b/tests/generate_benchmark_db.py @@ -53,6 +53,10 @@ async def run_match_benchmark(): async with HaikuRAG(db_path) as rag: for doc in corpus: doc_id = doc["document_id"] # type: ignore + expected_answer = doc["answer"] # type: ignore + if expected_answer == "The answer is not found in the document.": + progress.advance(task) + continue matches = await rag.search( query=doc["question"], # type: ignore limit=3, @@ -144,7 +148,7 @@ async def main(): await populate_db() console.print("Running retrieval benchmarks...", style="bold blue") - # await run_match_benchmark() + await run_match_benchmark() console.print("\nRunning QA benchmarks...", style="bold yellow") await run_qa_benchmark()