From 6f181e07ff806a9c62c483d4f037fd25ff386e4e Mon Sep 17 00:00:00 2001
From: Yiorgis Gozadinos <ggozadinos@gmail.com>
Date: Mon, 6 Oct 2025 14:53:23 +0300
Subject: [PATCH] Add Success@K metrics to retrieval benchmarks

---
 docs/benchmarks.md           | 28 ++++++++++++++++++++--------
 src/evaluations/benchmark.py | 30 ++++++++++++++++++++++++------
 2 files changed, 44 insertions(+), 14 deletions(-)

diff --git a/docs/benchmarks.md b/docs/benchmarks.md
index 33921382..a4d13d6d 100644
--- a/docs/benchmarks.md
+++ b/docs/benchmarks.md
@@ -46,17 +46,29 @@ Wix. The benchmark follows the evaluation protocol described in the
 [WixQA paper](https://arxiv.org/abs/2505.08643) and gives us a view into how the
 system handles conversational, product-specific support queries.
 
-For recall, we index the reference answer passages shipped with the dataset and
+For retrieval evaluation, we index the reference answer passages shipped with the dataset and
 run retrieval against each user question. Each sample supplies one or more
-relevant passage URIs; we count how many of those URIs land inside the top *k*
-retrieved documents, divide by the number of relevant passages for that query,
-and average across all queries.
+relevant passage URIs. We track two complementary metrics:
 
-The results for recall using the `WixQA` dataset are as follows:
+- **Recall@K**: Fraction of relevant documents retrieved in top K results. Measures coverage.
+- **Success@K**: Fraction of queries with at least one relevant document in top K. Most relevant for RAG, where finding one good document is often sufficient.
 
-| Embedding Model            | Document in top 1 | Document in top 3 | Reranker               |
-|----------------------------|-------------------|-------------------|------------------------|
-| `qwen3-embedding`          | 0.36              | 0.57              | `mxbai-rerank-base-v2` |
+### Recall@K Results
+
+| Embedding Model            | Recall@1 | Recall@3 | Recall@5 | Reranker               |
+|----------------------------|----------|----------|----------|------------------------|
+| `qwen3-embedding`          | 0.31     | 0.48     | 0.54     | None                   |
+| `qwen3-embedding`          | 0.36     | 0.57     | 0.68     | `mxbai-rerank-base-v2` |
+
+### Success@K Results
+
+| Embedding Model            | Success@1 | Success@3 | Success@5 | Reranker               |
+|----------------------------|-----------|-----------|-----------|------------------------|
+| `qwen3-embedding`          | 0.36      | 0.54      | 0.62      | None                   |
+| `qwen3-embedding`          | 0.42      | 0.66      | 0.76      | `mxbai-rerank-base-v2` |
+
+
+## QA Accuracy
 
 And for QA accuracy,
 
diff --git a/src/evaluations/benchmark.py b/src/evaluations/benchmark.py
index f309bd7e..4eb31835 100644
--- a/src/evaluations/benchmark.py
+++ b/src/evaluations/benchmark.py
@@ -80,6 +80,11 @@ async def run_retrieval_benchmark(spec: DatasetSpec) -> dict[str, float] | None:
         3: 0.0,
         5: 0.0,
     }
+    success_totals = {
+        1: 0.0,
+        3: 0.0,
+        5: 0.0,
+    }
     total_queries = 0
 
     with Progress() as progress:
@@ -109,15 +114,16 @@ async def run_retrieval_benchmark(spec: DatasetSpec) -> dict[str, float] | None:
                     if retrieved_doc and retrieved_doc.uri:
                         retrieved_uris.append(retrieved_doc.uri)
 
-                # Compute per-query recall@K by counting how many relevant
-                # documents are retrieved within the first K results and
-                # averaging these fractions across all queries.
+                # Compute metrics for each cutoff
                 for cutoff in (1, 3, 5):
                     top_k = set(retrieved_uris[:cutoff])
                     relevant = set(sample.expected_uris)
                     if relevant:
                         matched = len(top_k & relevant)
+                        # Recall: fraction of relevant docs retrieved
                         recall_totals[cutoff] += matched / len(relevant)
+                        # Success: binary - did we get at least one relevant doc?
+                        success_totals[cutoff] += 1.0 if matched > 0 else 0.0
 
                 progress.advance(task)
 
@@ -129,16 +135,28 @@ async def run_retrieval_benchmark(spec: DatasetSpec) -> dict[str, float] | None:
     recall_at_3 = recall_totals[3] / total_queries
     recall_at_5 = recall_totals[5] / total_queries
 
+    success_at_1 = success_totals[1] / total_queries
+    success_at_3 = success_totals[3] / total_queries
+    success_at_5 = success_totals[5] / total_queries
+
     console.print("\n=== Retrieval Benchmark Results ===", style="bold cyan")
     console.print(f"Total queries: {total_queries}")
-    console.print(f"Recall@1: {recall_at_1:.4f}")
-    console.print(f"Recall@3: {recall_at_3:.4f}")
-    console.print(f"Recall@5: {recall_at_5:.4f}")
+    console.print("\nRecall@K (fraction of relevant docs retrieved):")
+    console.print(f"  Recall@1: {recall_at_1:.4f}")
+    console.print(f"  Recall@3: {recall_at_3:.4f}")
+    console.print(f"  Recall@5: {recall_at_5:.4f}")
+    console.print("\nSuccess@K (queries with at least one relevant doc):")
+    console.print(f"  Success@1: {success_at_1:.4f} ({success_at_1 * 100:.1f}%)")
+    console.print(f"  Success@3: {success_at_3:.4f} ({success_at_3 * 100:.1f}%)")
+    console.print(f"  Success@5: {success_at_5:.4f} ({success_at_5 * 100:.1f}%)")
 
     return {
         "recall@1": recall_at_1,
         "recall@3": recall_at_3,
         "recall@5": recall_at_5,
+        "success@1": success_at_1,
+        "success@3": success_at_3,
+        "success@5": success_at_5,
     }