From b73949c784b4df25e62d7ef125894cc973e30127 Mon Sep 17 00:00:00 2001
From: Yiorgis Gozadinos <ggozadinos@gmail.com>
Date: Tue, 2 Sep 2025 14:52:49 +0300
Subject: [PATCH] update benchmarks

---
 docs/benchmarks.md             |  5 ++++-
 tests/generate_benchmark_db.py | 35 ++++++++++++++++++++--------------
 2 files changed, 25 insertions(+), 15 deletions(-)

diff --git a/docs/benchmarks.md b/docs/benchmarks.md
index 1b10b2b1..5f2d5ad8 100644
--- a/docs/benchmarks.md
+++ b/docs/benchmarks.md
@@ -28,6 +28,9 @@ Again using the same dataset, we use a QA agent to answer the question. In addit
 | Embedding Model                    | QA Model                          | Accuracy  | Reranker               |
 |------------------------------------|-----------------------------------|-----------|------------------------|
 | Ollama / `mxbai-embed-large`       | Ollama / `qwen3`                  | 0.85      | None                   |
-| Ollama / `mxbai-embed-large`       | Ollama / `qwen3`                  | 0.72      | `mxbai-rerank-base-v2` |
+| Ollama / `mxbai-embed-large`       | Ollama / `qwen3`                  | 0.87      | `mxbai-rerank-base-v2` |
+| Ollama / `mxbai-embed-large`       | Ollama / `qwen3:0.6b`             | 0.28      | None                   |
+
+Note the significant degradation when very small models are used such as `qwen3:0.6b`.
 <!-- | Ollama / `mxbai-embed-large`       | Anthropic / `Claude Sonnet 3.7`   | 0.79      | None                   |
 | OpenAI / `text-embeddings-3-small` | OpenAI / `gpt-4-turbo`            | 0.62      | None                   | -->
diff --git a/tests/generate_benchmark_db.py b/tests/generate_benchmark_db.py
index 90abd8c7..7318a27b 100644
--- a/tests/generate_benchmark_db.py
+++ b/tests/generate_benchmark_db.py
@@ -120,21 +120,28 @@ async def run_qa_benchmark(k: int | None = None):
                 question = doc["question"]  # type: ignore
                 expected_answer = doc["answer"]  # type: ignore
 
-                generated_answer = await qa.answer(question)
-                is_equivalent = await judge.judge_answers(
-                    question, generated_answer, expected_answer
-                )
-                console.print(f"Question: {question}")
-                console.print(f"Expected: {expected_answer}")
-                console.print(f"Generated: {generated_answer}")
-                console.print(f"Equivalent: {is_equivalent}\n")
+                # Really small models might fail, let's account for that in try/except
+                try:
+                    generated_answer = await qa.answer(question)
+                    is_equivalent = await judge.judge_answers(
+                        question, generated_answer, expected_answer
+                    )
+                    console.print(f"Question: {question}")
+                    console.print(f"Expected: {expected_answer}")
+                    console.print(f"Generated: {generated_answer}")
+                    console.print(f"Equivalent: {is_equivalent}\n")
 
-                if is_equivalent:
-                    correct_answers += 1
-                total_questions += 1
-                console.print("Current score:", correct_answers, "/", total_questions)
-
-                progress.advance(task)
+                    if is_equivalent:
+                        correct_answers += 1
+                except Exception as e:
+                    console.print(f"[red]Error processing question: {question}[/red]")
+                    console.print(f"[red]{e}[/red]")
+                finally:
+                    total_questions += 1
+                    console.print(
+                        "Current score:", correct_answers, "/", total_questions
+                    )
+                    progress.advance(task)
 
     accuracy = correct_answers / total_questions if total_questions > 0 else 0