From 3cb229d2e09cf1fc0f8d76f04d19d94a52de5238 Mon Sep 17 00:00:00 2001
From: Yiorgis Gozadinos <ggozadinos@gmail.com>
Date: Mon, 29 Jun 2026 10:52:23 +0300
Subject: [PATCH] Add t2_finqa pre-built evaluation database reference config
 and docs

---
 CHANGELOG.md                      |  3 ++-
 docs/benchmarks.md                |  1 +
 evaluations/configs/t2_finqa.yaml | 36 +++++++++++++++++++++++++++++++
 3 files changed, 39 insertions(+), 1 deletion(-)
 create mode 100644 evaluations/configs/t2_finqa.yaml

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 515bc5e4..d8b002ed 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -4,7 +4,8 @@
 ### Added
 
 - `orb_multimodal_nemotron` pre-built evaluation database (`nvidia/llama-nemotron-embed-vl-1b-v2` embedder).
-- Reference configs under `evaluations/configs/` for the pre-built evaluation databases (`wix`, `orb_text`, `orb_multimodal`, `orb_multimodal_nemotron`).
+- `t2_finqa` pre-built evaluation database (T²-RAGBench FinQA).
+- Reference configs under `evaluations/configs/` for the pre-built evaluation databases (`wix`, `orb_text`, `orb_multimodal`, `orb_multimodal_nemotron`, `t2_finqa`).
 
 ## [0.63.0] - 2026-06-28
 
diff --git a/docs/benchmarks.md b/docs/benchmarks.md
index 3e6f21d0..ca031adc 100644
--- a/docs/benchmarks.md
+++ b/docs/benchmarks.md
@@ -36,6 +36,7 @@ Active datasets:
 | `orb_text` — OpenRAG Bench, text embedder (`qwen3-embedding:4b`) with VLM picture descriptions baked into chunk content | ~18 GB |
 | `orb_multimodal` — OpenRAG Bench, multimodal embedder (`qwen3-vl-embedding-8b`); picture vectors live in the same space as text for cross-modal retrieval | ~16 GB |
 | `orb_multimodal_nemotron` — OpenRAG Bench, multimodal embedder (`nvidia/llama-nemotron-embed-vl-1b-v2`), the embedder behind the published headline results | ~16 GB |
+| `t2_finqa` — T²-RAGBench (FinQA) financial QA, text embedder (`qwen3-embedding:4b`); scored by exact numeric match, run with `--target analysis-skill` | ~2 GB |
 
 After downloading, run benchmarks with `--skip-db`. Each database is built with a specific embedder, so pass its reference config from `evaluations/configs/` (a database only opens against a config whose embedder matches):
 
diff --git a/evaluations/configs/t2_finqa.yaml b/evaluations/configs/t2_finqa.yaml
new file mode 100644
index 00000000..1c73eff7
--- /dev/null
+++ b/evaluations/configs/t2_finqa.yaml
@@ -0,0 +1,36 @@
+# Reference config for the `t2_finqa` pre-built evaluation database.
+# T²-RAGBench (FinQA) financial QA, scored by exact numeric match.
+# Run: evaluations run t2_finqa --skip-db --target analysis-skill --config configs/t2_finqa.yaml
+# base_url uses the `vllm` host serving each model over an OpenAI-compatible API.
+
+environment: development
+
+storage:
+  auto_vacuum: false
+
+embeddings:
+  model:
+    provider: openai
+    name: qwen3-embedding-4b
+    vector_dim: 2560
+    base_url: http://vllm:11431/v1
+
+reranking:
+  model:
+    provider: mxbai
+    name: mixedbread-ai/mxbai-rerank-base-v2
+
+qa:
+  model:
+    provider: openai
+    name: gemma4-26b
+    base_url: http://vllm:11432/v1
+
+prompts:
+  domain_preamble: |
+    Use search() to find the relevant documents. Do not iterate over all of
+    /documents or read every document's content, that will time out.
+
+    For questions with a numeric answer, end your response with a final line
+    formatted exactly as `ANSWER: <number>`, containing a single number. Keep a
+    percent sign if the answer is a percentage.