a17d9dee3d
Measures whether cross-database fusion reaches what a query needs, scored on retrieval alone so no model or judge sits between the fusion and the number. The corpus is MTRAG ClapNQ partitioned by article title, whole titles to a collection, so an article's passages never split and a query's gold stays concentrated in one collection, which is the condition a per-collection depth quota punishes. collection_of keys on sha256 rather than hash(), which is salted per process: the partition is never stored, and scoring recomputes it in a different process than the one that ingested. The 148 titles holding a gold passage carry 10,723 passages between them, so a budget near that floor leaves no cross-topic distractors and inflates recall. The default is 40,000 and the build reports the gold/distractor split, warning when there are none. build_databases opens each collection by configured name with a scope of one, since populate_db writes to a single database. The operator entry point emits the config for the partition it just built, so a config cannot search a differently-partitioned build. Claude-Session: https://claude.ai/code/session_01WhudUtZm6qqiuv8Y1sbwSc
56 lines
2 KiB
YAML
56 lines
2 KiB
YAML
# Reference config for `mtrag_federated`: IBM MTRAG ClapNQ, partitioned by
|
|
# article title into four collections, scored on retrieval only.
|
|
#
|
|
# Build the partition and emit the config that searches exactly it:
|
|
# uv run python -m evaluations.datasets.mtrag_federated \
|
|
# --config configs/mtrag_federated.yaml --n 4 --out ~/configs/fed-n4.yaml
|
|
# evaluations run mtrag_federated --config ~/configs/fed-n4.yaml \
|
|
# --skip-db --skip-qa
|
|
#
|
|
# The databases below are the canonical n=4 partition at seed 20260831. Sweep
|
|
# configs for other collection counts live outside the repo, because a config
|
|
# here must be named after a registered dataset.
|
|
#
|
|
# No reranking block on purpose: this eval measures the reciprocal-rank fusion
|
|
# path, where retrieval depth per collection is `limit // n`. Adding a reranker
|
|
# is the comparison arm, not the baseline.
|
|
# base_url uses the `vllm` host serving each model over an OpenAI-compatible API.
|
|
|
|
environment: development
|
|
|
|
storage:
|
|
auto_vacuum: false
|
|
|
|
lancedb:
|
|
# Declaration order is load-bearing: fusion resolves equal ranks to this
|
|
# order, so permuting these four keys is an arm.
|
|
databases:
|
|
clapnq_0: ${HOME}/.local/share/haiku.rag/evaluations/dbs/mtrag_federated_s20260831_n4_0.lancedb
|
|
clapnq_1: ${HOME}/.local/share/haiku.rag/evaluations/dbs/mtrag_federated_s20260831_n4_1.lancedb
|
|
clapnq_2: ${HOME}/.local/share/haiku.rag/evaluations/dbs/mtrag_federated_s20260831_n4_2.lancedb
|
|
clapnq_3: ${HOME}/.local/share/haiku.rag/evaluations/dbs/mtrag_federated_s20260831_n4_3.lancedb
|
|
|
|
embeddings:
|
|
model:
|
|
provider: openai
|
|
name: qwen3-embedding-4b
|
|
vector_dim: 2560
|
|
base_url: http://vllm:11431/v1
|
|
|
|
search:
|
|
# Matches the spec's retrieval_limit and the product default.
|
|
limit: 5
|
|
|
|
evaluations:
|
|
judge:
|
|
provider: openai
|
|
name: Inferact/Qwen3.8-27B-NVFP4
|
|
base_url: http://vllm:11439/v1
|
|
temperature: 0.6
|
|
max_tokens: 16384
|
|
extra_body:
|
|
top_p: 0.95
|
|
top_k: 20
|
|
min_p: 0
|
|
chat_template_kwargs:
|
|
reasoning_effort: low
|