d2b6b78d16
The build-time assertion fired on the first real build: the chunker keeps the whole twelve-row table in one chunk at chunk_size 256 and at 128, which would have let a single search hit answer the whole-document question. 64 splits it. The cost is a corpus chunked more finely than a real one. The alternative is a longer table at a realistic chunk size, which would change what the question asks.
72 lines
2.3 KiB
YAML
72 lines
2.3 KiB
YAML
# Reference config for the `multidb` behaviour families: the acceptance gate for
|
|
# lancedb.databases. Build the corpus first, then run with --skip-db:
|
|
# uv run python -m evaluations.datasets.multidb --config configs/multidb.yaml
|
|
# evaluations run multidb --config configs/multidb.yaml --skip-db --skip-retrieval
|
|
# For the no-reranker arm, copy this and drop the reranking block; a reference
|
|
# config's filename has to name a dataset, so that arm lives outside configs/.
|
|
|
|
environment: development
|
|
|
|
storage:
|
|
auto_vacuum: false
|
|
|
|
lancedb:
|
|
# Order matters: RRF ties resolve to the order listed here, so the first
|
|
# database wins ties. B2 rotates the order it passes per case.
|
|
databases:
|
|
northern: ${HOME}/.local/share/haiku.rag/evaluations/dbs/multidb_northern.lancedb
|
|
southern: ${HOME}/.local/share/haiku.rag/evaluations/dbs/multidb_southern.lancedb
|
|
equipment: ${HOME}/.local/share/haiku.rag/evaluations/dbs/multidb_equipment.lancedb
|
|
|
|
processing:
|
|
# Pinned, and small deliberately. The builder asserts that no single chunk
|
|
# holds all twelve monthly readings, so S3 cannot be answered from one search
|
|
# hit; measured, the chunker keeps the whole table in one chunk at 256 and at
|
|
# 128, and splits it at 64. The cost is a corpus chunked more finely than a
|
|
# real one. The alternative is a longer table at a realistic chunk size, which
|
|
# would change what S3 asks.
|
|
chunk_size: 64
|
|
|
|
search:
|
|
# Three databases at limit 5 means RRF fuses 15 candidates to 5, so the
|
|
# truncation bites and the no-reranker arm can lose evidence.
|
|
limit: 5
|
|
|
|
embeddings:
|
|
model:
|
|
provider: openai
|
|
name: qwen3-embedding-4b
|
|
vector_dim: 2560
|
|
base_url: http://vllm:11431/v1
|
|
|
|
reranking:
|
|
model:
|
|
provider: vllm
|
|
name: Qwen/Qwen3-Reranker-4B
|
|
base_url: http://vllm:11455
|
|
|
|
qa:
|
|
model:
|
|
provider: openai
|
|
name: Inferact/Qwen3.8-27B-NVFP4
|
|
base_url: http://vllm:11439/v1
|
|
max_tokens: 16384
|
|
extra_body:
|
|
chat_template_kwargs:
|
|
reasoning_effort: low
|
|
|
|
evaluations:
|
|
# Scoring is deterministic, but RefusalJudge runs on the B6 and B7 cases,
|
|
# which carry answerability labels.
|
|
judge:
|
|
provider: openai
|
|
name: Inferact/Qwen3.8-27B-NVFP4
|
|
base_url: http://vllm:11439/v1
|
|
temperature: 0.6
|
|
max_tokens: 16384
|
|
extra_body:
|
|
top_p: 0.95
|
|
top_k: 20
|
|
min_p: 0
|
|
chat_template_kwargs:
|
|
reasoning_effort: low
|