56 lines
1.7 KiB
YAML
56 lines
1.7 KiB
YAML
# Reference config for the `mtrag_clapnq` pre-built evaluation database.
|
|
# IBM MTRAG, ClapNQ (Wikipedia) domain: multi-turn retrieval and QA over
|
|
# 183,408 passages. Also serves mtrag_clapnq_rewrite, mtrag_clapnq_live and
|
|
# mtrag_clapnq_live_uncompacted.
|
|
# Run: evaluations run mtrag_clapnq --config configs/mtrag_clapnq.yaml
|
|
# base_url uses the `vllm` host serving each model over an OpenAI-compatible API.
|
|
# The corpus is text-only: no multimodal embedder, no vision paths. This eval
|
|
# cannot exercise image or vision turn-boundary behavior.
|
|
|
|
environment: development
|
|
|
|
storage:
|
|
auto_vacuum: false
|
|
|
|
embeddings:
|
|
model:
|
|
provider: openai
|
|
name: qwen3-embedding-4b
|
|
vector_dim: 2560
|
|
base_url: http://vllm:11431/v1
|
|
|
|
reranking:
|
|
model:
|
|
provider: vllm
|
|
name: Qwen/Qwen3-Reranker-4B
|
|
base_url: http://vllm:11455
|
|
|
|
qa:
|
|
model:
|
|
provider: openai
|
|
name: RedHatAI/Muse-Glimmer-30B-NVFP4
|
|
base_url: http://vllm:11450/v1
|
|
# vLLM enforces input + max_tokens <= max_model_len, so a large output
|
|
# budget silently shrinks the input budget. MTRAG answers are sentences.
|
|
max_tokens: 8192
|
|
extra_body:
|
|
chat_template_kwargs:
|
|
# Part of the measured baseline. vLLM's reasoning parser consumes
|
|
# enable_thinking before the chat template sees it; reasoning_strength
|
|
# is the knob Muse Glimmer templates honour, and a template that
|
|
# defaults it to low silently changes search behavior.
|
|
reasoning_strength: high
|
|
|
|
evaluations:
|
|
judge:
|
|
provider: openai
|
|
name: Inferact/Qwen3.8-27B-NVFP4
|
|
base_url: http://vllm:11439/v1
|
|
temperature: 0.6
|
|
max_tokens: 16384
|
|
extra_body:
|
|
top_p: 0.95
|
|
top_k: 20
|
|
min_p: 0
|
|
chat_template_kwargs:
|
|
reasoning_effort: low
|