# Reference config for the `frames` evaluation database.
# FRAMES (google/frames-benchmark): 824 multi-hop questions over a corpus of
# the ~2.5k Wikipedia articles linked per question, fetched at current
# revision (revid + fetch date recorded in the article cache).
# Run: evaluations run frames --config configs/frames.yaml
# base_url uses the `vllm` host serving each model over an OpenAI-compatible API.

environment: development

storage:
  auto_vacuum: false

embeddings:
  model:
    provider: openai
    name: qwen3-embedding-4b
    vector_dim: 2560
    base_url: http://vllm:11431/v1

reranking:
  model:
    provider: vllm
    name: Qwen/Qwen3-Reranker-4B
    base_url: http://vllm:11433

qa:
  model:
    provider: openai
    name: gemma4-26b
    base_url: http://vllm:11432/v1
    # vLLM reserves max_tokens out of max_model_len; a large value starves
    # the input budget and 400s long agentic contexts.
    max_tokens: 8192

evaluations:
  judge:
    provider: openai
    name: RedHatAI/Qwen3.6-35B-A3B-NVFP4
    base_url: http://vllm:11430/v1
    temperature: 0.0
    max_tokens: 16384