49 lines
1.5 KiB
YAML
49 lines
1.5 KiB
YAML
# Reference config for the `t2_finqa` pre-built evaluation database.
|
|
# T²-RAGBench (FinQA) financial QA, scored by exact numeric match.
|
|
# No `evaluations.judge` block: the spec sets `NumberMatchEvaluator`, which
|
|
# replaces the evaluator list, so no LLM judge is constructed for this dataset.
|
|
# Run: evaluations run t2_finqa --skip-db --target analysis-capability --config configs/t2_finqa.yaml
|
|
# base_url uses the `vllm` host serving each model over an OpenAI-compatible API.
|
|
|
|
environment: development
|
|
|
|
storage:
|
|
auto_vacuum: false
|
|
|
|
embeddings:
|
|
model:
|
|
provider: openai
|
|
name: qwen3-embedding-4b
|
|
vector_dim: 2560
|
|
base_url: http://vllm:11431/v1
|
|
|
|
processing:
|
|
converter: docling-local
|
|
pictures: none # ~4% of pages carry a figure/chart; dropped as non-essential to the numeric QA
|
|
chunking_use_markdown_tables: true
|
|
|
|
reranking:
|
|
model:
|
|
provider: vllm
|
|
name: Qwen/Qwen3-Reranker-4B
|
|
base_url: http://vllm:11455
|
|
|
|
qa:
|
|
model:
|
|
provider: openai
|
|
name: RedHatAI/Qwen3.6-35B-A3B-NVFP4
|
|
base_url: http://vllm:11430/v1
|
|
temperature: 0.3
|
|
max_tokens: 16384
|
|
extra_body:
|
|
chat_template_kwargs:
|
|
enable_thinking: true
|
|
|
|
prompts:
|
|
domain_preamble: |
|
|
Use search() to find the relevant documents. Do not iterate over all of
|
|
/documents or read every document's content — that will time out.
|
|
|
|
For questions with a numeric answer, end your response with a final line
|
|
formatted exactly as `ANSWER: <number>`, containing a single number. Keep a
|
|
percent sign if the answer is a percentage.
|