diff --git a/CHANGELOG.md b/CHANGELOG.md index 865fc18f..515bc5e4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,11 @@ # Changelog ## [Unreleased] +### Added + +- `orb_multimodal_nemotron` pre-built evaluation database (`nvidia/llama-nemotron-embed-vl-1b-v2` embedder). +- Reference configs under `evaluations/configs/` for the pre-built evaluation databases (`wix`, `orb_text`, `orb_multimodal`, `orb_multimodal_nemotron`). + ## [0.63.0] - 2026-06-28 ### Added diff --git a/docs/benchmarks.md b/docs/benchmarks.md index 70be679f..3e6f21d0 100644 --- a/docs/benchmarks.md +++ b/docs/benchmarks.md @@ -35,13 +35,16 @@ Active datasets: | `wix` | ~511MB | | `orb_text` — OpenRAG Bench, text embedder (`qwen3-embedding:4b`) with VLM picture descriptions baked into chunk content | ~18 GB | | `orb_multimodal` — OpenRAG Bench, multimodal embedder (`qwen3-vl-embedding-8b`); picture vectors live in the same space as text for cross-modal retrieval | ~16 GB | +| `orb_multimodal_nemotron` — OpenRAG Bench, multimodal embedder (`nvidia/llama-nemotron-embed-vl-1b-v2`), the embedder behind the published headline results | ~16 GB | -After downloading, run benchmarks with `--skip-db` to use the pre-built database: +After downloading, run benchmarks with `--skip-db`. Each database is built with a specific embedder, so pass its reference config from `evaluations/configs/` (a database only opens against a config whose embedder matches): ```bash -evaluations run wix --skip-db +evaluations run orb_multimodal_nemotron --skip-db --config configs/orb_multimodal_nemotron.yaml ``` +The configs use `vllm` as the model host. Point `base_url` at your own OpenAI-compatible endpoints to reproduce the numbers. + ### Configuration The benchmark script accepts several options: diff --git a/evaluations/configs/orb_multimodal.yaml b/evaluations/configs/orb_multimodal.yaml new file mode 100644 index 00000000..3fdff6a6 --- /dev/null +++ b/evaluations/configs/orb_multimodal.yaml @@ -0,0 +1,27 @@ +# Reference config for the `orb_multimodal` pre-built evaluation database. +# OpenRAG Bench with a multimodal embedder; picture vectors share the text space. +# Run: evaluations run orb_multimodal --skip-db --config configs/orb_multimodal.yaml +# base_url uses the `vllm` host serving each model over an OpenAI-compatible API. + +environment: development + +storage: + auto_vacuum: false + +embeddings: + model: + provider: vllm + name: qwen3-embedding-v-8b + vector_dim: 4096 + multimodal: true + base_url: http://vllm:11433/v1 + +reranking: + model: null + +qa: + model: + provider: openai + name: gemma4-26b + base_url: http://vllm:11432/v1 + vision: true diff --git a/evaluations/configs/orb_multimodal_nemotron.yaml b/evaluations/configs/orb_multimodal_nemotron.yaml new file mode 100644 index 00000000..8bef50bf --- /dev/null +++ b/evaluations/configs/orb_multimodal_nemotron.yaml @@ -0,0 +1,28 @@ +# Reference config for the `orb_multimodal_nemotron` pre-built evaluation database. +# OpenRAG Bench with the nvidia/llama-nemotron-embed-vl-1b-v2 multimodal embedder, +# the embedder behind the published headline benchmark numbers. +# Run: evaluations run orb_multimodal_nemotron --skip-db --config configs/orb_multimodal_nemotron.yaml +# base_url uses the `vllm` host serving each model over an OpenAI-compatible API. + +environment: development + +storage: + auto_vacuum: false + +embeddings: + model: + provider: vllm + name: nvidia/llama-nemotron-embed-vl-1b-v2 + vector_dim: 2048 + multimodal: true + base_url: http://vllm:11438/v1 + +reranking: + model: null + +qa: + model: + provider: openai + name: gemma4-26b + base_url: http://vllm:11432/v1 + vision: true diff --git a/evaluations/configs/orb_text.yaml b/evaluations/configs/orb_text.yaml new file mode 100644 index 00000000..3fa6ffae --- /dev/null +++ b/evaluations/configs/orb_text.yaml @@ -0,0 +1,28 @@ +# Reference config for the `orb_text` pre-built evaluation database. +# OpenRAG Bench with a text embedder and VLM picture descriptions baked into chunks. +# Run: evaluations run orb_text --skip-db --config configs/orb_text.yaml +# base_url uses the `vllm` host serving each model over an OpenAI-compatible API. + +environment: development + +storage: + auto_vacuum: false + +embeddings: + model: + provider: openai + name: qwen3-embedding-4b + vector_dim: 2560 + base_url: http://vllm:11431/v1 + +reranking: + model: + provider: mxbai + name: mixedbread-ai/mxbai-rerank-base-v2 + +qa: + model: + provider: openai + name: gemma4-26b + base_url: http://vllm:11432/v1 + vision: true diff --git a/evaluations/configs/wix.yaml b/evaluations/configs/wix.yaml new file mode 100644 index 00000000..a35d8843 --- /dev/null +++ b/evaluations/configs/wix.yaml @@ -0,0 +1,27 @@ +# Reference config for the `wix` pre-built evaluation database. +# Run: evaluations run wix --skip-db --config configs/wix.yaml +# base_url uses the `vllm` host serving each model over an OpenAI-compatible API. + +environment: development + +storage: + auto_vacuum: false + +embeddings: + model: + provider: openai + name: qwen3-embedding-4b + vector_dim: 2560 + base_url: http://vllm:11431/v1 + +reranking: + model: + provider: mxbai + name: mixedbread-ai/mxbai-rerank-base-v2 + +qa: + model: + provider: openai + name: gemma4-26b + base_url: http://vllm:11432/v1 + vision: true diff --git a/evaluations/evaluations/datasets/__init__.py b/evaluations/evaluations/datasets/__init__.py index c2ee4b84..1e132d04 100644 --- a/evaluations/evaluations/datasets/__init__.py +++ b/evaluations/evaluations/datasets/__init__.py @@ -1,6 +1,10 @@ from evaluations.config import DatasetSpec -from .open_rag_bench import ORB_MULTIMODAL_SPEC, ORB_TEXT_SPEC +from .open_rag_bench import ( + ORB_MULTIMODAL_NEMOTRON_SPEC, + ORB_MULTIMODAL_SPEC, + ORB_TEXT_SPEC, +) from .t2_ragbench import T2_FINQA_SPEC, T2_TATDQA_SPEC from .wix import WIX_SPEC @@ -10,6 +14,7 @@ DATASETS: dict[str, DatasetSpec] = { WIX_SPEC, ORB_TEXT_SPEC, ORB_MULTIMODAL_SPEC, + ORB_MULTIMODAL_NEMOTRON_SPEC, T2_FINQA_SPEC, T2_TATDQA_SPEC, ) diff --git a/evaluations/evaluations/datasets/open_rag_bench.py b/evaluations/evaluations/datasets/open_rag_bench.py index 5514627f..f20f48d9 100644 --- a/evaluations/evaluations/datasets/open_rag_bench.py +++ b/evaluations/evaluations/datasets/open_rag_bench.py @@ -239,3 +239,8 @@ ORB_MULTIMODAL_SPEC = _orb_spec( key="orb_multimodal", db_filename="open_rag_bench_multimodal.lancedb", ) + +ORB_MULTIMODAL_NEMOTRON_SPEC = _orb_spec( + key="orb_multimodal_nemotron", + db_filename="open_rag_bench_multimodal_nemotron.lancedb", +)