From f9392cee467789b8614888703f5f244f693203d4 Mon Sep 17 00:00:00 2001
From: Yiorgis Gozadinos <ggozadinos@gmail.com>
Date: Wed, 20 May 2026 12:56:22 +0300
Subject: [PATCH] evaluations: judge model moves to config.evaluations.judge

---
 CHANGELOG.md                              |  2 ++
 evaluations/evaluations/benchmark.py      |  7 +------
 haiku_rag_slim/haiku/rag/config/models.py | 16 ++++++++++++++++
 3 files changed, 19 insertions(+), 6 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 269f218b..04a1d3fe 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -13,6 +13,7 @@
 - BTree scalar indexes on `document_items.{document_id, position, self_ref}`. The 0.48.0 migration creates them on existing DBs. Per-doc lookups go from full-table scans (~100–300 ms) to point queries (~3–21 ms) on small/medium corpora.
 - Per-doc lazy cache for `items.jsonl` and `toc.json` in the analysis sandbox. First read fetches; subsequent reads of either file in the same `execute_code` session hit a serialized cache. One DB fetch per doc per session.
 - `cite` tool now accepts chunk_ids that resolve via the chunks table, not only chunk_ids from a prior `search()` result. Lets the model cite directly from `items.jsonl` / `toc.json` rows. The hallucination guard (`ModelRetry` on chunk_ids that don't exist in the DB) is preserved.
+- `AppConfig.evaluations` (`EvaluationsConfig`) with an optional `judge: ModelConfig`. Lets the eval CLI pin the LLM-as-judge per-yaml — including a custom `base_url` for any OpenAI-compatible endpoint (vLLM, LM Studio) without env-var routing.
 
 ### Removed
 
@@ -27,6 +28,7 @@
 - `prompts.qa` config field.
 - `evaluations optimize` subcommand and GEPA prompt-optimization. Drops `gepa` dep.
 - `--target qa` from `evaluations run`. Default is now `rag-skill`.
+- `--judge-model` flag from `evaluations run`. Set the judge in `config.evaluations.judge` instead.
 - `position` field on `toc.json` nodes (redundant with `item_range[0]`).
 - `position` and `tree_depth` from `items.jsonl` row serialization. Both fields are still persisted on `DocumentItem`; they are no longer surfaced to the sandbox.
 
diff --git a/evaluations/evaluations/benchmark.py b/evaluations/evaluations/benchmark.py
index 866287fc..254e0205 100644
--- a/evaluations/evaluations/benchmark.py
+++ b/evaluations/evaluations/benchmark.py
@@ -591,11 +591,6 @@ def run(
         "--multimodal-only",
         help="Only evaluate queries requiring image understanding.",
     ),
-    judge_model: str | None = typer.Option(
-        None,
-        "--judge-model",
-        help="Judge model as 'provider:name'. Defaults to ollama:qwen3.6.",
-    ),
     target: str = typer.Option(
         "rag-skill",
         "--target",
@@ -617,7 +612,7 @@ def run(
             f"Unknown target {target!r}. Choose from: {', '.join(TARGETS)}"
         )
     target_value = cast(Target, target)
-    judge_model_config = parse_model_option(judge_model) if judge_model else None
+    judge_model_config = app_config.evaluations.judge
     skill_model_config = parse_model_option(skill_model) if skill_model else None
 
     asyncio.run(
diff --git a/haiku_rag_slim/haiku/rag/config/models.py b/haiku_rag_slim/haiku/rag/config/models.py
index 387f19e6..a30c4524 100644
--- a/haiku_rag_slim/haiku/rag/config/models.py
+++ b/haiku_rag_slim/haiku/rag/config/models.py
@@ -236,6 +236,19 @@ class PromptsConfig(BaseModel):
     )
 
 
+class EvaluationsConfig(BaseModel):
+    """Settings consumed only by the `evaluations` package."""
+
+    judge: ModelConfig | None = Field(
+        default=None,
+        description=(
+            "Judge model for `evaluations run`'s LLM-as-judge step. "
+            "ModelConfig's base_url lets the judge point at any "
+            "OpenAI-compatible endpoint."
+        ),
+    )
+
+
 class AppConfig(BaseModel):
     environment: str = "production"
     storage: StorageConfig = Field(default_factory=StorageConfig)
@@ -249,3 +262,6 @@ class AppConfig(BaseModel):
     search: SearchConfig = Field(default_factory=SearchConfig)
     providers: ProvidersConfig = Field(default_factory=ProvidersConfig)
     prompts: PromptsConfig = Field(default_factory=PromptsConfig)
+    evaluations: "EvaluationsConfig" = Field(
+        default_factory=lambda: EvaluationsConfig()
+    )