Test evaluations

evaluations/tests/test_benchmark.py

@@ -0,0 +1,112 @@
+from pathlib import Path
+from unittest.mock import patch
+
+import pytest
+import typer
+
+from evaluations.benchmark import (
+    _load_config,
+    _resolve_dataset,
+    build_experiment_metadata,
+)
+from haiku.rag.config.models import AppConfig, ModelConfig
+
+
+class TestBuildExperimentMetadata:
+    def test_basic_metadata(self) -> None:
+        config = AppConfig()
+        result = build_experiment_metadata(
+            dataset_key="test",
+            test_cases=42,
+            config=config,
+        )
+
+        assert result["dataset"] == "test"
+        assert result["test_cases"] == 42
+        assert result["embedder_provider"] == config.embeddings.model.provider
+        assert result["embedder_model"] == config.embeddings.model.name
+        assert result["embedder_dim"] == config.embeddings.model.vector_dim
+        assert result["chunk_size"] == config.processing.chunk_size
+        assert result["search_limit"] == config.search.limit
+        assert result["context_radius"] == config.search.context_radius
+        assert result["qa_provider"] == config.qa.model.provider
+        assert result["qa_model"] == config.qa.model.name
+        assert "judge_provider" not in result
+
+    def test_with_judge_config(self) -> None:
+        config = AppConfig()
+        judge = ModelConfig(
+            provider="ollama", name="gpt-oss", enable_thinking=False, temperature=0.0
+        )
+        result = build_experiment_metadata(
+            dataset_key="test",
+            test_cases=10,
+            config=config,
+            judge_config=judge,
+        )
+
+        assert result["judge_provider"] == "ollama"
+        assert result["judge_model"] == "gpt-oss"
+        assert result["judge_temperature"] == 0.0
+        assert result["judge_enable_thinking"] is False
+
+    def test_no_reranker(self) -> None:
+        config = AppConfig()
+        result = build_experiment_metadata(
+            dataset_key="test", test_cases=1, config=config
+        )
+        assert result["rerank_provider"] is None
+        assert result["rerank_model"] is None
+
+    def test_with_reranker(self) -> None:
+        config = AppConfig()
+        config.reranking.model = ModelConfig(
+            provider="mxbai", name="mixedbread-ai/mxbai-rerank-base-v2"
+        )
+        result = build_experiment_metadata(
+            dataset_key="test", test_cases=1, config=config
+        )
+        assert result["rerank_provider"] == "mxbai"
+        assert result["rerank_model"] == "mixedbread-ai/mxbai-rerank-base-v2"
+
+
+class TestResolveDataset:
+    def test_valid_dataset(self) -> None:
+        spec = _resolve_dataset("repliqa")
+        assert spec.key == "repliqa"
+
+    def test_case_insensitive(self) -> None:
+        spec = _resolve_dataset("REPLIQA")
+        assert spec.key == "repliqa"
+
+    def test_unknown_dataset_raises(self) -> None:
+        with pytest.raises(typer.BadParameter, match="Unknown dataset 'nonexistent'"):
+            _resolve_dataset("nonexistent")
+
+    def test_error_lists_valid_datasets(self) -> None:
+        with pytest.raises(typer.BadParameter, match="repliqa"):
+            _resolve_dataset("nonexistent")
+
+
+class TestLoadConfig:
+    def test_explicit_path(self, tmp_path: Path) -> None:
+        config_file = tmp_path / "test.yaml"
+        config_file.write_text("search:\n  limit: 42\n")
+        config = _load_config(config_file)
+        assert config.search.limit == 42
+
+    def test_explicit_path_not_found(self, tmp_path: Path) -> None:
+        with pytest.raises(typer.BadParameter, match="Config file not found"):
+            _load_config(tmp_path / "nonexistent.yaml")
+
+    def test_none_falls_back_to_find_config(self, tmp_path: Path) -> None:
+        config_file = tmp_path / "haiku.rag.yaml"
+        config_file.write_text("search:\n  limit: 99\n")
+        with patch("evaluations.benchmark.find_config_file", return_value=config_file):
+            config = _load_config(None)
+        assert config.search.limit == 99
+
+    def test_none_no_config_uses_defaults(self) -> None:
+        with patch("evaluations.benchmark.find_config_file", return_value=None):
+            config = _load_config(None)
+        assert config == AppConfig()

evaluations/tests/test_config.py

@@ -0,0 +1,94 @@
+from pathlib import Path
+from unittest.mock import patch
+
+from evaluations.config import DatasetSpec, DocumentPayload, RetrievalSample
+
+
+def _make_spec(**kwargs: object) -> DatasetSpec:
+    defaults: dict[str, object] = {
+        "key": "test",
+        "db_filename": "test.lancedb",
+        "document_loader": lambda: None,
+        "document_mapper": lambda doc: None,
+        "qa_loader": lambda: None,
+        "qa_case_builder": lambda idx, doc: None,
+    }
+    defaults.update(kwargs)
+    return DatasetSpec(**defaults)  # type: ignore[arg-type]
+
+
+class TestDatasetSpecDbPath:
+    def test_override_path_takes_precedence(self) -> None:
+        spec = _make_spec()
+        override = Path("/tmp/custom.lancedb")
+        assert spec.db_path(override) == override
+
+    def test_default_uses_data_dir(self) -> None:
+        spec = _make_spec(db_filename="mydb.lancedb")
+        with patch(
+            "haiku.rag.utils.get_default_data_dir",
+            return_value=Path("/home/user/.local/share/haiku.rag"),
+        ):
+            result = spec.db_path()
+        assert result == Path(
+            "/home/user/.local/share/haiku.rag/evaluations/dbs/mydb.lancedb"
+        )
+
+    def test_none_override_uses_default(self) -> None:
+        spec = _make_spec(db_filename="other.lancedb")
+        with patch(
+            "haiku.rag.utils.get_default_data_dir",
+            return_value=Path("/data"),
+        ):
+            result = spec.db_path(None)
+        assert result == Path("/data/evaluations/dbs/other.lancedb")
+
+
+class TestDatasetSpecDefaults:
+    def test_optional_fields_default_to_none(self) -> None:
+        spec = _make_spec()
+        assert spec.retrieval_loader is None
+        assert spec.retrieval_mapper is None
+        assert spec.retrieval_evaluator is None
+        assert spec.document_limit is None
+        assert spec.system_prompt is None
+
+
+class TestDocumentPayload:
+    def test_defaults(self) -> None:
+        payload = DocumentPayload(uri="test://doc")
+        assert payload.content is None
+        assert payload.title is None
+        assert payload.metadata is None
+        assert payload.format == "md"
+        assert payload.source_path is None
+
+    def test_all_fields(self) -> None:
+        payload = DocumentPayload(
+            uri="test://doc",
+            content="hello",
+            title="Title",
+            metadata={"k": "v"},
+            format="html",
+            source_path=Path("/tmp/doc.pdf"),
+        )
+        assert payload.uri == "test://doc"
+        assert payload.content == "hello"
+        assert payload.source_path == Path("/tmp/doc.pdf")
+
+
+class TestRetrievalSample:
+    def test_defaults(self) -> None:
+        sample = RetrievalSample(question="q?", expected_uris=("u1",))
+        assert sample.skip is False
+        assert sample.source_type is None
+
+    def test_all_fields(self) -> None:
+        sample = RetrievalSample(
+            question="q?",
+            expected_uris=("u1", "u2"),
+            skip=True,
+            source_type="image",
+        )
+        assert sample.skip is True
+        assert sample.source_type == "image"

evaluations/tests/test_datasets.py

@@ -0,0 +1,287 @@
+from pathlib import Path
+
+from evaluations.datasets.hotpotqa import (
+    build_hotpotqa_case,
+    extract_unique_documents,
+    map_hotpotqa_document,
+    map_hotpotqa_retrieval,
+)
+from evaluations.datasets.open_rag_bench import (
+    build_orb_case,
+    download_pdf,
+    is_multimodal_query,
+    map_orb_document,
+    map_orb_retrieval,
+)
+from evaluations.datasets.repliqa import (
+    build_repliqa_case,
+    map_repliqa_document,
+    map_repliqa_retrieval,
+)
+from evaluations.datasets.wix import (
+    build_wix_case,
+    map_wix_document,
+    map_wix_retrieval,
+)
+
+
+class TestRepliqa:
+    def test_map_document(self) -> None:
+        doc = {"document_id": "doc-42", "document_extracted": "Some content here."}
+        payload = map_repliqa_document(doc)
+        assert payload.uri == "doc-42"
+        assert payload.content == "Some content here."
+
+    def test_map_retrieval(self) -> None:
+        doc = {
+            "question": "What happened?",
+            "answer": "Something happened.",
+            "document_id": "doc-42",
+        }
+        sample = map_repliqa_retrieval(doc)
+        assert sample is not None
+        assert sample.question == "What happened?"
+        assert sample.expected_uris == ("doc-42",)
+
+    def test_map_retrieval_skips_unanswerable(self) -> None:
+        doc = {
+            "question": "What?",
+            "answer": "The answer is not found in the document.",
+            "document_id": "doc-1",
+        }
+        assert map_repliqa_retrieval(doc) is None
+
+    def test_build_case(self) -> None:
+        doc = {
+            "document_id": "doc-7",
+            "question": "Why?",
+            "answer": "Because.",
+        }
+        case = build_repliqa_case(3, doc)
+        assert case.name == "3_doc-7"
+        assert case.inputs == "Why?"
+        assert case.expected_output == "Because."
+        assert case.metadata == {"document_id": "doc-7", "case_index": "3"}
+
+    def test_build_case_none_document_id(self) -> None:
+        doc = {"document_id": None, "question": "Q?", "answer": "A."}
+        case = build_repliqa_case(1, doc)
+        assert case.name == "case_1"
+
+
+class TestWix:
+    def test_map_document_with_all_fields(self) -> None:
+        doc = {
+            "id": 123,
+            "url": "https://wix.com/article",
+            "html_content": "<p>Content</p>",
+            "title": "My Article",
+        }
+        payload = map_wix_document(doc)
+        assert payload.uri == "123"
+        assert payload.content == "<p>Content</p>"
+        assert payload.title == "My Article"
+        assert payload.format == "html"
+        assert payload.metadata == {
+            "article_id": "123",
+            "url": "https://wix.com/article",
+        }
+
+    def test_map_document_no_id(self) -> None:
+        doc = {
+            "id": None,
+            "url": "https://wix.com/page",
+            "html_content": "<p>Text</p>",
+            "title": None,
+        }
+        payload = map_wix_document(doc)
+        assert payload.uri == "https://wix.com/page"
+
+    def test_map_document_no_metadata(self) -> None:
+        doc = {"id": None, "url": None, "html_content": "<p>X</p>", "title": None}
+        payload = map_wix_document(doc)
+        assert payload.metadata is None
+
+    def test_map_retrieval(self) -> None:
+        doc = {"question": "How to add a page?", "article_ids": [10, 20]}
+        sample = map_wix_retrieval(doc)
+        assert sample is not None
+        assert sample.question == "How to add a page?"
+        assert sample.expected_uris == ("10", "20")
+
+    def test_map_retrieval_no_article_ids(self) -> None:
+        doc = {"question": "Q?", "article_ids": None}
+        assert map_wix_retrieval(doc) is None
+
+    def test_map_retrieval_empty_article_ids(self) -> None:
+        doc = {"question": "Q?", "article_ids": []}
+        assert map_wix_retrieval(doc) is None
+
+    def test_build_case(self) -> None:
+        doc = {
+            "question": "How?",
+            "answer": "Like this.",
+            "article_ids": [5, 10],
+        }
+        case = build_wix_case(2, doc)
+        assert case.name == "2_5-10"
+        assert case.inputs == "How?"
+        assert case.expected_output == "Like this."
+        assert case.metadata is not None
+        assert case.metadata["case_index"] == "2"
+
+    def test_build_case_no_article_ids(self) -> None:
+        doc = {"question": "Q?", "answer": "A.", "article_ids": None}
+        case = build_wix_case(1, doc)
+        assert case.name == "case_1"
+
+
+class TestHotpotQA:
+    def test_map_document(self) -> None:
+        doc = {"title": "Albert Einstein", "content": "Was a physicist."}
+        payload = map_hotpotqa_document(doc)
+        assert payload.uri == "Albert Einstein"
+        assert payload.content == "Was a physicist."
+        assert payload.title == "Albert Einstein"
+
+    def test_map_retrieval(self) -> None:
+        doc = {
+            "question": "Who was Einstein?",
+            "supporting_facts": {"title": ["Albert Einstein", "Physics"]},
+        }
+        sample = map_hotpotqa_retrieval(doc)
+        assert sample is not None
+        assert sample.expected_uris == ("Albert Einstein", "Physics")
+
+    def test_map_retrieval_deduplicates_titles(self) -> None:
+        doc = {
+            "question": "Q?",
+            "supporting_facts": {"title": ["A", "B", "A"]},
+        }
+        sample = map_hotpotqa_retrieval(doc)
+        assert sample is not None
+        assert sample.expected_uris == ("A", "B")
+
+    def test_map_retrieval_no_titles(self) -> None:
+        doc = {"question": "Q?", "supporting_facts": {"title": []}}
+        assert map_hotpotqa_retrieval(doc) is None
+
+    def test_build_case(self) -> None:
+        doc = {
+            "id": "abc123",
+            "question": "What is X?",
+            "answer": "X is Y.",
+            "type": "comparison",
+            "level": "hard",
+        }
+        case = build_hotpotqa_case(5, doc)
+        assert case.name == "5_abc123"
+        assert case.inputs == "What is X?"
+        assert case.expected_output == "X is Y."
+        assert case.metadata == {
+            "question_id": "abc123",
+            "type": "comparison",
+            "level": "hard",
+            "case_index": "5",
+        }
+
+    def test_extract_unique_documents(self) -> None:
+        # Simulate a minimal dataset with context
+        dataset = [
+            {
+                "context": {
+                    "title": ["Doc A", "Doc B"],
+                    "sentences": [["Sentence 1."], ["Sentence 2.", " More."]],
+                }
+            },
+            {
+                "context": {
+                    "title": ["Doc A", "Doc C"],
+                    "sentences": [["Dupe."], ["Sentence 3."]],
+                }
+            },
+        ]
+        docs = extract_unique_documents(dataset)  # type: ignore[arg-type]
+        assert len(docs) == 3
+        titles = [d["title"] for d in docs]
+        assert titles == ["Doc A", "Doc B", "Doc C"]
+        assert docs[1]["content"] == "Sentence 2.  More."
+
+
+class TestOpenRAGBench:
+    def test_map_document(self, tmp_path: Path) -> None:
+        # Pre-create a cached PDF
+        cache_dir = tmp_path / "pdfs"
+        cache_dir.mkdir()
+        pdf_path = cache_dir / "paper1.pdf"
+        pdf_path.write_bytes(b"%PDF-fake")
+
+        doc = {"paper_id": "paper1", "pdf_url": "https://example.com/paper1.pdf"}
+        # Patch get_cache_dir to use our tmp_path
+        from unittest.mock import patch
+
+        with patch(
+            "evaluations.datasets.open_rag_bench.get_cache_dir", return_value=cache_dir
+        ):
+            payload = map_orb_document(doc)
+
+        assert payload is not None
+        assert payload.uri == "paper1"
+        assert payload.title == "paper1"
+        assert payload.source_path == pdf_path
+        assert payload.metadata == {"arxiv_id": "paper1"}
+
+    def test_map_document_download_fails(self, tmp_path: Path) -> None:
+        cache_dir = tmp_path / "pdfs"
+        cache_dir.mkdir()
+
+        doc = {"paper_id": "missing", "pdf_url": "https://example.com/missing.pdf"}
+        from unittest.mock import patch
+
+        with patch(
+            "evaluations.datasets.open_rag_bench.get_cache_dir", return_value=cache_dir
+        ):
+            with patch(
+                "evaluations.datasets.open_rag_bench.download_pdf", return_value=None
+            ):
+                payload = map_orb_document(doc)
+
+        assert payload is None
+
+    def test_map_retrieval(self) -> None:
+        doc = {
+            "query": "What is attention?",
+            "doc_id": "1706.03762",
+            "source": "text",
+        }
+        sample = map_orb_retrieval(doc)
+        assert sample is not None
+        assert sample.question == "What is attention?"
+        assert sample.expected_uris == ("1706.03762",)
+        assert sample.source_type == "text"
+
+    def test_build_case(self) -> None:
+        doc = {
+            "query_id": "q_abcdef12",
+            "query": "Explain transformers.",
+            "answer": "Transformers are...",
+            "type": "factual",
+            "source": "text",
+        }
+        case = build_orb_case(1, doc)
+        assert case.name == "1_q_abcdef"
+        assert case.inputs == "Explain transformers."
+        assert case.expected_output == "Transformers are..."
+        assert case.metadata is not None
+        assert case.metadata["query_id"] == "q_abcdef12"
+
+    def test_download_pdf_uses_cache(self, tmp_path: Path) -> None:
+        pdf_path = tmp_path / "cached.pdf"
+        pdf_path.write_bytes(b"%PDF-cached")
+        result = download_pdf("cached", "https://example.com/cached.pdf", tmp_path)
+        assert result == pdf_path
+
+    def test_is_multimodal_query(self) -> None:
+        assert is_multimodal_query("image") is True
+        assert is_multimodal_query("image_table") is True
+        assert is_multimodal_query("text") is False

evaluations/tests/test_evaluators.py

@@ -0,0 +1,104 @@
+from unittest.mock import MagicMock
+
+import pytest
+
+from evaluations.evaluators.map import MAPEvaluator
+from evaluations.evaluators.mrr import MRREvaluator
+
+
+class TestMRREvaluator:
+    def setup_method(self) -> None:
+        self.evaluator = MRREvaluator()
+
+    def _make_ctx(
+        self, relevant_uris: list[str], retrieved_uris: list[str]
+    ) -> MagicMock:
+        ctx = MagicMock()
+        ctx.metadata = {"relevant_uris": relevant_uris}
+        ctx.output = retrieved_uris
+        return ctx
+
+    def test_first_result_relevant(self) -> None:
+        ctx = self._make_ctx(["doc1"], ["doc1", "doc2", "doc3"])
+        assert self.evaluator.evaluate(ctx) == 1.0
+
+    def test_second_result_relevant(self) -> None:
+        ctx = self._make_ctx(["doc2"], ["doc1", "doc2", "doc3"])
+        assert self.evaluator.evaluate(ctx) == 0.5
+
+    def test_third_result_relevant(self) -> None:
+        ctx = self._make_ctx(["doc3"], ["doc1", "doc2", "doc3"])
+        assert self.evaluator.evaluate(ctx) == pytest.approx(1 / 3)
+
+    def test_no_relevant_found(self) -> None:
+        ctx = self._make_ctx(["doc_x"], ["doc1", "doc2", "doc3"])
+        assert self.evaluator.evaluate(ctx) == 0.0
+
+    def test_empty_retrieved(self) -> None:
+        ctx = self._make_ctx(["doc1"], [])
+        assert self.evaluator.evaluate(ctx) == 0.0
+
+    def test_multiple_relevant_returns_first_match(self) -> None:
+        ctx = self._make_ctx(["doc2", "doc3"], ["doc1", "doc2", "doc3"])
+        assert self.evaluator.evaluate(ctx) == 0.5
+
+    def test_none_metadata(self) -> None:
+        ctx = MagicMock()
+        ctx.metadata = None
+        ctx.output = ["doc1"]
+        assert self.evaluator.evaluate(ctx) == 0.0
+
+    def test_empty_relevant_uris(self) -> None:
+        ctx = self._make_ctx([], ["doc1", "doc2"])
+        assert self.evaluator.evaluate(ctx) == 0.0
+
+
+class TestMAPEvaluator:
+    def setup_method(self) -> None:
+        self.evaluator = MAPEvaluator()
+
+    def _make_ctx(
+        self, relevant_uris: list[str], retrieved_uris: list[str]
+    ) -> MagicMock:
+        ctx = MagicMock()
+        ctx.metadata = {"relevant_uris": relevant_uris}
+        ctx.output = retrieved_uris
+        return ctx
+
+    def test_perfect_single_doc(self) -> None:
+        ctx = self._make_ctx(["doc1"], ["doc1", "doc2", "doc3"])
+        assert self.evaluator.evaluate(ctx) == 1.0
+
+    def test_perfect_two_docs(self) -> None:
+        # Both relevant at positions 1 and 2: P@1=1/1, P@2=2/2 → AP = (1+1)/2 = 1.0
+        ctx = self._make_ctx(["doc1", "doc2"], ["doc1", "doc2", "doc3"])
+        assert self.evaluator.evaluate(ctx) == 1.0
+
+    def test_one_relevant_at_second_position(self) -> None:
+        # 1 relevant doc at position 2: P@2=1/2 → AP = 0.5/1 = 0.5
+        ctx = self._make_ctx(["doc2"], ["doc1", "doc2", "doc3"])
+        assert self.evaluator.evaluate(ctx) == 0.5
+
+    def test_two_relevant_with_gap(self) -> None:
+        # Relevant at positions 1 and 3: P@1=1/1, P@3=2/3 → AP = (1 + 2/3)/2
+        ctx = self._make_ctx(["doc1", "doc3"], ["doc1", "doc2", "doc3"])
+        expected = (1.0 + 2 / 3) / 2
+        assert self.evaluator.evaluate(ctx) == pytest.approx(expected)
+
+    def test_no_relevant_found(self) -> None:
+        ctx = self._make_ctx(["doc_x"], ["doc1", "doc2", "doc3"])
+        assert self.evaluator.evaluate(ctx) == 0.0
+
+    def test_empty_retrieved(self) -> None:
+        ctx = self._make_ctx(["doc1"], [])
+        assert self.evaluator.evaluate(ctx) == 0.0
+
+    def test_none_metadata(self) -> None:
+        ctx = MagicMock()
+        ctx.metadata = None
+        ctx.output = ["doc1"]
+        assert self.evaluator.evaluate(ctx) == 0.0
+
+    def test_empty_relevant_uris(self) -> None:
+        ctx = self._make_ctx([], ["doc1", "doc2"])
+        assert self.evaluator.evaluate(ctx) == 0.0

evaluations/tests/test_optimization.py

@@ -7,10 +7,12 @@ from pydantic_evals import Case
 
 from gepa.core.adapter import EvaluationBatch
 
+from evaluations.config import DatasetSpec
 from evaluations.optimization import (
     EvalTrajectory,
     QAPromptAdapter,
     ReflectionLM,
+    run_optimization,
 )
 from haiku.rag.config.models import AppConfig
 
@@ -261,6 +263,163 @@ class TestReflectionLM:
         assert "user: Hello" in call_arg
 
 
+class TestEvaluateSync:
+    def test_delegates_to_evaluate_async(
+        self,
+        adapter: QAPromptAdapter,
+        sample_cases: list[Case[str, str, dict[str, str]]],
+    ) -> None:
+        expected_batch: EvaluationBatch[EvalTrajectory, str | None] = EvaluationBatch(
+            outputs=["answer1", "answer2"],
+            scores=[0.9, 0.8],
+            trajectories=None,
+        )
+
+        with patch.object(
+            adapter,
+            "_evaluate_async",
+            new_callable=AsyncMock,
+            return_value=expected_batch,
+        ) as mock_eval:
+            result = adapter.evaluate(
+                sample_cases, {"instructions": "my prompt"}, capture_traces=True
+            )
+
+        mock_eval.assert_called_once_with(sample_cases, "my prompt", True)
+        assert result is expected_batch
+
+
 class TestProposalAttribute:
     def test_propose_new_texts_is_none(self, adapter: QAPromptAdapter) -> None:
         assert adapter.propose_new_texts is None
+
+
+class TestRunOptimization:
+    def _make_spec(self, db_path: Path) -> DatasetSpec:
+        return DatasetSpec(
+            key="test",
+            db_filename="test.lancedb",
+            document_loader=lambda: None,  # type: ignore[return-value]
+            document_mapper=lambda doc: None,
+            qa_loader=lambda: None,  # type: ignore[return-value]
+            qa_case_builder=lambda idx, doc: None,  # type: ignore[return-value]
+            system_prompt="You are a test assistant.",
+        )
+
+    def test_returns_results(self, tmp_path: Path) -> None:
+        spec = self._make_spec(tmp_path / "test.lancedb")
+        cases: list[Case[str, str, dict[str, str]]] = [
+            Case(
+                name="q1",
+                inputs="Q?",
+                expected_output="A.",
+                metadata={"case_index": "1"},
+            ),
+        ]
+
+        mock_result = MagicMock()
+        mock_result.best_idx = 0
+        mock_result.val_aggregate_scores = [0.95]
+        mock_result.best_candidate = {"instructions": "optimized prompt"}
+        mock_result.total_metric_calls = 10
+        mock_result.num_candidates = 3
+
+        with (
+            patch("evaluations.optimization.get_model"),
+            patch("evaluations.optimization.ReflectionLM"),
+            patch("gepa.optimize", return_value=mock_result),
+        ):
+            result = run_optimization(
+                spec=spec,
+                config=AppConfig(),
+                cases=cases,
+                max_calls=10,
+                db_path=tmp_path / "test.lancedb",
+            )
+
+        assert result["best_score"] == 0.95
+        assert result["best_prompt"] == "optimized prompt"
+        assert result["total_calls"] == 10
+        assert result["num_candidates"] == 3
+
+    def test_saves_output_file(self, tmp_path: Path) -> None:
+        spec = self._make_spec(tmp_path / "test.lancedb")
+        cases: list[Case[str, str, dict[str, str]]] = [
+            Case(
+                name="q1",
+                inputs="Q?",
+                expected_output="A.",
+                metadata={"case_index": "1"},
+            ),
+        ]
+        output_path = tmp_path / "prompt.txt"
+
+        mock_result = MagicMock()
+        mock_result.best_idx = 0
+        mock_result.val_aggregate_scores = [0.85]
+        mock_result.best_candidate = {"instructions": "saved prompt"}
+        mock_result.total_metric_calls = 5
+        mock_result.num_candidates = 2
+
+        with (
+            patch("evaluations.optimization.get_model"),
+            patch("evaluations.optimization.ReflectionLM"),
+            patch("gepa.optimize", return_value=mock_result),
+        ):
+            run_optimization(
+                spec=spec,
+                config=AppConfig(),
+                cases=cases,
+                max_calls=5,
+                db_path=tmp_path / "test.lancedb",
+                output=output_path,
+            )
+
+        assert output_path.read_text() == "saved prompt"
+
+    def test_uses_default_prompt_when_spec_has_none(self, tmp_path: Path) -> None:
+        spec = DatasetSpec(
+            key="test",
+            db_filename="test.lancedb",
+            document_loader=lambda: None,  # type: ignore[return-value]
+            document_mapper=lambda doc: None,
+            qa_loader=lambda: None,  # type: ignore[return-value]
+            qa_case_builder=lambda idx, doc: None,  # type: ignore[return-value]
+        )
+        cases: list[Case[str, str, dict[str, str]]] = [
+            Case(
+                name="q1",
+                inputs="Q?",
+                expected_output="A.",
+                metadata={"case_index": "1"},
+            ),
+        ]
+
+        mock_result = MagicMock()
+        mock_result.best_idx = 0
+        mock_result.val_aggregate_scores = [0.5]
+        mock_result.best_candidate = "fallback prompt"
+        mock_result.total_metric_calls = 1
+        mock_result.num_candidates = 1
+
+        with (
+            patch("evaluations.optimization.get_model"),
+            patch("evaluations.optimization.ReflectionLM"),
+            patch("gepa.optimize", return_value=mock_result) as mock_gepa,
+        ):
+            result = run_optimization(
+                spec=spec,
+                config=AppConfig(),
+                cases=cases,
+                max_calls=1,
+                db_path=tmp_path / "test.lancedb",
+            )
+
+        # When best_candidate is a string (not dict), it should be used directly
+        assert result["best_prompt"] == "fallback prompt"
+
+        # Verify seed_candidate used QA_SYSTEM_PROMPT (not None)
+        call_kwargs = mock_gepa.call_args[1]
+        seed = call_kwargs["seed_candidate"]
+        assert seed["instructions"] is not None
+        assert len(seed["instructions"]) > 0