"""Tests for `api.reasoning` — the live `docling-agent` RAG runner endpoint.
docling-agent + mellea are NOT installed in the CI test env (heavy deps).
The endpoint does a lazy import inside the handler; we stub the modules via
`sys.modules` injection so the tests cover the real code path without
bringing in Ollama, mellea, or LLM clients.
"""
from __future__ import annotations
import sys
import types
from dataclasses import replace
from unittest.mock import AsyncMock, MagicMock
import pytest
from fastapi import FastAPI
from fastapi.testclient import TestClient
from api import reasoning as reasoning_module
from api.reasoning import router
from domain.models import AnalysisJob
def _patched_settings(monkeypatch, **overrides):
"""Replace `api.reasoning.settings` with a frozen dataclass copy carrying
the given overrides. `Settings` is frozen, so attribute-level monkeypatch
doesn't work — we swap the whole instance on the module.
"""
new_settings = replace(reasoning_module.settings, **overrides)
monkeypatch.setattr(reasoning_module, "settings", new_settings)
return new_settings
def _job_with_doc_json() -> AnalysisJob:
job = AnalysisJob(document_id="doc-1")
job.document_filename = "hello.pdf"
job.mark_running()
job.mark_completed(
markdown="# Hello",
html="
Hello
",
pages_json="[]",
# Minimal placeholder — the test stubs `DoclingDocument.model_validate_json`
# so the content doesn't need to be a real DoclingDocument.
document_json='{"stub": true}',
chunks_json="[]",
)
return job
@pytest.fixture
def mock_analysis_repo() -> AsyncMock:
repo = AsyncMock()
repo.find_latest_completed_by_document.return_value = _job_with_doc_json()
return repo
@pytest.fixture
def stub_docling_agent(monkeypatch):
"""Inject fake `docling_agent.agents` + `docling_core.types.doc.document`
modules so the endpoint's lazy imports resolve to our stubs.
Returns the `DoclingRAGAgent` stub class so tests can assert on its calls
/ configure its `_rag_loop` return value.
"""
fake_result = MagicMock()
fake_result.answer = "stub answer"
fake_result.converged = True
fake_result.iterations = [
MagicMock(
model_dump=lambda: {
"iteration": 1,
"section_ref": "#/texts/0",
"reason": "looks relevant",
"section_text_length": 42,
"can_answer": True,
"response": "stub answer",
}
)
]
agent_instance = MagicMock()
agent_instance._rag_loop.return_value = fake_result
agent_class = MagicMock(return_value=agent_instance)
fake_agents_mod = types.ModuleType("docling_agent.agents")
fake_agents_mod.DoclingRAGAgent = agent_class
fake_root_mod = types.ModuleType("docling_agent")
fake_root_mod.agents = fake_agents_mod
fake_doc_class = MagicMock()
fake_doc_class.model_validate_json = MagicMock(return_value="fake-doc-instance")
fake_doc_mod = types.ModuleType("docling_core.types.doc.document")
fake_doc_mod.DoclingDocument = fake_doc_class
# Stub `mellea.backends.model_ids.ModelIdentifier` — the endpoint wraps
# the string model_id in this dataclass before handing to DoclingRAGAgent.
# Identity-like: stores the kwargs so tests can assert on `ollama_name`.
def fake_model_identifier(**kwargs):
m = MagicMock()
m.ollama_name = kwargs.get("ollama_name")
m.openai_name = kwargs.get("openai_name")
return m
fake_model_ids_mod = types.ModuleType("mellea.backends.model_ids")
fake_model_ids_mod.ModelIdentifier = fake_model_identifier
fake_backends_mod = types.ModuleType("mellea.backends")
fake_backends_mod.model_ids = fake_model_ids_mod
fake_mellea_mod = types.ModuleType("mellea")
fake_mellea_mod.backends = fake_backends_mod
monkeypatch.setitem(sys.modules, "docling_agent", fake_root_mod)
monkeypatch.setitem(sys.modules, "docling_agent.agents", fake_agents_mod)
monkeypatch.setitem(sys.modules, "docling_core.types.doc.document", fake_doc_mod)
monkeypatch.setitem(sys.modules, "mellea", fake_mellea_mod)
monkeypatch.setitem(sys.modules, "mellea.backends", fake_backends_mod)
monkeypatch.setitem(sys.modules, "mellea.backends.model_ids", fake_model_ids_mod)
return agent_class, agent_instance, fake_result
@pytest.fixture
def client(mock_analysis_repo: AsyncMock) -> TestClient:
app = FastAPI()
app.include_router(router)
app.state.analysis_repo = mock_analysis_repo
return TestClient(app)
class TestRagDisabled:
def test_503_when_flag_off(self, client: TestClient, monkeypatch) -> None:
_patched_settings(monkeypatch, rag_enabled=False)
resp = client.post("/api/documents/doc-1/rag", json={"query": "Q"})
assert resp.status_code == 503
assert "RAG_ENABLED" in resp.json()["detail"]
class TestRagValidation:
def test_400_when_query_empty(self, client: TestClient, monkeypatch) -> None:
_patched_settings(monkeypatch, rag_enabled=True)
resp = client.post("/api/documents/doc-1/rag", json={"query": " "})
assert resp.status_code == 400
def test_404_when_no_completed_analysis(
self, client: TestClient, mock_analysis_repo: AsyncMock, monkeypatch
) -> None:
_patched_settings(monkeypatch, rag_enabled=True)
mock_analysis_repo.find_latest_completed_by_document.return_value = None
resp = client.post("/api/documents/doc-1/rag", json={"query": "Q"})
assert resp.status_code == 404
class TestRagSuccess:
def test_returns_rag_result_shape(
self, client: TestClient, stub_docling_agent, monkeypatch
) -> None:
_patched_settings(monkeypatch, rag_enabled=True)
_agent_class, _agent_instance, _fake_result = stub_docling_agent
resp = client.post("/api/documents/doc-1/rag", json={"query": "What is this?"})
assert resp.status_code == 200
data = resp.json()
assert data["answer"] == "stub answer"
assert data["converged"] is True
assert len(data["iterations"]) == 1
it = data["iterations"][0]
assert it["iteration"] == 1
assert it["section_ref"] == "#/texts/0"
assert it["can_answer"] is True
def test_calls_rag_loop_with_query_and_doc(
self, client: TestClient, stub_docling_agent, monkeypatch
) -> None:
_patched_settings(monkeypatch, rag_enabled=True)
_agent_class, agent_instance, _ = stub_docling_agent
client.post("/api/documents/doc-1/rag", json={"query": "Hello?"})
agent_instance._rag_loop.assert_called_once()
kwargs = agent_instance._rag_loop.call_args.kwargs
assert kwargs["query"] == "Hello?"
# The stub returns the string "fake-doc-instance" from model_validate_json
# and we pass it straight through to `doc=`.
assert kwargs["doc"] == "fake-doc-instance"
def test_uses_default_model_id_when_not_overridden(
self, client: TestClient, stub_docling_agent, monkeypatch
) -> None:
_patched_settings(monkeypatch, rag_enabled=True, rag_model_id="custom-model:7b")
agent_class, _, _ = stub_docling_agent
client.post("/api/documents/doc-1/rag", json={"query": "Q"})
agent_class.assert_called_once()
# model_id is wrapped in a ModelIdentifier(ollama_name=...) dataclass
# before reaching the agent — the stub exposes the field for assertion.
passed = agent_class.call_args.kwargs["model_id"]
assert passed.ollama_name == "custom-model:7b"
def test_per_request_model_id_override_wins(
self, client: TestClient, stub_docling_agent, monkeypatch
) -> None:
_patched_settings(monkeypatch, rag_enabled=True, rag_model_id="default:7b")
agent_class, _, _ = stub_docling_agent
client.post("/api/documents/doc-1/rag", json={"query": "Q", "model_id": "override:13b"})
passed = agent_class.call_args.kwargs["model_id"]
assert passed.ollama_name == "override:13b"
def test_sets_ollama_host_env_from_settings(
self, client: TestClient, stub_docling_agent, monkeypatch
) -> None:
import os
_patched_settings(monkeypatch, rag_enabled=True, ollama_host="http://ollama:11434")
client.post("/api/documents/doc-1/rag", json={"query": "Q"})
assert os.environ["OLLAMA_HOST"] == "http://ollama:11434"
class TestRagDepsMissing:
def test_503_when_docling_agent_not_installed(self, client: TestClient, monkeypatch) -> None:
_patched_settings(monkeypatch, rag_enabled=True)
# Simulate the import failing: remove any stub and ensure the name
# resolves to a module that raises on attribute access.
monkeypatch.setitem(sys.modules, "docling_agent.agents", None)
resp = client.post("/api/documents/doc-1/rag", json={"query": "Q"})
assert resp.status_code == 503
assert "docling-agent" in resp.json()["detail"]
class TestRagUpstreamFailure:
def test_502_when_docling_agent_raises_indexerror(
self, client: TestClient, stub_docling_agent, monkeypatch
) -> None:
"""Known docling-agent bug: `find_json_dicts(answer.value)[0]` raises
`IndexError` when the model fails to produce parseable JSON after
retries. Our endpoint must surface a 502 with a human-readable
message, not a 500 stack trace."""
_patched_settings(monkeypatch, rag_enabled=True, rag_model_id="granite4:micro-h")
_agent_class, agent_instance, _ = stub_docling_agent
agent_instance._rag_loop.side_effect = IndexError("list index out of range")
resp = client.post("/api/documents/doc-1/rag", json={"query": "Quelle tarification ?"})
assert resp.status_code == 502
detail = resp.json()["detail"]
assert "granite4:micro-h" in detail
assert "parseable" in detail or "rephrase" in detail
def test_500_for_other_unexpected_errors(
self, client: TestClient, stub_docling_agent, monkeypatch
) -> None:
_patched_settings(monkeypatch, rag_enabled=True)
_agent_class, agent_instance, _ = stub_docling_agent
agent_instance._rag_loop.side_effect = RuntimeError("Ollama unreachable")
resp = client.post("/api/documents/doc-1/rag", json={"query": "Q"})
assert resp.status_code == 500
assert "Ollama unreachable" in resp.json()["detail"]