diff --git a/haiku_rag_slim/haiku/rag/ingester/api/routes/providers.py b/haiku_rag_slim/haiku/rag/ingester/api/routes/providers.py new file mode 100644 index 00000000..b2b9dc73 --- /dev/null +++ b/haiku_rag_slim/haiku/rag/ingester/api/routes/providers.py @@ -0,0 +1,38 @@ +import asyncio + +import httpx +from fastapi import APIRouter, Depends + +from haiku.rag.ingester.api.schemas import ProviderEndpoint, ProvidersResponse +from haiku.rag.ingester.api.server import APIState, get_state + +router = APIRouter(tags=["providers"]) + +# Short timeout — operators care that the endpoint is reachable right now, +# not that it might respond if we wait. A docling-serve that takes longer +# than this to answer /health is effectively down for ingest purposes. +_PROBE_TIMEOUT_S = 2.0 + + +async def _probe(client: httpx.AsyncClient, base_url: str) -> ProviderEndpoint: + url = f"{base_url.rstrip('/')}/health" + try: + response = await client.get(url) + except httpx.HTTPError as exc: + return ProviderEndpoint(base_url=base_url, reachable=False, error=str(exc)) + return ProviderEndpoint( + base_url=base_url, + reachable=response.is_success, + status_code=response.status_code, + ) + + +@router.get("/providers", response_model=ProvidersResponse) +async def providers(state: APIState = Depends(get_state)) -> ProvidersResponse: + """Probe configured external providers (currently docling-serve) and + return their reachability. Dashboard polls this to surface a downstream + outage that the ingester itself can only see via worker job failures.""" + base_urls = state.config.providers.docling_serve.base_urls + async with httpx.AsyncClient(timeout=_PROBE_TIMEOUT_S) as client: + results = await asyncio.gather(*(_probe(client, u) for u in base_urls)) + return ProvidersResponse(docling_serve=list(results)) diff --git a/haiku_rag_slim/haiku/rag/ingester/api/schemas.py b/haiku_rag_slim/haiku/rag/ingester/api/schemas.py index cc421fd1..9833923d 100644 --- a/haiku_rag_slim/haiku/rag/ingester/api/schemas.py +++ b/haiku_rag_slim/haiku/rag/ingester/api/schemas.py @@ -20,6 +20,24 @@ class HealthResponse(BaseModel): worker_breaker_consecutive_failures: int = 0 +class ProviderEndpoint(BaseModel): + base_url: str + reachable: bool + # Status code from the probe (200 on success, the HTTP code on a non-2xx + # response, or None when the probe failed before getting a response). + status_code: int | None = None + # Error string when reachable is False and we have one (DNS failure, + # connection refused, timeout). None on success or unknown failure. + error: str | None = None + + +class ProvidersResponse(BaseModel): + """Reachability snapshot of configured external providers. Probed + on demand when /providers is hit; not cached.""" + + docling_serve: list[ProviderEndpoint] + + class SourceSummary(BaseModel): source_id: str type: Literal["fs", "http", "s3", "webdav"] diff --git a/haiku_rag_slim/haiku/rag/ingester/api/server.py b/haiku_rag_slim/haiku/rag/ingester/api/server.py index f29f981c..2ef34bf0 100644 --- a/haiku_rag_slim/haiku/rag/ingester/api/server.py +++ b/haiku_rag_slim/haiku/rag/ingester/api/server.py @@ -39,6 +39,7 @@ def build_app( dlq, health, jobs, + providers, sources, stats, ) @@ -61,5 +62,6 @@ def build_app( app.include_router(sources.router, dependencies=auth_dep) app.include_router(dlq.router, dependencies=auth_dep) app.include_router(stats.router, dependencies=auth_dep) + app.include_router(providers.router, dependencies=auth_dep) return app diff --git a/haiku_rag_slim/haiku/rag/ingester/api/static/index.html b/haiku_rag_slim/haiku/rag/ingester/api/static/index.html index c1ed1bf5..8b598bf2 100644 --- a/haiku_rag_slim/haiku/rag/ingester/api/static/index.html +++ b/haiku_rag_slim/haiku/rag/ingester/api/static/index.html @@ -86,6 +86,7 @@ tr:last-child td { border-bottom: none; } tr:hover td { background: rgba(255, 255, 255, 0.02); } td.id, td.worker { font-family: ui-monospace, SFMono-Regular, Menlo, monospace; color: var(--muted); } + td.retried { color: var(--warn); font-weight: 600; cursor: help; } td.uri { font-family: ui-monospace, SFMono-Regular, Menlo, monospace; max-width: 0; white-space: nowrap; overflow: hidden; text-overflow: ellipsis; @@ -95,6 +96,10 @@ color: var(--dead); font-size: 12px; max-width: 0; white-space: nowrap; overflow: hidden; text-overflow: ellipsis; } + td.ok { + color: var(--succeeded); font-size: 12px; max-width: 0; + white-space: nowrap; overflow: hidden; text-overflow: ellipsis; + } td.actions { width: 1%; white-space: nowrap; } td.actions button { background: var(--bg); color: var(--text); border: 1px solid var(--border); @@ -141,8 +146,6 @@ never - -
@@ -199,6 +202,13 @@
+
+

Providers

+
+
loading…
+
+
+

Active jobs

@@ -207,7 +217,7 @@
-

Recent failures

+

Dead letter

loading…
@@ -228,14 +238,12 @@ const POLL_MS = 3000; let token = localStorage.getItem(TOKEN_KEY) || ""; - let paused = false; - let timer = null; let lastRefreshAt = null; const $ = (id) => document.getElementById(id); async function fetchJson(path) { - const headers = token ? { Authorization: "Bearer " + token } : {}; + const headers = token ? { Authorization: `Bearer ${token}` } : {}; const res = await fetch(path, { headers }); if (res.status === 401) { const entered = prompt( @@ -249,21 +257,21 @@ } throw new Error("unauthorized"); } - if (!res.ok) throw new Error(path + " → " + res.status); + if (!res.ok) throw new Error(`${path} → ${res.status}`); return res.json(); } async function postJson(path) { - const headers = token ? { Authorization: "Bearer " + token } : {}; + const headers = token ? { Authorization: `Bearer ${token}` } : {}; const res = await fetch(path, { method: "POST", headers }); - if (!res.ok) throw new Error(path + " → " + res.status); + if (!res.ok) throw new Error(`${path} → ${res.status}`); return res.json(); } async function deleteJson(path) { - const headers = token ? { Authorization: "Bearer " + token } : {}; + const headers = token ? { Authorization: `Bearer ${token}` } : {}; const res = await fetch(path, { method: "DELETE", headers }); - if (!res.ok) throw new Error(path + " → " + res.status); + if (!res.ok) throw new Error(`${path} → ${res.status}`); return res.json(); } @@ -271,16 +279,16 @@ if (!iso) return "—"; const ms = Date.now() - Date.parse(iso); if (ms < 0) return "in the future"; - return formatDuration(ms / 1000) + " ago"; + return `${formatDuration(ms / 1000)} ago`; } function formatDuration(s) { - if (s == null || isNaN(s)) return "—"; + if (s == null || Number.isNaN(s)) return "—"; if (s < 1) return "<1s"; - if (s < 60) return Math.floor(s) + "s"; - if (s < 3600) return Math.floor(s / 60) + "m " + Math.floor(s % 60) + "s"; - if (s < 86400) return Math.floor(s / 3600) + "h " + Math.floor((s % 3600) / 60) + "m"; - return Math.floor(s / 86400) + "d " + Math.floor((s % 86400) / 3600) + "h"; + if (s < 60) return `${Math.floor(s)}s`; + if (s < 3600) return `${Math.floor(s / 60)}m ${Math.floor(s % 60)}s`; + if (s < 86400) return `${Math.floor(s / 3600)}h ${Math.floor((s % 3600) / 60)}m`; + return `${Math.floor(s / 86400)}d ${Math.floor((s % 86400) / 3600)}h`; } function shortId(id) { @@ -310,7 +318,7 @@ $("stat-30m").textContent = stats.throughput.succeeded_30m; $("stat-1h").textContent = stats.throughput.succeeded_1h; $("stat-workers").textContent = - stats.workers.busy + " / " + stats.workers.total; + `${stats.workers.busy} / ${stats.workers.total}`; $("stat-oldest").textContent = stats.oldest_queued_age_s == null ? "—" @@ -344,7 +352,7 @@ }; function renderSources(sources) { - $("sources-count").textContent = "(" + sources.length + ")"; + $("sources-count").textContent = `(${sources.length})`; if (!sources.length) { $("sources-body").innerHTML = '
no sources configured
'; @@ -395,11 +403,11 @@ `; } - function truncateMiddle(s, n) { + function _truncateMiddle(s, n) { if (!s || s.length <= n) return s; const head = Math.ceil((n - 1) / 2); const tail = Math.floor((n - 1) / 2); - return s.slice(0, head) + "…" + s.slice(-tail); + return `${s.slice(0, head)}…${s.slice(-tail)}`; } function opBadge(op) { @@ -408,26 +416,68 @@ return `${label}`; } + function renderProviders(providers) { + const entries = providers.docling_serve || []; + $("providers-count").textContent = `(${entries.length})`; + if (!entries.length) { + $("providers-body").innerHTML = + '
no external providers configured
'; + return; + } + const rows = entries + .map((p) => { + const badge = p.reachable + ? 'REACHABLE' + : 'UNREACHABLE'; + const detail = p.reachable + ? p.status_code != null + ? `HTTP ${p.status_code}` + : "" + : escapeHtml(p.error ?? "no response"); + const detailClass = p.reachable ? "ok" : "err"; + return ` + docling-serve + ${escapeHtml(p.base_url)} + ${badge} + ${detail} + `; + }) + .join(""); + $("providers-body").innerHTML = ` + + + + ${rows} +
KindURLStatusDetail
`; + } + function renderActive(jobs) { - $("active-count").textContent = "(" + jobs.length + ")"; + $("active-count").textContent = `(${jobs.length})`; if (!jobs.length) { $("active-body").innerHTML = '
nothing running
'; return; } const rows = jobs - .map( - (j) => ` + .map((j) => { + const tryTitle = j.last_error + ? ` title="Previous attempt failed: ${escapeHtml(j.last_error)}"` + : ""; + const tryCell = + j.attempts > 1 + ? `${j.attempts}/${j.max_attempts}` + : `${j.attempts}/${j.max_attempts}`; + return ` ${shortId(j.id)} ${opBadge(j.op)} ${escapeHtml(j.uri)} ${escapeHtml(j.claimed_by ?? "—")} - ${j.attempts}/${j.max_attempts} + ${tryCell} ${relTime(j.claimed_at)} - + - `, - ) + `; + }) .join(""); $("active-body").innerHTML = ` @@ -438,9 +488,10 @@ } function renderDead(jobs) { - $("dead-count").textContent = "(" + jobs.length + ")"; + $("dead-count").textContent = `(${jobs.length})`; if (!jobs.length) { - $("dead-body").innerHTML = '
no recent failures
'; + $("dead-body").innerHTML = + '
no jobs in the dead letter queue
'; return; } const rows = jobs @@ -465,7 +516,7 @@ } function renderRecent(jobs) { - $("recent-count").textContent = "(" + jobs.length + ")"; + $("recent-count").textContent = `(${jobs.length})`; if (!jobs.length) { $("recent-body").innerHTML = '
nothing completed yet
'; return; @@ -496,20 +547,27 @@ async function refresh() { try { - const [health, sources, stats, active, dead, recent] = await Promise.all([ - fetchJson("/health"), - fetchJson("/sources"), - fetchJson("/stats"), - fetchJson("/jobs?status=claimed&limit=20"), - fetchJson("/jobs?status=dead&limit=10"), - fetchJson("/jobs?status=succeeded&limit=10"), - ]); + const [health, sources, providers, stats, active, dead, recent] = + await Promise.all([ + fetchJson("/health"), + fetchJson("/sources"), + fetchJson("/providers"), + fetchJson("/stats"), + fetchJson("/jobs?status=claimed&limit=20"), + fetchJson("/jobs?status=dead&limit=10"), + fetchJson("/jobs?status=succeeded&limit=10"), + ]); setHealthDot(false); - $("health-detail").textContent = - health.worker_count + " workers · " + health.poller_count + " pollers"; + let detail = `${health.worker_count} workers · ${health.poller_count} pollers`; + if (health.worker_breaker_open) { + const fails = health.worker_breaker_consecutive_failures; + detail += ` · WORKER BREAKER OPEN`; + } + $("health-detail").innerHTML = detail; renderChips(health.queue_counts); renderStats(stats); renderSources(sources); + renderProviders(providers); renderActive(active); renderDead(dead); renderRecent(recent); @@ -517,7 +575,7 @@ updateLastRefresh(); } catch (e) { setHealthDot(true); - $("health-detail").textContent = "error: " + e.message; + $("health-detail").textContent = `error: ${e.message}`; } } @@ -527,45 +585,29 @@ return; } const ms = Date.now() - lastRefreshAt.getTime(); - $("last-refresh").textContent = formatDuration(ms / 1000) + " ago"; - } - - function setPolling(on) { - paused = !on; - $("pause-btn").textContent = on ? "Pause" : "Resume"; - if (on) { - if (!timer) timer = setInterval(refresh, POLL_MS); - } else { - if (timer) { - clearInterval(timer); - timer = null; - } - } + $("last-refresh").textContent = `${formatDuration(ms / 1000)} ago`; } // Action handlers (called from inline onclick — kept on window). window.cancelJob = async (id) => { - if (!confirm("Cancel job " + id.slice(0, 8) + "…?")) return; - try { await deleteJson("/jobs/" + encodeURIComponent(id)); } catch (e) {} + if (!confirm(`Cancel job ${id.slice(0, 8)}…?`)) return; + try { await deleteJson(`/jobs/${encodeURIComponent(id)}`); } catch (e) {} refresh(); }; window.retryJob = async (id) => { - try { await postJson("/jobs/" + encodeURIComponent(id) + "/retry"); } catch (e) {} + try { await postJson(`/jobs/${encodeURIComponent(id)}/retry`); } catch (e) {} refresh(); }; window.refreshSource = async (sid) => { try { - await postJson("/sources/" + encodeURIComponent(sid) + "/refresh"); + await postJson(`/sources/${encodeURIComponent(sid)}/refresh`); } catch (e) {} refresh(); }; - $("pause-btn").onclick = () => setPolling(paused); - $("refresh-btn").onclick = () => refresh(); setInterval(updateLastRefresh, 1000); - refresh(); - setPolling(true); + setInterval(refresh, POLL_MS); diff --git a/tests/ingester/test_api.py b/tests/ingester/test_api.py index 13385ec7..c8bdfadf 100644 --- a/tests/ingester/test_api.py +++ b/tests/ingester/test_api.py @@ -511,6 +511,51 @@ async def test_stats_requires_auth(state): assert resp.status_code == 401 +# --- providers --- + + +@pytest.mark.asyncio +async def test_providers_probes_each_docling_serve_url(state, monkeypatch): + """Reachable URLs come back with status_code from the probe; unreachable + URLs come back with reachable=False and the httpx error message.""" + from haiku.rag.ingester.api.routes import providers as providers_mod + from haiku.rag.ingester.api.schemas import ProviderEndpoint + + async def _fake_probe(client, base_url): + if "down" in base_url: + return ProviderEndpoint( + base_url=base_url, + reachable=False, + error="Name or service not known", + ) + return ProviderEndpoint(base_url=base_url, reachable=True, status_code=200) + + monkeypatch.setattr(providers_mod, "_probe", _fake_probe) + state.config.providers.docling_serve.base_url = [ + "http://docling-serve-up:5001", + "http://docling-serve-down:5001", + ] + async with _client(state) as client: + resp = await client.get("/providers") + assert resp.status_code == 200 + body = resp.json() + assert [d["base_url"] for d in body["docling_serve"]] == [ + "http://docling-serve-up:5001", + "http://docling-serve-down:5001", + ] + assert body["docling_serve"][0]["reachable"] is True + assert body["docling_serve"][0]["status_code"] == 200 + assert body["docling_serve"][1]["reachable"] is False + assert "Name or service not known" in body["docling_serve"][1]["error"] + + +@pytest.mark.asyncio +async def test_providers_requires_auth(state): + async with _client(state, auth_token="secret") as client: + resp = await client.get("/providers") + assert resp.status_code == 401 + + # --- dashboard ---