From 4c186b01c75b36685f1116bf0094a554fc641493 Mon Sep 17 00:00:00 2001 From: Yiorgis Gozadinos Date: Tue, 9 Jun 2026 12:28:13 +0300 Subject: [PATCH] Add Database and Configuration panels to the ingester dashboard --- CHANGELOG.md | 1 + docs/ingester.md | 6 +- .../haiku/rag/ingester/api/static/index.html | 166 ++++++++++++++++++ tests/ingester/test_api.py | 15 ++ 4 files changed, 187 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0236c0df..e044dfd5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ### Added +- Ingester control plane gains `GET /database` (LanceDB snapshot: stored version, embeddings, per-table counts/sizes, vector index, pending migrations, package versions — the data `haiku-rag info` prints) and `GET /config` (full effective config as YAML, secrets redacted). The dashboard surfaces both as on-demand collapsible Database and Configuration panels. - `HaikuRAG.import_documents(imports)` batch-imports prepared documents (`DocumentImport`), writing the `documents`, `chunks`, and `document_items` tables once each regardless of batch size. `DocumentRepository.create` accepts `Document | list[Document]`. ### Fixed diff --git a/docs/ingester.md b/docs/ingester.md index 12019bd4..b99492dc 100644 --- a/docs/ingester.md +++ b/docs/ingester.md @@ -305,13 +305,17 @@ token; without one the API stays open and the service logs a warning. | `GET` | `/dlq` | dead jobs | | `POST` | `/dlq/{id}/retry` | resurrect from DLQ | | `GET` | `/stats` | rolling throughput (5m / 30m / 1h succeeded), worker occupancy, oldest queued age, per-source DLQ + backlog | +| `GET` | `/database` | LanceDB snapshot — stored version, embeddings, per-table row counts/sizes, vector index status, pending migrations, package versions (same data as `haiku-rag info`) | +| `GET` | `/config` | full effective configuration (defaults filled in) as YAML, with secrets redacted | OpenAPI docs at `http://localhost:8765/docs`. The dashboard at `/` polls the JSON endpoints above every few seconds and surfaces the same data visually — queue depth chips, per-source health with a `queue busy` badge when sweeps are skipped, throughput counters, active jobs with a Cancel button, recent failures with a Retry button, and the last-completed -feed. +feed. The Database and Configuration panels are collapsed and load on +demand (the Database panel has a Refresh button) rather than on the poll +loop. ![Ingester dashboard mid-ingest: queue depth, per-source health, active and recent jobs](img/ingester-dashboard.png) diff --git a/haiku_rag_slim/haiku/rag/ingester/api/static/index.html b/haiku_rag_slim/haiku/rag/ingester/api/static/index.html index 7f740f5b..42fcae7b 100644 --- a/haiku_rag_slim/haiku/rag/ingester/api/static/index.html +++ b/haiku_rag_slim/haiku/rag/ingester/api/static/index.html @@ -161,6 +161,33 @@ } #tooltip.visible { opacity: 1; } [data-tip] { cursor: help; } + + details.panel { padding: 0; } + details.panel > summary { + list-style: none; cursor: pointer; user-select: none; + display: flex; align-items: center; gap: 8px; padding: 16px; + font-size: 13px; font-weight: 600; letter-spacing: 0.04em; + text-transform: uppercase; color: var(--muted); + } + details.panel > summary::-webkit-details-marker { display: none; } + details.panel > summary::before { content: "▸"; color: var(--muted); } + details.panel[open] > summary::before { content: "▾"; } + details.panel > summary .spacer { flex: 1; } + details.panel > summary button { + background: var(--bg); color: var(--text); border: 1px solid var(--border); + padding: 4px 10px; border-radius: 4px; font: inherit; font-size: 11px; + text-transform: none; letter-spacing: 0; cursor: pointer; + } + details.panel > summary button:hover { background: var(--border); } + details.panel > .body { padding: 0 16px 16px; } + + pre.config { + margin: 0; max-height: 600px; overflow: auto; + background: var(--bg); border: 1px solid var(--border); border-radius: 6px; + padding: 12px; font-family: ui-monospace, SFMono-Regular, Menlo, monospace; + font-size: 12px; line-height: 1.5; color: var(--text); + white-space: pre; word-break: normal; + } @@ -255,6 +282,26 @@
loading…
+ +
+ + Database + + + +
+
expand to load…
+
+
+ +
+ + Configuration + +
+
expand to load…
+
+
@@ -321,6 +368,18 @@ return id ? id.slice(0, 8) : "—"; } + function formatBytes(n) { + if (n == null) return "—"; + const units = ["B", "KB", "MB", "GB", "TB"]; + let i = 0; + let v = n; + while (v >= 1024 && i < units.length - 1) { + v /= 1024; + i++; + } + return `${i === 0 ? v : v.toFixed(1)} ${units[i]}`; + } + function escapeHtml(s) { if (s == null) return ""; return String(s) @@ -668,6 +727,113 @@ }); })(); + // Database and Configuration panels are on-demand, not part of the + // POLL_MS loop: their data is database state, not live queue telemetry, + // and gathering it re-reads table manifests. Loaded lazily on first + // expand; the Database panel can be re-fetched with its Refresh button. + + function renderDatabase(info) { + if (!info.exists) { + $("db-body").innerHTML = + '
database empty or not initialized
'; + return; + } + const emb = info.embeddings || {}; + const vi = info.vector_index || {}; + const pkgs = info.packages || {}; + const stat = (label, value) => + `
${label}
${value}
`; + const stats = [ + stat("Stored version", escapeHtml(info.stored_version)), + stat("Vector dim", emb.vector_dim ?? "—"), + stat( + "Vector index", + vi.exists ? `✓ ${vi.indexed_rows}` : "✗ none", + ), + stat("haiku.rag", escapeHtml(pkgs.haiku_rag ?? "—")), + stat("lancedb", escapeHtml(pkgs.lancedb ?? "—")), + ].join(""); + + const tableRows = (info.tables || []) + .map((t) => { + if (!t.exists) { + return `${escapeHtml(t.name)}absent`; + } + return ` + ${escapeHtml(t.name)} + ${t.num_rows} + ${formatBytes(t.total_bytes)} + ${t.num_versions} + `; + }) + .join(""); + + let migration = 'up to date'; + const pending = info.pending_migrations || []; + if (pending.length) { + const items = pending + .map( + (m) => + `${escapeHtml(m.version)}`, + ) + .join(" "); + migration = `${pending.length} pending ${items}`; + } + let unindexed = ""; + if (vi.exists && vi.unindexed_rows > 0) { + unindexed = ` ${vi.unindexed_rows} unindexed`; + } + + $("db-body").innerHTML = ` +
${stats}
+
+ embeddings: ${escapeHtml(emb.provider ?? "?")}/${escapeHtml(emb.name ?? "?")} + path: ${escapeHtml(info.path)} +
+ + + ${tableRows} +
TableRowsSizeVersions
+
Migrations: ${migration}${unindexed}
`; + } + + let dbLoaded = false; + async function loadDatabase() { + $("db-body").innerHTML = '
loading…
'; + try { + renderDatabase(await fetchJson("/database")); + dbLoaded = true; + } catch (e) { + $("db-body").innerHTML = `
error: ${escapeHtml(e.message)}
`; + } + } + + let configLoaded = false; + async function loadConfig() { + $("config-body").innerHTML = '
loading…
'; + try { + const data = await fetchJson("/config"); + $("config-body").innerHTML = '
';
+          $("config-body").querySelector("pre").textContent = data.yaml;
+          configLoaded = true;
+        } catch (e) {
+          $("config-body").innerHTML = `
error: ${escapeHtml(e.message)}
`;
+        }
+      }
+
+      $("db-panel").addEventListener("toggle", (e) => {
+        if (e.target.open && !dbLoaded) loadDatabase();
+      });
+      $("db-refresh").addEventListener("click", (e) => {
+        e.preventDefault();
+        e.stopPropagation();
+        $("db-panel").open = true;
+        loadDatabase();
+      });
+      $("config-panel").addEventListener("toggle", (e) => {
+        if (e.target.open && !configLoaded) loadConfig();
+      });
+
       setInterval(updateLastRefresh, 1000);
       refresh();
       setInterval(refresh, POLL_MS);
diff --git a/tests/ingester/test_api.py b/tests/ingester/test_api.py
index 634db285..6d9c0aa2 100644
--- a/tests/ingester/test_api.py
+++ b/tests/ingester/test_api.py
@@ -635,6 +635,21 @@ async def test_dashboard_served_unauthenticated(state):
     assert "opBadge" in body
 
 
+@pytest.mark.asyncio
+async def test_dashboard_wires_database_and_config_panels(state):
+    """The on-demand Database and Configuration panels are present and call
+    their endpoints lazily (not in the POLL_MS loop)."""
+    async with _client(state) as client:
+        resp = await client.get("/")
+    body = resp.text
+    assert 'id="db-panel"' in body
+    assert 'id="config-panel"' in body
+    assert "/database" in body
+    assert "/config" in body
+    assert "loadDatabase" in body
+    assert "loadConfig" in body
+
+
 # --- config ---