Add Database and Configuration panels to the ingester dashboard
This commit is contained in:
Yiorgis Gozadinos
parent
c62166b26e
commit
4c186b01c7
4 changed files with 187 additions and 1 deletions
|
|
@ -3,6 +3,7 @@
|
||||||
|
|
||||||
### Added
|
### Added
|
||||||
|
|
||||||
|
- Ingester control plane gains `GET /database` (LanceDB snapshot: stored version, embeddings, per-table counts/sizes, vector index, pending migrations, package versions — the data `haiku-rag info` prints) and `GET /config` (full effective config as YAML, secrets redacted). The dashboard surfaces both as on-demand collapsible Database and Configuration panels.
|
||||||
- `HaikuRAG.import_documents(imports)` batch-imports prepared documents (`DocumentImport`), writing the `documents`, `chunks`, and `document_items` tables once each regardless of batch size. `DocumentRepository.create` accepts `Document | list[Document]`.
|
- `HaikuRAG.import_documents(imports)` batch-imports prepared documents (`DocumentImport`), writing the `documents`, `chunks`, and `document_items` tables once each regardless of batch size. `DocumentRepository.create` accepts `Document | list[Document]`.
|
||||||
|
|
||||||
### Fixed
|
### Fixed
|
||||||
|
|
|
||||||
|
|
@ -305,13 +305,17 @@ token; without one the API stays open and the service logs a warning.
|
||||||
| `GET` | `/dlq` | dead jobs |
|
| `GET` | `/dlq` | dead jobs |
|
||||||
| `POST` | `/dlq/{id}/retry` | resurrect from DLQ |
|
| `POST` | `/dlq/{id}/retry` | resurrect from DLQ |
|
||||||
| `GET` | `/stats` | rolling throughput (5m / 30m / 1h succeeded), worker occupancy, oldest queued age, per-source DLQ + backlog |
|
| `GET` | `/stats` | rolling throughput (5m / 30m / 1h succeeded), worker occupancy, oldest queued age, per-source DLQ + backlog |
|
||||||
|
| `GET` | `/database` | LanceDB snapshot — stored version, embeddings, per-table row counts/sizes, vector index status, pending migrations, package versions (same data as `haiku-rag info`) |
|
||||||
|
| `GET` | `/config` | full effective configuration (defaults filled in) as YAML, with secrets redacted |
|
||||||
|
|
||||||
OpenAPI docs at `http://localhost:8765/docs`. The dashboard at `/` polls
|
OpenAPI docs at `http://localhost:8765/docs`. The dashboard at `/` polls
|
||||||
the JSON endpoints above every few seconds and surfaces the same data
|
the JSON endpoints above every few seconds and surfaces the same data
|
||||||
visually — queue depth chips, per-source health with a `queue busy` badge
|
visually — queue depth chips, per-source health with a `queue busy` badge
|
||||||
when sweeps are skipped, throughput counters, active jobs with a Cancel
|
when sweeps are skipped, throughput counters, active jobs with a Cancel
|
||||||
button, recent failures with a Retry button, and the last-completed
|
button, recent failures with a Retry button, and the last-completed
|
||||||
feed.
|
feed. The Database and Configuration panels are collapsed and load on
|
||||||
|
demand (the Database panel has a Refresh button) rather than on the poll
|
||||||
|
loop.
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -161,6 +161,33 @@
|
||||||
}
|
}
|
||||||
#tooltip.visible { opacity: 1; }
|
#tooltip.visible { opacity: 1; }
|
||||||
[data-tip] { cursor: help; }
|
[data-tip] { cursor: help; }
|
||||||
|
|
||||||
|
details.panel { padding: 0; }
|
||||||
|
details.panel > summary {
|
||||||
|
list-style: none; cursor: pointer; user-select: none;
|
||||||
|
display: flex; align-items: center; gap: 8px; padding: 16px;
|
||||||
|
font-size: 13px; font-weight: 600; letter-spacing: 0.04em;
|
||||||
|
text-transform: uppercase; color: var(--muted);
|
||||||
|
}
|
||||||
|
details.panel > summary::-webkit-details-marker { display: none; }
|
||||||
|
details.panel > summary::before { content: "▸"; color: var(--muted); }
|
||||||
|
details.panel[open] > summary::before { content: "▾"; }
|
||||||
|
details.panel > summary .spacer { flex: 1; }
|
||||||
|
details.panel > summary button {
|
||||||
|
background: var(--bg); color: var(--text); border: 1px solid var(--border);
|
||||||
|
padding: 4px 10px; border-radius: 4px; font: inherit; font-size: 11px;
|
||||||
|
text-transform: none; letter-spacing: 0; cursor: pointer;
|
||||||
|
}
|
||||||
|
details.panel > summary button:hover { background: var(--border); }
|
||||||
|
details.panel > .body { padding: 0 16px 16px; }
|
||||||
|
|
||||||
|
pre.config {
|
||||||
|
margin: 0; max-height: 600px; overflow: auto;
|
||||||
|
background: var(--bg); border: 1px solid var(--border); border-radius: 6px;
|
||||||
|
padding: 12px; font-family: ui-monospace, SFMono-Regular, Menlo, monospace;
|
||||||
|
font-size: 12px; line-height: 1.5; color: var(--text);
|
||||||
|
white-space: pre; word-break: normal;
|
||||||
|
}
|
||||||
</style>
|
</style>
|
||||||
</head>
|
</head>
|
||||||
<body>
|
<body>
|
||||||
|
|
@ -255,6 +282,26 @@
|
||||||
<div class="empty">loading…</div>
|
<div class="empty">loading…</div>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
<details class="panel full" id="db-panel">
|
||||||
|
<summary>
|
||||||
|
<span>Database</span>
|
||||||
|
<span class="spacer"></span>
|
||||||
|
<button type="button" id="db-refresh">Refresh</button>
|
||||||
|
</summary>
|
||||||
|
<div class="body" id="db-body">
|
||||||
|
<div class="empty">expand to load…</div>
|
||||||
|
</div>
|
||||||
|
</details>
|
||||||
|
|
||||||
|
<details class="panel full" id="config-panel">
|
||||||
|
<summary>
|
||||||
|
<span>Configuration</span>
|
||||||
|
</summary>
|
||||||
|
<div class="body" id="config-body">
|
||||||
|
<div class="empty">expand to load…</div>
|
||||||
|
</div>
|
||||||
|
</details>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
|
@ -321,6 +368,18 @@
|
||||||
return id ? id.slice(0, 8) : "—";
|
return id ? id.slice(0, 8) : "—";
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function formatBytes(n) {
|
||||||
|
if (n == null) return "—";
|
||||||
|
const units = ["B", "KB", "MB", "GB", "TB"];
|
||||||
|
let i = 0;
|
||||||
|
let v = n;
|
||||||
|
while (v >= 1024 && i < units.length - 1) {
|
||||||
|
v /= 1024;
|
||||||
|
i++;
|
||||||
|
}
|
||||||
|
return `${i === 0 ? v : v.toFixed(1)} ${units[i]}`;
|
||||||
|
}
|
||||||
|
|
||||||
function escapeHtml(s) {
|
function escapeHtml(s) {
|
||||||
if (s == null) return "";
|
if (s == null) return "";
|
||||||
return String(s)
|
return String(s)
|
||||||
|
|
@ -668,6 +727,113 @@
|
||||||
});
|
});
|
||||||
})();
|
})();
|
||||||
|
|
||||||
|
// Database and Configuration panels are on-demand, not part of the
|
||||||
|
// POLL_MS loop: their data is database state, not live queue telemetry,
|
||||||
|
// and gathering it re-reads table manifests. Loaded lazily on first
|
||||||
|
// expand; the Database panel can be re-fetched with its Refresh button.
|
||||||
|
|
||||||
|
function renderDatabase(info) {
|
||||||
|
if (!info.exists) {
|
||||||
|
$("db-body").innerHTML =
|
||||||
|
'<div class="empty">database empty or not initialized</div>';
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const emb = info.embeddings || {};
|
||||||
|
const vi = info.vector_index || {};
|
||||||
|
const pkgs = info.packages || {};
|
||||||
|
const stat = (label, value) =>
|
||||||
|
`<div class="stat"><div class="label">${label}</div><div class="value">${value}</div></div>`;
|
||||||
|
const stats = [
|
||||||
|
stat("Stored version", escapeHtml(info.stored_version)),
|
||||||
|
stat("Vector dim", emb.vector_dim ?? "—"),
|
||||||
|
stat(
|
||||||
|
"Vector index",
|
||||||
|
vi.exists ? `✓ ${vi.indexed_rows}` : "✗ none",
|
||||||
|
),
|
||||||
|
stat("haiku.rag", escapeHtml(pkgs.haiku_rag ?? "—")),
|
||||||
|
stat("lancedb", escapeHtml(pkgs.lancedb ?? "—")),
|
||||||
|
].join("");
|
||||||
|
|
||||||
|
const tableRows = (info.tables || [])
|
||||||
|
.map((t) => {
|
||||||
|
if (!t.exists) {
|
||||||
|
return `<tr><td>${escapeHtml(t.name)}</td><td colspan="3"><span class="badge warn">absent</span></td></tr>`;
|
||||||
|
}
|
||||||
|
return `<tr>
|
||||||
|
<td>${escapeHtml(t.name)}</td>
|
||||||
|
<td>${t.num_rows}</td>
|
||||||
|
<td>${formatBytes(t.total_bytes)}</td>
|
||||||
|
<td>${t.num_versions}</td>
|
||||||
|
</tr>`;
|
||||||
|
})
|
||||||
|
.join("");
|
||||||
|
|
||||||
|
let migration = '<span class="badge ok">up to date</span>';
|
||||||
|
const pending = info.pending_migrations || [];
|
||||||
|
if (pending.length) {
|
||||||
|
const items = pending
|
||||||
|
.map(
|
||||||
|
(m) =>
|
||||||
|
`<span data-tip="${escapeHtml(m.description)}">${escapeHtml(m.version)}</span>`,
|
||||||
|
)
|
||||||
|
.join(" ");
|
||||||
|
migration = `<span class="badge warn" data-tip="run: haiku-rag migrate">${pending.length} pending</span> ${items}`;
|
||||||
|
}
|
||||||
|
let unindexed = "";
|
||||||
|
if (vi.exists && vi.unindexed_rows > 0) {
|
||||||
|
unindexed = ` <span class="badge warn" data-tip="run: haiku-rag create-index">${vi.unindexed_rows} unindexed</span>`;
|
||||||
|
}
|
||||||
|
|
||||||
|
$("db-body").innerHTML = `
|
||||||
|
<div class="stats-grid">${stats}</div>
|
||||||
|
<div class="breakdown">
|
||||||
|
<span>embeddings: ${escapeHtml(emb.provider ?? "?")}/${escapeHtml(emb.name ?? "?")}</span>
|
||||||
|
<span>path: ${escapeHtml(info.path)}</span>
|
||||||
|
</div>
|
||||||
|
<table>
|
||||||
|
<thead><tr><th>Table</th><th>Rows</th><th>Size</th><th>Versions</th></tr></thead>
|
||||||
|
<tbody>${tableRows}</tbody>
|
||||||
|
</table>
|
||||||
|
<div class="breakdown">Migrations: ${migration}${unindexed}</div>`;
|
||||||
|
}
|
||||||
|
|
||||||
|
let dbLoaded = false;
|
||||||
|
async function loadDatabase() {
|
||||||
|
$("db-body").innerHTML = '<div class="empty">loading…</div>';
|
||||||
|
try {
|
||||||
|
renderDatabase(await fetchJson("/database"));
|
||||||
|
dbLoaded = true;
|
||||||
|
} catch (e) {
|
||||||
|
$("db-body").innerHTML = `<div class="empty">error: ${escapeHtml(e.message)}</div>`;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let configLoaded = false;
|
||||||
|
async function loadConfig() {
|
||||||
|
$("config-body").innerHTML = '<div class="empty">loading…</div>';
|
||||||
|
try {
|
||||||
|
const data = await fetchJson("/config");
|
||||||
|
$("config-body").innerHTML = '<pre class="config"></pre>';
|
||||||
|
$("config-body").querySelector("pre").textContent = data.yaml;
|
||||||
|
configLoaded = true;
|
||||||
|
} catch (e) {
|
||||||
|
$("config-body").innerHTML = `<div class="empty">error: ${escapeHtml(e.message)}</div>`;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
$("db-panel").addEventListener("toggle", (e) => {
|
||||||
|
if (e.target.open && !dbLoaded) loadDatabase();
|
||||||
|
});
|
||||||
|
$("db-refresh").addEventListener("click", (e) => {
|
||||||
|
e.preventDefault();
|
||||||
|
e.stopPropagation();
|
||||||
|
$("db-panel").open = true;
|
||||||
|
loadDatabase();
|
||||||
|
});
|
||||||
|
$("config-panel").addEventListener("toggle", (e) => {
|
||||||
|
if (e.target.open && !configLoaded) loadConfig();
|
||||||
|
});
|
||||||
|
|
||||||
setInterval(updateLastRefresh, 1000);
|
setInterval(updateLastRefresh, 1000);
|
||||||
refresh();
|
refresh();
|
||||||
setInterval(refresh, POLL_MS);
|
setInterval(refresh, POLL_MS);
|
||||||
|
|
|
||||||
|
|
@ -635,6 +635,21 @@ async def test_dashboard_served_unauthenticated(state):
|
||||||
assert "opBadge" in body
|
assert "opBadge" in body
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_dashboard_wires_database_and_config_panels(state):
|
||||||
|
"""The on-demand Database and Configuration panels are present and call
|
||||||
|
their endpoints lazily (not in the POLL_MS loop)."""
|
||||||
|
async with _client(state) as client:
|
||||||
|
resp = await client.get("/")
|
||||||
|
body = resp.text
|
||||||
|
assert 'id="db-panel"' in body
|
||||||
|
assert 'id="config-panel"' in body
|
||||||
|
assert "/database" in body
|
||||||
|
assert "/config" in body
|
||||||
|
assert "loadDatabase" in body
|
||||||
|
assert "loadConfig" in body
|
||||||
|
|
||||||
|
|
||||||
# --- config ---
|
# --- config ---
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue