Add Database and Configuration panels to the ingester dashboard
This commit is contained in:
Yiorgis Gozadinos
parent
c62166b26e
commit
4c186b01c7
4 changed files with 187 additions and 1 deletions
|
|
@ -3,6 +3,7 @@
|
|||
|
||||
### Added
|
||||
|
||||
- Ingester control plane gains `GET /database` (LanceDB snapshot: stored version, embeddings, per-table counts/sizes, vector index, pending migrations, package versions — the data `haiku-rag info` prints) and `GET /config` (full effective config as YAML, secrets redacted). The dashboard surfaces both as on-demand collapsible Database and Configuration panels.
|
||||
- `HaikuRAG.import_documents(imports)` batch-imports prepared documents (`DocumentImport`), writing the `documents`, `chunks`, and `document_items` tables once each regardless of batch size. `DocumentRepository.create` accepts `Document | list[Document]`.
|
||||
|
||||
### Fixed
|
||||
|
|
|
|||
|
|
@ -305,13 +305,17 @@ token; without one the API stays open and the service logs a warning.
|
|||
| `GET` | `/dlq` | dead jobs |
|
||||
| `POST` | `/dlq/{id}/retry` | resurrect from DLQ |
|
||||
| `GET` | `/stats` | rolling throughput (5m / 30m / 1h succeeded), worker occupancy, oldest queued age, per-source DLQ + backlog |
|
||||
| `GET` | `/database` | LanceDB snapshot — stored version, embeddings, per-table row counts/sizes, vector index status, pending migrations, package versions (same data as `haiku-rag info`) |
|
||||
| `GET` | `/config` | full effective configuration (defaults filled in) as YAML, with secrets redacted |
|
||||
|
||||
OpenAPI docs at `http://localhost:8765/docs`. The dashboard at `/` polls
|
||||
the JSON endpoints above every few seconds and surfaces the same data
|
||||
visually — queue depth chips, per-source health with a `queue busy` badge
|
||||
when sweeps are skipped, throughput counters, active jobs with a Cancel
|
||||
button, recent failures with a Retry button, and the last-completed
|
||||
feed.
|
||||
feed. The Database and Configuration panels are collapsed and load on
|
||||
demand (the Database panel has a Refresh button) rather than on the poll
|
||||
loop.
|
||||
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -161,6 +161,33 @@
|
|||
}
|
||||
#tooltip.visible { opacity: 1; }
|
||||
[data-tip] { cursor: help; }
|
||||
|
||||
details.panel { padding: 0; }
|
||||
details.panel > summary {
|
||||
list-style: none; cursor: pointer; user-select: none;
|
||||
display: flex; align-items: center; gap: 8px; padding: 16px;
|
||||
font-size: 13px; font-weight: 600; letter-spacing: 0.04em;
|
||||
text-transform: uppercase; color: var(--muted);
|
||||
}
|
||||
details.panel > summary::-webkit-details-marker { display: none; }
|
||||
details.panel > summary::before { content: "▸"; color: var(--muted); }
|
||||
details.panel[open] > summary::before { content: "▾"; }
|
||||
details.panel > summary .spacer { flex: 1; }
|
||||
details.panel > summary button {
|
||||
background: var(--bg); color: var(--text); border: 1px solid var(--border);
|
||||
padding: 4px 10px; border-radius: 4px; font: inherit; font-size: 11px;
|
||||
text-transform: none; letter-spacing: 0; cursor: pointer;
|
||||
}
|
||||
details.panel > summary button:hover { background: var(--border); }
|
||||
details.panel > .body { padding: 0 16px 16px; }
|
||||
|
||||
pre.config {
|
||||
margin: 0; max-height: 600px; overflow: auto;
|
||||
background: var(--bg); border: 1px solid var(--border); border-radius: 6px;
|
||||
padding: 12px; font-family: ui-monospace, SFMono-Regular, Menlo, monospace;
|
||||
font-size: 12px; line-height: 1.5; color: var(--text);
|
||||
white-space: pre; word-break: normal;
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
|
|
@ -255,6 +282,26 @@
|
|||
<div class="empty">loading…</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<details class="panel full" id="db-panel">
|
||||
<summary>
|
||||
<span>Database</span>
|
||||
<span class="spacer"></span>
|
||||
<button type="button" id="db-refresh">Refresh</button>
|
||||
</summary>
|
||||
<div class="body" id="db-body">
|
||||
<div class="empty">expand to load…</div>
|
||||
</div>
|
||||
</details>
|
||||
|
||||
<details class="panel full" id="config-panel">
|
||||
<summary>
|
||||
<span>Configuration</span>
|
||||
</summary>
|
||||
<div class="body" id="config-body">
|
||||
<div class="empty">expand to load…</div>
|
||||
</div>
|
||||
</details>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
|
@ -321,6 +368,18 @@
|
|||
return id ? id.slice(0, 8) : "—";
|
||||
}
|
||||
|
||||
function formatBytes(n) {
|
||||
if (n == null) return "—";
|
||||
const units = ["B", "KB", "MB", "GB", "TB"];
|
||||
let i = 0;
|
||||
let v = n;
|
||||
while (v >= 1024 && i < units.length - 1) {
|
||||
v /= 1024;
|
||||
i++;
|
||||
}
|
||||
return `${i === 0 ? v : v.toFixed(1)} ${units[i]}`;
|
||||
}
|
||||
|
||||
function escapeHtml(s) {
|
||||
if (s == null) return "";
|
||||
return String(s)
|
||||
|
|
@ -668,6 +727,113 @@
|
|||
});
|
||||
})();
|
||||
|
||||
// Database and Configuration panels are on-demand, not part of the
|
||||
// POLL_MS loop: their data is database state, not live queue telemetry,
|
||||
// and gathering it re-reads table manifests. Loaded lazily on first
|
||||
// expand; the Database panel can be re-fetched with its Refresh button.
|
||||
|
||||
function renderDatabase(info) {
|
||||
if (!info.exists) {
|
||||
$("db-body").innerHTML =
|
||||
'<div class="empty">database empty or not initialized</div>';
|
||||
return;
|
||||
}
|
||||
const emb = info.embeddings || {};
|
||||
const vi = info.vector_index || {};
|
||||
const pkgs = info.packages || {};
|
||||
const stat = (label, value) =>
|
||||
`<div class="stat"><div class="label">${label}</div><div class="value">${value}</div></div>`;
|
||||
const stats = [
|
||||
stat("Stored version", escapeHtml(info.stored_version)),
|
||||
stat("Vector dim", emb.vector_dim ?? "—"),
|
||||
stat(
|
||||
"Vector index",
|
||||
vi.exists ? `✓ ${vi.indexed_rows}` : "✗ none",
|
||||
),
|
||||
stat("haiku.rag", escapeHtml(pkgs.haiku_rag ?? "—")),
|
||||
stat("lancedb", escapeHtml(pkgs.lancedb ?? "—")),
|
||||
].join("");
|
||||
|
||||
const tableRows = (info.tables || [])
|
||||
.map((t) => {
|
||||
if (!t.exists) {
|
||||
return `<tr><td>${escapeHtml(t.name)}</td><td colspan="3"><span class="badge warn">absent</span></td></tr>`;
|
||||
}
|
||||
return `<tr>
|
||||
<td>${escapeHtml(t.name)}</td>
|
||||
<td>${t.num_rows}</td>
|
||||
<td>${formatBytes(t.total_bytes)}</td>
|
||||
<td>${t.num_versions}</td>
|
||||
</tr>`;
|
||||
})
|
||||
.join("");
|
||||
|
||||
let migration = '<span class="badge ok">up to date</span>';
|
||||
const pending = info.pending_migrations || [];
|
||||
if (pending.length) {
|
||||
const items = pending
|
||||
.map(
|
||||
(m) =>
|
||||
`<span data-tip="${escapeHtml(m.description)}">${escapeHtml(m.version)}</span>`,
|
||||
)
|
||||
.join(" ");
|
||||
migration = `<span class="badge warn" data-tip="run: haiku-rag migrate">${pending.length} pending</span> ${items}`;
|
||||
}
|
||||
let unindexed = "";
|
||||
if (vi.exists && vi.unindexed_rows > 0) {
|
||||
unindexed = ` <span class="badge warn" data-tip="run: haiku-rag create-index">${vi.unindexed_rows} unindexed</span>`;
|
||||
}
|
||||
|
||||
$("db-body").innerHTML = `
|
||||
<div class="stats-grid">${stats}</div>
|
||||
<div class="breakdown">
|
||||
<span>embeddings: ${escapeHtml(emb.provider ?? "?")}/${escapeHtml(emb.name ?? "?")}</span>
|
||||
<span>path: ${escapeHtml(info.path)}</span>
|
||||
</div>
|
||||
<table>
|
||||
<thead><tr><th>Table</th><th>Rows</th><th>Size</th><th>Versions</th></tr></thead>
|
||||
<tbody>${tableRows}</tbody>
|
||||
</table>
|
||||
<div class="breakdown">Migrations: ${migration}${unindexed}</div>`;
|
||||
}
|
||||
|
||||
let dbLoaded = false;
|
||||
async function loadDatabase() {
|
||||
$("db-body").innerHTML = '<div class="empty">loading…</div>';
|
||||
try {
|
||||
renderDatabase(await fetchJson("/database"));
|
||||
dbLoaded = true;
|
||||
} catch (e) {
|
||||
$("db-body").innerHTML = `<div class="empty">error: ${escapeHtml(e.message)}</div>`;
|
||||
}
|
||||
}
|
||||
|
||||
let configLoaded = false;
|
||||
async function loadConfig() {
|
||||
$("config-body").innerHTML = '<div class="empty">loading…</div>';
|
||||
try {
|
||||
const data = await fetchJson("/config");
|
||||
$("config-body").innerHTML = '<pre class="config"></pre>';
|
||||
$("config-body").querySelector("pre").textContent = data.yaml;
|
||||
configLoaded = true;
|
||||
} catch (e) {
|
||||
$("config-body").innerHTML = `<div class="empty">error: ${escapeHtml(e.message)}</div>`;
|
||||
}
|
||||
}
|
||||
|
||||
$("db-panel").addEventListener("toggle", (e) => {
|
||||
if (e.target.open && !dbLoaded) loadDatabase();
|
||||
});
|
||||
$("db-refresh").addEventListener("click", (e) => {
|
||||
e.preventDefault();
|
||||
e.stopPropagation();
|
||||
$("db-panel").open = true;
|
||||
loadDatabase();
|
||||
});
|
||||
$("config-panel").addEventListener("toggle", (e) => {
|
||||
if (e.target.open && !configLoaded) loadConfig();
|
||||
});
|
||||
|
||||
setInterval(updateLastRefresh, 1000);
|
||||
refresh();
|
||||
setInterval(refresh, POLL_MS);
|
||||
|
|
|
|||
|
|
@ -635,6 +635,21 @@ async def test_dashboard_served_unauthenticated(state):
|
|||
assert "opBadge" in body
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_dashboard_wires_database_and_config_panels(state):
|
||||
"""The on-demand Database and Configuration panels are present and call
|
||||
their endpoints lazily (not in the POLL_MS loop)."""
|
||||
async with _client(state) as client:
|
||||
resp = await client.get("/")
|
||||
body = resp.text
|
||||
assert 'id="db-panel"' in body
|
||||
assert 'id="config-panel"' in body
|
||||
assert "/database" in body
|
||||
assert "/config" in body
|
||||
assert "loadDatabase" in body
|
||||
assert "loadConfig" in body
|
||||
|
||||
|
||||
# --- config ---
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue