4aee18dcbe
Self-contained HTML status page served from the ingester's FastAPI app.
Polls /health, /sources, /stats, /jobs?status={claimed,dead,succeeded}
every 3s from the browser and renders queue chips, sources with
last-poll/skip-reason/circuit state, active jobs with cancel, recent
failures with retry, and recently-completed feed with op badges so
DELETE rows are visually distinct from UPSERTs. Zero external deps —
single static HTML, no CDN, no fonts, no images. Works offline.
To support the dashboard:
- New /stats endpoint exposing rolling throughput (5m/30m/1h), worker
occupancy, oldest-queued age, and per-source DLQ + queue-depth
breakdowns. Each field is a single SQL aggregation against the queue.
- JobRepo gains count_succeeded_since, oldest_queued_age_seconds,
counts_by_source.
- SourceSummary gains last_skip_reason. BasePoller now records the
reason the most recent sweep attempt was skipped ("pending_work" /
"circuit_open"), cleared on the next successful poll. Closes the
gap where operators couldn't tell from /sources alone why a source
wasn't picking up new work.
Auth: dashboard route is unauthenticated (markup only). The JS attaches
the bearer to its own JSON fetches; on 401 it prompts once and stashes
the token in localStorage.
Two Logfire fixes that landed alongside:
- Drop logfire.instrument_fastapi() and the [fastapi] extra. The control
plane is polled frequently (dashboard + docker healthcheck), so every
endpoint became a span and drowned the useful traces. logfire itself
stays — pulled in transitively via pydantic-ai-slim[logfire] — so
ingester.poller.* / ingester.job / document.* spans keep emitting.
- Wrap FSPoller._handle_watch_change in an ingester.poller.watch_event
span and pass _enqueue_extra. Without this, the watchfiles callback
ran with no active context, the _otel carrier in job.extra was empty,
and the worker's ingester.job span surfaced as an orphan trace root
instead of nesting under the FS event that caused it.
65 lines
2 KiB
Python
65 lines
2 KiB
Python
from dataclasses import dataclass
|
|
from typing import TYPE_CHECKING
|
|
|
|
from fastapi import Depends, FastAPI, Request
|
|
|
|
from haiku.rag.ingester.api.auth import require_auth
|
|
|
|
if TYPE_CHECKING:
|
|
from haiku.rag.config import AppConfig
|
|
from haiku.rag.ingester.pollers.manager import PollerManager
|
|
from haiku.rag.ingester.queue.repository import JobRepo, SyncStateRepo
|
|
from haiku.rag.ingester.workers.pool import WorkerPool
|
|
|
|
|
|
@dataclass
|
|
class APIState:
|
|
"""Everything the API handlers need to read or act on. Pool/pollers are
|
|
optional so the FastAPI app can be tested in isolation."""
|
|
|
|
config: "AppConfig"
|
|
job_repo: "JobRepo"
|
|
sync_repo: "SyncStateRepo"
|
|
pool: "WorkerPool | None" = None
|
|
pollers: "PollerManager | None" = None
|
|
|
|
|
|
def get_state(request: Request) -> APIState:
|
|
return request.app.state.api_state
|
|
|
|
|
|
def build_app(
|
|
state: APIState,
|
|
*,
|
|
auth_token: str | None = None,
|
|
) -> FastAPI:
|
|
"""Construct the ingester's FastAPI control plane."""
|
|
from haiku.rag.ingester.api.routes import (
|
|
dashboard,
|
|
dlq,
|
|
health,
|
|
jobs,
|
|
sources,
|
|
stats,
|
|
)
|
|
|
|
app = FastAPI(
|
|
title="haiku-ingester",
|
|
description="Control plane for the haiku.rag production ingester.",
|
|
version="1",
|
|
)
|
|
app.state.api_state = state
|
|
app.state.auth_token = auth_token
|
|
|
|
auth_dep = [Depends(require_auth)]
|
|
app.include_router(health.router) # /health is unauthenticated by design
|
|
# Dashboard route is markup-only; the JS it serves attaches the bearer
|
|
# token to its own fetches. Keeping the route unauthenticated lets an
|
|
# operator open it in a browser and paste the token on demand.
|
|
app.include_router(dashboard.router)
|
|
app.include_router(jobs.router, dependencies=auth_dep)
|
|
app.include_router(sources.router, dependencies=auth_dep)
|
|
app.include_router(dlq.router, dependencies=auth_dep)
|
|
app.include_router(stats.router, dependencies=auth_dep)
|
|
|
|
return app
|