4aee18dcbe
Self-contained HTML status page served from the ingester's FastAPI app.
Polls /health, /sources, /stats, /jobs?status={claimed,dead,succeeded}
every 3s from the browser and renders queue chips, sources with
last-poll/skip-reason/circuit state, active jobs with cancel, recent
failures with retry, and recently-completed feed with op badges so
DELETE rows are visually distinct from UPSERTs. Zero external deps —
single static HTML, no CDN, no fonts, no images. Works offline.
To support the dashboard:
- New /stats endpoint exposing rolling throughput (5m/30m/1h), worker
occupancy, oldest-queued age, and per-source DLQ + queue-depth
breakdowns. Each field is a single SQL aggregation against the queue.
- JobRepo gains count_succeeded_since, oldest_queued_age_seconds,
counts_by_source.
- SourceSummary gains last_skip_reason. BasePoller now records the
reason the most recent sweep attempt was skipped ("pending_work" /
"circuit_open"), cleared on the next successful poll. Closes the
gap where operators couldn't tell from /sources alone why a source
wasn't picking up new work.
Auth: dashboard route is unauthenticated (markup only). The JS attaches
the bearer to its own JSON fetches; on 401 it prompts once and stashes
the token in localStorage.
Two Logfire fixes that landed alongside:
- Drop logfire.instrument_fastapi() and the [fastapi] extra. The control
plane is polled frequently (dashboard + docker healthcheck), so every
endpoint became a span and drowned the useful traces. logfire itself
stays — pulled in transitively via pydantic-ai-slim[logfire] — so
ingester.poller.* / ingester.job / document.* spans keep emitting.
- Wrap FSPoller._handle_watch_change in an ingester.poller.watch_event
span and pass _enqueue_extra. Without this, the watchfiles callback
ran with no active context, the _otel carrier in job.extra was empty,
and the worker's ingester.job span surfaced as an orphan trace root
instead of nesting under the FS event that caused it.
92 lines
2.7 KiB
TOML
92 lines
2.7 KiB
TOML
[project]
|
|
|
|
name = "haiku.rag-slim"
|
|
description = "Opinionated agentic RAG powered by LanceDB, Pydantic AI, and Docling - Minimal dependencies"
|
|
version = "0.50.0"
|
|
authors = [{ name = "Yiorgis Gozadinos", email = "ggozadinos@gmail.com" }]
|
|
license = { text = "MIT" }
|
|
readme = { file = "README.md", content-type = "text/markdown" }
|
|
requires-python = ">=3.12"
|
|
keywords = ["RAG", "lancedb", "vector-database", "ml", "mcp"]
|
|
classifiers = [
|
|
"Development Status :: 4 - Beta",
|
|
"Environment :: Console",
|
|
"Intended Audience :: Developers",
|
|
"Operating System :: Microsoft :: Windows :: Windows 10",
|
|
"Operating System :: Microsoft :: Windows :: Windows 11",
|
|
"Operating System :: MacOS",
|
|
"Operating System :: POSIX :: Linux",
|
|
"Programming Language :: Python :: 3.12",
|
|
"Programming Language :: Python :: 3.13",
|
|
"Typing :: Typed",
|
|
]
|
|
|
|
dependencies = [
|
|
"docling-core>=2.75.0",
|
|
"haiku.skills>=0.17.1",
|
|
"httpx>=0.28.1",
|
|
"jinja2>=3.1.0",
|
|
"jsonpatch>=1.33",
|
|
"lancedb==0.30.2",
|
|
"pathspec>=1.0.4",
|
|
"pydantic>=2.12.5",
|
|
"pydantic-ai-slim[openai,fastmcp,logfire,ag-ui]>=1.100.0",
|
|
"pydantic-monty>=0.0.17",
|
|
"pypdfium2>=5.0",
|
|
"python-dotenv>=1.2.2",
|
|
"pyyaml>=6.0.3",
|
|
"rich>=14.3.3",
|
|
"typer>=0.21.0,<0.22.0",
|
|
"watchfiles>=1.1.1",
|
|
"zstandard>=0.23.0; python_version<'3.14'",
|
|
]
|
|
|
|
[project.optional-dependencies]
|
|
# Document processing
|
|
docling = ["docling>=2.93.0", "opencv-python-headless>=4.13.0.92"]
|
|
# S3 / object-storage monitoring
|
|
s3 = ["obstore>=0.9,<0.10"]
|
|
# Embedding providers
|
|
voyageai = ["pydantic-ai-slim[voyageai]"]
|
|
# Rerankers
|
|
mxbai = ["mxbai-rerank>=0.1.6", "transformers>=4.49.0,<5.0.0"]
|
|
cohere = ["cohere>=5.21.1"]
|
|
zeroentropy = ["zeroentropy>=0.1.0a11"]
|
|
jina = ["transformers>=4.40.0", "torch>=2.0.0"]
|
|
cross-encoder = ["sentence-transformers>=3.0.0"]
|
|
# Production ingester (queue, workers, API)
|
|
ingester = [
|
|
"fastapi>=0.125",
|
|
"uvicorn[standard]>=0.32",
|
|
"aiosqlite>=0.20",
|
|
"haiku.rag-slim[s3]",
|
|
]
|
|
# TUI (chat and inspect commands)
|
|
tui = [
|
|
"textual>=8.2.4",
|
|
"textual-image>=0.8.5",
|
|
"tree-sitter>=0.25.2",
|
|
"tree-sitter-json>=0.24.8",
|
|
]
|
|
# Model providers (delegated to pydantic-ai-slim)
|
|
anthropic = ["pydantic-ai-slim[anthropic]"]
|
|
groq = ["pydantic-ai-slim[groq]"]
|
|
google = ["pydantic-ai-slim[google]"]
|
|
mistral = ["pydantic-ai-slim[mistral]"]
|
|
bedrock = ["pydantic-ai-slim[bedrock]"]
|
|
vertexai = ["pydantic-ai-slim[vertexai]"]
|
|
|
|
[project.entry-points."haiku.skills"]
|
|
rag = "haiku.rag.skills.rag:create_skill"
|
|
rag-analysis = "haiku.rag.skills.analysis:create_skill"
|
|
|
|
[project.scripts]
|
|
haiku-rag = "haiku.rag.cli:cli"
|
|
haiku-ingester = "haiku.rag.ingester.cli:cli"
|
|
|
|
[build-system]
|
|
requires = ["hatchling"]
|
|
build-backend = "hatchling.build"
|
|
|
|
[tool.hatch.build.targets.wheel]
|
|
packages = ["haiku"]
|