diff --git a/CHANGELOG.md b/CHANGELOG.md index 189b5fc8..ae707cfa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,14 +1,33 @@ # Changelog ## [Unreleased] +## [0.52.0] - 2026-06-01 + ### Added - `haiku-ingester run-batch`: one discover sweep across every configured source, drains the queue, then exits. Orphan deletion requires a persisted `ingester.db`. +- `max_file_size` (bytes) on every source type rejects oversized files before they are read into memory; they dead-letter as a permanent `FileTooLargeError`. FS and S3 check the size from metadata before fetching; HTTP and WebDAV rely on a `Content-Length` response header, so a server that omits it is not enforced. + +### Changed + +- Idle workers wake on enqueue via an `asyncio.Condition` instead of polling on `poll_idle_interval_s`. +- `sync_state` writes are batched into one transaction per sweep (`SyncStateRepo.batch_upsert`) instead of one commit per file. +- HTTP and WebDAV sources reuse a single `httpx.AsyncClient` across `head`/`fetch`/`discover` instead of opening one per request. +- Periodic poll sweeps start after a random jitter of 0–25% of `poll_interval_s`. +- New partial index `idx_jobs_succeeded_completed` backs the dashboard rolling-throughput query. ### Removed - `haiku-ingester run-once`. Use `run-batch` to ingest configured sources, or `serve` for continuous operation. +### Fixed + +- Revision-less HTTP/S3/WebDAV resources (no ETag or Last-Modified) emit `UNCHANGED` once known instead of re-ingesting on every sweep. +- Deleted, unreadable, and directory paths (`FileNotFoundError`, `PermissionError`, `IsADirectoryError`, `NotADirectoryError`) classify as permanent failures instead of exhausting retries first. +- FS `discover()` and the watch loop no longer crash when a file is removed between listing and `stat()`. +- HTTP `discover()` catches only `httpx.TransportError`; other exceptions propagate instead of being emitted as UPSERT. +- A `sync_state` write that fails after a job is marked succeeded is logged instead of crashing the worker. + ## [0.51.0] - 2026-05-29 ### Added @@ -1637,7 +1656,8 @@ Existing documents without DoclingDocument data will work but won't have provena - Initial version tracking -[Unreleased]: https://github.com/ggozad/haiku.rag/compare/0.51.0...HEAD +[Unreleased]: https://github.com/ggozad/haiku.rag/compare/0.52.0...HEAD +[0.52.0]: https://github.com/ggozad/haiku.rag/compare/0.51.0...0.52.0 [0.51.0]: https://github.com/ggozad/haiku.rag/compare/0.50.0...0.51.0 [0.50.0]: https://github.com/ggozad/haiku.rag/compare/0.48.1...0.50.0 [0.48.1]: https://github.com/ggozad/haiku.rag/compare/0.48.0...0.48.1 diff --git a/app/backend/pyproject.toml b/app/backend/pyproject.toml index 6b591c22..a5aecaff 100644 --- a/app/backend/pyproject.toml +++ b/app/backend/pyproject.toml @@ -8,7 +8,7 @@ dependencies = [ "uvicorn[standard]>=0.40.0", "pydantic-ai-slim[ag-ui,anthropic,openai]>=1.81.0", "python-dotenv>=1.2.1", - "haiku.rag-slim>=0.51.0", + "haiku.rag-slim>=0.52.0", "logfire[pydantic-ai]>=3.17.0", ] diff --git a/evaluations/pyproject.toml b/evaluations/pyproject.toml index e7360ad8..c9a706d4 100644 --- a/evaluations/pyproject.toml +++ b/evaluations/pyproject.toml @@ -2,7 +2,7 @@ name = "haiku.rag-evals" description = "Benchmarking and evaluation scripts for haiku.rag" -version = "0.51.0" +version = "0.52.0" authors = [{ name = "Yiorgis Gozadinos", email = "ggozadinos@gmail.com" }] license = { text = "MIT" } requires-python = ">=3.12" diff --git a/haiku_rag_slim/pyproject.toml b/haiku_rag_slim/pyproject.toml index b4bca2b5..3e4b9208 100644 --- a/haiku_rag_slim/pyproject.toml +++ b/haiku_rag_slim/pyproject.toml @@ -2,7 +2,7 @@ name = "haiku.rag-slim" description = "Opinionated agentic RAG powered by LanceDB, Pydantic AI, and Docling - Minimal dependencies" -version = "0.51.0" +version = "0.52.0" authors = [{ name = "Yiorgis Gozadinos", email = "ggozadinos@gmail.com" }] license = { text = "MIT" } readme = { file = "README.md", content-type = "text/markdown" } diff --git a/pyproject.toml b/pyproject.toml index 94245c67..57ce1454 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,7 +2,7 @@ name = "haiku.rag" description = "Opinionated agentic RAG powered by LanceDB, Pydantic AI, and Docling" -version = "0.51.0" +version = "0.52.0" authors = [{ name = "Yiorgis Gozadinos", email = "ggozadinos@gmail.com" }] license = { text = "MIT" } readme = { file = "README.md", content-type = "text/markdown" } @@ -30,7 +30,7 @@ classifiers = [ ] dependencies = [ - "haiku.rag-slim[docling,voyageai,mxbai,cohere,zeroentropy,tui,cross-encoder]==0.51.0", + "haiku.rag-slim[docling,voyageai,mxbai,cohere,zeroentropy,tui,cross-encoder]==0.52.0", ] [project.scripts] @@ -38,9 +38,9 @@ haiku-rag = "haiku.rag.cli:cli" [project.optional-dependencies] tui = ["textual>=8.2.4"] -s3 = ["haiku.rag-slim[s3]==0.51.0"] -cross-encoder = ["haiku.rag-slim[cross-encoder]==0.51.0"] -ingester = ["haiku.rag-slim[ingester]==0.51.0"] +s3 = ["haiku.rag-slim[s3]==0.52.0"] +cross-encoder = ["haiku.rag-slim[cross-encoder]==0.52.0"] +ingester = ["haiku.rag-slim[ingester]==0.52.0"] [build-system] requires = ["hatchling"] diff --git a/uv.lock b/uv.lock index d9df4240..27ec8522 100644 --- a/uv.lock +++ b/uv.lock @@ -1457,7 +1457,7 @@ wheels = [ [[package]] name = "haiku-rag" -version = "0.51.0" +version = "0.52.0" source = { editable = "." } dependencies = [ { name = "haiku-rag-slim", extra = ["cohere", "cross-encoder", "docling", "mxbai", "tui", "voyageai", "zeroentropy"] }, @@ -1524,7 +1524,7 @@ dev = [ [[package]] name = "haiku-rag-evals" -version = "0.51.0" +version = "0.52.0" source = { editable = "evaluations" } dependencies = [ { name = "datasets" }, @@ -1547,7 +1547,7 @@ requires-dist = [ [[package]] name = "haiku-rag-slim" -version = "0.51.0" +version = "0.52.0" source = { editable = "haiku_rag_slim" } dependencies = [ { name = "docling-core" },