From 66d5fee682ffb727f6371e477987096e06af7e55 Mon Sep 17 00:00:00 2001 From: Yiorgis Gozadinos Date: Fri, 22 May 2026 17:10:33 +0300 Subject: [PATCH] Docker & docs updates --- README.md | 14 ++++---- docker/Dockerfile | 17 +++++---- docker/Dockerfile.slim | 17 +++++---- docker/README.md | 27 ++++++++++---- docs/cli.md | 10 +++--- docs/configuration/index.md | 3 +- docs/installation.md | 2 +- docs/overview.md | 4 +-- examples/README.md | 2 +- examples/docker/README.md | 43 ++++++++++++++++------ examples/docker/docker-compose.yml | 50 ++++++++++++++++++++++---- examples/docker/haiku.rag.yaml.example | 3 ++ 12 files changed, 136 insertions(+), 56 deletions(-) diff --git a/README.md b/README.md index 22290658..ec7a5f5f 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ Agentic RAG built on [LanceDB](https://lancedb.com/), [Pydantic AI](https://ai.p - **CLI & Python API** — Full functionality from command line or code - **MCP server** — Expose as tools for AI assistants (Claude Desktop, etc.) - **Visual grounding** — View chunks highlighted on original page images -- **File monitoring** — Watch directories and auto-index on changes +- **Production ingester** — Long-lived `haiku-ingester` service with persistent SQLite queue, async worker pool with retries and a dead-letter queue, FS / HTTP / S3 / WebDAV source adapters, and a FastAPI control plane. See [docs/ingester.md](docs/ingester.md). - **Time travel** — Query the database at any historical point with `--before` - **Inspector** — TUI for browsing documents, chunks, and search results @@ -68,8 +68,8 @@ haiku-rag analyze "How many documents mention transformers?" # Interactive chat — multi-turn conversations with memory haiku-rag chat -# Watch a directory for changes -haiku-rag serve --monitor +# Continuously ingest from configured sources (FS, HTTP, S3, WebDAV) +haiku-ingester serve ``` See [Configuration](https://ggozad.github.io/haiku.rag/configuration/) for customization options. @@ -103,7 +103,7 @@ For details on the skills the client wraps, see the [Skills docs](https://ggozad Use with AI assistants like Claude Desktop: ```bash -haiku-rag serve --mcp --stdio +haiku-rag mcp --stdio ``` Add to your Claude Desktop configuration: @@ -113,7 +113,7 @@ Add to your Claude Desktop configuration: "mcpServers": { "haiku-rag": { "command": "haiku-rag", - "args": ["serve", "--mcp", "--stdio"] + "args": ["mcp", "--stdio"] } } } @@ -125,7 +125,7 @@ Provides tools for document management, search, QA, and analysis directly in you See the [examples directory](examples/) for working examples: -- **[Docker Setup](examples/docker/)** - Complete Docker deployment with file monitoring and MCP server +- **[Docker Setup](examples/docker/)** - Complete Docker deployment with continuous ingestion (`haiku-ingester`) and MCP server - **[Web Application](app/)** - Full-stack conversational RAG with CopilotKit frontend ## Documentation @@ -139,7 +139,7 @@ Full documentation at: https://ggozad.github.io/haiku.rag/ - [Skills](https://ggozad.github.io/haiku.rag/skills/) - The RAG and analysis skills the client wraps - [Analysis skill](https://ggozad.github.io/haiku.rag/skills/analysis/) - Complex analytical tasks via code execution - [Applications](https://ggozad.github.io/haiku.rag/apps/) - Chat TUI, web app, and inspector -- [Server](https://ggozad.github.io/haiku.rag/server/) - File monitoring and MCP +- [Ingester](https://ggozad.github.io/haiku.rag/ingester/) - Production ingester service for continuous indexing from FS, HTTP, S3, and WebDAV - [MCP](https://ggozad.github.io/haiku.rag/mcp/) - Model Context Protocol integration - [Benchmarks](https://ggozad.github.io/haiku.rag/benchmarks/) - Performance benchmarks - [Changelog](https://ggozad.github.io/haiku.rag/changelog/) - Version history diff --git a/docker/Dockerfile b/docker/Dockerfile index 852a176b..51e4f336 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -13,13 +13,13 @@ ENV UV_COMPILE_BYTECODE=1 \ COPY pyproject.toml uv.lock ./ COPY haiku_rag_slim/pyproject.toml haiku_rag_slim/README.md haiku_rag_slim/LICENSE haiku_rag_slim/ RUN --mount=type=cache,target=/root/.cache/uv \ - uv sync --frozen --no-install-project --no-dev + uv sync --frozen --no-install-project --no-dev --extra ingester -# Install the project itself +# Install the project itself (with the ingester extra so haiku-ingester is on PATH) COPY haiku_rag_slim haiku_rag_slim/ COPY README.md LICENSE ./ RUN --mount=type=cache,target=/root/.cache/uv \ - uv sync --frozen --no-editable --no-dev + uv sync --frozen --no-editable --no-dev --extra ingester # Final layer FROM python:3.13-slim @@ -33,8 +33,11 @@ ENV DEFAULT_DATA_DIR=/data ENV PATH="/app/.venv/bin:$PATH" -# Expose port for MCP server -EXPOSE 8001 +# Expose MCP server (8001) and ingester control plane (8765) ports. +# docker-compose overrides this image's default command to run either the +# MCP server (read-only) or the ingester service. +EXPOSE 8001 8765 -# Run all services (monitoring, MCP) -CMD ["python", "-m", "haiku.rag.cli", "--config", "/app/haiku.rag.yaml", "serve", "--monitor", "--mcp", "--mcp-port", "8001", "--db", "/data/haiku.rag.lancedb"] +# Default command: read-only MCP server. The companion ingester service is +# launched via docker-compose against the same image. +CMD ["haiku-rag", "--config", "/app/haiku.rag.yaml", "--read-only", "mcp", "--port", "8001"] diff --git a/docker/Dockerfile.slim b/docker/Dockerfile.slim index d1b24bf0..529cb6c0 100644 --- a/docker/Dockerfile.slim +++ b/docker/Dockerfile.slim @@ -12,13 +12,13 @@ ENV UV_COMPILE_BYTECODE=1 \ COPY pyproject.toml uv.lock ./ COPY haiku_rag_slim/pyproject.toml haiku_rag_slim/README.md haiku_rag_slim/LICENSE haiku_rag_slim/ RUN --mount=type=cache,target=/root/.cache/uv \ - uv sync --frozen --no-install-project --no-dev --package haiku.rag-slim + uv sync --frozen --no-install-project --no-dev --extra ingester --package haiku.rag-slim -# Install the project itself +# Install the project itself (with the ingester extra so haiku-ingester is on PATH) COPY haiku_rag_slim haiku_rag_slim/ COPY README.md LICENSE ./ RUN --mount=type=cache,target=/root/.cache/uv \ - uv sync --frozen --no-editable --no-dev --package haiku.rag-slim + uv sync --frozen --no-editable --no-dev --extra ingester --package haiku.rag-slim # Final layer FROM python:3.13-slim @@ -32,8 +32,11 @@ ENV DEFAULT_DATA_DIR=/data ENV PATH="/app/.venv/bin:$PATH" -# Expose port for MCP server -EXPOSE 8001 +# Expose MCP server (8001) and ingester control plane (8765) ports. +# docker-compose overrides this image's default command to run either the +# MCP server (read-only) or the ingester service. +EXPOSE 8001 8765 -# Run all services (monitoring, MCP) -CMD ["python", "-m", "haiku.rag.cli", "--config", "/app/haiku.rag.yaml", "serve", "--monitor", "--mcp", "--mcp-port", "8001", "--db", "/data/haiku.rag.lancedb"] +# Default command: read-only MCP server. The companion ingester service is +# launched via docker-compose against the same image. +CMD ["haiku-rag", "--config", "/app/haiku.rag.yaml", "--read-only", "mcp", "--port", "8001"] diff --git a/docker/README.md b/docker/README.md index 03053f38..ecee9e01 100644 --- a/docker/README.md +++ b/docker/README.md @@ -49,24 +49,37 @@ docker run -p 8001:8001 \ haiku-rag ``` -To enable file monitoring, also mount a documents directory: +For continuous ingestion of a watched directory, run `haiku-ingester` in a +separate container against the same data volume: ```bash -docker run -p 8001:8001 \ +docker run \ -v /path/to/haiku.rag.yaml:/app/haiku.rag.yaml \ -v /path/to/data:/data \ -v /path/to/docs:/docs \ - haiku-rag haiku-rag serve --mcp --monitor + -p 8765:8765 \ + haiku-rag haiku-ingester --config /app/haiku.rag.yaml serve ``` -Your `haiku.rag.yaml` must reference the **container path** for monitoring: +Configure the watched directory in `haiku.rag.yaml` using the **container +path**: ```yaml -monitor: - directories: - - /docs # Container path, not host path +ingester: + queue: + path: /data/ingester.db # persist queue in the data volume + sources: + - type: fs + id: docs + root: /docs # container path, not host path + delete_orphans: true ``` +The MCP server running in the first container must be started with +`--read-only` when an ingester is writing to the same database — LanceDB +allows one writer and N readers per URI. See +`examples/docker/docker-compose.yml` for a working two-service setup. + For API keys (OpenAI, Anthropic, etc.), pass them as environment variables: ```bash diff --git a/docs/cli.md b/docs/cli.md index 1bf36259..4e98f7df 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -57,7 +57,7 @@ From directory (recursively adds all supported files): haiku-rag add-src /path/to/documents/ ``` -From an S3 bucket (requires the `[s3]` extra, see [Server Mode → S3 / Object Storage Monitoring](server.md#s3-object-storage-monitoring)): +From an S3 bucket (requires the `[s3]` extra, see the [ingester docs](ingester.md) for continuous S3 polling): ```bash # AWS S3 with credentials in the default chain (env vars, IAM role, AWS profile) haiku-rag add-src s3://my-bucket/path/to/document.pdf @@ -69,7 +69,7 @@ AWS_ACCESS_KEY_ID=key AWS_SECRET_ACCESS_KEY=secret AWS_REGION=us-east-1 \ ``` !!! note - When adding a directory, the same content filters configured for [file monitoring](configuration/processing.md#filtering-monitored-files) are applied. This means `ignore_patterns` and `include_patterns` from your configuration will be used to filter which files are added. + When adding a directory, the converter's supported extensions filter applies. For pattern-based ignore/include filtering (e.g. `**/.git/**`), use the [ingester](ingester.md) with a filesystem source. !!! note As you add documents to `haiku.rag` the database keeps growing. By default, LanceDB supports versioning @@ -413,9 +413,9 @@ haiku-rag mcp --port 9000 haiku-rag --read-only mcp ``` -See [MCP Server](server.md) for details. For continuous document -ingestion (filesystem watch, S3 polling, HTTP sources), use -`haiku-ingester serve`. +See [MCP](mcp.md) for details. For continuous document ingestion +(filesystem watch, S3 polling, HTTP / WebDAV sources), use the +[ingester](ingester.md). ## Settings diff --git a/docs/configuration/index.md b/docs/configuration/index.md index 3305d3c8..20c01a2d 100644 --- a/docs/configuration/index.md +++ b/docs/configuration/index.md @@ -182,6 +182,7 @@ For detailed configuration of specific topics, see: - **[Providers](providers.md)** - Model settings and provider-specific configuration (embeddings, reranking) - **[Search and Question Answering](qa.md)** - Search settings and question answering -- **[Document Processing](processing.md)** - Document conversion, chunking, and file monitoring +- **[Document Processing](processing.md)** - Document conversion and chunking +- **[Ingester](../ingester.md)** - Continuous ingestion from filesystem, HTTP, S3, and WebDAV sources - **[Storage](storage.md)** - Database, remote storage, and vector indexing - **[Prompts](prompts.md)** - Customize agent prompts for your domain diff --git a/docs/installation.md b/docs/installation.md index ee44e629..8afdbce2 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -98,4 +98,4 @@ docker run -p 8001:8001 \ haiku-rag ``` -See `docker/README.md` for complete build and configuration instructions, including how to enable file monitoring. +See `docker/README.md` for complete build and configuration instructions, including how to run the [ingester](ingester.md) service for continuous document ingestion. diff --git a/docs/overview.md b/docs/overview.md index a2374302..80463ede 100644 --- a/docs/overview.md +++ b/docs/overview.md @@ -23,7 +23,7 @@ The chat TUI is one way to interact with the database. `haiku-rag ask` and `haik ## What it does -**Ingest.** PDFs, DOCX, HTML, images, and 40+ formats via Docling. Add files, URLs, or whole directories. Monitor folders and reindex on change. +**Ingest.** PDFs, DOCX, HTML, images, and 40+ formats via Docling. Add files, URLs, or whole directories with `haiku-rag add-src`, or run the [`haiku-ingester`](ingester.md) service for continuous, queue-backed ingestion from filesystem, HTTP, S3, or WebDAV sources. **Search.** Hybrid retrieval (vector + full-text with reciprocal rank fusion), optional cross-encoder reranking, structure-aware context expansion. Image-as-query and cross-modal retrieval when configured with a multimodal embedder. @@ -31,7 +31,7 @@ The chat TUI is one way to interact with the database. `haiku-rag ask` and `haik **Integrate.** Use it from Python, the CLI, the [MCP server](mcp.md), or as composable [skills](skills/index.md) built on haiku.skills. Skills bundle tools, prompts, and state for use inside any Pydantic AI agent. -**Operate.** Embedded LanceDB by default. Also runs on S3, GCS, Azure, or LanceDB Cloud. Time-travel queries via LanceDB versioning. File-monitoring mode for production deployments. +**Operate.** Embedded LanceDB by default. Also runs on S3, GCS, Azure, or LanceDB Cloud. Time-travel queries via LanceDB versioning. The [`haiku-ingester`](ingester.md) service runs continuously for production deployments. ## Where to go next diff --git a/examples/README.md b/examples/README.md index 1cae68b0..b99a9f5e 100644 --- a/examples/README.md +++ b/examples/README.md @@ -7,7 +7,7 @@ This directory contains example scripts demonstrating various features of haiku. **Directory:** `docker/` Complete Docker setup for running haiku.rag with all services: -- File monitoring for automatic document indexing +- Continuous ingestion from a watched directory via `haiku-ingester` - MCP server for AI assistant integration See `docker/README.md` for setup instructions. diff --git a/examples/docker/README.md b/examples/docker/README.md index a5d3fb52..ea19ac26 100644 --- a/examples/docker/README.md +++ b/examples/docker/README.md @@ -1,13 +1,22 @@ # haiku.rag Docker Compose Example -Run haiku.rag with docling-serve for remote document processing, file monitoring, and MCP server. +Run haiku.rag with docling-serve for remote document processing, continuous ingestion via `haiku-ingester`, and a read-only MCP server. ## Architecture -This example demonstrates remote processing with two services: +LanceDB allows exactly one writer + N readers per database URI, so the +example runs the ingester and the MCP server as **two separate containers** +sharing the same data volume: - **docling-serve** - Document conversion and chunking service -- **haiku-rag** - MCP server and file monitoring (using slim image) +- **haiku-ingester** - Long-lived writer. Watches `/docs`, ingests new and + changed files, queues retries, exposes the control plane on port 8765. +- **haiku-rag** - Read-only MCP server on port 8001 for AI assistant + integration. Cannot write to the database — the ingester owns writes. + +Both haiku.* services share the same slim image (built once) and the same +config file; docker-compose overrides the image's default command to give +each container its role. This setup showcases the minimal haiku.rag-slim image combined with external document processing, ideal for production deployments. @@ -28,13 +37,11 @@ Place documents in `docs/` for automatic indexing. ## Volume Mounts -The docker-compose.yml mounts three volumes: - -| Host Path | Container Path | Purpose | -|-----------|---------------|---------| -| `./data` | `/data` | Persistent LanceDB database | -| `./docs` | `/docs` | Documents to monitor and index | -| `./haiku.rag.yaml` | `/app/haiku.rag.yaml` | Configuration file | +| Host Path | Container Path | Mounted on | Purpose | +|-----------|----------------|------------|---------| +| `./data` | `/data` | both haiku containers | Persistent LanceDB + ingester queue | +| `./docs` | `/docs` | `haiku-ingester` only | Documents to ingest (watched by the FS source) | +| `./haiku.rag.yaml` | `/app/haiku.rag.yaml` | both haiku containers | Configuration file | **Important:** The `haiku.rag.yaml` config file must exist before running `docker compose up`. Copy it from the example: @@ -46,6 +53,11 @@ The example config sets `ingester.sources[0].root: /docs` - this is the **contai ## Usage +Add documents by dropping files into `./docs/` on the host — the ingester +picks them up automatically (watchfiles + periodic sweep). + +The `haiku-rag` container runs in read-only mode, so use it for queries: + ```bash # List documents docker compose exec haiku-rag haiku-rag list @@ -57,10 +69,19 @@ docker compose exec haiku-rag haiku-rag search "your query" docker compose exec haiku-rag haiku-rag ask "What is haiku.rag?" ``` +Check ingester progress via its control plane: + +```bash +curl http://localhost:8765/health +curl http://localhost:8765/jobs?status=queued +curl http://localhost:8765/dlq +``` + ## Ports - `5001` - docling-serve API (with UI enabled) -- `8001` - MCP server +- `8001` - MCP server (read-only) +- `8765` - ingester control plane (`/health`, `/jobs`, `/sources`, `/dlq`) ## Configuration diff --git a/examples/docker/docker-compose.yml b/examples/docker/docker-compose.yml index cbcb2d1a..d4e65246 100644 --- a/examples/docker/docker-compose.yml +++ b/examples/docker/docker-compose.yml @@ -15,19 +15,28 @@ services: start_period: 40s start_interval: 5s - haiku-rag: + # LanceDB allows one writer + N readers per database URI; the ingester is + # the writer, MCP runs read-only. + haiku-ingester: build: context: ../.. dockerfile: docker/Dockerfile.slim - container_name: haiku-rag + image: haiku-rag-slim:local + container_name: haiku-ingester + command: + [ + "haiku-ingester", + "--config", + "/app/haiku.rag.yaml", + "serve", + ] ports: - - "8001:8001" # MCP server + - "8765:8765" # ingester control plane (/health, /jobs, /sources, /dlq) volumes: - - ./data:/data # Persist database - - ./docs:/docs # Mount documents directory for monitoring - - ./haiku.rag.yaml:/app/haiku.rag.yaml:ro # Mount config file + - ./data:/data + - ./docs:/docs # Documents directory ingested by the FS source + - ./haiku.rag.yaml:/app/haiku.rag.yaml:ro environment: - # API keys (set as needed) - OPENAI_API_KEY=${OPENAI_API_KEY} - ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY} - VOYAGE_API_KEY=${VOYAGE_API_KEY} @@ -36,3 +45,30 @@ services: docling-serve: condition: service_healthy restart: unless-stopped + + haiku-rag: + image: haiku-rag-slim:local + container_name: haiku-rag + command: + [ + "haiku-rag", + "--config", + "/app/haiku.rag.yaml", + "--read-only", + "mcp", + "--port", + "8001", + ] + ports: + - "8001:8001" # MCP server (read-only against the same LanceDB) + volumes: + - ./data:/data # Shared with the ingester (read-only at this process) + - ./haiku.rag.yaml:/app/haiku.rag.yaml:ro + environment: + - OPENAI_API_KEY=${OPENAI_API_KEY} + - ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY} + - VOYAGE_API_KEY=${VOYAGE_API_KEY} + - CO_API_KEY=${CO_API_KEY} + depends_on: + - haiku-ingester + restart: unless-stopped diff --git a/examples/docker/haiku.rag.yaml.example b/examples/docker/haiku.rag.yaml.example index 465e6b4a..215daf6d 100644 --- a/examples/docker/haiku.rag.yaml.example +++ b/examples/docker/haiku.rag.yaml.example @@ -7,6 +7,9 @@ storage: data_dir: /data ingester: + # Queue lives next to the LanceDB so both persist in the data volume. + queue: + path: /data/ingester.db sources: - type: fs id: docs