Docker & docs updates
This commit is contained in:
Yiorgis Gozadinos
parent
055fd23d5d
commit
66d5fee682
12 changed files with 136 additions and 56 deletions
14
README.md
14
README.md
|
|
@ -22,7 +22,7 @@ Agentic RAG built on [LanceDB](https://lancedb.com/), [Pydantic AI](https://ai.p
|
|||
- **CLI & Python API** — Full functionality from command line or code
|
||||
- **MCP server** — Expose as tools for AI assistants (Claude Desktop, etc.)
|
||||
- **Visual grounding** — View chunks highlighted on original page images
|
||||
- **File monitoring** — Watch directories and auto-index on changes
|
||||
- **Production ingester** — Long-lived `haiku-ingester` service with persistent SQLite queue, async worker pool with retries and a dead-letter queue, FS / HTTP / S3 / WebDAV source adapters, and a FastAPI control plane. See [docs/ingester.md](docs/ingester.md).
|
||||
- **Time travel** — Query the database at any historical point with `--before`
|
||||
- **Inspector** — TUI for browsing documents, chunks, and search results
|
||||
|
||||
|
|
@ -68,8 +68,8 @@ haiku-rag analyze "How many documents mention transformers?"
|
|||
# Interactive chat — multi-turn conversations with memory
|
||||
haiku-rag chat
|
||||
|
||||
# Watch a directory for changes
|
||||
haiku-rag serve --monitor
|
||||
# Continuously ingest from configured sources (FS, HTTP, S3, WebDAV)
|
||||
haiku-ingester serve
|
||||
```
|
||||
|
||||
See [Configuration](https://ggozad.github.io/haiku.rag/configuration/) for customization options.
|
||||
|
|
@ -103,7 +103,7 @@ For details on the skills the client wraps, see the [Skills docs](https://ggozad
|
|||
Use with AI assistants like Claude Desktop:
|
||||
|
||||
```bash
|
||||
haiku-rag serve --mcp --stdio
|
||||
haiku-rag mcp --stdio
|
||||
```
|
||||
|
||||
Add to your Claude Desktop configuration:
|
||||
|
|
@ -113,7 +113,7 @@ Add to your Claude Desktop configuration:
|
|||
"mcpServers": {
|
||||
"haiku-rag": {
|
||||
"command": "haiku-rag",
|
||||
"args": ["serve", "--mcp", "--stdio"]
|
||||
"args": ["mcp", "--stdio"]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -125,7 +125,7 @@ Provides tools for document management, search, QA, and analysis directly in you
|
|||
|
||||
See the [examples directory](examples/) for working examples:
|
||||
|
||||
- **[Docker Setup](examples/docker/)** - Complete Docker deployment with file monitoring and MCP server
|
||||
- **[Docker Setup](examples/docker/)** - Complete Docker deployment with continuous ingestion (`haiku-ingester`) and MCP server
|
||||
- **[Web Application](app/)** - Full-stack conversational RAG with CopilotKit frontend
|
||||
|
||||
## Documentation
|
||||
|
|
@ -139,7 +139,7 @@ Full documentation at: https://ggozad.github.io/haiku.rag/
|
|||
- [Skills](https://ggozad.github.io/haiku.rag/skills/) - The RAG and analysis skills the client wraps
|
||||
- [Analysis skill](https://ggozad.github.io/haiku.rag/skills/analysis/) - Complex analytical tasks via code execution
|
||||
- [Applications](https://ggozad.github.io/haiku.rag/apps/) - Chat TUI, web app, and inspector
|
||||
- [Server](https://ggozad.github.io/haiku.rag/server/) - File monitoring and MCP
|
||||
- [Ingester](https://ggozad.github.io/haiku.rag/ingester/) - Production ingester service for continuous indexing from FS, HTTP, S3, and WebDAV
|
||||
- [MCP](https://ggozad.github.io/haiku.rag/mcp/) - Model Context Protocol integration
|
||||
- [Benchmarks](https://ggozad.github.io/haiku.rag/benchmarks/) - Performance benchmarks
|
||||
- [Changelog](https://ggozad.github.io/haiku.rag/changelog/) - Version history
|
||||
|
|
|
|||
|
|
@ -13,13 +13,13 @@ ENV UV_COMPILE_BYTECODE=1 \
|
|||
COPY pyproject.toml uv.lock ./
|
||||
COPY haiku_rag_slim/pyproject.toml haiku_rag_slim/README.md haiku_rag_slim/LICENSE haiku_rag_slim/
|
||||
RUN --mount=type=cache,target=/root/.cache/uv \
|
||||
uv sync --frozen --no-install-project --no-dev
|
||||
uv sync --frozen --no-install-project --no-dev --extra ingester
|
||||
|
||||
# Install the project itself
|
||||
# Install the project itself (with the ingester extra so haiku-ingester is on PATH)
|
||||
COPY haiku_rag_slim haiku_rag_slim/
|
||||
COPY README.md LICENSE ./
|
||||
RUN --mount=type=cache,target=/root/.cache/uv \
|
||||
uv sync --frozen --no-editable --no-dev
|
||||
uv sync --frozen --no-editable --no-dev --extra ingester
|
||||
|
||||
# Final layer
|
||||
FROM python:3.13-slim
|
||||
|
|
@ -33,8 +33,11 @@ ENV DEFAULT_DATA_DIR=/data
|
|||
|
||||
ENV PATH="/app/.venv/bin:$PATH"
|
||||
|
||||
# Expose port for MCP server
|
||||
EXPOSE 8001
|
||||
# Expose MCP server (8001) and ingester control plane (8765) ports.
|
||||
# docker-compose overrides this image's default command to run either the
|
||||
# MCP server (read-only) or the ingester service.
|
||||
EXPOSE 8001 8765
|
||||
|
||||
# Run all services (monitoring, MCP)
|
||||
CMD ["python", "-m", "haiku.rag.cli", "--config", "/app/haiku.rag.yaml", "serve", "--monitor", "--mcp", "--mcp-port", "8001", "--db", "/data/haiku.rag.lancedb"]
|
||||
# Default command: read-only MCP server. The companion ingester service is
|
||||
# launched via docker-compose against the same image.
|
||||
CMD ["haiku-rag", "--config", "/app/haiku.rag.yaml", "--read-only", "mcp", "--port", "8001"]
|
||||
|
|
|
|||
|
|
@ -12,13 +12,13 @@ ENV UV_COMPILE_BYTECODE=1 \
|
|||
COPY pyproject.toml uv.lock ./
|
||||
COPY haiku_rag_slim/pyproject.toml haiku_rag_slim/README.md haiku_rag_slim/LICENSE haiku_rag_slim/
|
||||
RUN --mount=type=cache,target=/root/.cache/uv \
|
||||
uv sync --frozen --no-install-project --no-dev --package haiku.rag-slim
|
||||
uv sync --frozen --no-install-project --no-dev --extra ingester --package haiku.rag-slim
|
||||
|
||||
# Install the project itself
|
||||
# Install the project itself (with the ingester extra so haiku-ingester is on PATH)
|
||||
COPY haiku_rag_slim haiku_rag_slim/
|
||||
COPY README.md LICENSE ./
|
||||
RUN --mount=type=cache,target=/root/.cache/uv \
|
||||
uv sync --frozen --no-editable --no-dev --package haiku.rag-slim
|
||||
uv sync --frozen --no-editable --no-dev --extra ingester --package haiku.rag-slim
|
||||
|
||||
# Final layer
|
||||
FROM python:3.13-slim
|
||||
|
|
@ -32,8 +32,11 @@ ENV DEFAULT_DATA_DIR=/data
|
|||
|
||||
ENV PATH="/app/.venv/bin:$PATH"
|
||||
|
||||
# Expose port for MCP server
|
||||
EXPOSE 8001
|
||||
# Expose MCP server (8001) and ingester control plane (8765) ports.
|
||||
# docker-compose overrides this image's default command to run either the
|
||||
# MCP server (read-only) or the ingester service.
|
||||
EXPOSE 8001 8765
|
||||
|
||||
# Run all services (monitoring, MCP)
|
||||
CMD ["python", "-m", "haiku.rag.cli", "--config", "/app/haiku.rag.yaml", "serve", "--monitor", "--mcp", "--mcp-port", "8001", "--db", "/data/haiku.rag.lancedb"]
|
||||
# Default command: read-only MCP server. The companion ingester service is
|
||||
# launched via docker-compose against the same image.
|
||||
CMD ["haiku-rag", "--config", "/app/haiku.rag.yaml", "--read-only", "mcp", "--port", "8001"]
|
||||
|
|
|
|||
|
|
@ -49,24 +49,37 @@ docker run -p 8001:8001 \
|
|||
haiku-rag
|
||||
```
|
||||
|
||||
To enable file monitoring, also mount a documents directory:
|
||||
For continuous ingestion of a watched directory, run `haiku-ingester` in a
|
||||
separate container against the same data volume:
|
||||
|
||||
```bash
|
||||
docker run -p 8001:8001 \
|
||||
docker run \
|
||||
-v /path/to/haiku.rag.yaml:/app/haiku.rag.yaml \
|
||||
-v /path/to/data:/data \
|
||||
-v /path/to/docs:/docs \
|
||||
haiku-rag haiku-rag serve --mcp --monitor
|
||||
-p 8765:8765 \
|
||||
haiku-rag haiku-ingester --config /app/haiku.rag.yaml serve
|
||||
```
|
||||
|
||||
Your `haiku.rag.yaml` must reference the **container path** for monitoring:
|
||||
Configure the watched directory in `haiku.rag.yaml` using the **container
|
||||
path**:
|
||||
|
||||
```yaml
|
||||
monitor:
|
||||
directories:
|
||||
- /docs # Container path, not host path
|
||||
ingester:
|
||||
queue:
|
||||
path: /data/ingester.db # persist queue in the data volume
|
||||
sources:
|
||||
- type: fs
|
||||
id: docs
|
||||
root: /docs # container path, not host path
|
||||
delete_orphans: true
|
||||
```
|
||||
|
||||
The MCP server running in the first container must be started with
|
||||
`--read-only` when an ingester is writing to the same database — LanceDB
|
||||
allows one writer and N readers per URI. See
|
||||
`examples/docker/docker-compose.yml` for a working two-service setup.
|
||||
|
||||
For API keys (OpenAI, Anthropic, etc.), pass them as environment variables:
|
||||
|
||||
```bash
|
||||
|
|
|
|||
10
docs/cli.md
10
docs/cli.md
|
|
@ -57,7 +57,7 @@ From directory (recursively adds all supported files):
|
|||
haiku-rag add-src /path/to/documents/
|
||||
```
|
||||
|
||||
From an S3 bucket (requires the `[s3]` extra, see [Server Mode → S3 / Object Storage Monitoring](server.md#s3-object-storage-monitoring)):
|
||||
From an S3 bucket (requires the `[s3]` extra, see the [ingester docs](ingester.md) for continuous S3 polling):
|
||||
```bash
|
||||
# AWS S3 with credentials in the default chain (env vars, IAM role, AWS profile)
|
||||
haiku-rag add-src s3://my-bucket/path/to/document.pdf
|
||||
|
|
@ -69,7 +69,7 @@ AWS_ACCESS_KEY_ID=key AWS_SECRET_ACCESS_KEY=secret AWS_REGION=us-east-1 \
|
|||
```
|
||||
|
||||
!!! note
|
||||
When adding a directory, the same content filters configured for [file monitoring](configuration/processing.md#filtering-monitored-files) are applied. This means `ignore_patterns` and `include_patterns` from your configuration will be used to filter which files are added.
|
||||
When adding a directory, the converter's supported extensions filter applies. For pattern-based ignore/include filtering (e.g. `**/.git/**`), use the [ingester](ingester.md) with a filesystem source.
|
||||
|
||||
!!! note
|
||||
As you add documents to `haiku.rag` the database keeps growing. By default, LanceDB supports versioning
|
||||
|
|
@ -413,9 +413,9 @@ haiku-rag mcp --port 9000
|
|||
haiku-rag --read-only mcp
|
||||
```
|
||||
|
||||
See [MCP Server](server.md) for details. For continuous document
|
||||
ingestion (filesystem watch, S3 polling, HTTP sources), use
|
||||
`haiku-ingester serve`.
|
||||
See [MCP](mcp.md) for details. For continuous document ingestion
|
||||
(filesystem watch, S3 polling, HTTP / WebDAV sources), use the
|
||||
[ingester](ingester.md).
|
||||
|
||||
## Settings
|
||||
|
||||
|
|
|
|||
|
|
@ -182,6 +182,7 @@ For detailed configuration of specific topics, see:
|
|||
|
||||
- **[Providers](providers.md)** - Model settings and provider-specific configuration (embeddings, reranking)
|
||||
- **[Search and Question Answering](qa.md)** - Search settings and question answering
|
||||
- **[Document Processing](processing.md)** - Document conversion, chunking, and file monitoring
|
||||
- **[Document Processing](processing.md)** - Document conversion and chunking
|
||||
- **[Ingester](../ingester.md)** - Continuous ingestion from filesystem, HTTP, S3, and WebDAV sources
|
||||
- **[Storage](storage.md)** - Database, remote storage, and vector indexing
|
||||
- **[Prompts](prompts.md)** - Customize agent prompts for your domain
|
||||
|
|
|
|||
|
|
@ -98,4 +98,4 @@ docker run -p 8001:8001 \
|
|||
haiku-rag
|
||||
```
|
||||
|
||||
See `docker/README.md` for complete build and configuration instructions, including how to enable file monitoring.
|
||||
See `docker/README.md` for complete build and configuration instructions, including how to run the [ingester](ingester.md) service for continuous document ingestion.
|
||||
|
|
|
|||
|
|
@ -23,7 +23,7 @@ The chat TUI is one way to interact with the database. `haiku-rag ask` and `haik
|
|||
|
||||
## What it does
|
||||
|
||||
**Ingest.** PDFs, DOCX, HTML, images, and 40+ formats via Docling. Add files, URLs, or whole directories. Monitor folders and reindex on change.
|
||||
**Ingest.** PDFs, DOCX, HTML, images, and 40+ formats via Docling. Add files, URLs, or whole directories with `haiku-rag add-src`, or run the [`haiku-ingester`](ingester.md) service for continuous, queue-backed ingestion from filesystem, HTTP, S3, or WebDAV sources.
|
||||
|
||||
**Search.** Hybrid retrieval (vector + full-text with reciprocal rank fusion), optional cross-encoder reranking, structure-aware context expansion. Image-as-query and cross-modal retrieval when configured with a multimodal embedder.
|
||||
|
||||
|
|
@ -31,7 +31,7 @@ The chat TUI is one way to interact with the database. `haiku-rag ask` and `haik
|
|||
|
||||
**Integrate.** Use it from Python, the CLI, the [MCP server](mcp.md), or as composable [skills](skills/index.md) built on haiku.skills. Skills bundle tools, prompts, and state for use inside any Pydantic AI agent.
|
||||
|
||||
**Operate.** Embedded LanceDB by default. Also runs on S3, GCS, Azure, or LanceDB Cloud. Time-travel queries via LanceDB versioning. File-monitoring mode for production deployments.
|
||||
**Operate.** Embedded LanceDB by default. Also runs on S3, GCS, Azure, or LanceDB Cloud. Time-travel queries via LanceDB versioning. The [`haiku-ingester`](ingester.md) service runs continuously for production deployments.
|
||||
|
||||
## Where to go next
|
||||
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ This directory contains example scripts demonstrating various features of haiku.
|
|||
**Directory:** `docker/`
|
||||
|
||||
Complete Docker setup for running haiku.rag with all services:
|
||||
- File monitoring for automatic document indexing
|
||||
- Continuous ingestion from a watched directory via `haiku-ingester`
|
||||
- MCP server for AI assistant integration
|
||||
|
||||
See `docker/README.md` for setup instructions.
|
||||
|
|
|
|||
|
|
@ -1,13 +1,22 @@
|
|||
# haiku.rag Docker Compose Example
|
||||
|
||||
Run haiku.rag with docling-serve for remote document processing, file monitoring, and MCP server.
|
||||
Run haiku.rag with docling-serve for remote document processing, continuous ingestion via `haiku-ingester`, and a read-only MCP server.
|
||||
|
||||
## Architecture
|
||||
|
||||
This example demonstrates remote processing with two services:
|
||||
LanceDB allows exactly one writer + N readers per database URI, so the
|
||||
example runs the ingester and the MCP server as **two separate containers**
|
||||
sharing the same data volume:
|
||||
|
||||
- **docling-serve** - Document conversion and chunking service
|
||||
- **haiku-rag** - MCP server and file monitoring (using slim image)
|
||||
- **haiku-ingester** - Long-lived writer. Watches `/docs`, ingests new and
|
||||
changed files, queues retries, exposes the control plane on port 8765.
|
||||
- **haiku-rag** - Read-only MCP server on port 8001 for AI assistant
|
||||
integration. Cannot write to the database — the ingester owns writes.
|
||||
|
||||
Both haiku.* services share the same slim image (built once) and the same
|
||||
config file; docker-compose overrides the image's default command to give
|
||||
each container its role.
|
||||
|
||||
This setup showcases the minimal haiku.rag-slim image combined with external document processing, ideal for production deployments.
|
||||
|
||||
|
|
@ -28,13 +37,11 @@ Place documents in `docs/` for automatic indexing.
|
|||
|
||||
## Volume Mounts
|
||||
|
||||
The docker-compose.yml mounts three volumes:
|
||||
|
||||
| Host Path | Container Path | Purpose |
|
||||
|-----------|---------------|---------|
|
||||
| `./data` | `/data` | Persistent LanceDB database |
|
||||
| `./docs` | `/docs` | Documents to monitor and index |
|
||||
| `./haiku.rag.yaml` | `/app/haiku.rag.yaml` | Configuration file |
|
||||
| Host Path | Container Path | Mounted on | Purpose |
|
||||
|-----------|----------------|------------|---------|
|
||||
| `./data` | `/data` | both haiku containers | Persistent LanceDB + ingester queue |
|
||||
| `./docs` | `/docs` | `haiku-ingester` only | Documents to ingest (watched by the FS source) |
|
||||
| `./haiku.rag.yaml` | `/app/haiku.rag.yaml` | both haiku containers | Configuration file |
|
||||
|
||||
**Important:** The `haiku.rag.yaml` config file must exist before running `docker compose up`. Copy it from the example:
|
||||
|
||||
|
|
@ -46,6 +53,11 @@ The example config sets `ingester.sources[0].root: /docs` - this is the **contai
|
|||
|
||||
## Usage
|
||||
|
||||
Add documents by dropping files into `./docs/` on the host — the ingester
|
||||
picks them up automatically (watchfiles + periodic sweep).
|
||||
|
||||
The `haiku-rag` container runs in read-only mode, so use it for queries:
|
||||
|
||||
```bash
|
||||
# List documents
|
||||
docker compose exec haiku-rag haiku-rag list
|
||||
|
|
@ -57,10 +69,19 @@ docker compose exec haiku-rag haiku-rag search "your query"
|
|||
docker compose exec haiku-rag haiku-rag ask "What is haiku.rag?"
|
||||
```
|
||||
|
||||
Check ingester progress via its control plane:
|
||||
|
||||
```bash
|
||||
curl http://localhost:8765/health
|
||||
curl http://localhost:8765/jobs?status=queued
|
||||
curl http://localhost:8765/dlq
|
||||
```
|
||||
|
||||
## Ports
|
||||
|
||||
- `5001` - docling-serve API (with UI enabled)
|
||||
- `8001` - MCP server
|
||||
- `8001` - MCP server (read-only)
|
||||
- `8765` - ingester control plane (`/health`, `/jobs`, `/sources`, `/dlq`)
|
||||
|
||||
## Configuration
|
||||
|
||||
|
|
|
|||
|
|
@ -15,19 +15,28 @@ services:
|
|||
start_period: 40s
|
||||
start_interval: 5s
|
||||
|
||||
haiku-rag:
|
||||
# LanceDB allows one writer + N readers per database URI; the ingester is
|
||||
# the writer, MCP runs read-only.
|
||||
haiku-ingester:
|
||||
build:
|
||||
context: ../..
|
||||
dockerfile: docker/Dockerfile.slim
|
||||
container_name: haiku-rag
|
||||
image: haiku-rag-slim:local
|
||||
container_name: haiku-ingester
|
||||
command:
|
||||
[
|
||||
"haiku-ingester",
|
||||
"--config",
|
||||
"/app/haiku.rag.yaml",
|
||||
"serve",
|
||||
]
|
||||
ports:
|
||||
- "8001:8001" # MCP server
|
||||
- "8765:8765" # ingester control plane (/health, /jobs, /sources, /dlq)
|
||||
volumes:
|
||||
- ./data:/data # Persist database
|
||||
- ./docs:/docs # Mount documents directory for monitoring
|
||||
- ./haiku.rag.yaml:/app/haiku.rag.yaml:ro # Mount config file
|
||||
- ./data:/data
|
||||
- ./docs:/docs # Documents directory ingested by the FS source
|
||||
- ./haiku.rag.yaml:/app/haiku.rag.yaml:ro
|
||||
environment:
|
||||
# API keys (set as needed)
|
||||
- OPENAI_API_KEY=${OPENAI_API_KEY}
|
||||
- ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY}
|
||||
- VOYAGE_API_KEY=${VOYAGE_API_KEY}
|
||||
|
|
@ -36,3 +45,30 @@ services:
|
|||
docling-serve:
|
||||
condition: service_healthy
|
||||
restart: unless-stopped
|
||||
|
||||
haiku-rag:
|
||||
image: haiku-rag-slim:local
|
||||
container_name: haiku-rag
|
||||
command:
|
||||
[
|
||||
"haiku-rag",
|
||||
"--config",
|
||||
"/app/haiku.rag.yaml",
|
||||
"--read-only",
|
||||
"mcp",
|
||||
"--port",
|
||||
"8001",
|
||||
]
|
||||
ports:
|
||||
- "8001:8001" # MCP server (read-only against the same LanceDB)
|
||||
volumes:
|
||||
- ./data:/data # Shared with the ingester (read-only at this process)
|
||||
- ./haiku.rag.yaml:/app/haiku.rag.yaml:ro
|
||||
environment:
|
||||
- OPENAI_API_KEY=${OPENAI_API_KEY}
|
||||
- ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY}
|
||||
- VOYAGE_API_KEY=${VOYAGE_API_KEY}
|
||||
- CO_API_KEY=${CO_API_KEY}
|
||||
depends_on:
|
||||
- haiku-ingester
|
||||
restart: unless-stopped
|
||||
|
|
|
|||
|
|
@ -7,6 +7,9 @@ storage:
|
|||
data_dir: /data
|
||||
|
||||
ingester:
|
||||
# Queue lives next to the LanceDB so both persist in the data volume.
|
||||
queue:
|
||||
path: /data/ingester.db
|
||||
sources:
|
||||
- type: fs
|
||||
id: docs
|
||||
|
|
|
|||
Loading…
Reference in a new issue