From 2f9c9070316862b22650edb7447abd19bd78f32f Mon Sep 17 00:00:00 2001 From: Yiorgis Gozadinos Date: Tue, 4 Nov 2025 17:55:57 +0200 Subject: [PATCH 01/17] Restructure into uv workspace to support minimal and full installations --- .dockerignore | 2 +- .gitignore | 2 +- docs/benchmarks.md | 4 +- evaluations/LICENSE | 7 + evaluations/README.md | 13 ++ {src => evaluations}/evaluations/__init__.py | 0 {src => evaluations}/evaluations/benchmark.py | 0 {src => evaluations}/evaluations/config.py | 0 .../evaluations/datasets/__init__.py | 0 .../evaluations/datasets/repliqa.py | 0 .../evaluations/datasets/wix.py | 0 {src => evaluations}/evaluations/llm_judge.py | 0 {src => evaluations}/evaluations/prompts.py | 0 evaluations/pyproject.toml | 23 ++ haiku_rag_slim/LICENSE | 7 + haiku_rag_slim/README.md | 204 ++++++++++++++++++ {src => haiku_rag_slim}/haiku/rag/__init__.py | 0 .../haiku/rag/a2a/__init__.py | 0 .../haiku/rag/a2a/client.py | 0 .../haiku/rag/a2a/context.py | 0 .../haiku/rag/a2a/models.py | 0 .../haiku/rag/a2a/prompts.py | 0 .../haiku/rag/a2a/skills.py | 0 .../haiku/rag/a2a/storage.py | 0 .../haiku/rag/a2a/worker.py | 0 {src => haiku_rag_slim}/haiku/rag/app.py | 0 {src => haiku_rag_slim}/haiku/rag/chunker.py | 0 {src => haiku_rag_slim}/haiku/rag/cli.py | 0 {src => haiku_rag_slim}/haiku/rag/client.py | 0 .../haiku/rag/config/__init__.py | 0 .../haiku/rag/config/loader.py | 0 .../haiku/rag/config/models.py | 0 .../haiku/rag/embeddings/__init__.py | 0 .../haiku/rag/embeddings/base.py | 0 .../haiku/rag/embeddings/ollama.py | 0 .../haiku/rag/embeddings/openai.py | 0 .../haiku/rag/embeddings/vllm.py | 0 .../haiku/rag/embeddings/voyageai.py | 0 .../haiku/rag/graph/__init__.py | 0 .../haiku/rag/graph/base.py | 0 .../haiku/rag/graph/common.py | 0 .../haiku/rag/graph/models.py | 0 .../haiku/rag/graph/nodes/__init__.py | 0 .../haiku/rag/graph/nodes/analysis.py | 0 .../haiku/rag/graph/nodes/plan.py | 0 .../haiku/rag/graph/nodes/search.py | 0 .../haiku/rag/graph/nodes/synthesize.py | 0 .../haiku/rag/graph/prompts.py | 0 {src => haiku_rag_slim}/haiku/rag/logging.py | 0 {src => haiku_rag_slim}/haiku/rag/mcp.py | 0 {src => haiku_rag_slim}/haiku/rag/monitor.py | 0 .../haiku/rag/qa/__init__.py | 0 {src => haiku_rag_slim}/haiku/rag/qa/agent.py | 0 .../haiku/rag/qa/deep/__init__.py | 0 .../haiku/rag/qa/deep/dependencies.py | 0 .../haiku/rag/qa/deep/graph.py | 0 .../haiku/rag/qa/deep/models.py | 0 .../haiku/rag/qa/deep/nodes.py | 0 .../haiku/rag/qa/deep/prompts.py | 0 .../haiku/rag/qa/deep/state.py | 0 .../haiku/rag/qa/prompts.py | 0 {src => haiku_rag_slim}/haiku/rag/reader.py | 0 .../haiku/rag/reranking/__init__.py | 0 .../haiku/rag/reranking/base.py | 0 .../haiku/rag/reranking/cohere.py | 0 .../haiku/rag/reranking/mxbai.py | 0 .../haiku/rag/reranking/vllm.py | 0 .../haiku/rag/reranking/zeroentropy.py | 0 .../haiku/rag/research/__init__.py | 0 .../haiku/rag/research/common.py | 0 .../haiku/rag/research/dependencies.py | 0 .../haiku/rag/research/graph.py | 0 .../haiku/rag/research/models.py | 0 .../haiku/rag/research/prompts.py | 0 .../haiku/rag/research/state.py | 0 .../haiku/rag/research/stream.py | 0 .../haiku/rag/store/__init__.py | 0 .../haiku/rag/store/engine.py | 0 .../haiku/rag/store/models/__init__.py | 0 .../haiku/rag/store/models/chunk.py | 0 .../haiku/rag/store/models/document.py | 0 .../haiku/rag/store/repositories/__init__.py | 0 .../haiku/rag/store/repositories/chunk.py | 0 .../haiku/rag/store/repositories/document.py | 0 .../haiku/rag/store/repositories/settings.py | 0 .../haiku/rag/store/upgrades/__init__.py | 0 .../haiku/rag/store/upgrades/v0_10_1.py | 0 .../haiku/rag/store/upgrades/v0_9_3.py | 0 {src => haiku_rag_slim}/haiku/rag/utils.py | 0 haiku_rag_slim/pyproject.toml | 56 +++++ pyproject.toml | 33 +-- uv.lock | 107 ++++++--- 92 files changed, 398 insertions(+), 60 deletions(-) create mode 100644 evaluations/LICENSE create mode 100644 evaluations/README.md rename {src => evaluations}/evaluations/__init__.py (100%) rename {src => evaluations}/evaluations/benchmark.py (100%) rename {src => evaluations}/evaluations/config.py (100%) rename {src => evaluations}/evaluations/datasets/__init__.py (100%) rename {src => evaluations}/evaluations/datasets/repliqa.py (100%) rename {src => evaluations}/evaluations/datasets/wix.py (100%) rename {src => evaluations}/evaluations/llm_judge.py (100%) rename {src => evaluations}/evaluations/prompts.py (100%) create mode 100644 evaluations/pyproject.toml create mode 100644 haiku_rag_slim/LICENSE create mode 100644 haiku_rag_slim/README.md rename {src => haiku_rag_slim}/haiku/rag/__init__.py (100%) rename {src => haiku_rag_slim}/haiku/rag/a2a/__init__.py (100%) rename {src => haiku_rag_slim}/haiku/rag/a2a/client.py (100%) rename {src => haiku_rag_slim}/haiku/rag/a2a/context.py (100%) rename {src => haiku_rag_slim}/haiku/rag/a2a/models.py (100%) rename {src => haiku_rag_slim}/haiku/rag/a2a/prompts.py (100%) rename {src => haiku_rag_slim}/haiku/rag/a2a/skills.py (100%) rename {src => haiku_rag_slim}/haiku/rag/a2a/storage.py (100%) rename {src => haiku_rag_slim}/haiku/rag/a2a/worker.py (100%) rename {src => haiku_rag_slim}/haiku/rag/app.py (100%) rename {src => haiku_rag_slim}/haiku/rag/chunker.py (100%) rename {src => haiku_rag_slim}/haiku/rag/cli.py (100%) rename {src => haiku_rag_slim}/haiku/rag/client.py (100%) rename {src => haiku_rag_slim}/haiku/rag/config/__init__.py (100%) rename {src => haiku_rag_slim}/haiku/rag/config/loader.py (100%) rename {src => haiku_rag_slim}/haiku/rag/config/models.py (100%) rename {src => haiku_rag_slim}/haiku/rag/embeddings/__init__.py (100%) rename {src => haiku_rag_slim}/haiku/rag/embeddings/base.py (100%) rename {src => haiku_rag_slim}/haiku/rag/embeddings/ollama.py (100%) rename {src => haiku_rag_slim}/haiku/rag/embeddings/openai.py (100%) rename {src => haiku_rag_slim}/haiku/rag/embeddings/vllm.py (100%) rename {src => haiku_rag_slim}/haiku/rag/embeddings/voyageai.py (100%) rename {src => haiku_rag_slim}/haiku/rag/graph/__init__.py (100%) rename {src => haiku_rag_slim}/haiku/rag/graph/base.py (100%) rename {src => haiku_rag_slim}/haiku/rag/graph/common.py (100%) rename {src => haiku_rag_slim}/haiku/rag/graph/models.py (100%) rename {src => haiku_rag_slim}/haiku/rag/graph/nodes/__init__.py (100%) rename {src => haiku_rag_slim}/haiku/rag/graph/nodes/analysis.py (100%) rename {src => haiku_rag_slim}/haiku/rag/graph/nodes/plan.py (100%) rename {src => haiku_rag_slim}/haiku/rag/graph/nodes/search.py (100%) rename {src => haiku_rag_slim}/haiku/rag/graph/nodes/synthesize.py (100%) rename {src => haiku_rag_slim}/haiku/rag/graph/prompts.py (100%) rename {src => haiku_rag_slim}/haiku/rag/logging.py (100%) rename {src => haiku_rag_slim}/haiku/rag/mcp.py (100%) rename {src => haiku_rag_slim}/haiku/rag/monitor.py (100%) rename {src => haiku_rag_slim}/haiku/rag/qa/__init__.py (100%) rename {src => haiku_rag_slim}/haiku/rag/qa/agent.py (100%) rename {src => haiku_rag_slim}/haiku/rag/qa/deep/__init__.py (100%) rename {src => haiku_rag_slim}/haiku/rag/qa/deep/dependencies.py (100%) rename {src => haiku_rag_slim}/haiku/rag/qa/deep/graph.py (100%) rename {src => haiku_rag_slim}/haiku/rag/qa/deep/models.py (100%) rename {src => haiku_rag_slim}/haiku/rag/qa/deep/nodes.py (100%) rename {src => haiku_rag_slim}/haiku/rag/qa/deep/prompts.py (100%) rename {src => haiku_rag_slim}/haiku/rag/qa/deep/state.py (100%) rename {src => haiku_rag_slim}/haiku/rag/qa/prompts.py (100%) rename {src => haiku_rag_slim}/haiku/rag/reader.py (100%) rename {src => haiku_rag_slim}/haiku/rag/reranking/__init__.py (100%) rename {src => haiku_rag_slim}/haiku/rag/reranking/base.py (100%) rename {src => haiku_rag_slim}/haiku/rag/reranking/cohere.py (100%) rename {src => haiku_rag_slim}/haiku/rag/reranking/mxbai.py (100%) rename {src => haiku_rag_slim}/haiku/rag/reranking/vllm.py (100%) rename {src => haiku_rag_slim}/haiku/rag/reranking/zeroentropy.py (100%) rename {src => haiku_rag_slim}/haiku/rag/research/__init__.py (100%) rename {src => haiku_rag_slim}/haiku/rag/research/common.py (100%) rename {src => haiku_rag_slim}/haiku/rag/research/dependencies.py (100%) rename {src => haiku_rag_slim}/haiku/rag/research/graph.py (100%) rename {src => haiku_rag_slim}/haiku/rag/research/models.py (100%) rename {src => haiku_rag_slim}/haiku/rag/research/prompts.py (100%) rename {src => haiku_rag_slim}/haiku/rag/research/state.py (100%) rename {src => haiku_rag_slim}/haiku/rag/research/stream.py (100%) rename {src => haiku_rag_slim}/haiku/rag/store/__init__.py (100%) rename {src => haiku_rag_slim}/haiku/rag/store/engine.py (100%) rename {src => haiku_rag_slim}/haiku/rag/store/models/__init__.py (100%) rename {src => haiku_rag_slim}/haiku/rag/store/models/chunk.py (100%) rename {src => haiku_rag_slim}/haiku/rag/store/models/document.py (100%) rename {src => haiku_rag_slim}/haiku/rag/store/repositories/__init__.py (100%) rename {src => haiku_rag_slim}/haiku/rag/store/repositories/chunk.py (100%) rename {src => haiku_rag_slim}/haiku/rag/store/repositories/document.py (100%) rename {src => haiku_rag_slim}/haiku/rag/store/repositories/settings.py (100%) rename {src => haiku_rag_slim}/haiku/rag/store/upgrades/__init__.py (100%) rename {src => haiku_rag_slim}/haiku/rag/store/upgrades/v0_10_1.py (100%) rename {src => haiku_rag_slim}/haiku/rag/store/upgrades/v0_9_3.py (100%) rename {src => haiku_rag_slim}/haiku/rag/utils.py (100%) create mode 100644 haiku_rag_slim/pyproject.toml diff --git a/.dockerignore b/.dockerignore index 404602b3..9def18fa 100644 --- a/.dockerignore +++ b/.dockerignore @@ -60,7 +60,7 @@ tests/ .pytest_cache/ .coverage htmlcov/ -src/evaluations/ +evaluations/ server.json # Examples examples/ diff --git a/.gitignore b/.gitignore index 1cd04da7..952f8b27 100644 --- a/.gitignore +++ b/.gitignore @@ -11,7 +11,7 @@ wheels/ # tests .coverage* -src/evaluations/data/ +evaluations/evaluations/data/ tests/data/ .pytest_cache/ .ruff_cache/ diff --git a/docs/benchmarks.md b/docs/benchmarks.md index 4eb31abd..0a6b5bbb 100644 --- a/docs/benchmarks.md +++ b/docs/benchmarks.md @@ -3,7 +3,7 @@ We use the [repliqa](https://huggingface.co/datasets/ServiceNow/repliqa) dataset for the evaluation of `haiku.rag`. You can perform your own evaluations with the Typer CLI in -`src/evaluations/benchmark.py`, for example `cd src && python -m evaluations.benchmark repliqa`. +`evaluations/evaluations/benchmark.py`, for example `python -m evaluations.benchmark repliqa`. The evaluation flow is orchestrated with [`pydantic-evals`](https://github.com/pydantic/pydantic-ai/tree/main/libs/pydantic-evals), which we leverage for dataset management, scoring, and report generation. @@ -13,7 +13,7 @@ which we leverage for dataset management, scoring, and report generation. The benchmark script accepts a `--config` option to specify a custom `haiku.rag.yaml` configuration file: ```bash -cd src && python -m evaluations.benchmark repliqa --config /path/to/haiku.rag.yaml +python -m evaluations.benchmark repliqa --config /path/to/haiku.rag.yaml ``` If no config file is specified, the script will search for a config file in the standard locations: diff --git a/evaluations/LICENSE b/evaluations/LICENSE new file mode 100644 index 00000000..806d932e --- /dev/null +++ b/evaluations/LICENSE @@ -0,0 +1,7 @@ +Copyright 2025 Yiorgis Gozadinos + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/evaluations/README.md b/evaluations/README.md new file mode 100644 index 00000000..2e758dd1 --- /dev/null +++ b/evaluations/README.md @@ -0,0 +1,13 @@ +# Haiku RAG - Evaluations + +Internal benchmarking and evaluation scripts for haiku.rag. + +This package is not published to PyPI and is only used for development and testing purposes. + +## Overview + +Contains evaluation scripts for benchmarking RAG performance using datasets like: +- RepliQA +- WiX + +Uses `pydantic-evals` for evaluation framework and `logfire` for observability. diff --git a/src/evaluations/__init__.py b/evaluations/evaluations/__init__.py similarity index 100% rename from src/evaluations/__init__.py rename to evaluations/evaluations/__init__.py diff --git a/src/evaluations/benchmark.py b/evaluations/evaluations/benchmark.py similarity index 100% rename from src/evaluations/benchmark.py rename to evaluations/evaluations/benchmark.py diff --git a/src/evaluations/config.py b/evaluations/evaluations/config.py similarity index 100% rename from src/evaluations/config.py rename to evaluations/evaluations/config.py diff --git a/src/evaluations/datasets/__init__.py b/evaluations/evaluations/datasets/__init__.py similarity index 100% rename from src/evaluations/datasets/__init__.py rename to evaluations/evaluations/datasets/__init__.py diff --git a/src/evaluations/datasets/repliqa.py b/evaluations/evaluations/datasets/repliqa.py similarity index 100% rename from src/evaluations/datasets/repliqa.py rename to evaluations/evaluations/datasets/repliqa.py diff --git a/src/evaluations/datasets/wix.py b/evaluations/evaluations/datasets/wix.py similarity index 100% rename from src/evaluations/datasets/wix.py rename to evaluations/evaluations/datasets/wix.py diff --git a/src/evaluations/llm_judge.py b/evaluations/evaluations/llm_judge.py similarity index 100% rename from src/evaluations/llm_judge.py rename to evaluations/evaluations/llm_judge.py diff --git a/src/evaluations/prompts.py b/evaluations/evaluations/prompts.py similarity index 100% rename from src/evaluations/prompts.py rename to evaluations/evaluations/prompts.py diff --git a/evaluations/pyproject.toml b/evaluations/pyproject.toml new file mode 100644 index 00000000..7b6f8922 --- /dev/null +++ b/evaluations/pyproject.toml @@ -0,0 +1,23 @@ +[project] + +name = "haiku.rag-evals" +description = "Internal benchmarking and evaluation scripts for haiku.rag" +version = "0.13.3" +authors = [{ name = "Yiorgis Gozadinos", email = "ggozadinos@gmail.com" }] +license = { text = "MIT" } +requires-python = ">=3.12" + +dependencies = [ + "haiku.rag-slim", + "pydantic-evals>=1.7.0", + "datasets>=4.3.0", + "logfire>=4.14.2", + "typer>=0.19.2,<0.20.0", +] + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +packages = ["evaluations"] diff --git a/haiku_rag_slim/LICENSE b/haiku_rag_slim/LICENSE new file mode 100644 index 00000000..806d932e --- /dev/null +++ b/haiku_rag_slim/LICENSE @@ -0,0 +1,7 @@ +Copyright 2025 Yiorgis Gozadinos + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/haiku_rag_slim/README.md b/haiku_rag_slim/README.md new file mode 100644 index 00000000..5f5c9a8e --- /dev/null +++ b/haiku_rag_slim/README.md @@ -0,0 +1,204 @@ +# Haiku RAG - Slim + +mcp-name: io.github.ggozad/haiku-rag + +Retrieval-Augmented Generation (RAG) library built on LanceDB - Minimal dependencies. + +`haiku.rag-slim` is the core package with minimal dependencies. Document processing via docling is optional and can be installed separately. + +For a batteries-included experience with all extras, see [`haiku.rag`](https://pypi.org/project/haiku.rag/). + +## Installation + +```bash +# Minimal installation (no docling) +uv pip install haiku.rag-slim + +# With docling support for document processing +uv pip install haiku.rag-slim[docling] + +# With specific extras +uv pip install haiku.rag-slim[docling,voyageai,mxbai] +``` + +> **Note**: Configuration now uses YAML files instead of environment variables. If you're upgrading from an older version, run `haiku-rag init-config --from-env` to migrate your `.env` file to `haiku.rag.yaml`. See [Configuration](https://ggozad.github.io/haiku.rag/configuration/) for details. + +## Features + +- **Local LanceDB**: No external servers required, supports also LanceDB cloud storage, S3, Google Cloud & Azure +- **Multiple embedding providers**: Ollama, VoyageAI, OpenAI, vLLM +- **Multiple QA providers**: Any provider/model supported by Pydantic AI +- **Research graph (multi‑agent)**: Plan → Search → Evaluate → Synthesize with agentic AI +- **Native hybrid search**: Vector + full-text search with native LanceDB RRF reranking +- **Reranking**: Default search result reranking with MixedBread AI, Cohere, Zero Entropy, or vLLM +- **Question answering**: Built-in QA agents on your documents +- **File monitoring**: Auto-index files when run as server +- **40+ file formats**: PDF, DOCX, HTML, Markdown, code files, URLs +- **MCP server**: Expose as tools for AI assistants +- **A2A agent**: Conversational agent with context and multi-turn dialogue +- **CLI & Python API**: Use from command line or Python + +## Quick Start + +```bash +# Install +# Python 3.12 or newer required +uv pip install haiku.rag-slim[docling] + +# Add documents +haiku-rag add "Your content here" +haiku-rag add "Your content here" --meta author=alice --meta topic=notes +haiku-rag add-src document.pdf --meta source=manual + +# Search +haiku-rag search "query" + +# Search with filters +haiku-rag search "query" --filter "uri LIKE '%.pdf' AND title LIKE '%paper%'" + +# Ask questions +haiku-rag ask "Who is the author of haiku.rag?" + +# Ask questions with citations +haiku-rag ask "Who is the author of haiku.rag?" --cite + +# Deep QA (multi-agent question decomposition) +haiku-rag ask "Who is the author of haiku.rag?" --deep --cite + +# Deep QA with verbose output +haiku-rag ask "Who is the author of haiku.rag?" --deep --verbose + +# Multi‑agent research (iterative plan/search/evaluate) +haiku-rag research \ + "What are the main drivers and trends of global temperature anomalies since 1990?" \ + --max-iterations 2 \ + --confidence-threshold 0.8 \ + --max-concurrency 3 \ + --verbose + +# Rebuild database (re-chunk and re-embed all documents) +haiku-rag rebuild + +# Start server with file monitoring +haiku-rag serve --monitor +``` + +To customize settings, create a `haiku.rag.yaml` config file (see [Configuration](https://ggozad.github.io/haiku.rag/configuration/)). + +## Python Usage + +```python +from haiku.rag.client import HaikuRAG +from haiku.rag.research import ( + PlanNode, + ResearchContext, + ResearchDeps, + ResearchState, + build_research_graph, + stream_research_graph, +) + +async with HaikuRAG("database.lancedb") as client: + # Add document + doc = await client.create_document("Your content") + + # Search (reranking enabled by default) + results = await client.search("query") + for chunk, score in results: + print(f"{score:.3f}: {chunk.content}") + + # Ask questions + answer = await client.ask("Who is the author of haiku.rag?") + print(answer) + + # Ask questions with citations + answer = await client.ask("Who is the author of haiku.rag?", cite=True) + print(answer) + + # Multi‑agent research pipeline (Plan → Search → Evaluate → Synthesize) + graph = build_research_graph() + question = ( + "What are the main drivers and trends of global temperature " + "anomalies since 1990?" + ) + state = ResearchState( + context=ResearchContext(original_question=question), + max_iterations=2, + confidence_threshold=0.8, + max_concurrency=2, + ) + deps = ResearchDeps(client=client) + + # Blocking run (final result only) + result = await graph.run( + PlanNode(provider="openai", model="gpt-4o-mini"), + state=state, + deps=deps, + ) + print(result.output.title) + + # Streaming progress (log/report/error events) + async for event in stream_research_graph( + graph, + PlanNode(provider="openai", model="gpt-4o-mini"), + state, + deps, + ): + if event.type == "log": + iteration = event.state.iterations if event.state else state.iterations + print(f"[{iteration}] {event.message}") + elif event.type == "report": + print("\nResearch complete!\n") + print(event.report.title) + print(event.report.executive_summary) +``` + +## MCP Server + +Use with AI assistants like Claude Desktop: + +```bash +haiku-rag serve --stdio +``` + +Provides tools for document management and search directly in your AI assistant. + +## A2A Agent + +Run as a conversational agent with the Agent-to-Agent protocol: + +```bash +# Start the A2A server +haiku-rag serve --a2a + +# Connect with the interactive client (in another terminal) +haiku-rag a2aclient +``` + +The A2A agent provides: + +- Multi-turn dialogue with context +- Intelligent multi-search for complex questions +- Source citations with titles and URIs +- Full document retrieval on request + +## Examples + +See the [examples directory](examples/) for working examples: + +- **[Interactive Research Assistant](examples/ag-ui-research/)** - Full-stack research assistant with Pydantic AI and AG-UI featuring human-in-the-loop approval and real-time state synchronization +- **[Docker Setup](examples/docker/)** - Complete Docker deployment with file monitoring, MCP server, and A2A agent +- **[A2A Security](examples/a2a-security/)** - Authentication examples (API key, OAuth2, GitHub) + +## Documentation + +Full documentation at: https://ggozad.github.io/haiku.rag/ + +- [Installation](https://ggozad.github.io/haiku.rag/installation/) - Provider setup +- [Configuration](https://ggozad.github.io/haiku.rag/configuration/) - YAML configuration +- [CLI](https://ggozad.github.io/haiku.rag/cli/) - Command reference +- [Python API](https://ggozad.github.io/haiku.rag/python/) - Complete API docs +- [Agents](https://ggozad.github.io/haiku.rag/agents/) - QA agent and multi-agent research +- [MCP Server](https://ggozad.github.io/haiku.rag/mcp/) - Model Context Protocol integration +- [A2A Agent](https://ggozad.github.io/haiku.rag/a2a/) - Agent-to-Agent protocol support +- [Benchmarks](https://ggozad.github.io/haiku.rag/benchmarks/) - Performance Benchmarks diff --git a/src/haiku/rag/__init__.py b/haiku_rag_slim/haiku/rag/__init__.py similarity index 100% rename from src/haiku/rag/__init__.py rename to haiku_rag_slim/haiku/rag/__init__.py diff --git a/src/haiku/rag/a2a/__init__.py b/haiku_rag_slim/haiku/rag/a2a/__init__.py similarity index 100% rename from src/haiku/rag/a2a/__init__.py rename to haiku_rag_slim/haiku/rag/a2a/__init__.py diff --git a/src/haiku/rag/a2a/client.py b/haiku_rag_slim/haiku/rag/a2a/client.py similarity index 100% rename from src/haiku/rag/a2a/client.py rename to haiku_rag_slim/haiku/rag/a2a/client.py diff --git a/src/haiku/rag/a2a/context.py b/haiku_rag_slim/haiku/rag/a2a/context.py similarity index 100% rename from src/haiku/rag/a2a/context.py rename to haiku_rag_slim/haiku/rag/a2a/context.py diff --git a/src/haiku/rag/a2a/models.py b/haiku_rag_slim/haiku/rag/a2a/models.py similarity index 100% rename from src/haiku/rag/a2a/models.py rename to haiku_rag_slim/haiku/rag/a2a/models.py diff --git a/src/haiku/rag/a2a/prompts.py b/haiku_rag_slim/haiku/rag/a2a/prompts.py similarity index 100% rename from src/haiku/rag/a2a/prompts.py rename to haiku_rag_slim/haiku/rag/a2a/prompts.py diff --git a/src/haiku/rag/a2a/skills.py b/haiku_rag_slim/haiku/rag/a2a/skills.py similarity index 100% rename from src/haiku/rag/a2a/skills.py rename to haiku_rag_slim/haiku/rag/a2a/skills.py diff --git a/src/haiku/rag/a2a/storage.py b/haiku_rag_slim/haiku/rag/a2a/storage.py similarity index 100% rename from src/haiku/rag/a2a/storage.py rename to haiku_rag_slim/haiku/rag/a2a/storage.py diff --git a/src/haiku/rag/a2a/worker.py b/haiku_rag_slim/haiku/rag/a2a/worker.py similarity index 100% rename from src/haiku/rag/a2a/worker.py rename to haiku_rag_slim/haiku/rag/a2a/worker.py diff --git a/src/haiku/rag/app.py b/haiku_rag_slim/haiku/rag/app.py similarity index 100% rename from src/haiku/rag/app.py rename to haiku_rag_slim/haiku/rag/app.py diff --git a/src/haiku/rag/chunker.py b/haiku_rag_slim/haiku/rag/chunker.py similarity index 100% rename from src/haiku/rag/chunker.py rename to haiku_rag_slim/haiku/rag/chunker.py diff --git a/src/haiku/rag/cli.py b/haiku_rag_slim/haiku/rag/cli.py similarity index 100% rename from src/haiku/rag/cli.py rename to haiku_rag_slim/haiku/rag/cli.py diff --git a/src/haiku/rag/client.py b/haiku_rag_slim/haiku/rag/client.py similarity index 100% rename from src/haiku/rag/client.py rename to haiku_rag_slim/haiku/rag/client.py diff --git a/src/haiku/rag/config/__init__.py b/haiku_rag_slim/haiku/rag/config/__init__.py similarity index 100% rename from src/haiku/rag/config/__init__.py rename to haiku_rag_slim/haiku/rag/config/__init__.py diff --git a/src/haiku/rag/config/loader.py b/haiku_rag_slim/haiku/rag/config/loader.py similarity index 100% rename from src/haiku/rag/config/loader.py rename to haiku_rag_slim/haiku/rag/config/loader.py diff --git a/src/haiku/rag/config/models.py b/haiku_rag_slim/haiku/rag/config/models.py similarity index 100% rename from src/haiku/rag/config/models.py rename to haiku_rag_slim/haiku/rag/config/models.py diff --git a/src/haiku/rag/embeddings/__init__.py b/haiku_rag_slim/haiku/rag/embeddings/__init__.py similarity index 100% rename from src/haiku/rag/embeddings/__init__.py rename to haiku_rag_slim/haiku/rag/embeddings/__init__.py diff --git a/src/haiku/rag/embeddings/base.py b/haiku_rag_slim/haiku/rag/embeddings/base.py similarity index 100% rename from src/haiku/rag/embeddings/base.py rename to haiku_rag_slim/haiku/rag/embeddings/base.py diff --git a/src/haiku/rag/embeddings/ollama.py b/haiku_rag_slim/haiku/rag/embeddings/ollama.py similarity index 100% rename from src/haiku/rag/embeddings/ollama.py rename to haiku_rag_slim/haiku/rag/embeddings/ollama.py diff --git a/src/haiku/rag/embeddings/openai.py b/haiku_rag_slim/haiku/rag/embeddings/openai.py similarity index 100% rename from src/haiku/rag/embeddings/openai.py rename to haiku_rag_slim/haiku/rag/embeddings/openai.py diff --git a/src/haiku/rag/embeddings/vllm.py b/haiku_rag_slim/haiku/rag/embeddings/vllm.py similarity index 100% rename from src/haiku/rag/embeddings/vllm.py rename to haiku_rag_slim/haiku/rag/embeddings/vllm.py diff --git a/src/haiku/rag/embeddings/voyageai.py b/haiku_rag_slim/haiku/rag/embeddings/voyageai.py similarity index 100% rename from src/haiku/rag/embeddings/voyageai.py rename to haiku_rag_slim/haiku/rag/embeddings/voyageai.py diff --git a/src/haiku/rag/graph/__init__.py b/haiku_rag_slim/haiku/rag/graph/__init__.py similarity index 100% rename from src/haiku/rag/graph/__init__.py rename to haiku_rag_slim/haiku/rag/graph/__init__.py diff --git a/src/haiku/rag/graph/base.py b/haiku_rag_slim/haiku/rag/graph/base.py similarity index 100% rename from src/haiku/rag/graph/base.py rename to haiku_rag_slim/haiku/rag/graph/base.py diff --git a/src/haiku/rag/graph/common.py b/haiku_rag_slim/haiku/rag/graph/common.py similarity index 100% rename from src/haiku/rag/graph/common.py rename to haiku_rag_slim/haiku/rag/graph/common.py diff --git a/src/haiku/rag/graph/models.py b/haiku_rag_slim/haiku/rag/graph/models.py similarity index 100% rename from src/haiku/rag/graph/models.py rename to haiku_rag_slim/haiku/rag/graph/models.py diff --git a/src/haiku/rag/graph/nodes/__init__.py b/haiku_rag_slim/haiku/rag/graph/nodes/__init__.py similarity index 100% rename from src/haiku/rag/graph/nodes/__init__.py rename to haiku_rag_slim/haiku/rag/graph/nodes/__init__.py diff --git a/src/haiku/rag/graph/nodes/analysis.py b/haiku_rag_slim/haiku/rag/graph/nodes/analysis.py similarity index 100% rename from src/haiku/rag/graph/nodes/analysis.py rename to haiku_rag_slim/haiku/rag/graph/nodes/analysis.py diff --git a/src/haiku/rag/graph/nodes/plan.py b/haiku_rag_slim/haiku/rag/graph/nodes/plan.py similarity index 100% rename from src/haiku/rag/graph/nodes/plan.py rename to haiku_rag_slim/haiku/rag/graph/nodes/plan.py diff --git a/src/haiku/rag/graph/nodes/search.py b/haiku_rag_slim/haiku/rag/graph/nodes/search.py similarity index 100% rename from src/haiku/rag/graph/nodes/search.py rename to haiku_rag_slim/haiku/rag/graph/nodes/search.py diff --git a/src/haiku/rag/graph/nodes/synthesize.py b/haiku_rag_slim/haiku/rag/graph/nodes/synthesize.py similarity index 100% rename from src/haiku/rag/graph/nodes/synthesize.py rename to haiku_rag_slim/haiku/rag/graph/nodes/synthesize.py diff --git a/src/haiku/rag/graph/prompts.py b/haiku_rag_slim/haiku/rag/graph/prompts.py similarity index 100% rename from src/haiku/rag/graph/prompts.py rename to haiku_rag_slim/haiku/rag/graph/prompts.py diff --git a/src/haiku/rag/logging.py b/haiku_rag_slim/haiku/rag/logging.py similarity index 100% rename from src/haiku/rag/logging.py rename to haiku_rag_slim/haiku/rag/logging.py diff --git a/src/haiku/rag/mcp.py b/haiku_rag_slim/haiku/rag/mcp.py similarity index 100% rename from src/haiku/rag/mcp.py rename to haiku_rag_slim/haiku/rag/mcp.py diff --git a/src/haiku/rag/monitor.py b/haiku_rag_slim/haiku/rag/monitor.py similarity index 100% rename from src/haiku/rag/monitor.py rename to haiku_rag_slim/haiku/rag/monitor.py diff --git a/src/haiku/rag/qa/__init__.py b/haiku_rag_slim/haiku/rag/qa/__init__.py similarity index 100% rename from src/haiku/rag/qa/__init__.py rename to haiku_rag_slim/haiku/rag/qa/__init__.py diff --git a/src/haiku/rag/qa/agent.py b/haiku_rag_slim/haiku/rag/qa/agent.py similarity index 100% rename from src/haiku/rag/qa/agent.py rename to haiku_rag_slim/haiku/rag/qa/agent.py diff --git a/src/haiku/rag/qa/deep/__init__.py b/haiku_rag_slim/haiku/rag/qa/deep/__init__.py similarity index 100% rename from src/haiku/rag/qa/deep/__init__.py rename to haiku_rag_slim/haiku/rag/qa/deep/__init__.py diff --git a/src/haiku/rag/qa/deep/dependencies.py b/haiku_rag_slim/haiku/rag/qa/deep/dependencies.py similarity index 100% rename from src/haiku/rag/qa/deep/dependencies.py rename to haiku_rag_slim/haiku/rag/qa/deep/dependencies.py diff --git a/src/haiku/rag/qa/deep/graph.py b/haiku_rag_slim/haiku/rag/qa/deep/graph.py similarity index 100% rename from src/haiku/rag/qa/deep/graph.py rename to haiku_rag_slim/haiku/rag/qa/deep/graph.py diff --git a/src/haiku/rag/qa/deep/models.py b/haiku_rag_slim/haiku/rag/qa/deep/models.py similarity index 100% rename from src/haiku/rag/qa/deep/models.py rename to haiku_rag_slim/haiku/rag/qa/deep/models.py diff --git a/src/haiku/rag/qa/deep/nodes.py b/haiku_rag_slim/haiku/rag/qa/deep/nodes.py similarity index 100% rename from src/haiku/rag/qa/deep/nodes.py rename to haiku_rag_slim/haiku/rag/qa/deep/nodes.py diff --git a/src/haiku/rag/qa/deep/prompts.py b/haiku_rag_slim/haiku/rag/qa/deep/prompts.py similarity index 100% rename from src/haiku/rag/qa/deep/prompts.py rename to haiku_rag_slim/haiku/rag/qa/deep/prompts.py diff --git a/src/haiku/rag/qa/deep/state.py b/haiku_rag_slim/haiku/rag/qa/deep/state.py similarity index 100% rename from src/haiku/rag/qa/deep/state.py rename to haiku_rag_slim/haiku/rag/qa/deep/state.py diff --git a/src/haiku/rag/qa/prompts.py b/haiku_rag_slim/haiku/rag/qa/prompts.py similarity index 100% rename from src/haiku/rag/qa/prompts.py rename to haiku_rag_slim/haiku/rag/qa/prompts.py diff --git a/src/haiku/rag/reader.py b/haiku_rag_slim/haiku/rag/reader.py similarity index 100% rename from src/haiku/rag/reader.py rename to haiku_rag_slim/haiku/rag/reader.py diff --git a/src/haiku/rag/reranking/__init__.py b/haiku_rag_slim/haiku/rag/reranking/__init__.py similarity index 100% rename from src/haiku/rag/reranking/__init__.py rename to haiku_rag_slim/haiku/rag/reranking/__init__.py diff --git a/src/haiku/rag/reranking/base.py b/haiku_rag_slim/haiku/rag/reranking/base.py similarity index 100% rename from src/haiku/rag/reranking/base.py rename to haiku_rag_slim/haiku/rag/reranking/base.py diff --git a/src/haiku/rag/reranking/cohere.py b/haiku_rag_slim/haiku/rag/reranking/cohere.py similarity index 100% rename from src/haiku/rag/reranking/cohere.py rename to haiku_rag_slim/haiku/rag/reranking/cohere.py diff --git a/src/haiku/rag/reranking/mxbai.py b/haiku_rag_slim/haiku/rag/reranking/mxbai.py similarity index 100% rename from src/haiku/rag/reranking/mxbai.py rename to haiku_rag_slim/haiku/rag/reranking/mxbai.py diff --git a/src/haiku/rag/reranking/vllm.py b/haiku_rag_slim/haiku/rag/reranking/vllm.py similarity index 100% rename from src/haiku/rag/reranking/vllm.py rename to haiku_rag_slim/haiku/rag/reranking/vllm.py diff --git a/src/haiku/rag/reranking/zeroentropy.py b/haiku_rag_slim/haiku/rag/reranking/zeroentropy.py similarity index 100% rename from src/haiku/rag/reranking/zeroentropy.py rename to haiku_rag_slim/haiku/rag/reranking/zeroentropy.py diff --git a/src/haiku/rag/research/__init__.py b/haiku_rag_slim/haiku/rag/research/__init__.py similarity index 100% rename from src/haiku/rag/research/__init__.py rename to haiku_rag_slim/haiku/rag/research/__init__.py diff --git a/src/haiku/rag/research/common.py b/haiku_rag_slim/haiku/rag/research/common.py similarity index 100% rename from src/haiku/rag/research/common.py rename to haiku_rag_slim/haiku/rag/research/common.py diff --git a/src/haiku/rag/research/dependencies.py b/haiku_rag_slim/haiku/rag/research/dependencies.py similarity index 100% rename from src/haiku/rag/research/dependencies.py rename to haiku_rag_slim/haiku/rag/research/dependencies.py diff --git a/src/haiku/rag/research/graph.py b/haiku_rag_slim/haiku/rag/research/graph.py similarity index 100% rename from src/haiku/rag/research/graph.py rename to haiku_rag_slim/haiku/rag/research/graph.py diff --git a/src/haiku/rag/research/models.py b/haiku_rag_slim/haiku/rag/research/models.py similarity index 100% rename from src/haiku/rag/research/models.py rename to haiku_rag_slim/haiku/rag/research/models.py diff --git a/src/haiku/rag/research/prompts.py b/haiku_rag_slim/haiku/rag/research/prompts.py similarity index 100% rename from src/haiku/rag/research/prompts.py rename to haiku_rag_slim/haiku/rag/research/prompts.py diff --git a/src/haiku/rag/research/state.py b/haiku_rag_slim/haiku/rag/research/state.py similarity index 100% rename from src/haiku/rag/research/state.py rename to haiku_rag_slim/haiku/rag/research/state.py diff --git a/src/haiku/rag/research/stream.py b/haiku_rag_slim/haiku/rag/research/stream.py similarity index 100% rename from src/haiku/rag/research/stream.py rename to haiku_rag_slim/haiku/rag/research/stream.py diff --git a/src/haiku/rag/store/__init__.py b/haiku_rag_slim/haiku/rag/store/__init__.py similarity index 100% rename from src/haiku/rag/store/__init__.py rename to haiku_rag_slim/haiku/rag/store/__init__.py diff --git a/src/haiku/rag/store/engine.py b/haiku_rag_slim/haiku/rag/store/engine.py similarity index 100% rename from src/haiku/rag/store/engine.py rename to haiku_rag_slim/haiku/rag/store/engine.py diff --git a/src/haiku/rag/store/models/__init__.py b/haiku_rag_slim/haiku/rag/store/models/__init__.py similarity index 100% rename from src/haiku/rag/store/models/__init__.py rename to haiku_rag_slim/haiku/rag/store/models/__init__.py diff --git a/src/haiku/rag/store/models/chunk.py b/haiku_rag_slim/haiku/rag/store/models/chunk.py similarity index 100% rename from src/haiku/rag/store/models/chunk.py rename to haiku_rag_slim/haiku/rag/store/models/chunk.py diff --git a/src/haiku/rag/store/models/document.py b/haiku_rag_slim/haiku/rag/store/models/document.py similarity index 100% rename from src/haiku/rag/store/models/document.py rename to haiku_rag_slim/haiku/rag/store/models/document.py diff --git a/src/haiku/rag/store/repositories/__init__.py b/haiku_rag_slim/haiku/rag/store/repositories/__init__.py similarity index 100% rename from src/haiku/rag/store/repositories/__init__.py rename to haiku_rag_slim/haiku/rag/store/repositories/__init__.py diff --git a/src/haiku/rag/store/repositories/chunk.py b/haiku_rag_slim/haiku/rag/store/repositories/chunk.py similarity index 100% rename from src/haiku/rag/store/repositories/chunk.py rename to haiku_rag_slim/haiku/rag/store/repositories/chunk.py diff --git a/src/haiku/rag/store/repositories/document.py b/haiku_rag_slim/haiku/rag/store/repositories/document.py similarity index 100% rename from src/haiku/rag/store/repositories/document.py rename to haiku_rag_slim/haiku/rag/store/repositories/document.py diff --git a/src/haiku/rag/store/repositories/settings.py b/haiku_rag_slim/haiku/rag/store/repositories/settings.py similarity index 100% rename from src/haiku/rag/store/repositories/settings.py rename to haiku_rag_slim/haiku/rag/store/repositories/settings.py diff --git a/src/haiku/rag/store/upgrades/__init__.py b/haiku_rag_slim/haiku/rag/store/upgrades/__init__.py similarity index 100% rename from src/haiku/rag/store/upgrades/__init__.py rename to haiku_rag_slim/haiku/rag/store/upgrades/__init__.py diff --git a/src/haiku/rag/store/upgrades/v0_10_1.py b/haiku_rag_slim/haiku/rag/store/upgrades/v0_10_1.py similarity index 100% rename from src/haiku/rag/store/upgrades/v0_10_1.py rename to haiku_rag_slim/haiku/rag/store/upgrades/v0_10_1.py diff --git a/src/haiku/rag/store/upgrades/v0_9_3.py b/haiku_rag_slim/haiku/rag/store/upgrades/v0_9_3.py similarity index 100% rename from src/haiku/rag/store/upgrades/v0_9_3.py rename to haiku_rag_slim/haiku/rag/store/upgrades/v0_9_3.py diff --git a/src/haiku/rag/utils.py b/haiku_rag_slim/haiku/rag/utils.py similarity index 100% rename from src/haiku/rag/utils.py rename to haiku_rag_slim/haiku/rag/utils.py diff --git a/haiku_rag_slim/pyproject.toml b/haiku_rag_slim/pyproject.toml new file mode 100644 index 00000000..6d5ff663 --- /dev/null +++ b/haiku_rag_slim/pyproject.toml @@ -0,0 +1,56 @@ +[project] + +name = "haiku.rag-slim" +description = "Agentic Retrieval Augmented Generation (RAG) with LanceDB - Minimal dependencies" +version = "0.13.3" +authors = [{ name = "Yiorgis Gozadinos", email = "ggozadinos@gmail.com" }] +license = { text = "MIT" } +readme = { file = "README.md", content-type = "text/markdown" } +requires-python = ">=3.12" +keywords = ["RAG", "lancedb", "vector-database", "ml", "mcp"] +classifiers = [ + "Development Status :: 4 - Beta", + "Environment :: Console", + "Intended Audience :: Developers", + "Operating System :: Microsoft :: Windows :: Windows 10", + "Operating System :: Microsoft :: Windows :: Windows 11", + "Operating System :: MacOS", + "Operating System :: POSIX :: Linux", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Typing :: Typed", +] + +dependencies = [ + "fastmcp>=2.13.0.2", + "httpx>=0.28.1", + "lancedb>=0.25.2", + "pathspec>=0.12.1", + "pydantic>=2.12.3", + "pydantic-ai>=1.7.0", + "pydantic-graph>=1.7.0", + "python-dotenv>=1.2.1", + "pyyaml>=6.0.3", + "rich>=14.2.0", + "tiktoken>=0.12.0", + "typer>=0.19.2,<0.20.0", + "watchfiles>=1.1.1", +] + +[project.optional-dependencies] +docling = ["docling>=2.58.0"] +voyageai = ["voyageai>=0.3.5"] +mxbai = ["mxbai-rerank>=0.1.6"] +a2a = ["fasta2a>=0.1.0"] +cohere = ["cohere>=5.0.0"] +zeroentropy = ["zeroentropy>=0.1.0a6"] + +[project.scripts] +haiku-rag = "haiku.rag.cli:cli" + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +packages = ["haiku"] diff --git a/pyproject.toml b/pyproject.toml index b5671dca..fb74e604 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,29 +22,9 @@ classifiers = [ ] dependencies = [ - "docling>=2.58.0", - "fastmcp>=2.13.0.2", - "httpx>=0.28.1", - "lancedb>=0.25.2", - "pathspec>=0.12.1", - "pydantic>=2.12.3", - "pydantic-ai>=1.7.0", - "pydantic-graph>=1.7.0", - "python-dotenv>=1.2.1", - "pyyaml>=6.0.3", - "rich>=14.2.0", - "tiktoken>=0.12.0", - "typer>=0.19.2,<0.20.0", - "watchfiles>=1.1.1", + "haiku.rag-slim[docling,voyageai,mxbai,a2a,cohere,zeroentropy]", ] -[project.optional-dependencies] -voyageai = ["voyageai>=0.3.5"] -mxbai = ["mxbai-rerank>=0.1.6"] -a2a = ["fasta2a>=0.1.0"] -cohere = ["cohere>=5.0.0"] -zeroentropy = ["zeroentropy>=0.1.0a6"] - [project.scripts] haiku-rag = "haiku.rag.cli:cli" @@ -53,10 +33,17 @@ requires = ["hatchling"] build-backend = "hatchling.build" [tool.hatch.build] -exclude = ["/docs", "/examples", "/tests", "/docker", "/.github"] +exclude = ["/docs", "/examples", "/tests", "/docker", "/.github", "/haiku_rag_slim", "/evaluations"] [tool.hatch.build.targets.wheel] -packages = ["src/haiku"] +only-include = ["/README.md"] + +[tool.uv.workspace] +members = ["haiku_rag_slim", "evaluations"] + +[tool.uv.sources] +"haiku.rag-slim" = { workspace = true } +"haiku.rag-evals" = { workspace = true } [dependency-groups] dev = [ diff --git a/uv.lock b/uv.lock index 194b2dd3..8a5ed5e6 100644 --- a/uv.lock +++ b/uv.lock @@ -13,6 +13,13 @@ resolution-markers = [ "(python_full_version >= '3.14' and platform_machine != 'aarch64' and sys_platform == 'linux') or (python_full_version >= '3.14' and sys_platform != 'darwin' and sys_platform != 'linux')", ] +[manifest] +members = [ + "haiku-rag", + "haiku-rag-evals", + "haiku-rag-slim", +] + [[package]] name = "accelerate" version = "1.10.1" @@ -1153,7 +1160,68 @@ name = "haiku-rag" version = "0.13.3" source = { editable = "." } dependencies = [ - { name = "docling" }, + { name = "haiku-rag-slim", extra = ["a2a", "cohere", "docling", "mxbai", "voyageai", "zeroentropy"] }, +] + +[package.dev-dependencies] +dev = [ + { name = "datasets" }, + { name = "logfire" }, + { name = "mkdocs" }, + { name = "mkdocs-material" }, + { name = "pre-commit" }, + { name = "pydantic-evals" }, + { name = "pyright" }, + { name = "pytest" }, + { name = "pytest-asyncio" }, + { name = "pytest-cov" }, + { name = "ruff" }, +] + +[package.metadata] +requires-dist = [{ name = "haiku-rag-slim", extras = ["docling", "voyageai", "mxbai", "a2a", "cohere", "zeroentropy"], editable = "haiku_rag_slim" }] + +[package.metadata.requires-dev] +dev = [ + { name = "datasets", specifier = ">=4.3.0" }, + { name = "logfire", specifier = ">=4.14.2" }, + { name = "mkdocs", specifier = ">=1.6.1" }, + { name = "mkdocs-material", specifier = ">=9.6.22" }, + { name = "pre-commit", specifier = ">=4.3.0" }, + { name = "pydantic-evals", specifier = ">=1.7.0" }, + { name = "pyright", specifier = ">=1.1.407" }, + { name = "pytest", specifier = ">=8.4.2" }, + { name = "pytest-asyncio", specifier = ">=1.2.0" }, + { name = "pytest-cov", specifier = ">=7.0.0" }, + { name = "ruff", specifier = ">=0.14.2" }, +] + +[[package]] +name = "haiku-rag-evals" +version = "0.13.3" +source = { editable = "evaluations" } +dependencies = [ + { name = "datasets" }, + { name = "haiku-rag-slim" }, + { name = "logfire" }, + { name = "pydantic-evals" }, + { name = "typer" }, +] + +[package.metadata] +requires-dist = [ + { name = "datasets", specifier = ">=4.3.0" }, + { name = "haiku-rag-slim", editable = "haiku_rag_slim" }, + { name = "logfire", specifier = ">=4.14.2" }, + { name = "pydantic-evals", specifier = ">=1.7.0" }, + { name = "typer", specifier = ">=0.19.2,<0.20.0" }, +] + +[[package]] +name = "haiku-rag-slim" +version = "0.13.3" +source = { editable = "haiku_rag_slim" } +dependencies = [ { name = "fastmcp" }, { name = "httpx" }, { name = "lancedb" }, @@ -1176,6 +1244,9 @@ a2a = [ cohere = [ { name = "cohere" }, ] +docling = [ + { name = "docling" }, +] mxbai = [ { name = "mxbai-rerank" }, ] @@ -1186,25 +1257,10 @@ zeroentropy = [ { name = "zeroentropy" }, ] -[package.dev-dependencies] -dev = [ - { name = "datasets" }, - { name = "logfire" }, - { name = "mkdocs" }, - { name = "mkdocs-material" }, - { name = "pre-commit" }, - { name = "pydantic-evals" }, - { name = "pyright" }, - { name = "pytest" }, - { name = "pytest-asyncio" }, - { name = "pytest-cov" }, - { name = "ruff" }, -] - [package.metadata] requires-dist = [ { name = "cohere", marker = "extra == 'cohere'", specifier = ">=5.0.0" }, - { name = "docling", specifier = ">=2.58.0" }, + { name = "docling", marker = "extra == 'docling'", specifier = ">=2.58.0" }, { name = "fasta2a", marker = "extra == 'a2a'", specifier = ">=0.1.0" }, { name = "fastmcp", specifier = ">=2.13.0.2" }, { name = "httpx", specifier = ">=0.28.1" }, @@ -1223,22 +1279,7 @@ requires-dist = [ { name = "watchfiles", specifier = ">=1.1.1" }, { name = "zeroentropy", marker = "extra == 'zeroentropy'", specifier = ">=0.1.0a6" }, ] -provides-extras = ["voyageai", "mxbai", "a2a", "cohere", "zeroentropy"] - -[package.metadata.requires-dev] -dev = [ - { name = "datasets", specifier = ">=4.3.0" }, - { name = "logfire", specifier = ">=4.14.2" }, - { name = "mkdocs", specifier = ">=1.6.1" }, - { name = "mkdocs-material", specifier = ">=9.6.22" }, - { name = "pre-commit", specifier = ">=4.3.0" }, - { name = "pydantic-evals", specifier = ">=1.7.0" }, - { name = "pyright", specifier = ">=1.1.407" }, - { name = "pytest", specifier = ">=8.4.2" }, - { name = "pytest-asyncio", specifier = ">=1.2.0" }, - { name = "pytest-cov", specifier = ">=7.0.0" }, - { name = "ruff", specifier = ">=0.14.2" }, -] +provides-extras = ["docling", "voyageai", "mxbai", "a2a", "cohere", "zeroentropy"] [[package]] name = "hf-xet" From b02514dc82b8ccf207a68f98e8819f5fb7dac1f0 Mon Sep 17 00:00:00 2001 From: Yiorgis Gozadinos Date: Tue, 4 Nov 2025 18:13:56 +0200 Subject: [PATCH 02/17] Make docling optional, depend on docling-core --- haiku_rag_slim/haiku/rag/chunker.py | 17 +++++++++++++++-- haiku_rag_slim/haiku/rag/reader.py | 17 +++++++++++++++-- haiku_rag_slim/haiku/rag/utils.py | 23 +++++++++++++++++------ haiku_rag_slim/pyproject.toml | 1 + uv.lock | 2 ++ 5 files changed, 50 insertions(+), 10 deletions(-) diff --git a/haiku_rag_slim/haiku/rag/chunker.py b/haiku_rag_slim/haiku/rag/chunker.py index f478d64c..fce09df5 100644 --- a/haiku_rag_slim/haiku/rag/chunker.py +++ b/haiku_rag_slim/haiku/rag/chunker.py @@ -1,12 +1,18 @@ from typing import ClassVar import tiktoken -from docling.chunking import HybridChunker # type: ignore from docling_core.transforms.chunker.tokenizer.openai import OpenAITokenizer from docling_core.types.doc.document import DoclingDocument - from haiku.rag.config import Config +# Check if docling is available +try: + import docling # noqa: F401 + + DOCLING_AVAILABLE = True +except ImportError: + DOCLING_AVAILABLE = False + class Chunker: """A class that chunks text into smaller pieces for embedding and retrieval. @@ -24,6 +30,13 @@ class Chunker: self, chunk_size: int = Config.processing.chunk_size, ): + if not DOCLING_AVAILABLE: + raise ImportError( + "Docling is required for chunking. " + "Install with: pip install haiku.rag-slim[docling]" + ) + from docling.chunking import HybridChunker # type: ignore + self.chunk_size = chunk_size tokenizer = OpenAITokenizer( tokenizer=tiktoken.encoding_for_model("gpt-4o"), max_tokens=chunk_size diff --git a/haiku_rag_slim/haiku/rag/reader.py b/haiku_rag_slim/haiku/rag/reader.py index 8e7fba2c..c710be49 100644 --- a/haiku_rag_slim/haiku/rag/reader.py +++ b/haiku_rag_slim/haiku/rag/reader.py @@ -1,11 +1,17 @@ from pathlib import Path from typing import ClassVar -from docling.document_converter import DocumentConverter from docling_core.types.doc.document import DoclingDocument - from haiku.rag.utils import text_to_docling_document +# Check if docling is available +try: + import docling # noqa: F401 + + DOCLING_AVAILABLE = True +except ImportError: + DOCLING_AVAILABLE = False + class FileReader: # Extensions supported by docling @@ -95,6 +101,13 @@ class FileReader: if file_extension in FileReader.docling_extensions: # Use docling for complex document formats + if not DOCLING_AVAILABLE: + raise ImportError( + "Docling is required for processing this file type. " + "Install with: pip install haiku.rag-slim[docling]" + ) + from docling.document_converter import DocumentConverter + converter = DocumentConverter() result = converter.convert(path) return result.document diff --git a/haiku_rag_slim/haiku/rag/utils.py b/haiku_rag_slim/haiku/rag/utils.py index 228aa7c2..950cfe87 100644 --- a/haiku_rag_slim/haiku/rag/utils.py +++ b/haiku_rag_slim/haiku/rag/utils.py @@ -103,9 +103,16 @@ def text_to_docling_document(text: str, name: str = "content.md"): Returns: A DoclingDocument created from the text content. """ - # Lazy import docling deps to keep import-time light - from docling.document_converter import DocumentConverter # type: ignore - from docling_core.types.io import DocumentStream # type: ignore + try: + import docling # noqa: F401 + except ImportError as e: + raise ImportError( + "Docling is required for document conversion. " + "Install with: pip install haiku.rag-slim[docling]" + ) from e + + from docling.document_converter import DocumentConverter + from docling_core.types.io import DocumentStream bytes_io = BytesIO(text.encode("utf-8")) doc_stream = DocumentStream(name=name, stream=bytes_io) @@ -168,11 +175,15 @@ def load_callable(path: str): def prefetch_models(): """Prefetch runtime models (Docling + Ollama as configured).""" import httpx - from docling.utils.model_downloader import download_models - from haiku.rag.config import Config - download_models() + try: + from docling.utils.model_downloader import download_models + + download_models() + except ImportError: + # Docling not installed, skip downloading docling models + pass # Collect Ollama models from config required_models: set[str] = set() diff --git a/haiku_rag_slim/pyproject.toml b/haiku_rag_slim/pyproject.toml index 6d5ff663..1e867157 100644 --- a/haiku_rag_slim/pyproject.toml +++ b/haiku_rag_slim/pyproject.toml @@ -22,6 +22,7 @@ classifiers = [ ] dependencies = [ + "docling-core>=2.48.4", "fastmcp>=2.13.0.2", "httpx>=0.28.1", "lancedb>=0.25.2", diff --git a/uv.lock b/uv.lock index 8a5ed5e6..549468d5 100644 --- a/uv.lock +++ b/uv.lock @@ -1222,6 +1222,7 @@ name = "haiku-rag-slim" version = "0.13.3" source = { editable = "haiku_rag_slim" } dependencies = [ + { name = "docling-core" }, { name = "fastmcp" }, { name = "httpx" }, { name = "lancedb" }, @@ -1261,6 +1262,7 @@ zeroentropy = [ requires-dist = [ { name = "cohere", marker = "extra == 'cohere'", specifier = ">=5.0.0" }, { name = "docling", marker = "extra == 'docling'", specifier = ">=2.58.0" }, + { name = "docling-core", specifier = ">=2.48.4" }, { name = "fasta2a", marker = "extra == 'a2a'", specifier = ">=0.1.0" }, { name = "fastmcp", specifier = ">=2.13.0.2" }, { name = "httpx", specifier = ">=0.28.1" }, From 44ad5bf10171021e4c09dbd27c1383216ccec4b6 Mon Sep 17 00:00:00 2001 From: Yiorgis Gozadinos Date: Tue, 4 Nov 2025 18:23:01 +0200 Subject: [PATCH 03/17] Add evals as a dev dependency --- pyproject.toml | 1 + uv.lock | 2 ++ 2 files changed, 3 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index fb74e604..d014deb4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,6 +47,7 @@ members = ["haiku_rag_slim", "evaluations"] [dependency-groups] dev = [ + "haiku.rag-evals", "datasets>=4.3.0", "logfire>=4.14.2", "mkdocs>=1.6.1", diff --git a/uv.lock b/uv.lock index 549468d5..8e6a1187 100644 --- a/uv.lock +++ b/uv.lock @@ -1166,6 +1166,7 @@ dependencies = [ [package.dev-dependencies] dev = [ { name = "datasets" }, + { name = "haiku-rag-evals" }, { name = "logfire" }, { name = "mkdocs" }, { name = "mkdocs-material" }, @@ -1184,6 +1185,7 @@ requires-dist = [{ name = "haiku-rag-slim", extras = ["docling", "voyageai", "mx [package.metadata.requires-dev] dev = [ { name = "datasets", specifier = ">=4.3.0" }, + { name = "haiku-rag-evals", editable = "evaluations" }, { name = "logfire", specifier = ">=4.14.2" }, { name = "mkdocs", specifier = ">=1.6.1" }, { name = "mkdocs-material", specifier = ">=9.6.22" }, From ce79f8e0371cab738bcad16e6b328566e46d1f01 Mon Sep 17 00:00:00 2001 From: Yiorgis Gozadinos Date: Tue, 4 Nov 2025 18:37:01 +0200 Subject: [PATCH 04/17] Update github actions --- .github/workflows/build-publish-slim.yml | 18 ++++++++++++++++++ .github/workflows/build-publish.yml | 20 ++++++++++++-------- .github/workflows/docker-publish.yml | 4 ++-- .github/workflows/publish-mcp.yml | 4 ++-- 4 files changed, 34 insertions(+), 12 deletions(-) create mode 100644 .github/workflows/build-publish-slim.yml diff --git a/.github/workflows/build-publish-slim.yml b/.github/workflows/build-publish-slim.yml new file mode 100644 index 00000000..67d4bb0d --- /dev/null +++ b/.github/workflows/build-publish-slim.yml @@ -0,0 +1,18 @@ +name: Build & publish haiku.rag-slim to pypi +on: + release: + types: [published] + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Set up uv + run: curl -LsSf https://astral.sh/uv/0.3.0/install.sh | sh + - name: Set up Python 3.12 + run: uv python install 3.12 + - name: Build haiku.rag-slim + run: uv build --package haiku.rag-slim + - name: Publish haiku.rag-slim + run: uvx twine upload -u __token__ -p ${{ secrets.PYPI_API_TOKEN }} dist/* --non-interactive diff --git a/.github/workflows/build-publish.yml b/.github/workflows/build-publish.yml index 000f8208..805afecd 100644 --- a/.github/workflows/build-publish.yml +++ b/.github/workflows/build-publish.yml @@ -1,18 +1,22 @@ -name: Build & publish to pypi +name: Build & publish haiku.rag to pypi on: - release: - types: [published] + workflow_run: + workflows: ["Build & publish haiku.rag-slim to pypi"] + types: + - completed + workflow_dispatch: jobs: build: runs-on: ubuntu-latest + if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }} steps: - uses: actions/checkout@v4 - name: Set up uv run: curl -LsSf https://astral.sh/uv/0.3.0/install.sh | sh - - name: Set up Python 3.10 - run: uv python install 3.10 - - name: Build package - run: uvx --from build pyproject-build --installer uv - - name: Publish package + - name: Set up Python 3.12 + run: uv python install 3.12 + - name: Build haiku.rag + run: uv build + - name: Publish haiku.rag run: uvx twine upload -u __token__ -p ${{ secrets.PYPI_API_TOKEN }} dist/* --non-interactive diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 525b1f1f..4568dcc3 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -1,7 +1,7 @@ name: Build & publish Docker image on: workflow_run: - workflows: ["Build & publish to pypi"] + workflows: ["Build & publish haiku.rag to pypi"] types: - completed workflow_dispatch: @@ -40,7 +40,7 @@ jobs: - name: Extract version from pyproject.toml id: version run: | - VERSION=$(grep -oP '^version = "\K[^"]+' pyproject.toml) + VERSION=$(grep -oP '^version = "\K[^"]+' haiku_rag_slim/pyproject.toml) echo "version=$VERSION" >> $GITHUB_OUTPUT - name: Build and push Docker image diff --git a/.github/workflows/publish-mcp.yml b/.github/workflows/publish-mcp.yml index da9466ab..888251be 100644 --- a/.github/workflows/publish-mcp.yml +++ b/.github/workflows/publish-mcp.yml @@ -2,7 +2,7 @@ name: Publish to MCP Registry on: workflow_dispatch: workflow_run: - workflows: ["Build & publish to pypi"] + workflows: ["Build & publish haiku.rag to pypi"] types: - completed @@ -21,7 +21,7 @@ jobs: python-version: '3.12' - name: Generate server.json from template run: | - VERSION=$(python -c "import tomllib; print(tomllib.load(open('pyproject.toml', 'rb'))['project']['version'])") + VERSION=$(python -c "import tomllib; print(tomllib.load(open('haiku_rag_slim/pyproject.toml', 'rb'))['project']['version'])") sed "s/{{VERSION}}/$VERSION/g" server.json > server.json.tmp mv server.json.tmp server.json echo "Generated server.json with version: $VERSION" From 5ed8c9cf440f134f34e29729d7f396dcf6ff8b05 Mon Sep 17 00:00:00 2001 From: Yiorgis Gozadinos Date: Tue, 4 Nov 2025 18:43:41 +0200 Subject: [PATCH 05/17] Update docker build --- .dockerignore | 2 +- docker/Dockerfile | 8 ++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/.dockerignore b/.dockerignore index 9def18fa..e536f720 100644 --- a/.dockerignore +++ b/.dockerignore @@ -60,7 +60,7 @@ tests/ .pytest_cache/ .coverage htmlcov/ -evaluations/ +evaluations/evaluations/data/ server.json # Examples examples/ diff --git a/docker/Dockerfile b/docker/Dockerfile index 7c689b6d..71216b42 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -8,14 +8,18 @@ ENV UV_COMPILE_BYTECODE=1 \ UV_LINK_MODE=copy # Install dependencies into a venv +# In workspace: root package (haiku.rag) includes all extras via haiku.rag-slim +# Copy workspace member directories first (needed for workspace resolution) COPY pyproject.toml uv.lock ./ +COPY haiku_rag_slim/pyproject.toml haiku_rag_slim/README.md haiku_rag_slim/LICENSE haiku_rag_slim/ +COPY evaluations/pyproject.toml evaluations/ RUN --mount=type=cache,target=/root/.cache/uv \ - uv sync --frozen --no-install-project --extra voyageai --extra mxbai --extra a2a + uv sync --frozen --no-install-project # Install the project itself COPY . . RUN --mount=type=cache,target=/root/.cache/uv \ - uv sync --frozen --no-editable --extra voyageai --extra mxbai --extra a2a + uv sync --frozen --no-editable # Final layer FROM python:3.13-slim From 700e7c94e05e3ee81a4493f2a9c81b06b18d960e Mon Sep 17 00:00:00 2001 From: Yiorgis Gozadinos Date: Tue, 4 Nov 2025 18:49:57 +0200 Subject: [PATCH 06/17] Update docs --- README.md | 24 +++++++++++--- docs/configuration.md | 16 +++++----- docs/installation.md | 74 ++++++++++++++++++++++--------------------- 3 files changed, 66 insertions(+), 48 deletions(-) diff --git a/README.md b/README.md index 949bae79..c7b5f823 100644 --- a/README.md +++ b/README.md @@ -23,13 +23,29 @@ Retrieval-Augmented Generation (RAG) library built on LanceDB. - **A2A agent**: Conversational agent with context and multi-turn dialogue - **CLI & Python API**: Use from command line or Python +## Installation + +**Python 3.12 or newer required** + +### Full Package (Recommended) + +```bash +uv pip install haiku.rag +``` + +Includes all features: document processing, all embedding providers, rerankers, and A2A agent support. + +### Slim Package (Minimal Dependencies) + +```bash +uv pip install haiku.rag-slim +``` + +Install only the extras you need. See the [Installation](https://ggozad.github.io/haiku.rag/installation/) documentation for available options + ## Quick Start ```bash -# Install -# Python 3.12 or newer required -uv pip install haiku.rag - # Add documents haiku-rag add "Your content here" haiku-rag add "Your content here" --meta author=alice --meta topic=notes diff --git a/docs/configuration.md b/docs/configuration.md index d67e5bdf..8cb6c986 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -238,10 +238,10 @@ If neither is set, it defaults to `http://localhost:11434`. ### VoyageAI -If you want to use VoyageAI embeddings you will need to install `haiku.rag` with the VoyageAI extras: +If you installed `haiku.rag` (full package), VoyageAI is already included. If you installed `haiku.rag-slim`, install with VoyageAI extras: ```bash -uv pip install haiku.rag[voyageai] +uv pip install haiku.rag-slim[voyageai] ``` ```yaml @@ -396,10 +396,10 @@ Reranking is **disabled by default** (`provider: ""`) for faster searches. You c ### MixedBread AI -For MxBAI reranking, install with mxbai extras: +If you installed `haiku.rag` (full package), MxBAI is already included. If you installed `haiku.rag-slim`, add the mxbai extra: ```bash -uv pip install haiku.rag[mxbai] +uv pip install haiku.rag-slim[mxbai] ``` Then configure: @@ -412,10 +412,10 @@ reranking: ### Cohere -Install with cohere extras: +If you installed `haiku.rag` (full package), Cohere is already included. If you installed `haiku.rag-slim`, add the cohere extra: ```bash -uv pip install haiku.rag[cohere] +uv pip install haiku.rag-slim[cohere] ``` Then configure: @@ -434,10 +434,10 @@ export CO_API_KEY=your-api-key ### Zero Entropy -Install with zeroentropy extras: +If you installed `haiku.rag` (full package), Zero Entropy is already included. If you installed `haiku.rag-slim`, add the zeroentropy extra: ```bash -uv pip install haiku.rag[zeroentropy] +uv pip install haiku.rag-slim[zeroentropy] ``` Then configure: diff --git a/docs/installation.md b/docs/installation.md index 36cf89d2..d064e485 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -1,48 +1,50 @@ # Installation -## Basic Installation +## Choose Your Package + +**haiku.rag** is available in two packages: + +### Full Package (Recommended) ```bash uv pip install haiku.rag ``` -This includes support for: -- **Ollama** (default embedding provider using `mxbai-embed-large`) +The full package includes **all features and extras**: +- **Document processing** (Docling) - PDF, DOCX, PPTX, images, and 40+ file formats +- **All embedding providers** - Ollama, OpenAI, VoyageAI, Anthropic, vLLM +- **All rerankers** - MixedBread AI, Cohere, Zero Entropy, vLLM +- **A2A agent** - Agent-to-Agent protocol support + +This is the easiest way to get started with all features enabled. + +### Slim Package (Minimal Dependencies) + +```bash +# Minimal installation (no document processing) +uv pip install haiku.rag-slim + +# With document processing +uv pip install haiku.rag-slim[docling] + +# With specific providers +uv pip install haiku.rag-slim[docling,voyageai,mxbai] +``` + +The slim package has minimal dependencies and lets you install only what you need: + +- `docling` - PDF, DOCX, PPTX, images, and other document formats +- `voyageai` - VoyageAI embeddings +- `mxbai` - MixedBread AI reranking +- `a2a` - Agent-to-Agent protocol support +- `cohere` - Cohere reranking +- `zeroentropy` - Zero Entropy reranking + +**Built-in providers** (no extras needed): +- **Ollama** (default embedding provider) - **OpenAI** (GPT models for QA and embeddings) - **Anthropic** (Claude models for QA) -- **vLLM** (high-performance local inference for embeddings, QA, and reranking) - -## Provider-Specific Installation - -For additional providers, install with extras: - -### Embedding Providers - -#### VoyageAI - -```bash -uv pip install haiku.rag[voyageai] -``` - -### Reranking Providers - -#### MixedBread AI - -```bash -uv pip install haiku.rag[mxbai] -``` - -#### Cohere - -```bash -uv pip install haiku.rag[cohere] -``` - -#### Zero Entropy - -```bash -uv pip install haiku.rag[zeroentropy] -``` +- **vLLM** (high-performance local inference) ### vLLM Setup From 5cd951a5f3472d6af8e9c34a47b38cf27972e6fe Mon Sep 17 00:00:00 2001 From: Yiorgis Gozadinos Date: Wed, 5 Nov 2025 10:11:17 +0200 Subject: [PATCH 07/17] When checking for updates, use haiku.rag-slim as the package, since it is always installed --- haiku_rag_slim/haiku/rag/app.py | 12 +++++------- haiku_rag_slim/haiku/rag/cli.py | 3 +-- haiku_rag_slim/haiku/rag/store/engine.py | 12 +++++------- haiku_rag_slim/haiku/rag/utils.py | 2 +- 4 files changed, 12 insertions(+), 17 deletions(-) diff --git a/haiku_rag_slim/haiku/rag/app.py b/haiku_rag_slim/haiku/rag/app.py index 8e95ae7c..5b6dbcc0 100644 --- a/haiku_rag_slim/haiku/rag/app.py +++ b/haiku_rag_slim/haiku/rag/app.py @@ -4,10 +4,6 @@ import logging from importlib.metadata import version as pkg_version from pathlib import Path -from rich.console import Console -from rich.markdown import Markdown -from rich.progress import Progress - from haiku.rag.client import HaikuRAG from haiku.rag.config import Config from haiku.rag.mcp import create_mcp_server @@ -22,6 +18,9 @@ from haiku.rag.research.graph import ( from haiku.rag.research.stream import stream_research_graph from haiku.rag.store.models.chunk import Chunk from haiku.rag.store.models.document import Document +from rich.console import Console +from rich.markdown import Markdown +from rich.progress import Progress logger = logging.getLogger(__name__) @@ -59,7 +58,7 @@ class HaikuRAGApp: except Exception: ldb_version = "unknown" try: - hr_version = pkg_version("haiku.rag") + hr_version = pkg_version("haiku.rag-slim") except Exception: hr_version = "unknown" try: @@ -211,12 +210,11 @@ class HaikuRAGApp: async with HaikuRAG(db_path=self.db_path) as self.client: try: if deep: - from rich.console import Console - from haiku.rag.qa.deep.dependencies import DeepQAContext from haiku.rag.qa.deep.graph import build_deep_qa_graph from haiku.rag.qa.deep.nodes import DeepQAPlanNode from haiku.rag.qa.deep.state import DeepQADeps, DeepQAState + from rich.console import Console graph = build_deep_qa_graph() context = DeepQAContext( diff --git a/haiku_rag_slim/haiku/rag/cli.py b/haiku_rag_slim/haiku/rag/cli.py index 1d654717..004c0292 100644 --- a/haiku_rag_slim/haiku/rag/cli.py +++ b/haiku_rag_slim/haiku/rag/cli.py @@ -36,7 +36,7 @@ async def check_version(): def version_callback(value: bool): if value: - v = version("haiku.rag") + v = version("haiku.rag-slim") typer.echo(f"haiku.rag version {v}") raise typer.Exit() @@ -353,7 +353,6 @@ def init_config( ): """Generate a YAML configuration file with defaults or from .env.""" import yaml - from haiku.rag.config.loader import generate_default_config, load_config_from_env if output.exists(): diff --git a/haiku_rag_slim/haiku/rag/store/engine.py b/haiku_rag_slim/haiku/rag/store/engine.py index 6cbe328e..ab3ac71f 100644 --- a/haiku_rag_slim/haiku/rag/store/engine.py +++ b/haiku_rag_slim/haiku/rag/store/engine.py @@ -7,11 +7,10 @@ from pathlib import Path from uuid import uuid4 import lancedb -from lancedb.pydantic import LanceModel, Vector -from pydantic import Field - from haiku.rag.config import AppConfig, Config from haiku.rag.embeddings import get_embedder +from lancedb.pydantic import LanceModel, Vector +from pydantic import Field logger = logging.getLogger(__name__) @@ -189,7 +188,7 @@ class Store: try: from haiku.rag.store.upgrades import run_pending_upgrades - current_version = metadata.version("haiku.rag") + current_version = metadata.version("haiku.rag-slim") db_version = self.get_haiku_version() if db_version != "0.0.0": @@ -199,9 +198,8 @@ class Store: # to the greater of the installed package version and the # highest available upgrade step version in code. try: - from packaging.version import parse as _v - from haiku.rag.store.upgrades import upgrades as _steps + from packaging.version import parse as _v highest_step = max((_v(u.version) for u in _steps), default=None) effective_version = ( @@ -218,7 +216,7 @@ class Store: logger.warning( "Skipping upgrade due to error (db=%s -> pkg=%s): %s", self.get_haiku_version(), - metadata.version("haiku.rag") if hasattr(metadata, "version") else "", + metadata.version("haiku.rag-slim"), e, ) diff --git a/haiku_rag_slim/haiku/rag/utils.py b/haiku_rag_slim/haiku/rag/utils.py index 950cfe87..70667a45 100644 --- a/haiku_rag_slim/haiku/rag/utils.py +++ b/haiku_rag_slim/haiku/rag/utils.py @@ -82,7 +82,7 @@ async def is_up_to_date() -> tuple[bool, Version, Version]: import httpx async with httpx.AsyncClient() as client: - running_version = parse(metadata.version("haiku.rag")) + running_version = parse(metadata.version("haiku.rag-slim")) try: response = await client.get("https://pypi.org/pypi/haiku.rag/json") data = response.json() From be2d8ea8f46735a17c166cef3625e88d470ea322 Mon Sep 17 00:00:00 2001 From: Yiorgis Gozadinos Date: Wed, 5 Nov 2025 10:13:58 +0200 Subject: [PATCH 08/17] Catch import errors seperately to raise errors about missing dependencies. --- haiku_rag_slim/haiku/rag/reader.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/haiku_rag_slim/haiku/rag/reader.py b/haiku_rag_slim/haiku/rag/reader.py index c710be49..d6103b5f 100644 --- a/haiku_rag_slim/haiku/rag/reader.py +++ b/haiku_rag_slim/haiku/rag/reader.py @@ -126,5 +126,7 @@ class FileReader: # Fallback: try to read as text and convert to DoclingDocument content = path.read_text(encoding="utf-8") return text_to_docling_document(content, name=f"{path.stem}.md") + except ImportError: + raise except Exception: raise ValueError(f"Failed to parse file: {path}") From f06dac71026adca55fd9a0cee248c96634d8ea79 Mon Sep 17 00:00:00 2001 From: Yiorgis Gozadinos Date: Wed, 5 Nov 2025 10:47:00 +0200 Subject: [PATCH 09/17] Update READMEs --- README.md | 4 +- evaluations/README.md | 2 - haiku_rag_slim/README.md | 217 +++++++-------------------------------- 3 files changed, 38 insertions(+), 185 deletions(-) diff --git a/README.md b/README.md index c7b5f823..6c285fb6 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,5 @@ # Haiku RAG -mcp-name: io.github.ggozad/haiku-rag - Retrieval-Augmented Generation (RAG) library built on LanceDB. `haiku.rag` is a Retrieval-Augmented Generation (RAG) library built to work with LanceDB as a local vector database. It uses LanceDB for storing embeddings and performs semantic (vector) search as well as full-text search combined through native hybrid search with Reciprocal Rank Fusion. Both open-source (Ollama) as well as commercial (OpenAI, VoyageAI) embedding providers are supported. @@ -203,3 +201,5 @@ Full documentation at: https://ggozad.github.io/haiku.rag/ - [MCP Server](https://ggozad.github.io/haiku.rag/mcp/) - Model Context Protocol integration - [A2A Agent](https://ggozad.github.io/haiku.rag/a2a/) - Agent-to-Agent protocol support - [Benchmarks](https://ggozad.github.io/haiku.rag/benchmarks/) - Performance Benchmarks + +mcp-name: io.github.ggozad/haiku-rag diff --git a/evaluations/README.md b/evaluations/README.md index 2e758dd1..9cfde36f 100644 --- a/evaluations/README.md +++ b/evaluations/README.md @@ -9,5 +9,3 @@ This package is not published to PyPI and is only used for development and testi Contains evaluation scripts for benchmarking RAG performance using datasets like: - RepliQA - WiX - -Uses `pydantic-evals` for evaluation framework and `logfire` for observability. diff --git a/haiku_rag_slim/README.md b/haiku_rag_slim/README.md index 5f5c9a8e..02cebe8d 100644 --- a/haiku_rag_slim/README.md +++ b/haiku_rag_slim/README.md @@ -1,204 +1,59 @@ -# Haiku RAG - Slim +# haiku.rag-slim -mcp-name: io.github.ggozad/haiku-rag +Retrieval-Augmented Generation (RAG) library built on LanceDB - Core package with minimal dependencies. -Retrieval-Augmented Generation (RAG) library built on LanceDB - Minimal dependencies. +`haiku.rag-slim` is the core package for users who want to install only the dependencies they need. Document processing (docling), rerankers, and A2A support are all optional extras. -`haiku.rag-slim` is the core package with minimal dependencies. Document processing via docling is optional and can be installed separately. - -For a batteries-included experience with all extras, see [`haiku.rag`](https://pypi.org/project/haiku.rag/). +**For most users, we recommend installing [`haiku.rag`](https://pypi.org/project/haiku.rag/) instead**, which includes all features out of the box. ## Installation +**Python 3.12 or newer required** + +### Minimal Installation + ```bash -# Minimal installation (no docling) uv pip install haiku.rag-slim +``` -# With docling support for document processing +Basic functionality without document processing (docling). You can still use text input and URLs. + +### With Document Processing + +```bash uv pip install haiku.rag-slim[docling] +``` -# With specific extras +Adds support for 40+ file formats including PDF, DOCX, HTML, and more. + +### Available Extras + +- `docling` - Document processing for PDFs, DOCX, HTML, etc. +- `voyageai` - VoyageAI embedding provider +- `mxbai` - MixedBread AI reranker +- `cohere` - Cohere reranker +- `zeroentropy` - Zero Entropy reranker +- `a2a` - Agent-to-Agent protocol support + +```bash +# Multiple extras uv pip install haiku.rag-slim[docling,voyageai,mxbai] ``` -> **Note**: Configuration now uses YAML files instead of environment variables. If you're upgrading from an older version, run `haiku-rag init-config --from-env` to migrate your `.env` file to `haiku.rag.yaml`. See [Configuration](https://ggozad.github.io/haiku.rag/configuration/) for details. +## Usage -## Features - -- **Local LanceDB**: No external servers required, supports also LanceDB cloud storage, S3, Google Cloud & Azure -- **Multiple embedding providers**: Ollama, VoyageAI, OpenAI, vLLM -- **Multiple QA providers**: Any provider/model supported by Pydantic AI -- **Research graph (multi‑agent)**: Plan → Search → Evaluate → Synthesize with agentic AI -- **Native hybrid search**: Vector + full-text search with native LanceDB RRF reranking -- **Reranking**: Default search result reranking with MixedBread AI, Cohere, Zero Entropy, or vLLM -- **Question answering**: Built-in QA agents on your documents -- **File monitoring**: Auto-index files when run as server -- **40+ file formats**: PDF, DOCX, HTML, Markdown, code files, URLs -- **MCP server**: Expose as tools for AI assistants -- **A2A agent**: Conversational agent with context and multi-turn dialogue -- **CLI & Python API**: Use from command line or Python - -## Quick Start - -```bash -# Install -# Python 3.12 or newer required -uv pip install haiku.rag-slim[docling] - -# Add documents -haiku-rag add "Your content here" -haiku-rag add "Your content here" --meta author=alice --meta topic=notes -haiku-rag add-src document.pdf --meta source=manual - -# Search -haiku-rag search "query" - -# Search with filters -haiku-rag search "query" --filter "uri LIKE '%.pdf' AND title LIKE '%paper%'" - -# Ask questions -haiku-rag ask "Who is the author of haiku.rag?" - -# Ask questions with citations -haiku-rag ask "Who is the author of haiku.rag?" --cite - -# Deep QA (multi-agent question decomposition) -haiku-rag ask "Who is the author of haiku.rag?" --deep --cite - -# Deep QA with verbose output -haiku-rag ask "Who is the author of haiku.rag?" --deep --verbose - -# Multi‑agent research (iterative plan/search/evaluate) -haiku-rag research \ - "What are the main drivers and trends of global temperature anomalies since 1990?" \ - --max-iterations 2 \ - --confidence-threshold 0.8 \ - --max-concurrency 3 \ - --verbose - -# Rebuild database (re-chunk and re-embed all documents) -haiku-rag rebuild - -# Start server with file monitoring -haiku-rag serve --monitor -``` - -To customize settings, create a `haiku.rag.yaml` config file (see [Configuration](https://ggozad.github.io/haiku.rag/configuration/)). - -## Python Usage - -```python -from haiku.rag.client import HaikuRAG -from haiku.rag.research import ( - PlanNode, - ResearchContext, - ResearchDeps, - ResearchState, - build_research_graph, - stream_research_graph, -) - -async with HaikuRAG("database.lancedb") as client: - # Add document - doc = await client.create_document("Your content") - - # Search (reranking enabled by default) - results = await client.search("query") - for chunk, score in results: - print(f"{score:.3f}: {chunk.content}") - - # Ask questions - answer = await client.ask("Who is the author of haiku.rag?") - print(answer) - - # Ask questions with citations - answer = await client.ask("Who is the author of haiku.rag?", cite=True) - print(answer) - - # Multi‑agent research pipeline (Plan → Search → Evaluate → Synthesize) - graph = build_research_graph() - question = ( - "What are the main drivers and trends of global temperature " - "anomalies since 1990?" - ) - state = ResearchState( - context=ResearchContext(original_question=question), - max_iterations=2, - confidence_threshold=0.8, - max_concurrency=2, - ) - deps = ResearchDeps(client=client) - - # Blocking run (final result only) - result = await graph.run( - PlanNode(provider="openai", model="gpt-4o-mini"), - state=state, - deps=deps, - ) - print(result.output.title) - - # Streaming progress (log/report/error events) - async for event in stream_research_graph( - graph, - PlanNode(provider="openai", model="gpt-4o-mini"), - state, - deps, - ): - if event.type == "log": - iteration = event.state.iterations if event.state else state.iterations - print(f"[{iteration}] {event.message}") - elif event.type == "report": - print("\nResearch complete!\n") - print(event.report.title) - print(event.report.executive_summary) -``` - -## MCP Server - -Use with AI assistants like Claude Desktop: - -```bash -haiku-rag serve --stdio -``` - -Provides tools for document management and search directly in your AI assistant. - -## A2A Agent - -Run as a conversational agent with the Agent-to-Agent protocol: - -```bash -# Start the A2A server -haiku-rag serve --a2a - -# Connect with the interactive client (in another terminal) -haiku-rag a2aclient -``` - -The A2A agent provides: - -- Multi-turn dialogue with context -- Intelligent multi-search for complex questions -- Source citations with titles and URIs -- Full document retrieval on request - -## Examples - -See the [examples directory](examples/) for working examples: - -- **[Interactive Research Assistant](examples/ag-ui-research/)** - Full-stack research assistant with Pydantic AI and AG-UI featuring human-in-the-loop approval and real-time state synchronization -- **[Docker Setup](examples/docker/)** - Complete Docker deployment with file monitoring, MCP server, and A2A agent -- **[A2A Security](examples/a2a-security/)** - Authentication examples (API key, OAuth2, GitHub) +See the main [`haiku.rag`](https://github.com/ggozad/haiku.rag) repository for: +- Quick start guide +- CLI examples +- Python API usage +- MCP server setup +- A2A agent configuration ## Documentation -Full documentation at: https://ggozad.github.io/haiku.rag/ +Full documentation: https://ggozad.github.io/haiku.rag/ - [Installation](https://ggozad.github.io/haiku.rag/installation/) - Provider setup - [Configuration](https://ggozad.github.io/haiku.rag/configuration/) - YAML configuration - [CLI](https://ggozad.github.io/haiku.rag/cli/) - Command reference - [Python API](https://ggozad.github.io/haiku.rag/python/) - Complete API docs -- [Agents](https://ggozad.github.io/haiku.rag/agents/) - QA agent and multi-agent research -- [MCP Server](https://ggozad.github.io/haiku.rag/mcp/) - Model Context Protocol integration -- [A2A Agent](https://ggozad.github.io/haiku.rag/a2a/) - Agent-to-Agent protocol support -- [Benchmarks](https://ggozad.github.io/haiku.rag/benchmarks/) - Performance Benchmarks From ba8e503cb14e3dff98fc562e0711a70249052310 Mon Sep 17 00:00:00 2001 From: Yiorgis Gozadinos Date: Wed, 5 Nov 2025 11:35:55 +0200 Subject: [PATCH 10/17] Depend on pydantic-ai-slim with extras --- evaluations/pyproject.toml | 3 +- haiku_rag_slim/README.md | 35 ++++-- haiku_rag_slim/pyproject.toml | 20 +++- pyproject.toml | 2 - uv.lock | 213 +++++++++------------------------- 5 files changed, 97 insertions(+), 176 deletions(-) diff --git a/evaluations/pyproject.toml b/evaluations/pyproject.toml index 7b6f8922..6f56d9ce 100644 --- a/evaluations/pyproject.toml +++ b/evaluations/pyproject.toml @@ -9,9 +9,8 @@ requires-python = ">=3.12" dependencies = [ "haiku.rag-slim", - "pydantic-evals>=1.7.0", + "pydantic-ai-slim[evals,logfire]>=1.11.0", "datasets>=4.3.0", - "logfire>=4.14.2", "typer>=0.19.2,<0.20.0", ] diff --git a/haiku_rag_slim/README.md b/haiku_rag_slim/README.md index 02cebe8d..8e760b5c 100644 --- a/haiku_rag_slim/README.md +++ b/haiku_rag_slim/README.md @@ -16,7 +16,7 @@ Retrieval-Augmented Generation (RAG) library built on LanceDB - Core package wit uv pip install haiku.rag-slim ``` -Basic functionality without document processing (docling). You can still use text input and URLs. +Core functionality with OpenAI/Ollama support, MCP server, and Logfire observability. Document processing (docling) is optional. ### With Document Processing @@ -28,16 +28,33 @@ Adds support for 40+ file formats including PDF, DOCX, HTML, and more. ### Available Extras -- `docling` - Document processing for PDFs, DOCX, HTML, etc. -- `voyageai` - VoyageAI embedding provider -- `mxbai` - MixedBread AI reranker -- `cohere` - Cohere reranker -- `zeroentropy` - Zero Entropy reranker -- `a2a` - Agent-to-Agent protocol support +**Document Processing:** +- `docling` - PDF, DOCX, HTML, and 40+ file formats + +**Embedding Providers:** +- `voyageai` - VoyageAI embeddings + +**Rerankers:** +- `mxbai` - MixedBread AI +- `cohere` - Cohere +- `zeroentropy` - Zero Entropy + +**Model Providers:** +- OpenAI/Ollama - included in core (OpenAI-compatible APIs) +- `anthropic` - Anthropic Claude +- `groq` - Groq +- `google` - Google Gemini +- `mistral` - Mistral AI +- `bedrock` - AWS Bedrock +- `vertexai` - Google Vertex AI + +**Agent Protocol:** +- `a2a` - Agent-to-Agent protocol ```bash -# Multiple extras -uv pip install haiku.rag-slim[docling,voyageai,mxbai] +# Common combinations +uv pip install haiku.rag-slim[docling,anthropic,mxbai] +uv pip install haiku.rag-slim[docling,groq,logfire] ``` ## Usage diff --git a/haiku_rag_slim/pyproject.toml b/haiku_rag_slim/pyproject.toml index 1e867157..e3de412b 100644 --- a/haiku_rag_slim/pyproject.toml +++ b/haiku_rag_slim/pyproject.toml @@ -23,13 +23,11 @@ classifiers = [ dependencies = [ "docling-core>=2.48.4", - "fastmcp>=2.13.0.2", "httpx>=0.28.1", "lancedb>=0.25.2", "pathspec>=0.12.1", "pydantic>=2.12.3", - "pydantic-ai>=1.7.0", - "pydantic-graph>=1.7.0", + "pydantic-ai-slim[openai,fastmcp,logfire]>=1.11.0", "python-dotenv>=1.2.1", "pyyaml>=6.0.3", "rich>=14.2.0", @@ -39,12 +37,26 @@ dependencies = [ ] [project.optional-dependencies] +# Document processing docling = ["docling>=2.58.0"] +# Embedding providers voyageai = ["voyageai>=0.3.5"] +# Rerankers mxbai = ["mxbai-rerank>=0.1.6"] -a2a = ["fasta2a>=0.1.0"] cohere = ["cohere>=5.0.0"] zeroentropy = ["zeroentropy>=0.1.0a6"] +# Agent protocols +a2a = [ + "fasta2a>=0.1.0", + "pydantic-ai-slim[a2a]", +] +# Model providers (delegated to pydantic-ai-slim) +anthropic = ["pydantic-ai-slim[anthropic]"] +groq = ["pydantic-ai-slim[groq]"] +google = ["pydantic-ai-slim[google]"] +mistral = ["pydantic-ai-slim[mistral]"] +bedrock = ["pydantic-ai-slim[bedrock]"] +vertexai = ["pydantic-ai-slim[vertexai]"] [project.scripts] haiku-rag = "haiku.rag.cli:cli" diff --git a/pyproject.toml b/pyproject.toml index d014deb4..c6d9d88a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -49,10 +49,8 @@ members = ["haiku_rag_slim", "evaluations"] dev = [ "haiku.rag-evals", "datasets>=4.3.0", - "logfire>=4.14.2", "mkdocs>=1.6.1", "mkdocs-material>=9.6.22", - "pydantic-evals>=1.7.0", "pre-commit>=4.3.0", "pyright>=1.1.407", "pytest>=8.4.2", diff --git a/uv.lock b/uv.lock index 8e6a1187..74226e99 100644 --- a/uv.lock +++ b/uv.lock @@ -38,18 +38,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/5f/a0/d9ef19f780f319c21ee90ecfef4431cbeeca95bec7f14071785c17b6029b/accelerate-1.10.1-py3-none-any.whl", hash = "sha256:3621cff60b9a27ce798857ece05e2b9f56fcc71631cfb31ccf71f0359c311f11", size = 374909, upload-time = "2025-08-25T13:57:04.55Z" }, ] -[[package]] -name = "ag-ui-protocol" -version = "0.1.8" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pydantic" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/27/de/0bddf7f26d5f38274c99401735c82ad59df9cead6de42f4bb2ad837286fe/ag_ui_protocol-0.1.8.tar.gz", hash = "sha256:eb745855e9fc30964c77e953890092f8bd7d4bbe6550d6413845428dd0faac0b", size = 5323, upload-time = "2025-07-15T10:55:36.389Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c4/00/40c6b0313c25d1ab6fac2ecba1cd5b15b1cd3c3a71b3d267ad890e405889/ag_ui_protocol-0.1.8-py3-none-any.whl", hash = "sha256:1567ccb067b7b8158035b941a985e7bb185172d660d4542f3f9c6fff77b55c6e", size = 7066, upload-time = "2025-07-15T10:55:35.075Z" }, -] - [[package]] name = "aiohappyeyeballs" version = "2.6.1" @@ -180,15 +168,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/6f/12/e5e0282d673bb9746bacfb6e2dba8719989d3660cdb2ea79aee9a9651afb/anyio-4.10.0-py3-none-any.whl", hash = "sha256:60e474ac86736bbfd6f210f7a61218939c318f43f9972497381f1c5e930ed3d1", size = 107213, upload-time = "2025-08-04T08:54:24.882Z" }, ] -[[package]] -name = "argcomplete" -version = "3.6.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/16/0f/861e168fc813c56a78b35f3c30d91c6757d1fd185af1110f1aec784b35d0/argcomplete-3.6.2.tar.gz", hash = "sha256:d0519b1bc867f5f4f4713c41ad0aba73a4a5f007449716b16f385f2166dc6adf", size = 73403, upload-time = "2025-04-03T04:57:03.52Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/31/da/e42d7a9d8dd33fa775f467e4028a47936da2f01e4b0e561f9ba0d74cb0ca/argcomplete-3.6.2-py3-none-any.whl", hash = "sha256:65b3133a29ad53fb42c48cf5114752c7ab66c1c38544fdf6460f450c09b42591", size = 43708, upload-time = "2025-04-03T04:57:01.591Z" }, -] - [[package]] name = "attrs" version = "25.3.0" @@ -266,30 +245,30 @@ wheels = [ [[package]] name = "boto3" -version = "1.40.32" +version = "1.40.66" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "botocore" }, { name = "jmespath" }, { name = "s3transfer" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/c7/c7/39b10ce9e79fb40327c8e96074223cfca01b5b9165827c8a9b2e4c7a8935/boto3-1.40.32.tar.gz", hash = "sha256:6951aac75ce25611df55a31c844b118c288460648535ce9125fb15b490387ba4", size = 111562, upload-time = "2025-09-16T19:30:22.293Z" } +sdist = { url = "https://files.pythonhosted.org/packages/9a/95/db1f23bbc5cf1a9b66cb1828a0940305ea300162ae12c55522c738ab6f0e/boto3-1.40.66.tar.gz", hash = "sha256:f2038d9bac5154da7390c29bfd013546ac96609e7ce5a7f3cb6f99412be3f4c0", size = 111564, upload-time = "2025-11-04T20:28:59.274Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/5f/40/2ce4219d4df77c50d3de58fa33a9f6eea1ff183f01d1d42ae333c9789eb9/boto3-1.40.32-py3-none-any.whl", hash = "sha256:9edf07327f444ddd142f45120c6979902712921b6e90fdb05963996b97bd7689", size = 139344, upload-time = "2025-09-16T19:30:19.684Z" }, + { url = "https://files.pythonhosted.org/packages/bd/c2/3097e2492931b8fdcab47217b917c7964dbc8bfce31f89ace49568ed47f8/boto3-1.40.66-py3-none-any.whl", hash = "sha256:ee4fe21c5301cc0e11cc11a53e71e5ddd82d5fae42b10fa8e5403f3aa06434e3", size = 139361, upload-time = "2025-11-04T20:28:57.146Z" }, ] [[package]] name = "botocore" -version = "1.40.32" +version = "1.40.66" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "jmespath" }, { name = "python-dateutil" }, { name = "urllib3" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/b6/9f/679a62a91414907b7f27d1757238b95f821f769ca98b1418d4e32758d8b5/botocore-1.40.32.tar.gz", hash = "sha256:e027adff8d6e177b396dff68340dafa7548b84aeb2cff149d84844d56ad29bae", size = 14344643, upload-time = "2025-09-16T19:30:10.129Z" } +sdist = { url = "https://files.pythonhosted.org/packages/c3/f3/5dae6e3b06493f2ac769c6764543b84fa50a8de3fec1e33252271166b394/botocore-1.40.66.tar.gz", hash = "sha256:e49a55ad54426c4ea853a59ff9d8243023a90c935782d4c287e9b3424883c3fa", size = 14411853, upload-time = "2025-11-04T20:28:48.07Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/53/00/7598993d4838262316c75810e6a8d2e0bdeaefed8159125c949e26a643b4/botocore-1.40.32-py3-none-any.whl", hash = "sha256:a7cc409648aaf08a7d1aab76c675818868a0b76745b9deb478f91c1d8898aa8a", size = 14016094, upload-time = "2025-09-16T19:30:05.759Z" }, + { url = "https://files.pythonhosted.org/packages/9a/48/43f9335e28351f35a939dce366a3943296f381ecd4660bd1c8d2bb8f3006/botocore-1.40.66-py3-none-any.whl", hash = "sha256:98d5766e17e72110b1d08ab510a8475a6597c59d9560235e2d28ae1a4b043b92", size = 14076509, upload-time = "2025-11-04T20:28:44.233Z" }, ] [[package]] @@ -1167,11 +1146,9 @@ dependencies = [ dev = [ { name = "datasets" }, { name = "haiku-rag-evals" }, - { name = "logfire" }, { name = "mkdocs" }, { name = "mkdocs-material" }, { name = "pre-commit" }, - { name = "pydantic-evals" }, { name = "pyright" }, { name = "pytest" }, { name = "pytest-asyncio" }, @@ -1186,11 +1163,9 @@ requires-dist = [{ name = "haiku-rag-slim", extras = ["docling", "voyageai", "mx dev = [ { name = "datasets", specifier = ">=4.3.0" }, { name = "haiku-rag-evals", editable = "evaluations" }, - { name = "logfire", specifier = ">=4.14.2" }, { name = "mkdocs", specifier = ">=1.6.1" }, { name = "mkdocs-material", specifier = ">=9.6.22" }, { name = "pre-commit", specifier = ">=4.3.0" }, - { name = "pydantic-evals", specifier = ">=1.7.0" }, { name = "pyright", specifier = ">=1.1.407" }, { name = "pytest", specifier = ">=8.4.2" }, { name = "pytest-asyncio", specifier = ">=1.2.0" }, @@ -1205,8 +1180,7 @@ source = { editable = "evaluations" } dependencies = [ { name = "datasets" }, { name = "haiku-rag-slim" }, - { name = "logfire" }, - { name = "pydantic-evals" }, + { name = "pydantic-ai-slim", extra = ["evals", "logfire"] }, { name = "typer" }, ] @@ -1214,8 +1188,7 @@ dependencies = [ requires-dist = [ { name = "datasets", specifier = ">=4.3.0" }, { name = "haiku-rag-slim", editable = "haiku_rag_slim" }, - { name = "logfire", specifier = ">=4.14.2" }, - { name = "pydantic-evals", specifier = ">=1.7.0" }, + { name = "pydantic-ai-slim", extras = ["evals", "logfire"], specifier = ">=1.11.0" }, { name = "typer", specifier = ">=0.19.2,<0.20.0" }, ] @@ -1225,13 +1198,11 @@ version = "0.13.3" source = { editable = "haiku_rag_slim" } dependencies = [ { name = "docling-core" }, - { name = "fastmcp" }, { name = "httpx" }, { name = "lancedb" }, { name = "pathspec" }, { name = "pydantic" }, - { name = "pydantic-ai" }, - { name = "pydantic-graph" }, + { name = "pydantic-ai-slim", extra = ["fastmcp", "logfire", "openai"] }, { name = "python-dotenv" }, { name = "pyyaml" }, { name = "rich" }, @@ -1243,6 +1214,13 @@ dependencies = [ [package.optional-dependencies] a2a = [ { name = "fasta2a" }, + { name = "pydantic-ai-slim", extra = ["a2a"] }, +] +anthropic = [ + { name = "pydantic-ai-slim", extra = ["anthropic"] }, +] +bedrock = [ + { name = "pydantic-ai-slim", extra = ["bedrock"] }, ] cohere = [ { name = "cohere" }, @@ -1250,9 +1228,21 @@ cohere = [ docling = [ { name = "docling" }, ] +google = [ + { name = "pydantic-ai-slim", extra = ["google"] }, +] +groq = [ + { name = "pydantic-ai-slim", extra = ["groq"] }, +] +mistral = [ + { name = "pydantic-ai-slim", extra = ["mistral"] }, +] mxbai = [ { name = "mxbai-rerank" }, ] +vertexai = [ + { name = "pydantic-ai-slim", extra = ["vertexai"] }, +] voyageai = [ { name = "voyageai" }, ] @@ -1266,14 +1256,19 @@ requires-dist = [ { name = "docling", marker = "extra == 'docling'", specifier = ">=2.58.0" }, { name = "docling-core", specifier = ">=2.48.4" }, { name = "fasta2a", marker = "extra == 'a2a'", specifier = ">=0.1.0" }, - { name = "fastmcp", specifier = ">=2.13.0.2" }, { name = "httpx", specifier = ">=0.28.1" }, { name = "lancedb", specifier = ">=0.25.2" }, { name = "mxbai-rerank", marker = "extra == 'mxbai'", specifier = ">=0.1.6" }, { name = "pathspec", specifier = ">=0.12.1" }, { name = "pydantic", specifier = ">=2.12.3" }, - { name = "pydantic-ai", specifier = ">=1.7.0" }, - { name = "pydantic-graph", specifier = ">=1.7.0" }, + { name = "pydantic-ai-slim", extras = ["a2a"], marker = "extra == 'a2a'" }, + { name = "pydantic-ai-slim", extras = ["anthropic"], marker = "extra == 'anthropic'" }, + { name = "pydantic-ai-slim", extras = ["bedrock"], marker = "extra == 'bedrock'" }, + { name = "pydantic-ai-slim", extras = ["google"], marker = "extra == 'google'" }, + { name = "pydantic-ai-slim", extras = ["groq"], marker = "extra == 'groq'" }, + { name = "pydantic-ai-slim", extras = ["mistral"], marker = "extra == 'mistral'" }, + { name = "pydantic-ai-slim", extras = ["openai", "fastmcp", "logfire"], specifier = ">=1.11.0" }, + { name = "pydantic-ai-slim", extras = ["vertexai"], marker = "extra == 'vertexai'" }, { name = "python-dotenv", specifier = ">=1.2.1" }, { name = "pyyaml", specifier = ">=6.0.3" }, { name = "rich", specifier = ">=14.2.0" }, @@ -1283,7 +1278,7 @@ requires-dist = [ { name = "watchfiles", specifier = ">=1.1.1" }, { name = "zeroentropy", marker = "extra == 'zeroentropy'", specifier = ">=0.1.0a6" }, ] -provides-extras = ["docling", "voyageai", "mxbai", "a2a", "cohere", "zeroentropy"] +provides-extras = ["docling", "voyageai", "mxbai", "cohere", "zeroentropy", "a2a", "anthropic", "groq", "google", "mistral", "bedrock", "vertexai"] [[package]] name = "hf-xet" @@ -1356,11 +1351,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/fe/85/a18508becfa01f1e4351b5e18651b06d210dbd96debccd48a452acccb901/huggingface_hub-0.35.0-py3-none-any.whl", hash = "sha256:f2e2f693bca9a26530b1c0b9bcd4c1495644dad698e6a0060f90e22e772c31e9", size = 563436, upload-time = "2025-09-16T13:49:30.627Z" }, ] -[package.optional-dependencies] -inference = [ - { name = "aiohttp" }, -] - [[package]] name = "identify" version = "2.6.14" @@ -1402,11 +1392,11 @@ wheels = [ [[package]] name = "invoke" -version = "2.2.0" +version = "2.2.1" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/f9/42/127e6d792884ab860defc3f4d80a8f9812e48ace584ffc5a346de58cdc6c/invoke-2.2.0.tar.gz", hash = "sha256:ee6cbb101af1a859c7fe84f2a264c059020b0cb7fe3535f9424300ab568f6bd5", size = 299835, upload-time = "2023-07-12T18:05:17.998Z" } +sdist = { url = "https://files.pythonhosted.org/packages/de/bd/b461d3424a24c80490313fd77feeb666ca4f6a28c7e72713e3d9095719b4/invoke-2.2.1.tar.gz", hash = "sha256:515bf49b4a48932b79b024590348da22f39c4942dff991ad1fb8b8baea1be707", size = 304762, upload-time = "2025-10-11T00:36:35.172Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/0a/66/7f8c48009c72d73bc6bbe6eb87ac838d6a526146f7dab14af671121eb379/invoke-2.2.0-py3-none-any.whl", hash = "sha256:6ea924cc53d4f78e3d98bc436b08069a03077e6f85ad1ddaa8a116d7dad15820", size = 160274, upload-time = "2023-07-12T18:05:16.294Z" }, + { url = "https://files.pythonhosted.org/packages/32/4b/b99e37f88336009971405cbb7630610322ed6fbfa31e1d7ab3fbf3049a2d/invoke-2.2.1-py3-none-any.whl", hash = "sha256:2413bc441b376e5cd3f55bb5d364f973ad8bdd7bf87e53c79de3c11bf3feecc8", size = 160287, upload-time = "2025-10-11T00:36:33.703Z" }, ] [[package]] @@ -1915,7 +1905,7 @@ wheels = [ [[package]] name = "mistralai" -version = "1.9.10" +version = "1.9.11" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "eval-type-backport" }, @@ -1926,9 +1916,9 @@ dependencies = [ { name = "pyyaml" }, { name = "typing-inspection" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/6d/a3/1ae43c9db1fc612176d5d3418c12cd363852e954c5d12bf3a4477de2e4a6/mistralai-1.9.10.tar.gz", hash = "sha256:a95721276f035bf86c7fdc1373d7fb7d056d83510226f349426e0d522c0c0965", size = 205043, upload-time = "2025-09-02T07:44:38.859Z" } +sdist = { url = "https://files.pythonhosted.org/packages/5a/8d/d8b7af67a966b6f227024e1cb7287fc19901a434f87a5a391dcfe635d338/mistralai-1.9.11.tar.gz", hash = "sha256:3df9e403c31a756ec79e78df25ee73cea3eb15f86693773e16b16adaf59c9b8a", size = 208051, upload-time = "2025-10-02T15:53:40.473Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/29/40/646448b5ad66efec097471bd5ab25f5b08360e3f34aecbe5c4fcc6845c01/mistralai-1.9.10-py3-none-any.whl", hash = "sha256:cf0a2906e254bb4825209a26e1957e6e0bacbbe61875bd22128dc3d5d51a7b0a", size = 440538, upload-time = "2025-09-02T07:44:37.5Z" }, + { url = "https://files.pythonhosted.org/packages/fe/76/4ce12563aea5a76016f8643eff30ab731e6656c845e9e4d090ef10c7b925/mistralai-1.9.11-py3-none-any.whl", hash = "sha256:7a3dc2b8ef3fceaa3582220234261b5c4e3e03a972563b07afa150e44a25a6d3", size = 442796, upload-time = "2025-10-02T15:53:39.134Z" }, ] [[package]] @@ -2142,18 +2132,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/eb/8d/776adee7bbf76365fdd7f2552710282c79a4ead5d2a46408c9043a2b70ba/networkx-3.5-py3-none-any.whl", hash = "sha256:0030d386a9a06dee3565298b4a734b68589749a544acbb6c412dc9e2489ec6ec", size = 2034406, upload-time = "2025-05-29T11:35:04.961Z" }, ] -[[package]] -name = "nexus-rpc" -version = "1.1.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/ef/66/540687556bd28cf1ec370cc6881456203dfddb9dab047b8979c6865b5984/nexus_rpc-1.1.0.tar.gz", hash = "sha256:d65ad6a2f54f14e53ebe39ee30555eaeb894102437125733fb13034a04a44553", size = 77383, upload-time = "2025-07-07T19:03:58.368Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/bf/2f/9e9d0dcaa4c6ffa22b7aa31069a8a264c753ff8027b36af602cce038c92f/nexus_rpc-1.1.0-py3-none-any.whl", hash = "sha256:d1b007af2aba186a27e736f8eaae39c03aed05b488084ff6c3d1785c9ba2ad38", size = 27743, upload-time = "2025-07-07T19:03:57.556Z" }, -] - [[package]] name = "nodeenv" version = "1.9.1" @@ -2819,18 +2797,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/5b/a5/987a405322d78a73b66e39e4a90e4ef156fd7141bf71df987e50717c321b/pre_commit-4.3.0-py2.py3-none-any.whl", hash = "sha256:2b0747ad7e6e967169136edffee14c16e148a778a54e4f967921aa1ebf2308d8", size = 220965, upload-time = "2025-08-09T18:56:13.192Z" }, ] -[[package]] -name = "prompt-toolkit" -version = "3.0.52" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "wcwidth" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/a1/96/06e01a7b38dce6fe1db213e061a4602dd6032a8a97ef6c1a862537732421/prompt_toolkit-3.0.52.tar.gz", hash = "sha256:28cde192929c8e7321de85de1ddbe736f1375148b02f2e17edd840042b1be855", size = 434198, upload-time = "2025-08-27T15:24:02.057Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/84/03/0d3ce49e2505ae70cf43bc5bb3033955d2fc9f932163e84dc0779cc47f48/prompt_toolkit-3.0.52-py3-none-any.whl", hash = "sha256:9aac639a3bbd33284347de5ad8d68ecc044b91a762dc39b7c21095fcd6a19955", size = 391431, upload-time = "2025-08-27T15:23:59.498Z" }, -] - [[package]] name = "propcache" version = "0.3.2" @@ -3054,21 +3020,9 @@ email = [ { name = "email-validator" }, ] -[[package]] -name = "pydantic-ai" -version = "1.7.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pydantic-ai-slim", extra = ["ag-ui", "anthropic", "bedrock", "cli", "cohere", "evals", "fastmcp", "google", "groq", "huggingface", "logfire", "mcp", "mistral", "openai", "retries", "temporal", "vertexai"] }, -] -sdist = { url = "https://files.pythonhosted.org/packages/4e/2e/ceaa37fdeb084dfdea640fee2487e81a2fe16552421de8854d4b1346a233/pydantic_ai-1.7.0.tar.gz", hash = "sha256:3035e09360dc7758f6adeb77da27af68fffced46d18258b7b0269e9b432fe7f1", size = 100392908, upload-time = "2025-10-28T01:09:42.268Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/75/d8/37a65a0b8792ff47d87d66aded6a779f4fe1885965672dc4d8dbc0fe1fb5/pydantic_ai-1.7.0-py3-none-any.whl", hash = "sha256:c891b4bff7836fdcea968d3430c1992decd1015a15cabf2b74816cf6beb38a31", size = 11724, upload-time = "2025-10-28T01:09:31.326Z" }, -] - [[package]] name = "pydantic-ai-slim" -version = "1.7.0" +version = "1.11.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "genai-prices" }, @@ -3079,15 +3033,14 @@ dependencies = [ { name = "pydantic-graph" }, { name = "typing-inspection" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/c9/eb/ad788c802cd107321188a96f06b896a16a68bc8342a8034ef68584480503/pydantic_ai_slim-1.7.0.tar.gz", hash = "sha256:2bd0e637ecc311ac672758c163fa738235651cf8a593f0abcf8e833e6e89e014", size = 284460, upload-time = "2025-10-28T01:09:47.488Z" } +sdist = { url = "https://files.pythonhosted.org/packages/e0/bd/5092711d19da6ac8e7219307ee71a99f177a52cc9bfbcfe144c29b1f0469/pydantic_ai_slim-1.11.0.tar.gz", hash = "sha256:d03733e49774e62f5dcb1331160c3e0747c89038f4cb4d301ce6e982080bd5f6", size = 298121, upload-time = "2025-11-05T04:10:52.912Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/b1/1a/0a66c82f0c10fb3cd697f35d3fe99a3c27ca6328774ca619e49315aa557e/pydantic_ai_slim-1.7.0-py3-none-any.whl", hash = "sha256:76b03f1381be98e86bce2da1709ecbee16241b1a5f641aa19982fb583c3d6849", size = 375720, upload-time = "2025-10-28T01:09:34.692Z" }, + { url = "https://files.pythonhosted.org/packages/0b/34/38fd79467531188cbfbf4a16b56d962dc43778dcedea07d86963e91e64ba/pydantic_ai_slim-1.11.0-py3-none-any.whl", hash = "sha256:dcdaa6411f3e5b6669edff3ca344f8b0bf657a635df307ade88b90f030db1624", size = 397355, upload-time = "2025-11-05T04:10:38.108Z" }, ] [package.optional-dependencies] -ag-ui = [ - { name = "ag-ui-protocol" }, - { name = "starlette" }, +a2a = [ + { name = "fasta2a" }, ] anthropic = [ { name = "anthropic" }, @@ -3095,15 +3048,6 @@ anthropic = [ bedrock = [ { name = "boto3" }, ] -cli = [ - { name = "argcomplete" }, - { name = "prompt-toolkit" }, - { name = "pyperclip" }, - { name = "rich" }, -] -cohere = [ - { name = "cohere", marker = "sys_platform != 'emscripten'" }, -] evals = [ { name = "pydantic-evals" }, ] @@ -3116,27 +3060,15 @@ google = [ groq = [ { name = "groq" }, ] -huggingface = [ - { name = "huggingface-hub", extra = ["inference"] }, -] logfire = [ { name = "logfire", extra = ["httpx"] }, ] -mcp = [ - { name = "mcp" }, -] mistral = [ { name = "mistralai" }, ] openai = [ { name = "openai" }, ] -retries = [ - { name = "tenacity" }, -] -temporal = [ - { name = "temporalio" }, -] vertexai = [ { name = "google-auth" }, { name = "requests" }, @@ -3211,7 +3143,7 @@ wheels = [ [[package]] name = "pydantic-evals" -version = "1.7.0" +version = "1.11.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "anyio" }, @@ -3221,14 +3153,14 @@ dependencies = [ { name = "pyyaml" }, { name = "rich" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/1c/5a/8b6d16d8269ac69c00cc0f724b52419680ffaf2c3717c55a125816b01a4c/pydantic_evals-1.7.0.tar.gz", hash = "sha256:f36958ec2ebde50bba081747de05e307dad2c9aa884cbd61829d6283bd140c39", size = 45929, upload-time = "2025-10-28T01:09:48.876Z" } +sdist = { url = "https://files.pythonhosted.org/packages/0d/05/73f50ca4301f70da5e260d880be5bff1f1ae9de4e63e3ec96d80f7537926/pydantic_evals-1.11.0.tar.gz", hash = "sha256:cab5d742c6ccf28d67b0841e64a408c749ac0c820295b6fbaa8ed460d338ce8d", size = 46968, upload-time = "2025-11-05T04:10:54.374Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/4e/5b/3cdfc72191963089af72e3457806b923c75abfe6ce770f7e49be1fa212f6/pydantic_evals-1.7.0-py3-none-any.whl", hash = "sha256:ed5142d48ebbe52125ad11a78b0bd67426b421a2536f5975a1b3e79e3afeb103", size = 55021, upload-time = "2025-10-28T01:09:36.667Z" }, + { url = "https://files.pythonhosted.org/packages/d5/aa/73aa02f32f8fc050e69623be33c78707d9a13b1658e33b6f082d82d71693/pydantic_evals-1.11.0-py3-none-any.whl", hash = "sha256:79eee0dcc606cb4d7ca3c55e025221ce698224c4efe94a0701374d1740cd57d2", size = 56133, upload-time = "2025-11-05T04:10:39.841Z" }, ] [[package]] name = "pydantic-graph" -version = "1.7.0" +version = "1.11.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "httpx" }, @@ -3236,9 +3168,9 @@ dependencies = [ { name = "pydantic" }, { name = "typing-inspection" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/10/dd/a6707f30f2c2348dd52e8d80b6dfddd1d2984f6912fdc69b5f40eccafb18/pydantic_graph-1.7.0.tar.gz", hash = "sha256:eeaf857c608c2f6c6a49571bbbf4458a40b947a6c0ff60842fe564e13bf831bf", size = 56932, upload-time = "2025-10-28T01:09:49.812Z" } +sdist = { url = "https://files.pythonhosted.org/packages/eb/b6/74186f367c1af46e3002792fa68921f98db5c665f6f58597107b1138e751/pydantic_graph-1.11.0.tar.gz", hash = "sha256:9c651ce2c202f4c5d8ce36f09227eab0ea3d48ac0c3f31586e5fc4e15a616516", size = 57900, upload-time = "2025-11-05T04:10:56.082Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/24/da/e7e94d6601cbbd60216cd13d0b03410593a61d806dc2130f92fb52b2a444/pydantic_graph-1.7.0-py3-none-any.whl", hash = "sha256:ae51cd834517988a3e9331492143f4b6bd25a561caf4c607d7263be4f1028332", size = 70913, upload-time = "2025-10-28T01:09:38.333Z" }, + { url = "https://files.pythonhosted.org/packages/8c/78/fa42782a5436160827657f741dad64dc678df863a27926034e9552adb25f/pydantic_graph-1.11.0-py3-none-any.whl", hash = "sha256:0811a32d83e0b19f864ef34c9d0f06db944ca7eefb33b96fd63bceebe35029db", size = 71936, upload-time = "2025-11-05T04:10:42.193Z" }, ] [[package]] @@ -4141,25 +4073,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/40/44/4a5f08c96eb108af5cb50b41f76142f0afa346dfa99d5296fe7202a11854/tabulate-0.9.0-py3-none-any.whl", hash = "sha256:024ca478df22e9340661486f85298cff5f6dcdba14f3813e8830015b9ed1948f", size = 35252, upload-time = "2022-10-06T17:21:44.262Z" }, ] -[[package]] -name = "temporalio" -version = "1.18.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "nexus-rpc" }, - { name = "protobuf" }, - { name = "types-protobuf" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/7e/20/b52c96b37bf00ead6e8a4a197075770ebad516db765cc3abca8396de0689/temporalio-1.18.0.tar.gz", hash = "sha256:7ff7f833eb1e7697084b4ed9d86c3167cbff1ec77f1b40df774313a5d0fd5f6d", size = 1781572, upload-time = "2025-09-19T23:40:52.511Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/2f/28/c5a4ee259748450ac0765837f8c78cbfa36800264158d98bd2cde4496d87/temporalio-1.18.0-cp39-abi3-macosx_10_12_x86_64.whl", hash = "sha256:ac5d30d8b010c9b042065ea1259da7638db1a0a25e81ee4be0671a393ed329c5", size = 12734753, upload-time = "2025-09-19T23:40:06.575Z" }, - { url = "https://files.pythonhosted.org/packages/be/94/24bd903b5594420a4d131bfa3de965313f9a409af77b47e9a9a56d85bb9e/temporalio-1.18.0-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:19315d192247230c9bd7c60a566c2b3a80ad4d9de891c6aa13df63d72d3ec169", size = 12323141, upload-time = "2025-09-19T23:40:16.817Z" }, - { url = "https://files.pythonhosted.org/packages/6d/76/82415b43c68e2c6bb3a85e8800555d206767815088c8cad0ade9a06bd7ac/temporalio-1.18.0-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a023b25033e48b2e43f623a78737047a45b8cb553f69f457d09272fce5c723da", size = 12694061, upload-time = "2025-09-19T23:40:26.388Z" }, - { url = "https://files.pythonhosted.org/packages/41/60/176a3224c2739fee270052dd9224ae36370c4e13d2ab1bb96a2f9bbb513c/temporalio-1.18.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:695211dddbcffc20077d5b3b9a9b41bd09f60393c4ff211bcc7d6d895d607cc1", size = 12879404, upload-time = "2025-09-19T23:40:37.487Z" }, - { url = "https://files.pythonhosted.org/packages/e3/8d/e3809b356262d1d398d8cbb78df1e19d460c0a89e6ab64ca8d9c05d5fe5a/temporalio-1.18.0-cp39-abi3-win_amd64.whl", hash = "sha256:e3f691bd0a01a22c0fe40e87b6236cc8a292628e3a5a490880d1bf94709249c9", size = 13088041, upload-time = "2025-09-19T23:40:49.469Z" }, -] - [[package]] name = "tenacity" version = "9.1.2" @@ -4369,15 +4282,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/00/22/35617eee79080a5d071d0f14ad698d325ee6b3bf824fc0467c03b30e7fa8/typer-0.19.2-py3-none-any.whl", hash = "sha256:755e7e19670ffad8283db353267cb81ef252f595aa6834a0d1ca9312d9326cb9", size = 46748, upload-time = "2025-09-23T09:47:46.777Z" }, ] -[[package]] -name = "types-protobuf" -version = "6.30.2.20250914" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/36/d1/e12dad323fe6e2455b768828de288f60d5160f41dad5d31af8ef92a6acbb/types_protobuf-6.30.2.20250914.tar.gz", hash = "sha256:c2105326d0a52de3d33b84af0010d834ebbd4c17c50ff261fa82551ab75d9559", size = 62424, upload-time = "2025-09-14T02:56:00.798Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/6c/c4/3fcb1f8e03456a8a33a5dfb9f9788b0a91023e5fad6a37d46fc6831629a7/types_protobuf-6.30.2.20250914-py3-none-any.whl", hash = "sha256:cfc24977c0f38cf2896d918a59faed7650eb983be6070343a6204ac8ac0a297e", size = 76546, upload-time = "2025-09-14T02:55:59.489Z" }, -] - [[package]] name = "types-requests" version = "2.32.4.20250913" @@ -4570,15 +4474,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e3/bd/fa9bb053192491b3867ba07d2343d9f2252e00811567d30ae8d0f78136fe/watchfiles-1.1.1-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:a916a2932da8f8ab582f242c065f5c81bed3462849ca79ee357dd9551b0e9b01", size = 622112, upload-time = "2025-10-14T15:05:50.941Z" }, ] -[[package]] -name = "wcwidth" -version = "0.2.13" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/6c/63/53559446a878410fc5a5974feb13d31d78d752eb18aeba59c7fef1af7598/wcwidth-0.2.13.tar.gz", hash = "sha256:72ea0c06399eb286d978fdedb6923a9eb47e1c486ce63e9b4e64fc18303972b5", size = 101301, upload-time = "2024-01-06T02:10:57.829Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/fd/84/fd2ba7aafacbad3c4201d395674fc6348826569da3c0937e75505ead3528/wcwidth-0.2.13-py2.py3-none-any.whl", hash = "sha256:3da69048e4540d84af32131829ff948f1e022c1c6bdb8d6102117aac784f6859", size = 34166, upload-time = "2024-01-06T02:10:55.763Z" }, -] - [[package]] name = "websockets" version = "15.0.1" From 009e5298696fe6b787845748666f081891cfa87b Mon Sep 17 00:00:00 2001 From: Yiorgis Gozadinos Date: Wed, 5 Nov 2025 12:35:37 +0200 Subject: [PATCH 11/17] No need to have evaluations in the docker image --- docker/Dockerfile | 8 ++++---- evaluations/evaluations/benchmark.py | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 71216b42..5eac172a 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -12,14 +12,14 @@ ENV UV_COMPILE_BYTECODE=1 \ # Copy workspace member directories first (needed for workspace resolution) COPY pyproject.toml uv.lock ./ COPY haiku_rag_slim/pyproject.toml haiku_rag_slim/README.md haiku_rag_slim/LICENSE haiku_rag_slim/ -COPY evaluations/pyproject.toml evaluations/ RUN --mount=type=cache,target=/root/.cache/uv \ - uv sync --frozen --no-install-project + uv sync --frozen --no-install-project --no-dev # Install the project itself -COPY . . +COPY haiku_rag_slim haiku_rag_slim/ +COPY README.md LICENSE ./ RUN --mount=type=cache,target=/root/.cache/uv \ - uv sync --frozen --no-editable + uv sync --frozen --no-editable --no-dev # Final layer FROM python:3.13-slim diff --git a/evaluations/evaluations/benchmark.py b/evaluations/evaluations/benchmark.py index 9f3f4eb4..f75bd2bf 100644 --- a/evaluations/evaluations/benchmark.py +++ b/evaluations/evaluations/benchmark.py @@ -5,6 +5,10 @@ from typing import Any, cast import logfire import typer +from haiku.rag.client import HaikuRAG +from haiku.rag.config import AppConfig, find_config_file, load_yaml_config +from haiku.rag.logging import configure_cli_logging +from haiku.rag.qa import get_qa_agent from pydantic_ai.models.openai import OpenAIChatModel from pydantic_ai.providers.ollama import OllamaProvider from pydantic_evals import Dataset as EvalDataset @@ -17,10 +21,6 @@ from evaluations.config import DatasetSpec from evaluations.datasets import DATASETS from evaluations.llm_judge import ANSWER_EQUIVALENCE_RUBRIC from evaluations.prompts import WIX_SUPPORT_PROMPT -from haiku.rag.client import HaikuRAG -from haiku.rag.config import AppConfig, find_config_file, load_yaml_config -from haiku.rag.logging import configure_cli_logging -from haiku.rag.qa import get_qa_agent QA_JUDGE_MODEL = "qwen3" From 106945ddc006ad82a8189e977345972cf42d24ae Mon Sep 17 00:00:00 2001 From: Yiorgis Gozadinos Date: Wed, 5 Nov 2025 12:44:39 +0200 Subject: [PATCH 12/17] Update tutorial --- docs/tutorial.md | 34 ++++++++++++---------------------- 1 file changed, 12 insertions(+), 22 deletions(-) diff --git a/docs/tutorial.md b/docs/tutorial.md index e9b26df3..659b1ffb 100644 --- a/docs/tutorial.md +++ b/docs/tutorial.md @@ -1,12 +1,12 @@ # Tutorial -This tutorial quickstart instructions for getting familiar with `haiku.rag`. This tutorial is intended for people who are familiar with command line and Python, but not different AI ecosystem tools. +This tutorial provides quickstart instructions for getting familiar with `haiku.rag`. This tutorial is intended for people who are familiar with command line and Python, but not different AI ecosystem tools. The tutorial covers: - RAG and embeddings basics -- Install `haiku.rag` Python package -- Set up environment variables for running `haiku.rag` +- Installing `haiku.rag` Python package +- Configuring `haiku.rag` with YAML - Adding and retrieving items - Inspecting the database @@ -14,11 +14,9 @@ The tutorial uses OpenAI API service - no local installation needed and will wor ## Introduction -Embeddings serve as the foundational bridge between unstructured text data and computational efficiency in AI systems, particularly within Retrieval-Augmented Generation (RAG) frameworks that enhance Large Language Models (LLMs). At their core, embeddings are dense vector representations of words, sentences, or documents, created by models like those from OpenAI, which encode semantic and contextual meaning into numerical forms. In a RAG pipeline, these embeddings enable the system to index and store vast amounts of information in a vector database, allowing for rapid similarity searches. When a user query is posed, its embedding is generated and compared against the stored vectors using metrics like cosine similarity to retrieve the most relevant documents or chunks of text. This retrieval step addresses a key limitation of standalone LLMs, which rely on pre-trained knowledge that can be outdated, incomplete, or prone to hallucinations, by providing external, grounded context to inform the model's output. +Retrieval-Augmented Generation (RAG) lets you give AI models access to your own documents and data. Instead of relying solely on the model's training data, RAG finds relevant information from your documents and includes it in the AI's responses. -You technically can create vector embeddings yourself without relying on pre-built models like OpenAI's, but it's impractical for most users due to the complexity and resources involved in training or implementing from scratch. Generating embeddings requires a neural network architecture (e.g., transformer-based) trained on massive datasets to learn semantic relationships, which demands significant computational power (GPUs/TPUs), expertise in machine learning, and access to billions of text examples for fine-tuning. - -`haiku.rag` is a Python library allowing you to set up your own embeddings database, pipeline to feed into it and query it with different LLM providers and related services. +`haiku.rag` handles the mechanics: it converts your documents into searchable embeddings, stores them locally, and retrieves relevant chunks when you ask questions. You provide the documents and questions, and it coordinates between the embedding service (like OpenAI) and the AI model to give you accurate, grounded answers. ## Setup @@ -81,16 +79,16 @@ You should get the back the information: ``` haiku.rag database info path: /Users/moo/Library/Application Support/haiku.rag/haiku.rag.lancedb - haiku.rag version (db): 0.12.1 + haiku.rag version (db): 0.13.3 embeddings: openai/text-embedding-3-small (dim: 1536) - documents: 4 - versions (documents): 9 - versions (chunks): 10 + documents: 3 + versions (documents): 3 + versions (chunks): 3 ────────────────────────────────────────────────────────────────────────────────── Versions - haiku.rag: 0.12.1 + haiku.rag: 0.13.3 lancedb: 0.25.2 - docling: 2.57.0 + docling: 2.58.0 ``` ## Asking questions and retrieving information @@ -167,7 +165,7 @@ According to the document, Python is considered the best programming language in ## Complex documents -Haiku RAG can also handle types beyond plain text. +Haiku RAG can also handle types beyond plain text, including PDF, DOCX, HTML, and 40+ other file formats. Here we add research papers about Python from [arxiv](https://arxiv.org/search/?query=python&searchtype=all&source=header) using URL retriever. @@ -215,14 +213,6 @@ The following people are presenting talks at PyCon Finland 2025: 5 Andreas Jung (also presenting another talk) - Talk: Debugging Python ``` -## Reseting the embeddings database - -If you change your embeddings provider (OpenAI -> ollama) or its parameters, you need to delete the LanceDB database and add the documents again: - -```shell -rm -rf "/Users/moo/Library/Application Support/haiku.rag/haiku.rag.lancedb" -``` - ## Configuration See [Configuration page](./configuration.md) for complete documentation on YAML configuration and all available options. From ef5e829bdce2d92d83647724d0316c20b7de83d4 Mon Sep 17 00:00:00 2001 From: Yiorgis Gozadinos Date: Wed, 5 Nov 2025 12:56:03 +0200 Subject: [PATCH 13/17] Update docs --- docs/installation.md | 51 ++++---------------------------------------- mkdocs.yml | 4 ++-- 2 files changed, 6 insertions(+), 49 deletions(-) diff --git a/docs/installation.md b/docs/installation.md index d064e485..f088631c 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -12,8 +12,8 @@ uv pip install haiku.rag The full package includes **all features and extras**: - **Document processing** (Docling) - PDF, DOCX, PPTX, images, and 40+ file formats -- **All embedding providers** - Ollama, OpenAI, VoyageAI, Anthropic, vLLM -- **All rerankers** - MixedBread AI, Cohere, Zero Entropy, vLLM +- **All embedding providers** - VoyageAI +- **All rerankers** - MixedBread AI, Cohere, Zero Entropy - **A2A agent** - Agent-to-Agent protocol support This is the easiest way to get started with all features enabled. @@ -44,56 +44,13 @@ The slim package has minimal dependencies and lets you install only what you nee - **Ollama** (default embedding provider) - **OpenAI** (GPT models for QA and embeddings) - **Anthropic** (Claude models for QA) -- **vLLM** (high-performance local inference) -### vLLM Setup - -vLLM requires no additional installation - it works with the base haiku.rag package. However, you need to run vLLM servers separately: - -```bash -# Install vLLM -pip install vllm - -# Serve an embedding model -vllm serve mixedbread-ai/mxbai-embed-large-v1 --port 8000 - -# Serve a model for QA (requires tool calling support) -vllm serve Qwen/Qwen3-4B --port 8002 --enable-auto-tool-choice --tool-call-parser hermes - -# Serve a model for reranking -vllm serve mixedbread-ai/mxbai-rerank-base-v2 --hf_overrides '{"architectures": ["Qwen2ForSequenceClassification"],"classifier_from_token": ["0", "1"], "method": "from_2_way_softmax"}' --port 8001 -``` - -Then configure haiku.rag to use the vLLM servers. Create a `haiku.rag.yaml` file: - -```yaml -embeddings: - provider: vllm - model: mixedbread-ai/mxbai-embed-large-v1 - vector_dim: 512 - -qa: - provider: vllm - model: Qwen/Qwen3-4B - -reranking: - provider: vllm - model: mixedbread-ai/mxbai-rerank-base-v2 - -providers: - vllm: - embeddings_base_url: http://localhost:8000 - qa_base_url: http://localhost:8002 - rerank_base_url: http://localhost:8001 -``` - -See [Configuration](configuration.md) for all available options. +See [Configuration](configuration.md) for configuring providers including advanced options like vLLM. ## Requirements - Python 3.12+ -- Ollama (for default embeddings) -- vLLM server (for vLLM provider) +- Ollama (for default embeddings and QA) ## Pre-download Models (Optional) diff --git a/mkdocs.yml b/mkdocs.yml index 8b28324d..f7034495 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -61,9 +61,9 @@ nav: - Installation: installation.md - Configuration: configuration.md - CLI: cli.md - - Server: server.md - - Agents: agents.md - Python: python.md + - Agents: agents.md + - Server: server.md - MCP: mcp.md - A2A: a2a.md - Benchmarks: benchmarks.md From 74d9c60ba0f850c4c4f6fd9297ef0b32b4a1159f Mon Sep 17 00:00:00 2001 From: Yiorgis Gozadinos Date: Wed, 5 Nov 2025 13:11:30 +0200 Subject: [PATCH 14/17] Update docling --- haiku_rag_slim/pyproject.toml | 9 +++------ uv.lock | 16 ++++++++-------- 2 files changed, 11 insertions(+), 14 deletions(-) diff --git a/haiku_rag_slim/pyproject.toml b/haiku_rag_slim/pyproject.toml index e3de412b..3e8e83b9 100644 --- a/haiku_rag_slim/pyproject.toml +++ b/haiku_rag_slim/pyproject.toml @@ -22,7 +22,7 @@ classifiers = [ ] dependencies = [ - "docling-core>=2.48.4", + "docling-core>=2.50.1", "httpx>=0.28.1", "lancedb>=0.25.2", "pathspec>=0.12.1", @@ -38,7 +38,7 @@ dependencies = [ [project.optional-dependencies] # Document processing -docling = ["docling>=2.58.0"] +docling = ["docling>=2.60.1"] # Embedding providers voyageai = ["voyageai>=0.3.5"] # Rerankers @@ -46,10 +46,7 @@ mxbai = ["mxbai-rerank>=0.1.6"] cohere = ["cohere>=5.0.0"] zeroentropy = ["zeroentropy>=0.1.0a6"] # Agent protocols -a2a = [ - "fasta2a>=0.1.0", - "pydantic-ai-slim[a2a]", -] +a2a = ["fasta2a>=0.1.0", "pydantic-ai-slim[a2a]"] # Model providers (delegated to pydantic-ai-slim) anthropic = ["pydantic-ai-slim[anthropic]"] groq = ["pydantic-ai-slim[groq]"] diff --git a/uv.lock b/uv.lock index 74226e99..5e178aba 100644 --- a/uv.lock +++ b/uv.lock @@ -669,7 +669,7 @@ wheels = [ [[package]] name = "docling" -version = "2.58.0" +version = "2.60.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "accelerate" }, @@ -701,14 +701,14 @@ dependencies = [ { name = "tqdm" }, { name = "typer" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/26/96/435a8a5da619fe80f04eb33d7c597a971d9da1b197ed3edb0aa886742cf9/docling-2.58.0.tar.gz", hash = "sha256:c1d120d5179807e6fd72947f8709b7c2d716ae024df4b0df44c8abc0bb50476c", size = 231806, upload-time = "2025-10-22T11:32:52.913Z" } +sdist = { url = "https://files.pythonhosted.org/packages/05/eb/6db76ba07e5e57ed05aa1cf031d583ec6cd38c14e9f8c2c6395a872a1fb7/docling-2.60.1.tar.gz", hash = "sha256:7205feeea1148ef0d13a7c1184067be29595ef75606f66bc24881343c68fc16b", size = 235217, upload-time = "2025-11-04T11:27:39.795Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/b5/68/5128fa336866f425052816db2c20a777b6060db84ab5a70a85fd01b1c334/docling-2.58.0-py3-none-any.whl", hash = "sha256:bc28a4603a249f7a73d39f23345a096f08801fbf430b803e36c6f350e9e0848e", size = 251396, upload-time = "2025-10-22T11:32:51.446Z" }, + { url = "https://files.pythonhosted.org/packages/12/9a/55d04f94c8ba446b01c38c1eaa45b12f3d7508525d607ec62d44915af6f8/docling-2.60.1-py3-none-any.whl", hash = "sha256:0562cc45e7227b23ba1f7c82cbd5aa71ebfd38ab43c36cec37b433f4846f8822", size = 253120, upload-time = "2025-11-04T11:27:38.228Z" }, ] [[package]] name = "docling-core" -version = "2.48.4" +version = "2.50.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "jsonref" }, @@ -722,9 +722,9 @@ dependencies = [ { name = "typer" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/38/d8/f0c8034f87d6151eb955e56975b9f2374a54d57af2b56b1682d7c8ff5c71/docling_core-2.48.4.tar.gz", hash = "sha256:d87ce3021cdae3d073ce7572a2396b69be3cde82ebf9a74d4bad1e1cdfdfd524", size = 161377, upload-time = "2025-10-01T09:10:08.614Z" } +sdist = { url = "https://files.pythonhosted.org/packages/4e/aa/67810ed6f425c597bc5779560c3e550d23ac8cc76f5f62eddae8406cdaf7/docling_core-2.50.1.tar.gz", hash = "sha256:8afae348abb7f7622899d8664195a5e0bef4b2a872f0df0ed40bcd023970a995", size = 168008, upload-time = "2025-11-04T13:20:07.092Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/c8/2a/06e5f9d3083f830de8bef86f91acda994965f88d8b945ce3b257ea83e780/docling_core-2.48.4-py3-none-any.whl", hash = "sha256:367675c1165d0934ae498fa57ca2d27ef0468aad74dc44a5ab061f5d87882ea1", size = 164374, upload-time = "2025-10-01T09:10:06.034Z" }, + { url = "https://files.pythonhosted.org/packages/39/af/b1dd00c28bedcebef3f54b4e86d427a951f78818dbd7ea854c37a1a971fe/docling_core-2.50.1-py3-none-any.whl", hash = "sha256:92a34b77e02ed4faad451be36a56c37142e8ae240d87a0dcf58e89df41e256ae", size = 169285, upload-time = "2025-11-04T13:20:05.167Z" }, ] [package.optional-dependencies] @@ -1253,8 +1253,8 @@ zeroentropy = [ [package.metadata] requires-dist = [ { name = "cohere", marker = "extra == 'cohere'", specifier = ">=5.0.0" }, - { name = "docling", marker = "extra == 'docling'", specifier = ">=2.58.0" }, - { name = "docling-core", specifier = ">=2.48.4" }, + { name = "docling", marker = "extra == 'docling'", specifier = ">=2.60.1" }, + { name = "docling-core", specifier = ">=2.50.1" }, { name = "fasta2a", marker = "extra == 'a2a'", specifier = ">=0.1.0" }, { name = "httpx", specifier = ">=0.28.1" }, { name = "lancedb", specifier = ">=0.25.2" }, From 4129be5bf951aa1af7a94632e4e230f6b81f3616 Mon Sep 17 00:00:00 2001 From: Yiorgis Gozadinos Date: Wed, 5 Nov 2025 13:15:48 +0200 Subject: [PATCH 15/17] Version bump & Changelog --- CHANGELOG.md | 69 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 CHANGELOG.md diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 00000000..10edf68c --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,69 @@ +# Changelog +## [Unreleased] + +## [0.14.0] - 2024-11-05 + +### Added + +- New `haiku.rag-slim` package with minimal dependencies for users who want to install only what they need +- Evaluations package (`haiku.rag-evals`) for internal benchmarking and testing +- Improved search filtering performance by using pandas DataFrames for joins instead of SQL WHERE IN clauses + +### Changed + +- **BREAKING**: Restructured project into UV workspace with three packages: + - `haiku.rag-slim` - Core package with minimal dependencies + - `haiku.rag` - Full package with all extras (recommended for most users) + - `haiku.rag-evals` - Internal benchmarking and evaluation tools +- Migrated from `pydantic-ai` to `pydantic-ai-slim` with extras system +- Docling is now an optional dependency (install with `haiku.rag-slim[docling]`) +- Package metadata checks now use `haiku.rag-slim` (always present) instead of `haiku.rag` +- Docker image optimized: removed evaluations package, reducing installed packages from 307 to 259 +- Improved vector search performance through optimized score normalization + +### Fixed + +- ImportError now properly raised when optional docling dependency is missing + +## [0.13.3] - 2024-11-04 + +### Added + +- Support for Zero Entropy reranker +- Filter parameter to `search()` for filtering documents before search +- Filter parameter to CLI `search` command +- Filter parameter to CLI `list` command for filtering document listings +- Config option to pass custom configuration files to evaluation commands +- Document filtering now respects configured include/exclude patterns when using `add-src` with directories +- Max retries to insight_agent when producing structured output + +### Fixed + +- CLI now loads `.env` files at startup +- Info command no longer attempts to use deprecated `.env` settings +- Documentation typos + +## [0.13.2] - 2024-11-04 + +### Added + +- Gitignore-style pattern filtering for file monitoring using pathspec +- Include/exclude pattern documentation for FileMonitor + +### Changed + +- Moved monitor configuration to its own section in config +- Improved configuration documentation +- Updated dependencies + +## [0.13.1] - 2024-11-03 + +### Added + +- Initial version tracking + +[Unreleased]: https://github.com/ggozad/haiku.rag/compare/0.14.0...HEAD +[0.14.0]: https://github.com/ggozad/haiku.rag/compare/0.13.3...0.14.0 +[0.13.3]: https://github.com/ggozad/haiku.rag/compare/0.13.2...0.13.3 +[0.13.2]: https://github.com/ggozad/haiku.rag/compare/0.13.1...0.13.2 +[0.13.1]: https://github.com/ggozad/haiku.rag/releases/tag/0.13.1 From 21bb7b5aabd93b3ebf9d0ff319b128d1c003adec Mon Sep 17 00:00:00 2001 From: Yiorgis Gozadinos Date: Wed, 5 Nov 2025 13:16:03 +0200 Subject: [PATCH 16/17] Version bump & Changelog --- evaluations/pyproject.toml | 2 +- haiku_rag_slim/pyproject.toml | 2 +- pyproject.toml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/evaluations/pyproject.toml b/evaluations/pyproject.toml index 6f56d9ce..22ba5d4f 100644 --- a/evaluations/pyproject.toml +++ b/evaluations/pyproject.toml @@ -2,7 +2,7 @@ name = "haiku.rag-evals" description = "Internal benchmarking and evaluation scripts for haiku.rag" -version = "0.13.3" +version = "0.14.0" authors = [{ name = "Yiorgis Gozadinos", email = "ggozadinos@gmail.com" }] license = { text = "MIT" } requires-python = ">=3.12" diff --git a/haiku_rag_slim/pyproject.toml b/haiku_rag_slim/pyproject.toml index 3e8e83b9..ff8a9408 100644 --- a/haiku_rag_slim/pyproject.toml +++ b/haiku_rag_slim/pyproject.toml @@ -2,7 +2,7 @@ name = "haiku.rag-slim" description = "Agentic Retrieval Augmented Generation (RAG) with LanceDB - Minimal dependencies" -version = "0.13.3" +version = "0.14.0" authors = [{ name = "Yiorgis Gozadinos", email = "ggozadinos@gmail.com" }] license = { text = "MIT" } readme = { file = "README.md", content-type = "text/markdown" } diff --git a/pyproject.toml b/pyproject.toml index c6d9d88a..76ced0e1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,7 +2,7 @@ name = "haiku.rag" description = "Agentic Retrieval Augmented Generation (RAG) with LanceDB" -version = "0.13.3" +version = "0.14.0" authors = [{ name = "Yiorgis Gozadinos", email = "ggozadinos@gmail.com" }] license = { text = "MIT" } readme = { file = "README.md", content-type = "text/markdown" } From eb94f377dcc149ff9052389e2322fd394b39fb68 Mon Sep 17 00:00:00 2001 From: Yiorgis Gozadinos Date: Wed, 5 Nov 2025 13:16:24 +0200 Subject: [PATCH 17/17] version bump script --- scripts/bump_version.py | 131 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 131 insertions(+) create mode 100755 scripts/bump_version.py diff --git a/scripts/bump_version.py b/scripts/bump_version.py new file mode 100755 index 00000000..94d3a35f --- /dev/null +++ b/scripts/bump_version.py @@ -0,0 +1,131 @@ +#!/usr/bin/env python3 +""" +Version bumping script for haiku.rag workspace. + +Updates version in all pyproject.toml files and CHANGELOG.md. +""" + +import re +import sys +from datetime import date +from pathlib import Path + + +def get_current_version(file_path: Path) -> str: + """Extract current version from pyproject.toml.""" + content = file_path.read_text() + match = re.search(r'^version = "([^"]+)"', content, re.MULTILINE) + if not match: + raise ValueError(f"Could not find version in {file_path}") + return match.group(1) + + +def update_version_in_file(file_path: Path, new_version: str) -> None: + """Update version in a pyproject.toml file.""" + content = file_path.read_text() + updated = re.sub( + r'^version = "[^"]+"', f'version = "{new_version}"', content, flags=re.MULTILINE + ) + file_path.write_text(updated) + print(f"✓ Updated {file_path.relative_to(Path.cwd())}") + + +def update_changelog(changelog_path: Path, new_version: str) -> None: + """Update CHANGELOG.md with new version.""" + content = changelog_path.read_text() + today = date.today().isoformat() + + # Replace [Unreleased] with new version + updated = re.sub( + r"## \[Unreleased\]", + f"## [Unreleased]\n\n## [{new_version}] - {today}", + content, + count=1, + ) + + # Update comparison links + # Find the old [Unreleased] link + old_unreleased_match = re.search( + r"\[Unreleased\]: https://github\.com/ggozad/haiku\.rag/compare/([^.]+)\.\.\.HEAD", + updated, + ) + + if old_unreleased_match: + prev_version = old_unreleased_match.group(1) + + # Update [Unreleased] link + updated = re.sub( + r"\[Unreleased\]: https://github\.com/ggozad/haiku\.rag/compare/[^.]+\.\.\.HEAD", + f"[Unreleased]: https://github.com/ggozad/haiku.rag/compare/{new_version}...HEAD", + updated, + ) + + # Add new version link after [Unreleased] + updated = re.sub( + r"(\[Unreleased\]: https://github\.com/ggozad/haiku\.rag/compare/[^\n]+\n)", + f"\\1[{new_version}]: https://github.com/ggozad/haiku.rag/compare/{prev_version}...{new_version}\n", + updated, + ) + + changelog_path.write_text(updated) + print(f"✓ Updated {changelog_path.relative_to(Path.cwd())}") + + +def main(): + if len(sys.argv) != 2: + print("Usage: python scripts/bump_version.py ") + print("Example: python scripts/bump_version.py 0.14.0") + sys.exit(1) + + new_version = sys.argv[1] + + # Validate version format + if not re.match(r"^\d+\.\d+\.\d+$", new_version): + print(f"Error: Invalid version format '{new_version}'") + print("Version must be in format: X.Y.Z (e.g., 0.14.0)") + sys.exit(1) + + root = Path(__file__).parent.parent + + # Files to update + pyproject_files = [ + root / "pyproject.toml", + root / "haiku_rag_slim" / "pyproject.toml", + root / "evaluations" / "pyproject.toml", + ] + + changelog_file = root / "CHANGELOG.md" + + # Check all files exist + for file in pyproject_files + [changelog_file]: + if not file.exists(): + print(f"Error: {file} not found") + sys.exit(1) + + # Get current version from root pyproject.toml + current_version = get_current_version(pyproject_files[0]) + print(f"Current version: {current_version}") + print(f"New version: {new_version}") + print() + + # Confirm + response = input("Proceed with version bump? [y/N] ") + if response.lower() != "y": + print("Aborted.") + sys.exit(0) + + print() + + # Update all pyproject.toml files + for file in pyproject_files: + update_version_in_file(file, new_version) + + # Update CHANGELOG.md + update_changelog(changelog_file, new_version) + + print() + print(f"✓ Version bumped from {current_version} to {new_version}") + + +if __name__ == "__main__": + main()