From 94ff11ad25cfcb444683979eb4ec76d572c37790 Mon Sep 17 00:00:00 2001
From: Russ Ferriday <russ.ferriday@gmail.com>
Date: Tue, 24 Jun 2025 23:00:18 +0100
Subject: [PATCH 01/42] feat(tests): Add tests for app and cli modules

---
 tests/test_app.py | 203 ++++++++++++++++++++++++++++++++++++++++++++++
 tests/test_cli.py | 129 +++++++++++++++++++++++++++++
 2 files changed, 332 insertions(+)
 create mode 100644 tests/test_app.py
 create mode 100644 tests/test_cli.py

diff --git a/tests/test_app.py b/tests/test_app.py
new file mode 100644
index 00000000..fd8c604a
--- /dev/null
+++ b/tests/test_app.py
@@ -0,0 +1,203 @@
+import asyncio
+from pathlib import Path
+from unittest.mock import AsyncMock, MagicMock, patch
+
+import pytest
+from haiku.rag.app import HaikuRAGApp
+from haiku.rag.store.models.document import Document
+
+
+@pytest.fixture
+def app():
+    return HaikuRAGApp(db_path=Path(":memory:"))
+
+
+@pytest.fixture
+def app():
+    """Fixture for HaikuRAGApp."""
+    return HaikuRAGApp(db_path=Path(":memory:"))
+
+
+@pytest.mark.asyncio
+async def test_list_documents(app: HaikuRAGApp, monkeypatch):
+    """Test listing documents."""
+    mock_docs = [
+        Document(id=1, content="doc 1"),
+        Document(id=2, content="doc 2"),
+    ]
+    mock_client = AsyncMock()
+    mock_client.list_documents.return_value = mock_docs
+    # The async context manager should return the mock client itself
+    mock_client.__aenter__.return_value = mock_client
+
+    monkeypatch.setattr(app, "_rich_print_document", MagicMock())
+    monkeypatch.setattr(app.console, "print", MagicMock())
+
+    with patch("haiku.rag.app.HaikuRAG", return_value=mock_client):
+        await app.list_documents()
+
+    mock_client.list_documents.assert_called_once()
+    assert app._rich_print_document.call_count == len(mock_docs)
+    app._rich_print_document.assert_any_call(mock_docs[0], truncate=True)
+    app._rich_print_document.assert_any_call(mock_docs[1], truncate=True)
+
+
+@pytest.mark.asyncio
+async def test_add_document_from_text(app: HaikuRAGApp, monkeypatch):
+    """Test adding a document from text."""
+    mock_doc = Document(id=1, content="test document")
+    mock_client = AsyncMock()
+    mock_client.create_document.return_value = mock_doc
+    mock_client.__aenter__.return_value = mock_client
+
+    monkeypatch.setattr(app, "_rich_print_document", MagicMock())
+    mock_print = MagicMock()
+    monkeypatch.setattr(app.console, "print", mock_print)
+
+    with patch("haiku.rag.app.HaikuRAG", return_value=mock_client):
+        await app.add_document_from_text("test document")
+
+    mock_client.create_document.assert_called_once_with("test document")
+    app._rich_print_document.assert_called_once_with(mock_doc, truncate=True)
+    mock_print.assert_called_once_with(
+        "[b]Document with id [cyan]1[/cyan] added successfully.[/b]"
+    )
+
+
+@pytest.mark.asyncio
+async def test_add_document_from_source(app: HaikuRAGApp, monkeypatch):
+    """Test adding a document from a source path."""
+    mock_doc = Document(id=1, content="test document")
+    mock_client = AsyncMock()
+    mock_client.create_document_from_source.return_value = mock_doc
+    mock_client.__aenter__.return_value = mock_client
+
+    monkeypatch.setattr(app, "_rich_print_document", MagicMock())
+    mock_print = MagicMock()
+    monkeypatch.setattr(app.console, "print", mock_print)
+
+    file_path = Path("test.txt")
+    with patch("haiku.rag.app.HaikuRAG", return_value=mock_client):
+        await app.add_document_from_source(file_path)
+
+    mock_client.create_document_from_source.assert_called_once_with(file_path)
+    app._rich_print_document.assert_called_once_with(mock_doc, truncate=True)
+    mock_print.assert_called_once_with(
+        "[b]Document with id [cyan]1[/cyan] added successfully.[/b]"
+    )
+
+
+@pytest.mark.asyncio
+async def test_get_document(app: HaikuRAGApp, monkeypatch):
+    """Test getting a document."""
+    mock_doc = Document(id=1, content="test document")
+    mock_client = AsyncMock()
+    mock_client.get_document_by_id.return_value = mock_doc
+    mock_client.__aenter__.return_value = mock_client
+
+    monkeypatch.setattr(app, "_rich_print_document", MagicMock())
+
+    with patch("haiku.rag.app.HaikuRAG", return_value=mock_client):
+        await app.get_document(1)
+
+    mock_client.get_document_by_id.assert_called_once_with(1)
+    app._rich_print_document.assert_called_once_with(mock_doc, truncate=False)
+
+
+@pytest.mark.asyncio
+async def test_get_document_not_found(app: HaikuRAGApp, monkeypatch):
+    """Test getting a document that does not exist."""
+    mock_client = AsyncMock()
+    mock_client.get_document_by_id.return_value = None
+    mock_client.__aenter__.return_value = mock_client
+
+    mock_print = MagicMock()
+    monkeypatch.setattr(app.console, "print", mock_print)
+
+    with patch("haiku.rag.app.HaikuRAG", return_value=mock_client):
+        await app.get_document(1)
+
+    mock_client.get_document_by_id.assert_called_once_with(1)
+    mock_print.assert_called_once_with("[red]Document with id 1 not found.[/red]")
+
+
+@pytest.mark.asyncio
+async def test_delete_document(app: HaikuRAGApp, monkeypatch):
+    """Test deleting a document."""
+    mock_client = AsyncMock()
+    mock_client.__aenter__.return_value = mock_client
+
+    mock_print = MagicMock()
+    monkeypatch.setattr(app.console, "print", mock_print)
+
+    with patch("haiku.rag.app.HaikuRAG", return_value=mock_client):
+        await app.delete_document(1)
+
+    mock_client.delete_document.assert_called_once_with(1)
+    mock_print.assert_called_once_with("[b]Document 1 deleted successfully.[/b]")
+
+
+@pytest.mark.asyncio
+async def test_search(app: HaikuRAGApp, monkeypatch):
+    """Test searching for documents."""
+    mock_results = [("chunk1", 0.9), ("chunk2", 0.8)]
+    mock_client = AsyncMock()
+    mock_client.search.return_value = mock_results
+    mock_client.__aenter__.return_value = mock_client
+
+    monkeypatch.setattr(app, "_rich_print_search_result", MagicMock())
+
+    with patch("haiku.rag.app.HaikuRAG", return_value=mock_client):
+        await app.search("query")
+
+    mock_client.search.assert_called_once_with("query", limit=5, k=60)
+    assert app._rich_print_search_result.call_count == len(mock_results)
+
+
+@pytest.mark.asyncio
+async def test_search_no_results(app: HaikuRAGApp, monkeypatch):
+    """Test searching with no results."""
+    mock_client = AsyncMock()
+    mock_client.search.return_value = []
+    mock_client.__aenter__.return_value = mock_client
+
+    mock_print = MagicMock()
+    monkeypatch.setattr(app.console, "print", mock_print)
+
+    with patch("haiku.rag.app.HaikuRAG", return_value=mock_client):
+        await app.search("query")
+
+    mock_client.search.assert_called_once_with("query", limit=5, k=60)
+    mock_print.assert_called_once_with("[red]No results found.[/red]")
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("transport", ["stdio", "sse", "http", None])
+async def test_serve(app: HaikuRAGApp, monkeypatch, transport):
+    """Test the serve method with different transports."""
+    mock_server = AsyncMock()
+    mock_watcher = MagicMock()
+    mock_task = asyncio.create_task(asyncio.sleep(0))
+    mock_task.cancel = MagicMock()
+
+    monkeypatch.setattr("haiku.rag.app.create_mcp_server", MagicMock(return_value=mock_server))
+    monkeypatch.setattr("haiku.rag.app.FileWatcher", MagicMock(return_value=mock_watcher))
+    monkeypatch.setattr("asyncio.create_task", MagicMock(return_value=mock_task))
+
+    mock_client = AsyncMock()
+    mock_client.__aenter__.return_value = mock_client
+
+    with patch("haiku.rag.app.HaikuRAG", return_value=mock_client):
+        if transport:
+            await app.serve(transport=transport)
+        else:
+            await app.serve()
+
+    if transport == "stdio":
+        mock_server.run_stdio_async.assert_called_once()
+    elif transport == "sse":
+        mock_server.run_sse_async.assert_called_once_with("sse")
+    else:
+        mock_server.run_http_async.assert_called_once_with("streamable-http")
+
+    mock_task.cancel.assert_called_once()
\ No newline at end of file
diff --git a/tests/test_cli.py b/tests/test_cli.py
new file mode 100644
index 00000000..36289558
--- /dev/null
+++ b/tests/test_cli.py
@@ -0,0 +1,129 @@
+from unittest.mock import AsyncMock, MagicMock, patch
+
+from typer.testing import CliRunner
+
+from haiku.rag.cli import cli
+
+runner = CliRunner()
+
+
+def test_list_documents():
+    with patch("haiku.rag.cli.HaikuRAGApp") as mock_app:
+        mock_app_instance = MagicMock()
+        mock_app_instance.list_documents = AsyncMock()
+        mock_app.return_value = mock_app_instance
+
+        result = runner.invoke(cli, ["list"])
+
+        assert result.exit_code == 0
+        mock_app_instance.list_documents.assert_called_once()
+
+
+def test_add_document_text():
+    with patch("haiku.rag.cli.HaikuRAGApp") as mock_app:
+        mock_app_instance = MagicMock()
+        mock_app_instance.add_document_from_text = AsyncMock()
+        mock_app.return_value = mock_app_instance
+
+        result = runner.invoke(cli, ["add", "test document"])
+
+        assert result.exit_code == 0
+        mock_app_instance.add_document_from_text.assert_called_once_with(
+            text="test document"
+        )
+
+
+def test_add_document_src():
+    with patch("haiku.rag.cli.HaikuRAGApp") as mock_app:
+        mock_app_instance = MagicMock()
+        mock_app_instance.add_document_from_source = AsyncMock()
+        mock_app.return_value = mock_app_instance
+
+        result = runner.invoke(cli, ["add-src", "test.txt"])
+
+        assert result.exit_code == 0
+        mock_app_instance.add_document_from_source.assert_called_once()
+
+
+def test_get_document():
+    with patch("haiku.rag.cli.HaikuRAGApp") as mock_app:
+        mock_app_instance = MagicMock()
+        mock_app_instance.get_document = AsyncMock()
+        mock_app.return_value = mock_app_instance
+
+        result = runner.invoke(cli, ["get", "1"])
+
+        assert result.exit_code == 0
+        mock_app_instance.get_document.assert_called_once_with(doc_id=1)
+
+
+def test_delete_document():
+    with patch("haiku.rag.cli.HaikuRAGApp") as mock_app:
+        mock_app_instance = MagicMock()
+        mock_app_instance.delete_document = AsyncMock()
+        mock_app.return_value = mock_app_instance
+
+        result = runner.invoke(cli, ["delete", "1"])
+
+        assert result.exit_code == 0
+        mock_app_instance.delete_document.assert_called_once_with(doc_id=1)
+
+
+def test_search():
+    with patch("haiku.rag.cli.HaikuRAGApp") as mock_app:
+        mock_app_instance = MagicMock()
+        mock_app_instance.search = AsyncMock()
+        mock_app.return_value = mock_app_instance
+
+        result = runner.invoke(cli, ["search", "query"])
+
+        assert result.exit_code == 0
+        mock_app_instance.search.assert_called_once_with(query="query", limit=5, k=60)
+
+
+def test_serve():
+    with patch("haiku.rag.cli.HaikuRAGApp") as mock_app:
+        mock_app_instance = MagicMock()
+        mock_app_instance.serve = AsyncMock()
+        mock_app.return_value = mock_app_instance
+
+        result = runner.invoke(cli, ["serve"])
+
+        assert result.exit_code == 0
+        mock_app_instance.serve.assert_called_once_with(transport=None)
+
+
+def test_serve_stdio():
+    with patch("haiku.rag.cli.HaikuRAGApp") as mock_app:
+        mock_app_instance = MagicMock()
+        mock_app_instance.serve = AsyncMock()
+        mock_app.return_value = mock_app_instance
+
+        result = runner.invoke(cli, ["serve", "--stdio"])
+
+        assert result.exit_code == 0
+        mock_app_instance.serve.assert_called_once_with(transport="stdio")
+
+
+def test_serve_sse():
+    with patch("haiku.rag.cli.HaikuRAGApp") as mock_app:
+        mock_app_instance = MagicMock()
+        mock_app_instance.serve = AsyncMock()
+        mock_app.return_value = mock_app_instance
+
+        result = runner.invoke(cli, ["serve", "--sse"])
+
+        assert result.exit_code == 0
+        mock_app_instance.serve.assert_called_once_with(transport="sse")
+
+
+def test_serve_stdio_and_sse():
+    with patch("haiku.rag.cli.HaikuRAGApp") as mock_app:
+        mock_app_instance = MagicMock()
+        mock_app_instance.serve = AsyncMock()
+        mock_app.return_value = mock_app_instance
+
+        result = runner.invoke(cli, ["serve", "--stdio", "--sse"])
+
+        assert result.exit_code == 1
+        assert "Error: Cannot use both --stdio and --http options" in result.stdout
\ No newline at end of file

From c4c670bc405568b3d7dcb9548957e027e7686a1a Mon Sep 17 00:00:00 2001
From: Yiorgis Gozadinos <ggozadinos@gmail.com>
Date: Wed, 25 Jun 2025 17:50:19 +0300
Subject: [PATCH 02/42] Use proper mocks instead of setattr to satisfy typing.
 Fix minor typos

---
 tests/test_app.py | 49 ++++++++++++++++++++++++++---------------------
 1 file changed, 27 insertions(+), 22 deletions(-)

diff --git a/tests/test_app.py b/tests/test_app.py
index fd8c604a..f432d5ff 100644
--- a/tests/test_app.py
+++ b/tests/test_app.py
@@ -3,6 +3,7 @@ from pathlib import Path
 from unittest.mock import AsyncMock, MagicMock, patch
 
 import pytest
+
 from haiku.rag.app import HaikuRAGApp
 from haiku.rag.store.models.document import Document
 
@@ -12,12 +13,6 @@ def app():
     return HaikuRAGApp(db_path=Path(":memory:"))
 
 
-@pytest.fixture
-def app():
-    """Fixture for HaikuRAGApp."""
-    return HaikuRAGApp(db_path=Path(":memory:"))
-
-
 @pytest.mark.asyncio
 async def test_list_documents(app: HaikuRAGApp, monkeypatch):
     """Test listing documents."""
@@ -30,16 +25,18 @@ async def test_list_documents(app: HaikuRAGApp, monkeypatch):
     # The async context manager should return the mock client itself
     mock_client.__aenter__.return_value = mock_client
 
-    monkeypatch.setattr(app, "_rich_print_document", MagicMock())
-    monkeypatch.setattr(app.console, "print", MagicMock())
+    mock_rich_print = MagicMock()
+    mock_console_print = MagicMock()
+    monkeypatch.setattr(app, "_rich_print_document", mock_rich_print)
+    monkeypatch.setattr(app.console, "print", mock_console_print)
 
     with patch("haiku.rag.app.HaikuRAG", return_value=mock_client):
         await app.list_documents()
 
     mock_client.list_documents.assert_called_once()
-    assert app._rich_print_document.call_count == len(mock_docs)
-    app._rich_print_document.assert_any_call(mock_docs[0], truncate=True)
-    app._rich_print_document.assert_any_call(mock_docs[1], truncate=True)
+    assert mock_rich_print.call_count == len(mock_docs)
+    mock_rich_print.assert_any_call(mock_docs[0], truncate=True)
+    mock_rich_print.assert_any_call(mock_docs[1], truncate=True)
 
 
 @pytest.mark.asyncio
@@ -50,15 +47,16 @@ async def test_add_document_from_text(app: HaikuRAGApp, monkeypatch):
     mock_client.create_document.return_value = mock_doc
     mock_client.__aenter__.return_value = mock_client
 
-    monkeypatch.setattr(app, "_rich_print_document", MagicMock())
+    mock_rich_print = MagicMock()
     mock_print = MagicMock()
+    monkeypatch.setattr(app, "_rich_print_document", mock_rich_print)
     monkeypatch.setattr(app.console, "print", mock_print)
 
     with patch("haiku.rag.app.HaikuRAG", return_value=mock_client):
         await app.add_document_from_text("test document")
 
     mock_client.create_document.assert_called_once_with("test document")
-    app._rich_print_document.assert_called_once_with(mock_doc, truncate=True)
+    mock_rich_print.assert_called_once_with(mock_doc, truncate=True)
     mock_print.assert_called_once_with(
         "[b]Document with id [cyan]1[/cyan] added successfully.[/b]"
     )
@@ -72,8 +70,9 @@ async def test_add_document_from_source(app: HaikuRAGApp, monkeypatch):
     mock_client.create_document_from_source.return_value = mock_doc
     mock_client.__aenter__.return_value = mock_client
 
-    monkeypatch.setattr(app, "_rich_print_document", MagicMock())
+    mock_rich_print = MagicMock()
     mock_print = MagicMock()
+    monkeypatch.setattr(app, "_rich_print_document", mock_rich_print)
     monkeypatch.setattr(app.console, "print", mock_print)
 
     file_path = Path("test.txt")
@@ -81,7 +80,7 @@ async def test_add_document_from_source(app: HaikuRAGApp, monkeypatch):
         await app.add_document_from_source(file_path)
 
     mock_client.create_document_from_source.assert_called_once_with(file_path)
-    app._rich_print_document.assert_called_once_with(mock_doc, truncate=True)
+    mock_rich_print.assert_called_once_with(mock_doc, truncate=True)
     mock_print.assert_called_once_with(
         "[b]Document with id [cyan]1[/cyan] added successfully.[/b]"
     )
@@ -95,13 +94,14 @@ async def test_get_document(app: HaikuRAGApp, monkeypatch):
     mock_client.get_document_by_id.return_value = mock_doc
     mock_client.__aenter__.return_value = mock_client
 
-    monkeypatch.setattr(app, "_rich_print_document", MagicMock())
+    mock_rich_print = MagicMock()
+    monkeypatch.setattr(app, "_rich_print_document", mock_rich_print)
 
     with patch("haiku.rag.app.HaikuRAG", return_value=mock_client):
         await app.get_document(1)
 
     mock_client.get_document_by_id.assert_called_once_with(1)
-    app._rich_print_document.assert_called_once_with(mock_doc, truncate=False)
+    mock_rich_print.assert_called_once_with(mock_doc, truncate=False)
 
 
 @pytest.mark.asyncio
@@ -145,13 +145,14 @@ async def test_search(app: HaikuRAGApp, monkeypatch):
     mock_client.search.return_value = mock_results
     mock_client.__aenter__.return_value = mock_client
 
-    monkeypatch.setattr(app, "_rich_print_search_result", MagicMock())
+    mock_rich_print_search = MagicMock()
+    monkeypatch.setattr(app, "_rich_print_search_result", mock_rich_print_search)
 
     with patch("haiku.rag.app.HaikuRAG", return_value=mock_client):
         await app.search("query")
 
     mock_client.search.assert_called_once_with("query", limit=5, k=60)
-    assert app._rich_print_search_result.call_count == len(mock_results)
+    assert mock_rich_print_search.call_count == len(mock_results)
 
 
 @pytest.mark.asyncio
@@ -180,8 +181,12 @@ async def test_serve(app: HaikuRAGApp, monkeypatch, transport):
     mock_task = asyncio.create_task(asyncio.sleep(0))
     mock_task.cancel = MagicMock()
 
-    monkeypatch.setattr("haiku.rag.app.create_mcp_server", MagicMock(return_value=mock_server))
-    monkeypatch.setattr("haiku.rag.app.FileWatcher", MagicMock(return_value=mock_watcher))
+    monkeypatch.setattr(
+        "haiku.rag.app.create_mcp_server", MagicMock(return_value=mock_server)
+    )
+    monkeypatch.setattr(
+        "haiku.rag.app.FileWatcher", MagicMock(return_value=mock_watcher)
+    )
     monkeypatch.setattr("asyncio.create_task", MagicMock(return_value=mock_task))
 
     mock_client = AsyncMock()
@@ -200,4 +205,4 @@ async def test_serve(app: HaikuRAGApp, monkeypatch, transport):
     else:
         mock_server.run_http_async.assert_called_once_with("streamable-http")
 
-    mock_task.cancel.assert_called_once()
\ No newline at end of file
+    mock_task.cancel.assert_called_once()

From 26900d96879bd50668f5872aaa11ce7b172eb403 Mon Sep 17 00:00:00 2001
From: Yiorgis Gozadinos <ggozadinos@gmail.com>
Date: Tue, 24 Jun 2025 17:28:24 +0300
Subject: [PATCH 03/42] Run simple benchmarks

---
 BENCHMARKS.md                  |  9 ++++
 tests/generate_benchmark_db.py | 80 ++++++++++++++++++++++++++++++++++
 2 files changed, 89 insertions(+)
 create mode 100644 BENCHMARKS.md
 create mode 100644 tests/generate_benchmark_db.py

diff --git a/BENCHMARKS.md b/BENCHMARKS.md
new file mode 100644
index 00000000..bc66cab3
--- /dev/null
+++ b/BENCHMARKS.md
@@ -0,0 +1,9 @@
+# `haiku.rag` benchmarks
+
+We use [repliqa](https://huggingface.co/datasets/ServiceNow/repliqa) for the evaluation of `haiku.rag`
+
+* Recall
+
+We load the `News Stories` from `repliqa_3` which is 1035 documents, using `tests/generate_benchmark_db.py`, using the `mxbai-embed-large` Ollama embeddings.
+
+Subsequently, we run a search over the `question` for each row of the dataset and check whether we match the document that answers the question. The recall obtained is ~0.75 for matching in the top result, raising to ~0.75 for the top 3 results.
diff --git a/tests/generate_benchmark_db.py b/tests/generate_benchmark_db.py
new file mode 100644
index 00000000..01ce7c20
--- /dev/null
+++ b/tests/generate_benchmark_db.py
@@ -0,0 +1,80 @@
+from pathlib import Path
+
+from datasets import Dataset, load_dataset
+from tqdm import tqdm
+
+from haiku.rag.client import HaikuRAG
+
+
+async def populate_db():
+    if (Path(__file__).parent / "benchmark.sqlite").exists():
+        print("Benchmark database already exists. Skipping creation.")
+        return
+
+    ds: Dataset = load_dataset("ServiceNow/repliqa")["repliqa_3"]  # type: ignore
+    corpus = ds.filter(lambda doc: doc["document_topic"] == "News Stories")
+
+    async with HaikuRAG(Path(__file__).parent / "benchmark.sqlite") as rag:
+        for i, doc in enumerate(tqdm(corpus)):
+            await rag.create_document(
+                content=doc["document_extracted"],  # type: ignore
+                uri=doc["document_id"],  # type: ignore
+            )
+
+
+async def run_match_benchmark():
+    ds: Dataset = load_dataset("ServiceNow/repliqa")["repliqa_3"]  # type: ignore
+    corpus = ds.filter(lambda doc: doc["document_topic"] == "News Stories")
+
+    correct_at_1 = 0
+    correct_at_2 = 0
+    correct_at_3 = 0
+    total_queries = 0
+
+    async with HaikuRAG(Path(__file__).parent / "benchmark.sqlite") as rag:
+        for i, doc in enumerate(tqdm(corpus)):
+            doc_id = doc["document_id"]  # type: ignore
+            matches = await rag.search(
+                query=doc["question"],  # type: ignore
+                limit=3,
+            )
+
+            total_queries += 1
+
+            # Check position of correct document in results
+            for position, (chunk, _) in enumerate(matches):
+                retrieved = await rag.get_document_by_id(chunk.document_id)
+                if retrieved and retrieved.uri == doc_id:
+                    if position == 0:  # First position
+                        correct_at_1 += 1
+                        correct_at_2 += 1
+                        correct_at_3 += 1
+                    elif position == 1:  # Second position
+                        correct_at_2 += 1
+                        correct_at_3 += 1
+                    elif position == 2:  # Third position
+                        correct_at_3 += 1
+                    break
+
+    # Calculate recall metrics
+    recall_at_1 = correct_at_1 / total_queries
+    recall_at_2 = correct_at_2 / total_queries
+    recall_at_3 = correct_at_3 / total_queries
+
+    print(f"Total queries: {total_queries}")
+    print(f"Recall@1: {recall_at_1:.4f}")
+    print(f"Recall@2: {recall_at_2:.4f}")
+    print(f"Recall@3: {recall_at_3:.4f}")
+
+    return {"recall@1": recall_at_1, "recall@2": recall_at_2, "recall@3": recall_at_3}
+
+
+async def main():
+    await populate_db()
+    await run_match_benchmark()
+
+
+if __name__ == "__main__":
+    import asyncio
+
+    asyncio.run(main())

From 095d532a7a12801014a0a0210afcca6d90d0cd36 Mon Sep 17 00:00:00 2001
From: Yiorgis Gozadinos <ggozadinos@gmail.com>
Date: Tue, 24 Jun 2025 20:01:45 +0300
Subject: [PATCH 04/42] QA LLM, Ollama implementation

---
 src/haiku/rag/config.py                   |  3 +
 src/haiku/rag/monitor.py                  |  1 -
 src/haiku/rag/qa/__init__.py              |  0
 src/haiku/rag/qa/base.py                  | 16 ++++
 src/haiku/rag/qa/ollama.py                | 89 +++++++++++++++++++++++
 src/haiku/rag/qa/prompts.py               |  7 ++
 src/haiku/rag/store/repositories/chunk.py |  1 -
 7 files changed, 115 insertions(+), 2 deletions(-)
 create mode 100644 src/haiku/rag/qa/__init__.py
 create mode 100644 src/haiku/rag/qa/base.py
 create mode 100644 src/haiku/rag/qa/ollama.py
 create mode 100644 src/haiku/rag/qa/prompts.py

diff --git a/src/haiku/rag/config.py b/src/haiku/rag/config.py
index 26a29dfe..270c0aea 100644
--- a/src/haiku/rag/config.py
+++ b/src/haiku/rag/config.py
@@ -19,6 +19,9 @@ class AppConfig(BaseModel):
     EMBEDDINGS_MODEL: str = "mxbai-embed-large"
     EMBEDDINGS_VECTOR_DIM: int = 1024
 
+    QA_PROVIDER: str = "ollama"
+    QA_MODEL: str = "qwen3"
+
     CHUNK_SIZE: int = 256
     CHUNK_OVERLAP: int = 32
 
diff --git a/src/haiku/rag/monitor.py b/src/haiku/rag/monitor.py
index 97809e9e..618bb32a 100644
--- a/src/haiku/rag/monitor.py
+++ b/src/haiku/rag/monitor.py
@@ -49,7 +49,6 @@ class FileWatcher:
         try:
             uri = file.as_uri()
             existing_doc = await self.client.get_document_by_uri(uri)
-            print(uri)
             if existing_doc:
                 doc = await self.client.create_document_from_source(str(file))
                 logger.info(f"Updated document {existing_doc.id} from {file}")
diff --git a/src/haiku/rag/qa/__init__.py b/src/haiku/rag/qa/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/src/haiku/rag/qa/base.py b/src/haiku/rag/qa/base.py
new file mode 100644
index 00000000..6c8f8359
--- /dev/null
+++ b/src/haiku/rag/qa/base.py
@@ -0,0 +1,16 @@
+from haiku.rag.client import HaikuRAG
+from haiku.rag.qa.prompts import SYSTEM_PROMPT
+
+
+class QABase:
+    _model: str = ""
+    _system_prompt: str = SYSTEM_PROMPT
+
+    def __init__(self, client: HaikuRAG, model: str = ""):
+        self._model = model
+        self._client = client
+
+    async def answer(self, question: str) -> str:
+        raise NotImplementedError(
+            "QABase is an abstract class. Please implement the answer method in a subclass."
+        )
diff --git a/src/haiku/rag/qa/ollama.py b/src/haiku/rag/qa/ollama.py
new file mode 100644
index 00000000..273317b1
--- /dev/null
+++ b/src/haiku/rag/qa/ollama.py
@@ -0,0 +1,89 @@
+from ollama import AsyncClient
+
+from haiku.rag.client import HaikuRAG
+from haiku.rag.config import Config
+from haiku.rag.qa.base import QABase
+
+
+class QA(QABase):
+    def __init__(self, client: HaikuRAG, model: str = Config.QA_MODEL):
+        super().__init__(client, model or self._model)
+
+    async def answer(self, question: str) -> str:
+        ollama_client = AsyncClient(host=Config.OLLAMA_BASE_URL)
+
+        # Define the search tool
+        tools = [
+            {
+                "type": "function",
+                "function": {
+                    "name": "search_documents",
+                    "description": "Search the knowledge base for relevant documents",
+                    "parameters": {
+                        "type": "object",
+                        "properties": {
+                            "query": {
+                                "type": "string",
+                                "description": "The search query to find relevant documents",
+                            },
+                            "limit": {
+                                "type": "integer",
+                                "description": "Maximum number of results to return",
+                                "default": 3,
+                            },
+                        },
+                        "required": ["query"],
+                    },
+                },
+            }
+        ]
+
+        messages = [
+            {"role": "system", "content": self._system_prompt},
+            {"role": "user", "content": question},
+        ]
+
+        # Initial response with tool calling
+        response = await ollama_client.chat(
+            model=self._model,
+            messages=messages,
+            tools=tools,
+            options={"temperature": 0.0, "seed": 42},
+            think=False,
+        )
+
+        if response.get("message", {}).get("tool_calls"):
+            for tool_call in response["message"]["tool_calls"]:
+                if tool_call["function"]["name"] == "search_documents":
+                    args = tool_call["function"]["arguments"]
+                    query = args.get("query", question)
+                    limit = int(args.get("limit", 3))
+
+                    search_results = await self._client.search(query, limit=limit)
+
+                    context_chunks = []
+                    for chunk, score in search_results:
+                        context_chunks.append(
+                            f"Content: {chunk.content}\nScore: {score:.4f}"
+                        )
+
+                    context = "\n\n".join(context_chunks)
+
+                    messages.append(response["message"])
+                    messages.append(
+                        {
+                            "role": "tool",
+                            "content": context,
+                            "tool_call_id": tool_call.get("id", "search_tool"),
+                        }
+                    )
+
+            final_response = await ollama_client.chat(
+                model=self._model,
+                messages=messages,
+                think=False,
+                options={"temperature": 0.0, "seed": 42},
+            )
+            return final_response["message"]["content"]
+        else:
+            return response["message"]["content"]
diff --git a/src/haiku/rag/qa/prompts.py b/src/haiku/rag/qa/prompts.py
new file mode 100644
index 00000000..fc8f2c9b
--- /dev/null
+++ b/src/haiku/rag/qa/prompts.py
@@ -0,0 +1,7 @@
+SYSTEM_PROMPT = """
+You are a helpful assistant that uses a RAG library to answer the user's prompt.
+Your task is to provide a concise and accurate answer based on the provided context.
+You should ask the provided tools to find relevant documents and then use the content of those documents to answer the question.
+Never make up information, always use the context to answer the question.
+If the context does not contain enough information to answer the question, respond with "I cannot answer that based on the provided context."
+"""
diff --git a/src/haiku/rag/store/repositories/chunk.py b/src/haiku/rag/store/repositories/chunk.py
index b80e27db..1ffec47b 100644
--- a/src/haiku/rag/store/repositories/chunk.py
+++ b/src/haiku/rag/store/repositories/chunk.py
@@ -325,7 +325,6 @@ class ChunkRepository(BaseRepository[Chunk]):
         words = re.findall(r"\b\w+\b", query.lower())
         # Join with OR to find chunks containing any of the keywords
         fts_query = " OR ".join(words) if words else query
-
         # Perform hybrid search using RRF (Reciprocal Rank Fusion)
         cursor.execute(
             """

From 2855dd7c1341c2fd61d7586d79edf47311b5f255 Mon Sep 17 00:00:00 2001
From: Yiorgis Gozadinos <ggozadinos@gmail.com>
Date: Tue, 24 Jun 2025 20:02:07 +0300
Subject: [PATCH 05/42] Use LLM-as-a-judge to test QA

---
 tests/__init__.py              |  0
 tests/conftest.py              |  7 ++++
 tests/generate_benchmark_db.py | 46 +++++++++++++++++++++--
 tests/llm_judge.py             | 68 ++++++++++++++++++++++++++++++++++
 tests/test_qa.py               | 42 +++++++++++++++++++++
 5 files changed, 160 insertions(+), 3 deletions(-)
 create mode 100644 tests/__init__.py
 create mode 100644 tests/llm_judge.py
 create mode 100644 tests/test_qa.py

diff --git a/tests/__init__.py b/tests/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/tests/conftest.py b/tests/conftest.py
index 31ed812d..2dcea549 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -3,6 +3,8 @@ from pathlib import Path
 import pytest
 from datasets import Dataset, load_dataset, load_from_disk
 
+from .llm_judge import LLMJudge
+
 
 @pytest.fixture(scope="session")
 def qa_corpus() -> Dataset:
@@ -16,3 +18,8 @@ def qa_corpus() -> Dataset:
         corpus = ds.filter(lambda doc: doc["document_topic"] == "News Stories")
         corpus.save_to_disk(ds_path)
         return corpus
+
+
+@pytest.fixture(scope="session")
+def llm_judge() -> LLMJudge:
+    return LLMJudge()
diff --git a/tests/generate_benchmark_db.py b/tests/generate_benchmark_db.py
index 01ce7c20..b37519a8 100644
--- a/tests/generate_benchmark_db.py
+++ b/tests/generate_benchmark_db.py
@@ -1,13 +1,16 @@
+import asyncio
 from pathlib import Path
 
 from datasets import Dataset, load_dataset
+from llm_judge import LLMJudge
 from tqdm import tqdm
 
 from haiku.rag.client import HaikuRAG
+from haiku.rag.qa.ollama import QA
 
 
 async def populate_db():
-    if (Path(__file__).parent / "benchmark.sqlite").exists():
+    if (Path(__file__).parent / "data" / "benchmark.sqlite").exists():
         print("Benchmark database already exists. Skipping creation.")
         return
 
@@ -61,6 +64,7 @@ async def run_match_benchmark():
     recall_at_2 = correct_at_2 / total_queries
     recall_at_3 = correct_at_3 / total_queries
 
+    print("\n=== Retrieval Benchmark Results ===")
     print(f"Total queries: {total_queries}")
     print(f"Recall@1: {recall_at_1:.4f}")
     print(f"Recall@2: {recall_at_2:.4f}")
@@ -69,12 +73,48 @@ async def run_match_benchmark():
     return {"recall@1": recall_at_1, "recall@2": recall_at_2, "recall@3": recall_at_3}
 
 
+async def run_qa_benchmark():
+    """Run QA benchmarking on the corpus."""
+    ds: Dataset = load_dataset("ServiceNow/repliqa")["repliqa_3"]  # type: ignore
+    corpus = ds.filter(lambda doc: doc["document_topic"] == "News Stories")
+
+    judge = LLMJudge()
+    correct_answers = 0
+    total_questions = 0
+
+    async with HaikuRAG(Path(__file__).parent / "benchmark.sqlite") as rag:
+        qa = QA(rag)
+
+        for i, doc in enumerate(tqdm(corpus, desc="QA Benchmarking")):
+            question = doc["question"]  # type: ignore
+            expected_answer = doc["answer"]  # type: ignore
+
+            generated_answer = await qa.answer(question)
+            is_equivalent = await judge.judge_answers(
+                question, generated_answer, expected_answer
+            )
+
+            if is_equivalent:
+                correct_answers += 1
+            total_questions += 1
+
+    accuracy = correct_answers / total_questions if total_questions > 0 else 0
+
+    print("\n=== QA Benchmark Results ===")
+    print(f"Total questions: {total_questions}")
+    print(f"Correct answers: {correct_answers}")
+    print(f"QA Accuracy: {accuracy:.4f} ({accuracy * 100:.2f}%)")
+
+
 async def main():
     await populate_db()
+
+    print("Running retrieval benchmarks...")
     await run_match_benchmark()
 
+    print("\nRunning QA benchmarks...")
+    await run_qa_benchmark()
+
 
 if __name__ == "__main__":
-    import asyncio
-
     asyncio.run(main())
diff --git a/tests/llm_judge.py b/tests/llm_judge.py
new file mode 100644
index 00000000..66bfd2cb
--- /dev/null
+++ b/tests/llm_judge.py
@@ -0,0 +1,68 @@
+import json
+
+from ollama import AsyncClient
+from pydantic import BaseModel
+
+from haiku.rag.config import Config
+
+
+class LLMJudgeResponseSchema(BaseModel):
+    equivalent: bool
+
+
+class LLMJudge:
+    """LLM-as-judge for evaluating answer equivalence using Ollama."""
+
+    def __init__(self, model: str = "qwen3"):
+        self.model = model
+        self.client = AsyncClient(host=Config.OLLAMA_BASE_URL)
+
+    async def judge_answers(
+        self, question: str, answer: str, expected_answer: str
+    ) -> bool:
+        """
+        Judge whether two answers are equivalent for a given question.
+
+        Args:
+            question: The original question
+            answer: The generated answer to evaluate
+            expected_answer: The reference/expected answer
+
+        Returns:
+            Dictionary with judgment result:
+            - equivalent: bool indicating if answers are equivalent
+            - explanation: str explaining the reasoning
+            - score: str rating from 1-5
+        """
+
+        prompt = f"""
+        You are an expert judge evaluating the equivalence of two answers to the same question.
+
+        Question: {question}
+
+        Generated Answer: {answer}
+
+        Expected Answer: {expected_answer}
+
+        Your task is to determine if these two answers are equivalent in meaning and both correctly answer the question. Consider:
+
+        1. Do both answers provide the same answer?
+        2. Do both answers directly address the question asked?
+        3. Minor differences in wording or style are acceptable if the meaning of the answer is the same.
+
+        Be strict but fair in your evaluation. Focus on factual correctness and whether both answers would satisfy someone asking the question."""
+
+        response = await self.client.chat(
+            model=self.model,
+            messages=[{"role": "user", "content": prompt}],
+            format=LLMJudgeResponseSchema.model_json_schema(),
+            think=False,
+        )
+
+        answer = response["message"]["content"].strip()
+        try:
+            res = json.loads(answer)
+            assert "equivalent" in res, "Response must contain 'equivalent' key"
+            return res["equivalent"]
+        except json.JSONDecodeError:
+            assert False, "Response is not valid JSON"
diff --git a/tests/test_qa.py b/tests/test_qa.py
new file mode 100644
index 00000000..18496df7
--- /dev/null
+++ b/tests/test_qa.py
@@ -0,0 +1,42 @@
+from typing import TYPE_CHECKING
+
+import pytest
+from datasets import Dataset
+
+from haiku.rag.client import HaikuRAG
+from haiku.rag.qa.ollama import QA
+
+if TYPE_CHECKING:
+    import sys
+    from pathlib import Path
+
+    sys.path.append(str(Path(__file__).parent))
+    from llm_judge import LLMJudge
+
+
+@pytest.mark.asyncio
+async def test_qa_with_dataset_question(qa_corpus: Dataset, llm_judge: "LLMJudge"):
+    """Test QA with actual question from the dataset using LLM judge."""
+    client = HaikuRAG(":memory:")
+    qa = QA(client)
+
+    # Use the first document from the corpus
+    doc = qa_corpus[1]
+
+    # Add the document to database
+    await client.create_document(
+        content=doc["document_extracted"], uri=doc["document_id"]
+    )
+
+    question = doc["question"]
+    expected_answer = doc["answer"]
+
+    answer = await qa.answer(question)
+    # Use LLM judge to evaluate answer equivalence
+    is_equivalent = await llm_judge.judge_answers(question, answer, expected_answer)
+
+    assert isinstance(answer, str)
+    assert len(answer) > 0
+    assert is_equivalent, (
+        f"Generated answer not equivalent to expected answer.\nQuestion: {question}\nGenerated: {answer}\nExpected: {expected_answer}"
+    )

From ca45f8e5d405e0f688cf5617b63f1aa5e2210c5a Mon Sep 17 00:00:00 2001
From: Yiorgis Gozadinos <ggozadinos@gmail.com>
Date: Wed, 25 Jun 2025 19:01:57 +0300
Subject: [PATCH 06/42] Fix db path

---
 src/haiku/rag/qa/ollama.py     |  6 ++++--
 tests/generate_benchmark_db.py | 19 ++++++++++++++-----
 2 files changed, 18 insertions(+), 7 deletions(-)

diff --git a/src/haiku/rag/qa/ollama.py b/src/haiku/rag/qa/ollama.py
index 273317b1..021190ca 100644
--- a/src/haiku/rag/qa/ollama.py
+++ b/src/haiku/rag/qa/ollama.py
@@ -4,6 +4,8 @@ from haiku.rag.client import HaikuRAG
 from haiku.rag.config import Config
 from haiku.rag.qa.base import QABase
 
+OLLAMA_OPTIONS = {"temperature": 0.0, "seed": 42, "num_ctx": 64000}
+
 
 class QA(QABase):
     def __init__(self, client: HaikuRAG, model: str = Config.QA_MODEL):
@@ -48,7 +50,7 @@ class QA(QABase):
             model=self._model,
             messages=messages,
             tools=tools,
-            options={"temperature": 0.0, "seed": 42},
+            options=OLLAMA_OPTIONS,
             think=False,
         )
 
@@ -82,7 +84,7 @@ class QA(QABase):
                 model=self._model,
                 messages=messages,
                 think=False,
-                options={"temperature": 0.0, "seed": 42},
+                options=OLLAMA_OPTIONS,
             )
             return final_response["message"]["content"]
         else:
diff --git a/tests/generate_benchmark_db.py b/tests/generate_benchmark_db.py
index b37519a8..bbebb5eb 100644
--- a/tests/generate_benchmark_db.py
+++ b/tests/generate_benchmark_db.py
@@ -8,16 +8,18 @@ from tqdm import tqdm
 from haiku.rag.client import HaikuRAG
 from haiku.rag.qa.ollama import QA
 
+db_path = Path(__file__).parent / "data" / "benchmark.sqlite"
+
 
 async def populate_db():
-    if (Path(__file__).parent / "data" / "benchmark.sqlite").exists():
+    if (db_path).exists():
         print("Benchmark database already exists. Skipping creation.")
         return
 
     ds: Dataset = load_dataset("ServiceNow/repliqa")["repliqa_3"]  # type: ignore
     corpus = ds.filter(lambda doc: doc["document_topic"] == "News Stories")
 
-    async with HaikuRAG(Path(__file__).parent / "benchmark.sqlite") as rag:
+    async with HaikuRAG(db_path) as rag:
         for i, doc in enumerate(tqdm(corpus)):
             await rag.create_document(
                 content=doc["document_extracted"],  # type: ignore
@@ -34,7 +36,7 @@ async def run_match_benchmark():
     correct_at_3 = 0
     total_queries = 0
 
-    async with HaikuRAG(Path(__file__).parent / "benchmark.sqlite") as rag:
+    async with HaikuRAG(db_path) as rag:
         for i, doc in enumerate(tqdm(corpus)):
             doc_id = doc["document_id"]  # type: ignore
             matches = await rag.search(
@@ -73,16 +75,19 @@ async def run_match_benchmark():
     return {"recall@1": recall_at_1, "recall@2": recall_at_2, "recall@3": recall_at_3}
 
 
-async def run_qa_benchmark():
+async def run_qa_benchmark(k: int | None = None):
     """Run QA benchmarking on the corpus."""
     ds: Dataset = load_dataset("ServiceNow/repliqa")["repliqa_3"]  # type: ignore
     corpus = ds.filter(lambda doc: doc["document_topic"] == "News Stories")
 
+    if k is not None:
+        corpus = corpus.select(range(min(k, len(corpus))))
+
     judge = LLMJudge()
     correct_answers = 0
     total_questions = 0
 
-    async with HaikuRAG(Path(__file__).parent / "benchmark.sqlite") as rag:
+    async with HaikuRAG(db_path) as rag:
         qa = QA(rag)
 
         for i, doc in enumerate(tqdm(corpus, desc="QA Benchmarking")):
@@ -93,6 +98,10 @@ async def run_qa_benchmark():
             is_equivalent = await judge.judge_answers(
                 question, generated_answer, expected_answer
             )
+            print(f"Question: {question}")
+            print(f"Expected: {expected_answer}")
+            print(f"Generated: {generated_answer}")
+            print(f"Equivalent: {is_equivalent}\n")
 
             if is_equivalent:
                 correct_answers += 1

From a704f300bc8bb3a4f7d09a322ee1e1301afc96f0 Mon Sep 17 00:00:00 2001
From: Yiorgis Gozadinos <ggozadinos@gmail.com>
Date: Wed, 25 Jun 2025 20:16:08 +0300
Subject: [PATCH 07/42] Initial skeleton for docs

---
 docs/.DS_Store        | Bin 0 -> 6148 bytes
 docs/cli.md           |  74 ++++++++++++
 docs/configuration.md |  43 +++++++
 docs/img/.DS_Store    | Bin 0 -> 6148 bytes
 docs/index.md         |  52 ++++++++
 docs/installation.md  |  31 +++++
 docs/mcp.md           |  40 +++++++
 docs/python.md        | 101 ++++++++++++++++
 docs/server.md        |  41 +++++++
 mkdocs.yml            |  77 ++++++++++++
 pyproject.toml        |   2 +
 uv.lock               | 271 ++++++++++++++++++++++++++++++++++++++++++
 12 files changed, 732 insertions(+)
 create mode 100644 docs/.DS_Store
 create mode 100644 docs/cli.md
 create mode 100644 docs/configuration.md
 create mode 100644 docs/img/.DS_Store
 create mode 100644 docs/index.md
 create mode 100644 docs/installation.md
 create mode 100644 docs/mcp.md
 create mode 100644 docs/python.md
 create mode 100644 docs/server.md
 create mode 100644 mkdocs.yml

diff --git a/docs/.DS_Store b/docs/.DS_Store
new file mode 100644
index 0000000000000000000000000000000000000000..940e942902c4c95c53cc11d73aa9ce43019c31f2
GIT binary patch
literal 6148
zcmeH~J!%6%427R!7lt%jx}3%b$PET#pTHMLVK4#Pfk0Bv(ew1vWSu%J;R&QS(yZ9s
zuh>}uu>I%x1(*PA=&sm#n3*wO;SD!jzD^(a>-+t}idTWBh?%i6VYXk}5)lvq5fA|p
z5P<~|$Wt7f=LJ2J9z_I1U>OAb`_SmFy>z6;r-LCz0P33MFs@^kpf)d1d+A7Jg=RH9
zShZS=AzqJmYOCvd=}66XSPdUmcQ&75XqN4;#)M`)L_q{ZU`Ak-`Q+#Sk^bBKKWkAc
z0wVCw2-x~?I_&vUb+$gdp4VTi>gz$L#^nq@egc^IQM{#xaliS3+Dk_&D>VHG1O^2W
H_)`MkK;;pJ

literal 0
HcmV?d00001

diff --git a/docs/cli.md b/docs/cli.md
new file mode 100644
index 00000000..e5100062
--- /dev/null
+++ b/docs/cli.md
@@ -0,0 +1,74 @@
+# Command Line Interface
+
+The `haiku-rag` CLI provides complete document management functionality.
+
+## Document Management
+
+### List Documents
+
+```bash
+haiku-rag list
+```
+
+### Add Documents
+
+From text:
+```bash
+haiku-rag add "Your document content here"
+```
+
+From file or URL:
+```bash
+haiku-rag add-src /path/to/document.pdf
+haiku-rag add-src https://example.com/article.html
+```
+
+### Get Document
+
+```bash
+haiku-rag get 1
+```
+
+### Delete Document
+
+```bash
+haiku-rag delete 1
+```
+
+## Search
+
+Basic search:
+```bash
+haiku-rag search "machine learning"
+```
+
+With options:
+```bash
+haiku-rag search "python programming" --limit 10 --k 100
+```
+
+## Server
+
+Start the MCP server:
+```bash
+# HTTP transport (default)
+haiku-rag serve
+
+# stdio transport
+haiku-rag serve --stdio
+
+# SSE transport
+haiku-rag serve --sse
+```
+
+## Options
+
+All commands support:
+- `--db` - Specify custom database path
+- `-h` - Show help for specific command
+
+Example:
+```bash
+haiku-rag list --db /path/to/custom.db
+haiku-rag add -h
+```
diff --git a/docs/configuration.md b/docs/configuration.md
new file mode 100644
index 00000000..6e7c6610
--- /dev/null
+++ b/docs/configuration.md
@@ -0,0 +1,43 @@
+# Configuration
+
+Configuration is done through environment variables.
+
+## File Monitoring
+
+Set directories to monitor for automatic indexing:
+
+```bash
+# Monitor single directory
+export MONITOR_DIRECTORIES="/path/to/documents"
+
+# Monitor multiple directories
+export MONITOR_DIRECTORIES="/path/to/documents,/another_path/to/documents"
+```
+
+## Embedding Providers
+
+### Ollama (Default)
+
+```bash
+EMBEDDINGS_PROVIDER="ollama"
+EMBEDDINGS_MODEL="mxbai-embed-large"
+EMBEDDINGS_VECTOR_DIM=1024
+```
+
+### VoyageAI
+
+```bash
+EMBEDDINGS_PROVIDER="voyageai"
+EMBEDDINGS_MODEL="voyage-3.5"
+EMBEDDINGS_VECTOR_DIM=1024
+VOYAGE_API_KEY="your-api-key"
+```
+
+### OpenAI
+
+```bash
+EMBEDDINGS_PROVIDER="openai"
+EMBEDDINGS_MODEL="text-embedding-3-small"  # or text-embedding-3-large
+EMBEDDINGS_VECTOR_DIM=1536
+OPENAI_API_KEY="your-api-key"
+```
diff --git a/docs/img/.DS_Store b/docs/img/.DS_Store
new file mode 100644
index 0000000000000000000000000000000000000000..5008ddfcf53c02e82d7eee2e57c38e5672ef89f6
GIT binary patch
literal 6148
zcmeH~Jr2S!425mzP>H1@V-^m;4Wg<&0T*E43hX&L&p$$qDprKhvt+--jT7}7np#A3
zem<@ulZcFPQ@L2!n>{z**<q8>++&mCkOWA81W14cNZ<zv;LbK1Poaz?KmsK2CSc!(
z0ynLxE!0092;Krf2c+FF_Fe*7ECH>lEfg7;MkzE(HCqgga^y>{tEnwC%0;vJ&^%eQ
zLs35+`xjp>T0<F0fCPF1$Cyrb|F7^5{eNG?83~ZUUlGt@xh*qZDeu<Z%US-OSsOPv
j)R!Z4KLME7ReXlK;d!wEw5GODWMKRea10D2@KpjYNUI8I

literal 0
HcmV?d00001

diff --git a/docs/index.md b/docs/index.md
new file mode 100644
index 00000000..c81ab0f2
--- /dev/null
+++ b/docs/index.md
@@ -0,0 +1,52 @@
+# haiku.rag
+
+A Retrieval-Augmented Generation (RAG) library on SQLite.
+
+## Features
+
+- **Local SQLite**: No need to run additional servers
+- **Support for various embedding providers**: Ollama, VoyageAI, OpenAI or add your own
+- **Hybrid Search**: Vector search using `sqlite-vec` combined with full-text search `FTS5`, using Reciprocal Rank Fusion
+- **File monitoring**: Automatically index files when run as a server
+- **Extended file format support**: Parse 40+ file formats including PDF, DOCX, HTML, Markdown, audio and more. Or add a URL!
+- **MCP server**: Exposes functionality as MCP tools
+- **CLI commands**: Access all functionality from your terminal
+- **Python client**: Call `haiku.rag` from your own python applications
+
+## Quick Start
+
+Install haiku.rag:
+```bash
+uv pip install haiku.rag
+```
+
+Use from Python:
+```python
+from haiku.rag.client import HaikuRAG
+
+async with HaikuRAG("database.db") as client:
+    # Add a document
+    doc = await client.create_document("Your content here")
+
+    # Search documents
+    results = await client.search("query")
+```
+
+Or use the CLI:
+```bash
+haiku-rag add "Your document content"
+haiku-rag search "query"
+```
+
+## Documentation
+
+- [Installation](installation.md) - Install haiku.rag with different providers
+- [Configuration](configuration.md) - Environment variables and settings
+- [CLI](cli.md) - Command line interface usage
+- [Server](server.md) - File monitoring and server mode
+- [MCP](mcp.md) - Model Context Protocol integration
+- [Python](python.md) - Python API reference
+
+## License
+
+This project is licensed under the [MIT License](https://raw.githubusercontent.com/ggozad/haiku.rag/main/LICENSE).
diff --git a/docs/installation.md b/docs/installation.md
new file mode 100644
index 00000000..b3da3847
--- /dev/null
+++ b/docs/installation.md
@@ -0,0 +1,31 @@
+# Installation
+
+## Basic Installation
+
+```bash
+uv pip install haiku.rag
+```
+
+By default, Ollama (with the `mxbai-embed-large` model) is used for embeddings.
+
+## Provider-Specific Installation
+
+For other embedding providers, install with extras:
+
+### VoyageAI
+
+```bash
+uv pip install haiku.rag --extra voyageai
+```
+
+### OpenAI
+
+```bash
+uv pip install haiku.rag --extra openai
+```
+
+## Requirements
+
+- Python 3.10+
+- SQLite 3.38+
+- Ollama (for default embeddings)
diff --git a/docs/mcp.md b/docs/mcp.md
new file mode 100644
index 00000000..58adf7db
--- /dev/null
+++ b/docs/mcp.md
@@ -0,0 +1,40 @@
+# Model Context Protocol (MCP)
+
+The MCP server exposes RAG functionality as tools for AI assistants like Claude Desktop.
+
+## Available Tools
+
+### Document Management
+
+- `add_document_from_file` - Add documents from local file paths
+- `add_document_from_url` - Add documents from URLs
+- `add_document_from_text` - Add documents from raw text content
+- `get_document` - Retrieve specific documents by ID
+- `list_documents` - List all documents with pagination
+- `delete_document` - Delete documents by ID
+
+### Search
+
+- `search_documents` - Search documents using hybrid search (vector + full-text)
+
+## Starting MCP Server
+
+The MCP server starts automatically with the serve command:
+
+```bash
+# Default HTTP transport
+haiku-rag serve
+
+# stdio transport (for Claude Desktop)
+haiku-rag serve --stdio
+
+# SSE transport
+haiku-rag serve --sse
+```
+
+## Integration
+
+The MCP server follows the Model Context Protocol specification, making it compatible with:
+- Claude Desktop
+- Other MCP-compatible AI assistants
+- Custom MCP clients
diff --git a/docs/python.md b/docs/python.md
new file mode 100644
index 00000000..3b81c232
--- /dev/null
+++ b/docs/python.md
@@ -0,0 +1,101 @@
+# Python API
+
+Use `haiku.rag` directly in your Python applications.
+
+## Basic Usage
+
+```python
+from pathlib import Path
+from haiku.rag.client import HaikuRAG
+
+# Use as async context manager (recommended)
+async with HaikuRAG("path/to/database.db") as client:
+    # Your code here
+    pass
+```
+
+## Document Management
+
+### Creating Documents
+
+From text:
+```python
+doc = await client.create_document(
+    content="Your document content here",
+    uri="doc://example",
+    metadata={"source": "manual", "topic": "example"}
+)
+```
+
+From file:
+```python
+doc = await client.create_document_from_source("path/to/document.pdf")
+```
+
+From URL:
+```python
+doc = await client.create_document_from_source("https://example.com/article.html")
+```
+
+### Retrieving Documents
+
+By ID:
+```python
+doc = await client.get_document_by_id(1)
+```
+
+By URI:
+```python
+doc = await client.get_document_by_uri("file:///path/to/document.pdf")
+```
+
+List all documents:
+```python
+docs = await client.list_documents(limit=10, offset=0)
+```
+
+### Updating Documents
+
+```python
+doc.content = "Updated content"
+await client.update_document(doc)
+```
+
+### Deleting Documents
+
+```python
+await client.delete_document(doc.id)
+```
+
+## Searching Documents
+
+Basic search:
+```python
+results = await client.search("machine learning algorithms", limit=5)
+for chunk, score in results:
+    print(f"Score: {score:.3f}")
+    print(f"Content: {chunk.content}")
+    print(f"Document ID: {chunk.document_id}")
+```
+
+With options:
+```python
+results = await client.search(
+    query="machine learning",
+    limit=5,  # Maximum results to return
+    k=60      # RRF parameter for reciprocal rank fusion
+)
+
+# Process results
+for chunk, relevance_score in results:
+    print(f"Relevance: {relevance_score:.3f}")
+    print(f"Content: {chunk.content}")
+    print(f"From document: {chunk.document_id}")
+```
+
+## Search Technology
+
+`haiku.rag` uses hybrid search combining:
+- **Vector search** using `sqlite-vec` for semantic similarity
+- **Full-text search** using SQLite's `FTS5` for keyword matching
+- **Reciprocal Rank Fusion** to combine and rank results
diff --git a/docs/server.md b/docs/server.md
new file mode 100644
index 00000000..d9fc5d71
--- /dev/null
+++ b/docs/server.md
@@ -0,0 +1,41 @@
+# Server Mode
+
+The server provides automatic file monitoring and MCP functionality.
+
+## Starting the Server
+
+```bash
+haiku-rag serve
+```
+
+Transport options:
+- `--http` (default) - Streamable HTTP transport
+- `--stdio` - Standard input/output transport
+- `--sse` - Server-sent events transport
+
+## File Monitoring
+
+Set `MONITOR_DIRECTORIES` environment variable to enable automatic file monitoring:
+
+```bash
+export MONITOR_DIRECTORIES="/path/to/documents"
+haiku-rag serve
+```
+
+### Monitoring Features
+
+- **Startup**: Scans all monitored directories and adds new files
+- **File Added/Modified**: Automatically parses and updates documents
+- **File Deleted**: Removes corresponding documents from database
+
+### Supported Formats
+
+The server can parse 40+ file formats including:
+- PDF documents
+- Microsoft Office (DOCX, XLSX, PPTX)
+- HTML and Markdown
+- Plain text files
+- Audio files
+- And more...
+
+URLs are also supported for web content.
diff --git a/mkdocs.yml b/mkdocs.yml
new file mode 100644
index 00000000..e9767caa
--- /dev/null
+++ b/mkdocs.yml
@@ -0,0 +1,77 @@
+site_name: haiku.rag
+site_description: Retrieval-Augmented Generation (RAG) library on SQLite.
+site_url: https://ggozad.github.io/haiku.rag/
+theme:
+  name: material
+  palette:
+    - media: "(prefers-color-scheme)"
+      toggle:
+        icon: material/lightbulb-auto
+        name: Switch to light mode
+    - media: "(prefers-color-scheme: light)"
+      scheme: default
+      primary: deep purple
+      accent: amber
+      toggle:
+        icon: material/lightbulb
+        name: Switch to dark mode
+    - media: "(prefers-color-scheme: dark)"
+      scheme: slate
+      primary: deep purple
+      accent: amber
+      toggle:
+        icon: material/lightbulb-outline
+        name: Switch to system preference
+  features:
+    - content.code.annotate
+    - content.code.copy
+    - content.code.select
+    - content.footnote.tooltips
+    - content.tabs.link
+    - content.tooltips
+    - navigation.footer
+    - navigation.indexes
+    - navigation.instant
+    - navigation.instant.prefetch
+    - navigation.instant.progress
+    - navigation.path
+    - navigation.tabs
+    - navigation.tabs.sticky
+    - navigation.top
+    - navigation.tracking
+    - search.highlight
+    - search.share
+    - search.suggest
+    - toc.follow
+
+  icon:
+    repo: fontawesome/brands/github-alt
+  # logo: img/icon-white.svg
+  # favicon: img/favicon.png
+  language: en
+repo_name: ggozad/haiku.rag
+repo_url: https://github.com/ggozad/haiku.rag
+plugins:
+  # Material for MkDocs
+  search:
+nav:
+  - haiku.rag:
+      - index.md
+      - Installation: installation.md
+      - Configuration: configuration.md
+      - CLI: cli.md
+      - Server: server.md
+      - MCP: mcp.md
+      - Python: python.md
+markdown_extensions:
+  - admonition
+  - attr_list
+  - pymdownx.details
+  - pymdownx.highlight:
+      anchor_linenums: true
+      line_spans: __span
+      pygments_lang_class: true
+      use_pygments: true
+  - pymdownx.inlinehilite
+  - pymdownx.snippets
+  - pymdownx.superfences
diff --git a/pyproject.toml b/pyproject.toml
index 4cbfb8eb..f9e5534c 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -52,6 +52,8 @@ packages = ["src/haiku"]
 [dependency-groups]
 dev = [
     "datasets>=3.6.0",
+    "mkdocs>=1.6.1",
+    "mkdocs-material>=9.6.14",
     "pre-commit>=4.2.0",
     "pyright>=1.1.402",
     "pytest>=8.4.0",
diff --git a/uv.lock b/uv.lock
index ca5f3739..14f667c6 100644
--- a/uv.lock
+++ b/uv.lock
@@ -218,6 +218,29 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/84/29/587c189bbab1ccc8c86a03a5d0e13873df916380ef1be461ebe6acebf48d/authlib-1.6.0-py2.py3-none-any.whl", hash = "sha256:91685589498f79e8655e8a8947431ad6288831d643f11c55c2143ffcc738048d", size = 239981, upload-time = "2025-05-23T00:21:43.075Z" },
 ]
 
+[[package]]
+name = "babel"
+version = "2.17.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/7d/6b/d52e42361e1aa00709585ecc30b3f9684b3ab62530771402248b1b1d6240/babel-2.17.0.tar.gz", hash = "sha256:0c54cffb19f690cdcc52a3b50bcbf71e07a808d1c80d549f2459b9d2cf0afb9d", size = 9951852, upload-time = "2025-02-01T15:17:41.026Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/b7/b8/3fe70c75fe32afc4bb507f75563d39bc5642255d1d94f1f23604725780bf/babel-2.17.0-py3-none-any.whl", hash = "sha256:4d0b53093fdfb4b21c92b5213dba5a1b23885afa8383709427046b21c366e5f2", size = 10182537, upload-time = "2025-02-01T15:17:37.39Z" },
+]
+
+[[package]]
+name = "backrefs"
+version = "5.9"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/eb/a7/312f673df6a79003279e1f55619abbe7daebbb87c17c976ddc0345c04c7b/backrefs-5.9.tar.gz", hash = "sha256:808548cb708d66b82ee231f962cb36faaf4f2baab032f2fbb783e9c2fdddaa59", size = 5765857, upload-time = "2025-06-22T19:34:13.97Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/19/4d/798dc1f30468134906575156c089c492cf79b5a5fd373f07fe26c4d046bf/backrefs-5.9-py310-none-any.whl", hash = "sha256:db8e8ba0e9de81fcd635f440deab5ae5f2591b54ac1ebe0550a2ca063488cd9f", size = 380267, upload-time = "2025-06-22T19:34:05.252Z" },
+    { url = "https://files.pythonhosted.org/packages/55/07/f0b3375bf0d06014e9787797e6b7cc02b38ac9ff9726ccfe834d94e9991e/backrefs-5.9-py311-none-any.whl", hash = "sha256:6907635edebbe9b2dc3de3a2befff44d74f30a4562adbb8b36f21252ea19c5cf", size = 392072, upload-time = "2025-06-22T19:34:06.743Z" },
+    { url = "https://files.pythonhosted.org/packages/9d/12/4f345407259dd60a0997107758ba3f221cf89a9b5a0f8ed5b961aef97253/backrefs-5.9-py312-none-any.whl", hash = "sha256:7fdf9771f63e6028d7fee7e0c497c81abda597ea45d6b8f89e8ad76994f5befa", size = 397947, upload-time = "2025-06-22T19:34:08.172Z" },
+    { url = "https://files.pythonhosted.org/packages/10/bf/fa31834dc27a7f05e5290eae47c82690edc3a7b37d58f7fb35a1bdbf355b/backrefs-5.9-py313-none-any.whl", hash = "sha256:cc37b19fa219e93ff825ed1fed8879e47b4d89aa7a1884860e2db64ccd7c676b", size = 399843, upload-time = "2025-06-22T19:34:09.68Z" },
+    { url = "https://files.pythonhosted.org/packages/fc/24/b29af34b2c9c41645a9f4ff117bae860291780d73880f449e0b5d948c070/backrefs-5.9-py314-none-any.whl", hash = "sha256:df5e169836cc8acb5e440ebae9aad4bf9d15e226d3bad049cf3f6a5c20cc8dc9", size = 411762, upload-time = "2025-06-22T19:34:11.037Z" },
+    { url = "https://files.pythonhosted.org/packages/41/ff/392bff89415399a979be4a65357a41d92729ae8580a66073d8ec8d810f98/backrefs-5.9-py39-none-any.whl", hash = "sha256:f48ee18f6252b8f5777a22a00a09a85de0ca931658f1dd96d4406a34f3748c60", size = 380265, upload-time = "2025-06-22T19:34:12.405Z" },
+]
+
 [[package]]
 name = "beautifulsoup4"
 version = "4.13.4"
@@ -752,6 +775,18 @@ http = [
     { name = "aiohttp" },
 ]
 
+[[package]]
+name = "ghp-import"
+version = "2.1.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "python-dateutil" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/d9/29/d40217cbe2f6b1359e00c6c307bb3fc876ba74068cbab3dde77f03ca0dc4/ghp-import-2.1.0.tar.gz", hash = "sha256:9c535c4c61193c2df8871222567d7fd7e5014d835f97dc7b7439069e2413d343", size = 10943, upload-time = "2022-05-02T15:47:16.11Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/f7/ec/67fbef5d497f86283db54c22eec6f6140243aae73265799baaaa19cd17fb/ghp_import-2.1.0-py3-none-any.whl", hash = "sha256:8337dd7b50877f163d4c0289bc1f1c7f127550241988d568c1db512c4324a619", size = 11034, upload-time = "2022-05-02T15:47:14.552Z" },
+]
+
 [[package]]
 name = "h11"
 version = "0.16.0"
@@ -790,6 +825,8 @@ voyageai = [
 [package.dev-dependencies]
 dev = [
     { name = "datasets" },
+    { name = "mkdocs" },
+    { name = "mkdocs-material" },
     { name = "pre-commit" },
     { name = "pyright" },
     { name = "pytest" },
@@ -819,6 +856,8 @@ provides-extras = ["voyageai", "openai"]
 [package.metadata.requires-dev]
 dev = [
     { name = "datasets", specifier = ">=3.6.0" },
+    { name = "mkdocs", specifier = ">=1.6.1" },
+    { name = "mkdocs-material", specifier = ">=9.6.14" },
     { name = "pre-commit", specifier = ">=4.2.0" },
     { name = "pyright", specifier = ">=1.1.402" },
     { name = "pytest", specifier = ">=8.4.0" },
@@ -937,6 +976,18 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/2c/e1/e6716421ea10d38022b952c159d5161ca1193197fb744506875fbb87ea7b/iniconfig-2.1.0-py3-none-any.whl", hash = "sha256:9deba5723312380e77435581c6bf4935c94cbfab9b1ed33ef8d238ea168eb760", size = 6050, upload-time = "2025-03-19T20:10:01.071Z" },
 ]
 
+[[package]]
+name = "jinja2"
+version = "3.1.6"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "markupsafe" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/df/bf/f7da0350254c0ed7c72f3e33cef02e048281fec7ecec5f032d4aac52226b/jinja2-3.1.6.tar.gz", hash = "sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d", size = 245115, upload-time = "2025-03-05T20:05:02.478Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67", size = 134899, upload-time = "2025-03-05T20:05:00.369Z" },
+]
+
 [[package]]
 name = "jiter"
 version = "0.10.0"
@@ -1122,6 +1173,15 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/be/0c/3153f159b78e368ac473a00e955d69d976e4b69740ed07c76c9f72a161b8/mammoth-1.9.1-py2.py3-none-any.whl", hash = "sha256:f0569bd640cee6c77a07e7c75c5dc10d745dc4dc95d530cfcbb0a5d9536d636c", size = 52991, upload-time = "2025-05-28T19:17:54.62Z" },
 ]
 
+[[package]]
+name = "markdown"
+version = "3.8.2"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/d7/c2/4ab49206c17f75cb08d6311171f2d65798988db4360c4d1485bd0eedd67c/markdown-3.8.2.tar.gz", hash = "sha256:247b9a70dd12e27f67431ce62523e675b866d254f900c4fe75ce3dda62237c45", size = 362071, upload-time = "2025-06-19T17:12:44.483Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/96/2b/34cc11786bc00d0f04d0f5fdc3a2b1ae0b6239eef72d3d345805f9ad92a1/markdown-3.8.2-py3-none-any.whl", hash = "sha256:5c83764dbd4e00bdd94d85a19b8d55ccca20fe35b2e678a1422b380324dd5f24", size = 106827, upload-time = "2025-06-19T17:12:42.994Z" },
+]
+
 [[package]]
 name = "markdown-it-py"
 version = "3.0.0"
@@ -1184,6 +1244,64 @@ xlsx = [
     { name = "pandas" },
 ]
 
+[[package]]
+name = "markupsafe"
+version = "3.0.2"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/b2/97/5d42485e71dfc078108a86d6de8fa46db44a1a9295e89c5d6d4a06e23a62/markupsafe-3.0.2.tar.gz", hash = "sha256:ee55d3edf80167e48ea11a923c7386f4669df67d7994554387f84e7d8b0a2bf0", size = 20537, upload-time = "2024-10-18T15:21:54.129Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/04/90/d08277ce111dd22f77149fd1a5d4653eeb3b3eaacbdfcbae5afb2600eebd/MarkupSafe-3.0.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:7e94c425039cde14257288fd61dcfb01963e658efbc0ff54f5306b06054700f8", size = 14357, upload-time = "2024-10-18T15:20:51.44Z" },
+    { url = "https://files.pythonhosted.org/packages/04/e1/6e2194baeae0bca1fae6629dc0cbbb968d4d941469cbab11a3872edff374/MarkupSafe-3.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9e2d922824181480953426608b81967de705c3cef4d1af983af849d7bd619158", size = 12393, upload-time = "2024-10-18T15:20:52.426Z" },
+    { url = "https://files.pythonhosted.org/packages/1d/69/35fa85a8ece0a437493dc61ce0bb6d459dcba482c34197e3efc829aa357f/MarkupSafe-3.0.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:38a9ef736c01fccdd6600705b09dc574584b89bea478200c5fbf112a6b0d5579", size = 21732, upload-time = "2024-10-18T15:20:53.578Z" },
+    { url = "https://files.pythonhosted.org/packages/22/35/137da042dfb4720b638d2937c38a9c2df83fe32d20e8c8f3185dbfef05f7/MarkupSafe-3.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bbcb445fa71794da8f178f0f6d66789a28d7319071af7a496d4d507ed566270d", size = 20866, upload-time = "2024-10-18T15:20:55.06Z" },
+    { url = "https://files.pythonhosted.org/packages/29/28/6d029a903727a1b62edb51863232152fd335d602def598dade38996887f0/MarkupSafe-3.0.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:57cb5a3cf367aeb1d316576250f65edec5bb3be939e9247ae594b4bcbc317dfb", size = 20964, upload-time = "2024-10-18T15:20:55.906Z" },
+    { url = "https://files.pythonhosted.org/packages/cc/cd/07438f95f83e8bc028279909d9c9bd39e24149b0d60053a97b2bc4f8aa51/MarkupSafe-3.0.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:3809ede931876f5b2ec92eef964286840ed3540dadf803dd570c3b7e13141a3b", size = 21977, upload-time = "2024-10-18T15:20:57.189Z" },
+    { url = "https://files.pythonhosted.org/packages/29/01/84b57395b4cc062f9c4c55ce0df7d3108ca32397299d9df00fedd9117d3d/MarkupSafe-3.0.2-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:e07c3764494e3776c602c1e78e298937c3315ccc9043ead7e685b7f2b8d47b3c", size = 21366, upload-time = "2024-10-18T15:20:58.235Z" },
+    { url = "https://files.pythonhosted.org/packages/bd/6e/61ebf08d8940553afff20d1fb1ba7294b6f8d279df9fd0c0db911b4bbcfd/MarkupSafe-3.0.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:b424c77b206d63d500bcb69fa55ed8d0e6a3774056bdc4839fc9298a7edca171", size = 21091, upload-time = "2024-10-18T15:20:59.235Z" },
+    { url = "https://files.pythonhosted.org/packages/11/23/ffbf53694e8c94ebd1e7e491de185124277964344733c45481f32ede2499/MarkupSafe-3.0.2-cp310-cp310-win32.whl", hash = "sha256:fcabf5ff6eea076f859677f5f0b6b5c1a51e70a376b0579e0eadef8db48c6b50", size = 15065, upload-time = "2024-10-18T15:21:00.307Z" },
+    { url = "https://files.pythonhosted.org/packages/44/06/e7175d06dd6e9172d4a69a72592cb3f7a996a9c396eee29082826449bbc3/MarkupSafe-3.0.2-cp310-cp310-win_amd64.whl", hash = "sha256:6af100e168aa82a50e186c82875a5893c5597a0c1ccdb0d8b40240b1f28b969a", size = 15514, upload-time = "2024-10-18T15:21:01.122Z" },
+    { url = "https://files.pythonhosted.org/packages/6b/28/bbf83e3f76936960b850435576dd5e67034e200469571be53f69174a2dfd/MarkupSafe-3.0.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:9025b4018f3a1314059769c7bf15441064b2207cb3f065e6ea1e7359cb46db9d", size = 14353, upload-time = "2024-10-18T15:21:02.187Z" },
+    { url = "https://files.pythonhosted.org/packages/6c/30/316d194b093cde57d448a4c3209f22e3046c5bb2fb0820b118292b334be7/MarkupSafe-3.0.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:93335ca3812df2f366e80509ae119189886b0f3c2b81325d39efdb84a1e2ae93", size = 12392, upload-time = "2024-10-18T15:21:02.941Z" },
+    { url = "https://files.pythonhosted.org/packages/f2/96/9cdafba8445d3a53cae530aaf83c38ec64c4d5427d975c974084af5bc5d2/MarkupSafe-3.0.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2cb8438c3cbb25e220c2ab33bb226559e7afb3baec11c4f218ffa7308603c832", size = 23984, upload-time = "2024-10-18T15:21:03.953Z" },
+    { url = "https://files.pythonhosted.org/packages/f1/a4/aefb044a2cd8d7334c8a47d3fb2c9f328ac48cb349468cc31c20b539305f/MarkupSafe-3.0.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a123e330ef0853c6e822384873bef7507557d8e4a082961e1defa947aa59ba84", size = 23120, upload-time = "2024-10-18T15:21:06.495Z" },
+    { url = "https://files.pythonhosted.org/packages/8d/21/5e4851379f88f3fad1de30361db501300d4f07bcad047d3cb0449fc51f8c/MarkupSafe-3.0.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1e084f686b92e5b83186b07e8a17fc09e38fff551f3602b249881fec658d3eca", size = 23032, upload-time = "2024-10-18T15:21:07.295Z" },
+    { url = "https://files.pythonhosted.org/packages/00/7b/e92c64e079b2d0d7ddf69899c98842f3f9a60a1ae72657c89ce2655c999d/MarkupSafe-3.0.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:d8213e09c917a951de9d09ecee036d5c7d36cb6cb7dbaece4c71a60d79fb9798", size = 24057, upload-time = "2024-10-18T15:21:08.073Z" },
+    { url = "https://files.pythonhosted.org/packages/f9/ac/46f960ca323037caa0a10662ef97d0a4728e890334fc156b9f9e52bcc4ca/MarkupSafe-3.0.2-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:5b02fb34468b6aaa40dfc198d813a641e3a63b98c2b05a16b9f80b7ec314185e", size = 23359, upload-time = "2024-10-18T15:21:09.318Z" },
+    { url = "https://files.pythonhosted.org/packages/69/84/83439e16197337b8b14b6a5b9c2105fff81d42c2a7c5b58ac7b62ee2c3b1/MarkupSafe-3.0.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:0bff5e0ae4ef2e1ae4fdf2dfd5b76c75e5c2fa4132d05fc1b0dabcd20c7e28c4", size = 23306, upload-time = "2024-10-18T15:21:10.185Z" },
+    { url = "https://files.pythonhosted.org/packages/9a/34/a15aa69f01e2181ed8d2b685c0d2f6655d5cca2c4db0ddea775e631918cd/MarkupSafe-3.0.2-cp311-cp311-win32.whl", hash = "sha256:6c89876f41da747c8d3677a2b540fb32ef5715f97b66eeb0c6b66f5e3ef6f59d", size = 15094, upload-time = "2024-10-18T15:21:11.005Z" },
+    { url = "https://files.pythonhosted.org/packages/da/b8/3a3bd761922d416f3dc5d00bfbed11f66b1ab89a0c2b6e887240a30b0f6b/MarkupSafe-3.0.2-cp311-cp311-win_amd64.whl", hash = "sha256:70a87b411535ccad5ef2f1df5136506a10775d267e197e4cf531ced10537bd6b", size = 15521, upload-time = "2024-10-18T15:21:12.911Z" },
+    { url = "https://files.pythonhosted.org/packages/22/09/d1f21434c97fc42f09d290cbb6350d44eb12f09cc62c9476effdb33a18aa/MarkupSafe-3.0.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:9778bd8ab0a994ebf6f84c2b949e65736d5575320a17ae8984a77fab08db94cf", size = 14274, upload-time = "2024-10-18T15:21:13.777Z" },
+    { url = "https://files.pythonhosted.org/packages/6b/b0/18f76bba336fa5aecf79d45dcd6c806c280ec44538b3c13671d49099fdd0/MarkupSafe-3.0.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:846ade7b71e3536c4e56b386c2a47adf5741d2d8b94ec9dc3e92e5e1ee1e2225", size = 12348, upload-time = "2024-10-18T15:21:14.822Z" },
+    { url = "https://files.pythonhosted.org/packages/e0/25/dd5c0f6ac1311e9b40f4af06c78efde0f3b5cbf02502f8ef9501294c425b/MarkupSafe-3.0.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1c99d261bd2d5f6b59325c92c73df481e05e57f19837bdca8413b9eac4bd8028", size = 24149, upload-time = "2024-10-18T15:21:15.642Z" },
+    { url = "https://files.pythonhosted.org/packages/f3/f0/89e7aadfb3749d0f52234a0c8c7867877876e0a20b60e2188e9850794c17/MarkupSafe-3.0.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e17c96c14e19278594aa4841ec148115f9c7615a47382ecb6b82bd8fea3ab0c8", size = 23118, upload-time = "2024-10-18T15:21:17.133Z" },
+    { url = "https://files.pythonhosted.org/packages/d5/da/f2eeb64c723f5e3777bc081da884b414671982008c47dcc1873d81f625b6/MarkupSafe-3.0.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:88416bd1e65dcea10bc7569faacb2c20ce071dd1f87539ca2ab364bf6231393c", size = 22993, upload-time = "2024-10-18T15:21:18.064Z" },
+    { url = "https://files.pythonhosted.org/packages/da/0e/1f32af846df486dce7c227fe0f2398dc7e2e51d4a370508281f3c1c5cddc/MarkupSafe-3.0.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2181e67807fc2fa785d0592dc2d6206c019b9502410671cc905d132a92866557", size = 24178, upload-time = "2024-10-18T15:21:18.859Z" },
+    { url = "https://files.pythonhosted.org/packages/c4/f6/bb3ca0532de8086cbff5f06d137064c8410d10779c4c127e0e47d17c0b71/MarkupSafe-3.0.2-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:52305740fe773d09cffb16f8ed0427942901f00adedac82ec8b67752f58a1b22", size = 23319, upload-time = "2024-10-18T15:21:19.671Z" },
+    { url = "https://files.pythonhosted.org/packages/a2/82/8be4c96ffee03c5b4a034e60a31294daf481e12c7c43ab8e34a1453ee48b/MarkupSafe-3.0.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:ad10d3ded218f1039f11a75f8091880239651b52e9bb592ca27de44eed242a48", size = 23352, upload-time = "2024-10-18T15:21:20.971Z" },
+    { url = "https://files.pythonhosted.org/packages/51/ae/97827349d3fcffee7e184bdf7f41cd6b88d9919c80f0263ba7acd1bbcb18/MarkupSafe-3.0.2-cp312-cp312-win32.whl", hash = "sha256:0f4ca02bea9a23221c0182836703cbf8930c5e9454bacce27e767509fa286a30", size = 15097, upload-time = "2024-10-18T15:21:22.646Z" },
+    { url = "https://files.pythonhosted.org/packages/c1/80/a61f99dc3a936413c3ee4e1eecac96c0da5ed07ad56fd975f1a9da5bc630/MarkupSafe-3.0.2-cp312-cp312-win_amd64.whl", hash = "sha256:8e06879fc22a25ca47312fbe7c8264eb0b662f6db27cb2d3bbbc74b1df4b9b87", size = 15601, upload-time = "2024-10-18T15:21:23.499Z" },
+    { url = "https://files.pythonhosted.org/packages/83/0e/67eb10a7ecc77a0c2bbe2b0235765b98d164d81600746914bebada795e97/MarkupSafe-3.0.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:ba9527cdd4c926ed0760bc301f6728ef34d841f405abf9d4f959c478421e4efd", size = 14274, upload-time = "2024-10-18T15:21:24.577Z" },
+    { url = "https://files.pythonhosted.org/packages/2b/6d/9409f3684d3335375d04e5f05744dfe7e9f120062c9857df4ab490a1031a/MarkupSafe-3.0.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f8b3d067f2e40fe93e1ccdd6b2e1d16c43140e76f02fb1319a05cf2b79d99430", size = 12352, upload-time = "2024-10-18T15:21:25.382Z" },
+    { url = "https://files.pythonhosted.org/packages/d2/f5/6eadfcd3885ea85fe2a7c128315cc1bb7241e1987443d78c8fe712d03091/MarkupSafe-3.0.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:569511d3b58c8791ab4c2e1285575265991e6d8f8700c7be0e88f86cb0672094", size = 24122, upload-time = "2024-10-18T15:21:26.199Z" },
+    { url = "https://files.pythonhosted.org/packages/0c/91/96cf928db8236f1bfab6ce15ad070dfdd02ed88261c2afafd4b43575e9e9/MarkupSafe-3.0.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:15ab75ef81add55874e7ab7055e9c397312385bd9ced94920f2802310c930396", size = 23085, upload-time = "2024-10-18T15:21:27.029Z" },
+    { url = "https://files.pythonhosted.org/packages/c2/cf/c9d56af24d56ea04daae7ac0940232d31d5a8354f2b457c6d856b2057d69/MarkupSafe-3.0.2-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f3818cb119498c0678015754eba762e0d61e5b52d34c8b13d770f0719f7b1d79", size = 22978, upload-time = "2024-10-18T15:21:27.846Z" },
+    { url = "https://files.pythonhosted.org/packages/2a/9f/8619835cd6a711d6272d62abb78c033bda638fdc54c4e7f4272cf1c0962b/MarkupSafe-3.0.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:cdb82a876c47801bb54a690c5ae105a46b392ac6099881cdfb9f6e95e4014c6a", size = 24208, upload-time = "2024-10-18T15:21:28.744Z" },
+    { url = "https://files.pythonhosted.org/packages/f9/bf/176950a1792b2cd2102b8ffeb5133e1ed984547b75db47c25a67d3359f77/MarkupSafe-3.0.2-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:cabc348d87e913db6ab4aa100f01b08f481097838bdddf7c7a84b7575b7309ca", size = 23357, upload-time = "2024-10-18T15:21:29.545Z" },
+    { url = "https://files.pythonhosted.org/packages/ce/4f/9a02c1d335caabe5c4efb90e1b6e8ee944aa245c1aaaab8e8a618987d816/MarkupSafe-3.0.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:444dcda765c8a838eaae23112db52f1efaf750daddb2d9ca300bcae1039adc5c", size = 23344, upload-time = "2024-10-18T15:21:30.366Z" },
+    { url = "https://files.pythonhosted.org/packages/ee/55/c271b57db36f748f0e04a759ace9f8f759ccf22b4960c270c78a394f58be/MarkupSafe-3.0.2-cp313-cp313-win32.whl", hash = "sha256:bcf3e58998965654fdaff38e58584d8937aa3096ab5354d493c77d1fdd66d7a1", size = 15101, upload-time = "2024-10-18T15:21:31.207Z" },
+    { url = "https://files.pythonhosted.org/packages/29/88/07df22d2dd4df40aba9f3e402e6dc1b8ee86297dddbad4872bd5e7b0094f/MarkupSafe-3.0.2-cp313-cp313-win_amd64.whl", hash = "sha256:e6a2a455bd412959b57a172ce6328d2dd1f01cb2135efda2e4576e8a23fa3b0f", size = 15603, upload-time = "2024-10-18T15:21:32.032Z" },
+    { url = "https://files.pythonhosted.org/packages/62/6a/8b89d24db2d32d433dffcd6a8779159da109842434f1dd2f6e71f32f738c/MarkupSafe-3.0.2-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:b5a6b3ada725cea8a5e634536b1b01c30bcdcd7f9c6fff4151548d5bf6b3a36c", size = 14510, upload-time = "2024-10-18T15:21:33.625Z" },
+    { url = "https://files.pythonhosted.org/packages/7a/06/a10f955f70a2e5a9bf78d11a161029d278eeacbd35ef806c3fd17b13060d/MarkupSafe-3.0.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:a904af0a6162c73e3edcb969eeeb53a63ceeb5d8cf642fade7d39e7963a22ddb", size = 12486, upload-time = "2024-10-18T15:21:34.611Z" },
+    { url = "https://files.pythonhosted.org/packages/34/cf/65d4a571869a1a9078198ca28f39fba5fbb910f952f9dbc5220afff9f5e6/MarkupSafe-3.0.2-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4aa4e5faecf353ed117801a068ebab7b7e09ffb6e1d5e412dc852e0da018126c", size = 25480, upload-time = "2024-10-18T15:21:35.398Z" },
+    { url = "https://files.pythonhosted.org/packages/0c/e3/90e9651924c430b885468b56b3d597cabf6d72be4b24a0acd1fa0e12af67/MarkupSafe-3.0.2-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c0ef13eaeee5b615fb07c9a7dadb38eac06a0608b41570d8ade51c56539e509d", size = 23914, upload-time = "2024-10-18T15:21:36.231Z" },
+    { url = "https://files.pythonhosted.org/packages/66/8c/6c7cf61f95d63bb866db39085150df1f2a5bd3335298f14a66b48e92659c/MarkupSafe-3.0.2-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d16a81a06776313e817c951135cf7340a3e91e8c1ff2fac444cfd75fffa04afe", size = 23796, upload-time = "2024-10-18T15:21:37.073Z" },
+    { url = "https://files.pythonhosted.org/packages/bb/35/cbe9238ec3f47ac9a7c8b3df7a808e7cb50fe149dc7039f5f454b3fba218/MarkupSafe-3.0.2-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:6381026f158fdb7c72a168278597a5e3a5222e83ea18f543112b2662a9b699c5", size = 25473, upload-time = "2024-10-18T15:21:37.932Z" },
+    { url = "https://files.pythonhosted.org/packages/e6/32/7621a4382488aa283cc05e8984a9c219abad3bca087be9ec77e89939ded9/MarkupSafe-3.0.2-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:3d79d162e7be8f996986c064d1c7c817f6df3a77fe3d6859f6f9e7be4b8c213a", size = 24114, upload-time = "2024-10-18T15:21:39.799Z" },
+    { url = "https://files.pythonhosted.org/packages/0d/80/0985960e4b89922cb5a0bac0ed39c5b96cbc1a536a99f30e8c220a996ed9/MarkupSafe-3.0.2-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:131a3c7689c85f5ad20f9f6fb1b866f402c445b220c19fe4308c0b147ccd2ad9", size = 24098, upload-time = "2024-10-18T15:21:40.813Z" },
+    { url = "https://files.pythonhosted.org/packages/82/78/fedb03c7d5380df2427038ec8d973587e90561b2d90cd472ce9254cf348b/MarkupSafe-3.0.2-cp313-cp313t-win32.whl", hash = "sha256:ba8062ed2cf21c07a9e295d5b8a2a5ce678b913b45fdf68c32d95d6c1291e0b6", size = 15208, upload-time = "2024-10-18T15:21:41.814Z" },
+    { url = "https://files.pythonhosted.org/packages/4f/65/6079a46068dfceaeabb5dcad6d674f5f5c61a6fa5673746f42a9f4c233b3/MarkupSafe-3.0.2-cp313-cp313t-win_amd64.whl", hash = "sha256:e444a31f8db13eb18ada366ab3cf45fd4b31e4db1236a4448f68778c1d1a5a2f", size = 15739, upload-time = "2024-10-18T15:21:42.784Z" },
+]
+
 [[package]]
 name = "mcp"
 version = "1.9.4"
@@ -1213,6 +1331,84 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979, upload-time = "2022-08-14T12:40:09.779Z" },
 ]
 
+[[package]]
+name = "mergedeep"
+version = "1.3.4"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/3a/41/580bb4006e3ed0361b8151a01d324fb03f420815446c7def45d02f74c270/mergedeep-1.3.4.tar.gz", hash = "sha256:0096d52e9dad9939c3d975a774666af186eda617e6ca84df4c94dec30004f2a8", size = 4661, upload-time = "2021-02-05T18:55:30.623Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/2c/19/04f9b178c2d8a15b076c8b5140708fa6ffc5601fb6f1e975537072df5b2a/mergedeep-1.3.4-py3-none-any.whl", hash = "sha256:70775750742b25c0d8f36c55aed03d24c3384d17c951b3175d898bd778ef0307", size = 6354, upload-time = "2021-02-05T18:55:29.583Z" },
+]
+
+[[package]]
+name = "mkdocs"
+version = "1.6.1"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "click" },
+    { name = "colorama", marker = "sys_platform == 'win32'" },
+    { name = "ghp-import" },
+    { name = "jinja2" },
+    { name = "markdown" },
+    { name = "markupsafe" },
+    { name = "mergedeep" },
+    { name = "mkdocs-get-deps" },
+    { name = "packaging" },
+    { name = "pathspec" },
+    { name = "pyyaml" },
+    { name = "pyyaml-env-tag" },
+    { name = "watchdog" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/bc/c6/bbd4f061bd16b378247f12953ffcb04786a618ce5e904b8c5a01a0309061/mkdocs-1.6.1.tar.gz", hash = "sha256:7b432f01d928c084353ab39c57282f29f92136665bdd6abf7c1ec8d822ef86f2", size = 3889159, upload-time = "2024-08-30T12:24:06.899Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/22/5b/dbc6a8cddc9cfa9c4971d59fb12bb8d42e161b7e7f8cc89e49137c5b279c/mkdocs-1.6.1-py3-none-any.whl", hash = "sha256:db91759624d1647f3f34aa0c3f327dd2601beae39a366d6e064c03468d35c20e", size = 3864451, upload-time = "2024-08-30T12:24:05.054Z" },
+]
+
+[[package]]
+name = "mkdocs-get-deps"
+version = "0.2.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "mergedeep" },
+    { name = "platformdirs" },
+    { name = "pyyaml" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/98/f5/ed29cd50067784976f25ed0ed6fcd3c2ce9eb90650aa3b2796ddf7b6870b/mkdocs_get_deps-0.2.0.tar.gz", hash = "sha256:162b3d129c7fad9b19abfdcb9c1458a651628e4b1dea628ac68790fb3061c60c", size = 10239, upload-time = "2023-11-20T17:51:09.981Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/9f/d4/029f984e8d3f3b6b726bd33cafc473b75e9e44c0f7e80a5b29abc466bdea/mkdocs_get_deps-0.2.0-py3-none-any.whl", hash = "sha256:2bf11d0b133e77a0dd036abeeb06dec8775e46efa526dc70667d8863eefc6134", size = 9521, upload-time = "2023-11-20T17:51:08.587Z" },
+]
+
+[[package]]
+name = "mkdocs-material"
+version = "9.6.14"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "babel" },
+    { name = "backrefs" },
+    { name = "colorama" },
+    { name = "jinja2" },
+    { name = "markdown" },
+    { name = "mkdocs" },
+    { name = "mkdocs-material-extensions" },
+    { name = "paginate" },
+    { name = "pygments" },
+    { name = "pymdown-extensions" },
+    { name = "requests" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/b3/fa/0101de32af88f87cf5cc23ad5f2e2030d00995f74e616306513431b8ab4b/mkdocs_material-9.6.14.tar.gz", hash = "sha256:39d795e90dce6b531387c255bd07e866e027828b7346d3eba5ac3de265053754", size = 3951707, upload-time = "2025-05-13T13:27:57.173Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/3a/a1/7fdb959ad592e013c01558822fd3c22931a95a0f08cf0a7c36da13a5b2b5/mkdocs_material-9.6.14-py3-none-any.whl", hash = "sha256:3b9cee6d3688551bf7a8e8f41afda97a3c39a12f0325436d76c86706114b721b", size = 8703767, upload-time = "2025-05-13T13:27:54.089Z" },
+]
+
+[[package]]
+name = "mkdocs-material-extensions"
+version = "1.3.1"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/79/9b/9b4c96d6593b2a541e1cb8b34899a6d021d208bb357042823d4d2cabdbe7/mkdocs_material_extensions-1.3.1.tar.gz", hash = "sha256:10c9511cea88f568257f960358a467d12b970e1f7b2c0e5fb2bb48cab1928443", size = 11847, upload-time = "2023-11-22T19:09:45.208Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/5b/54/662a4743aa81d9582ee9339d4ffa3c8fd40a4965e033d77b9da9774d3960/mkdocs_material_extensions-1.3.1-py3-none-any.whl", hash = "sha256:adff8b62700b25cb77b53358dad940f3ef973dd6db797907c49e3c2ef3ab4e31", size = 8728, upload-time = "2023-11-22T19:09:43.465Z" },
+]
+
 [[package]]
 name = "mpmath"
 version = "1.3.0"
@@ -1573,6 +1769,15 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/20/12/38679034af332785aac8774540895e234f4d07f7545804097de4b666afd8/packaging-25.0-py3-none-any.whl", hash = "sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484", size = 66469, upload-time = "2025-04-19T11:48:57.875Z" },
 ]
 
+[[package]]
+name = "paginate"
+version = "0.5.7"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/ec/46/68dde5b6bc00c1296ec6466ab27dddede6aec9af1b99090e1107091b3b84/paginate-0.5.7.tar.gz", hash = "sha256:22bd083ab41e1a8b4f3690544afb2c60c25e5c9a63a30fa2f483f6c60c8e5945", size = 19252, upload-time = "2024-08-25T14:17:24.139Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/90/96/04b8e52da071d28f5e21a805b19cb9390aa17a47462ac87f5e2696b9566d/paginate-0.5.7-py2.py3-none-any.whl", hash = "sha256:b885e2af73abcf01d9559fd5216b57ef722f8c42affbb63942377668e35c7591", size = 13746, upload-time = "2024-08-25T14:17:22.55Z" },
+]
+
 [[package]]
 name = "pandas"
 version = "2.3.0"
@@ -1622,6 +1827,15 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/39/c2/646d2e93e0af70f4e5359d870a63584dacbc324b54d73e6b3267920ff117/pandas-2.3.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:bb3be958022198531eb7ec2008cfc78c5b1eed51af8600c6c5d9160d89d8d249", size = 13231847, upload-time = "2025-06-05T03:27:51.465Z" },
 ]
 
+[[package]]
+name = "pathspec"
+version = "0.12.1"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/ca/bc/f35b8446f4531a7cb215605d100cd88b7ac6f44ab3fc94870c120ab3adbf/pathspec-0.12.1.tar.gz", hash = "sha256:a482d51503a1ab33b1c67a6c3813a26953dbdc71c31dacaef9a838c4e29f5712", size = 51043, upload-time = "2023-12-10T22:30:45Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/cc/20/ff623b09d963f88bfde16306a54e12ee5ea43e9b597108672ff3a408aad6/pathspec-0.12.1-py3-none-any.whl", hash = "sha256:a0d503e138a4c123b27490a4f7beda6a01c6f288df0e4a8b79c7eb0dc7b4cc08", size = 31191, upload-time = "2023-12-10T22:30:43.14Z" },
+]
+
 [[package]]
 name = "pdfminer-six"
 version = "20250506"
@@ -2045,6 +2259,19 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/8a/0b/9fcc47d19c48b59121088dd6da2488a49d5f72dacf8262e2790a1d2c7d15/pygments-2.19.1-py3-none-any.whl", hash = "sha256:9ea1544ad55cecf4b8242fab6dd35a93bbce657034b0611ee383099054ab6d8c", size = 1225293, upload-time = "2025-01-06T17:26:25.553Z" },
 ]
 
+[[package]]
+name = "pymdown-extensions"
+version = "10.16"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "markdown" },
+    { name = "pyyaml" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/1a/0a/c06b542ac108bfc73200677309cd9188a3a01b127a63f20cadc18d873d88/pymdown_extensions-10.16.tar.gz", hash = "sha256:71dac4fca63fabeffd3eb9038b756161a33ec6e8d230853d3cecf562155ab3de", size = 853197, upload-time = "2025-06-21T17:56:36.974Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/98/d4/10bb14004d3c792811e05e21b5e5dcae805aacb739bd12a0540967b99592/pymdown_extensions-10.16-py3-none-any.whl", hash = "sha256:f5dd064a4db588cb2d95229fc4ee63a1b16cc8b4d0e6145c0899ed8723da1df2", size = 266143, upload-time = "2025-06-21T17:56:35.356Z" },
+]
+
 [[package]]
 name = "pyreadline3"
 version = "3.5.4"
@@ -2209,6 +2436,18 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/fa/de/02b54f42487e3d3c6efb3f89428677074ca7bf43aae402517bc7cca949f3/PyYAML-6.0.2-cp313-cp313-win_amd64.whl", hash = "sha256:8388ee1976c416731879ac16da0aff3f63b286ffdd57cdeb95f3f2e085687563", size = 156446, upload-time = "2024-08-06T20:33:04.33Z" },
 ]
 
+[[package]]
+name = "pyyaml-env-tag"
+version = "1.1"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "pyyaml" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/eb/2e/79c822141bfd05a853236b504869ebc6b70159afc570e1d5a20641782eaa/pyyaml_env_tag-1.1.tar.gz", hash = "sha256:2eb38b75a2d21ee0475d6d97ec19c63287a7e140231e4214969d0eac923cd7ff", size = 5737, upload-time = "2025-05-13T15:24:01.64Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/04/11/432f32f8097b03e3cd5fe57e88efb685d964e2e5178a48ed61e841f7fdce/pyyaml_env_tag-1.1-py3-none-any.whl", hash = "sha256:17109e1a528561e32f026364712fee1264bc2ea6715120891174ed1b980d2e04", size = 4722, upload-time = "2025-05-13T15:23:59.629Z" },
+]
+
 [[package]]
 name = "regex"
 version = "2024.11.6"
@@ -2675,6 +2914,38 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/86/e1/0b2defa3a83aabe67db05d5f494d617dd4764b2a043d83ddc26be5e6e0db/voyageai-0.3.2-py3-none-any.whl", hash = "sha256:1398d6c6bfb1dd3b484f400713e538f00ce8a335250442b0902c21116d9705a8", size = 25518, upload-time = "2024-12-03T00:33:51.927Z" },
 ]
 
+[[package]]
+name = "watchdog"
+version = "6.0.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/db/7d/7f3d619e951c88ed75c6037b246ddcf2d322812ee8ea189be89511721d54/watchdog-6.0.0.tar.gz", hash = "sha256:9ddf7c82fda3ae8e24decda1338ede66e1c99883db93711d8fb941eaa2d8c282", size = 131220, upload-time = "2024-11-01T14:07:13.037Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/0c/56/90994d789c61df619bfc5ce2ecdabd5eeff564e1eb47512bd01b5e019569/watchdog-6.0.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:d1cdb490583ebd691c012b3d6dae011000fe42edb7a82ece80965b42abd61f26", size = 96390, upload-time = "2024-11-01T14:06:24.793Z" },
+    { url = "https://files.pythonhosted.org/packages/55/46/9a67ee697342ddf3c6daa97e3a587a56d6c4052f881ed926a849fcf7371c/watchdog-6.0.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:bc64ab3bdb6a04d69d4023b29422170b74681784ffb9463ed4870cf2f3e66112", size = 88389, upload-time = "2024-11-01T14:06:27.112Z" },
+    { url = "https://files.pythonhosted.org/packages/44/65/91b0985747c52064d8701e1075eb96f8c40a79df889e59a399453adfb882/watchdog-6.0.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c897ac1b55c5a1461e16dae288d22bb2e412ba9807df8397a635d88f671d36c3", size = 89020, upload-time = "2024-11-01T14:06:29.876Z" },
+    { url = "https://files.pythonhosted.org/packages/e0/24/d9be5cd6642a6aa68352ded4b4b10fb0d7889cb7f45814fb92cecd35f101/watchdog-6.0.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:6eb11feb5a0d452ee41f824e271ca311a09e250441c262ca2fd7ebcf2461a06c", size = 96393, upload-time = "2024-11-01T14:06:31.756Z" },
+    { url = "https://files.pythonhosted.org/packages/63/7a/6013b0d8dbc56adca7fdd4f0beed381c59f6752341b12fa0886fa7afc78b/watchdog-6.0.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ef810fbf7b781a5a593894e4f439773830bdecb885e6880d957d5b9382a960d2", size = 88392, upload-time = "2024-11-01T14:06:32.99Z" },
+    { url = "https://files.pythonhosted.org/packages/d1/40/b75381494851556de56281e053700e46bff5b37bf4c7267e858640af5a7f/watchdog-6.0.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:afd0fe1b2270917c5e23c2a65ce50c2a4abb63daafb0d419fde368e272a76b7c", size = 89019, upload-time = "2024-11-01T14:06:34.963Z" },
+    { url = "https://files.pythonhosted.org/packages/39/ea/3930d07dafc9e286ed356a679aa02d777c06e9bfd1164fa7c19c288a5483/watchdog-6.0.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:bdd4e6f14b8b18c334febb9c4425a878a2ac20efd1e0b231978e7b150f92a948", size = 96471, upload-time = "2024-11-01T14:06:37.745Z" },
+    { url = "https://files.pythonhosted.org/packages/12/87/48361531f70b1f87928b045df868a9fd4e253d9ae087fa4cf3f7113be363/watchdog-6.0.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:c7c15dda13c4eb00d6fb6fc508b3c0ed88b9d5d374056b239c4ad1611125c860", size = 88449, upload-time = "2024-11-01T14:06:39.748Z" },
+    { url = "https://files.pythonhosted.org/packages/5b/7e/8f322f5e600812e6f9a31b75d242631068ca8f4ef0582dd3ae6e72daecc8/watchdog-6.0.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6f10cb2d5902447c7d0da897e2c6768bca89174d0c6e1e30abec5421af97a5b0", size = 89054, upload-time = "2024-11-01T14:06:41.009Z" },
+    { url = "https://files.pythonhosted.org/packages/68/98/b0345cabdce2041a01293ba483333582891a3bd5769b08eceb0d406056ef/watchdog-6.0.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:490ab2ef84f11129844c23fb14ecf30ef3d8a6abafd3754a6f75ca1e6654136c", size = 96480, upload-time = "2024-11-01T14:06:42.952Z" },
+    { url = "https://files.pythonhosted.org/packages/85/83/cdf13902c626b28eedef7ec4f10745c52aad8a8fe7eb04ed7b1f111ca20e/watchdog-6.0.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:76aae96b00ae814b181bb25b1b98076d5fc84e8a53cd8885a318b42b6d3a5134", size = 88451, upload-time = "2024-11-01T14:06:45.084Z" },
+    { url = "https://files.pythonhosted.org/packages/fe/c4/225c87bae08c8b9ec99030cd48ae9c4eca050a59bf5c2255853e18c87b50/watchdog-6.0.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a175f755fc2279e0b7312c0035d52e27211a5bc39719dd529625b1930917345b", size = 89057, upload-time = "2024-11-01T14:06:47.324Z" },
+    { url = "https://files.pythonhosted.org/packages/30/ad/d17b5d42e28a8b91f8ed01cb949da092827afb9995d4559fd448d0472763/watchdog-6.0.0-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:c7ac31a19f4545dd92fc25d200694098f42c9a8e391bc00bdd362c5736dbf881", size = 87902, upload-time = "2024-11-01T14:06:53.119Z" },
+    { url = "https://files.pythonhosted.org/packages/5c/ca/c3649991d140ff6ab67bfc85ab42b165ead119c9e12211e08089d763ece5/watchdog-6.0.0-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:9513f27a1a582d9808cf21a07dae516f0fab1cf2d7683a742c498b93eedabb11", size = 88380, upload-time = "2024-11-01T14:06:55.19Z" },
+    { url = "https://files.pythonhosted.org/packages/a9/c7/ca4bf3e518cb57a686b2feb4f55a1892fd9a3dd13f470fca14e00f80ea36/watchdog-6.0.0-py3-none-manylinux2014_aarch64.whl", hash = "sha256:7607498efa04a3542ae3e05e64da8202e58159aa1fa4acddf7678d34a35d4f13", size = 79079, upload-time = "2024-11-01T14:06:59.472Z" },
+    { url = "https://files.pythonhosted.org/packages/5c/51/d46dc9332f9a647593c947b4b88e2381c8dfc0942d15b8edc0310fa4abb1/watchdog-6.0.0-py3-none-manylinux2014_armv7l.whl", hash = "sha256:9041567ee8953024c83343288ccc458fd0a2d811d6a0fd68c4c22609e3490379", size = 79078, upload-time = "2024-11-01T14:07:01.431Z" },
+    { url = "https://files.pythonhosted.org/packages/d4/57/04edbf5e169cd318d5f07b4766fee38e825d64b6913ca157ca32d1a42267/watchdog-6.0.0-py3-none-manylinux2014_i686.whl", hash = "sha256:82dc3e3143c7e38ec49d61af98d6558288c415eac98486a5c581726e0737c00e", size = 79076, upload-time = "2024-11-01T14:07:02.568Z" },
+    { url = "https://files.pythonhosted.org/packages/ab/cc/da8422b300e13cb187d2203f20b9253e91058aaf7db65b74142013478e66/watchdog-6.0.0-py3-none-manylinux2014_ppc64.whl", hash = "sha256:212ac9b8bf1161dc91bd09c048048a95ca3a4c4f5e5d4a7d1b1a7d5752a7f96f", size = 79077, upload-time = "2024-11-01T14:07:03.893Z" },
+    { url = "https://files.pythonhosted.org/packages/2c/3b/b8964e04ae1a025c44ba8e4291f86e97fac443bca31de8bd98d3263d2fcf/watchdog-6.0.0-py3-none-manylinux2014_ppc64le.whl", hash = "sha256:e3df4cbb9a450c6d49318f6d14f4bbc80d763fa587ba46ec86f99f9e6876bb26", size = 79078, upload-time = "2024-11-01T14:07:05.189Z" },
+    { url = "https://files.pythonhosted.org/packages/62/ae/a696eb424bedff7407801c257d4b1afda455fe40821a2be430e173660e81/watchdog-6.0.0-py3-none-manylinux2014_s390x.whl", hash = "sha256:2cce7cfc2008eb51feb6aab51251fd79b85d9894e98ba847408f662b3395ca3c", size = 79077, upload-time = "2024-11-01T14:07:06.376Z" },
+    { url = "https://files.pythonhosted.org/packages/b5/e8/dbf020b4d98251a9860752a094d09a65e1b436ad181faf929983f697048f/watchdog-6.0.0-py3-none-manylinux2014_x86_64.whl", hash = "sha256:20ffe5b202af80ab4266dcd3e91aae72bf2da48c0d33bdb15c66658e685e94e2", size = 79078, upload-time = "2024-11-01T14:07:07.547Z" },
+    { url = "https://files.pythonhosted.org/packages/07/f6/d0e5b343768e8bcb4cda79f0f2f55051bf26177ecd5651f84c07567461cf/watchdog-6.0.0-py3-none-win32.whl", hash = "sha256:07df1fdd701c5d4c8e55ef6cf55b8f0120fe1aef7ef39a1c6fc6bc2e606d517a", size = 79065, upload-time = "2024-11-01T14:07:09.525Z" },
+    { url = "https://files.pythonhosted.org/packages/db/d9/c495884c6e548fce18a8f40568ff120bc3a4b7b99813081c8ac0c936fa64/watchdog-6.0.0-py3-none-win_amd64.whl", hash = "sha256:cbafb470cf848d93b5d013e2ecb245d4aa1c8fd0504e863ccefa32445359d680", size = 79070, upload-time = "2024-11-01T14:07:10.686Z" },
+    { url = "https://files.pythonhosted.org/packages/33/e8/e40370e6d74ddba47f002a32919d91310d6074130fe4e17dabcafc15cbf1/watchdog-6.0.0-py3-none-win_ia64.whl", hash = "sha256:a1914259fa9e1454315171103c6a30961236f508b9b623eae470268bbcc6a22f", size = 79067, upload-time = "2024-11-01T14:07:11.845Z" },
+]
+
 [[package]]
 name = "watchfiles"
 version = "1.1.0"

From 7d2d895ff0fc60383dd254463cd46587ecce5b34 Mon Sep 17 00:00:00 2001
From: Yiorgis Gozadinos <ggozadinos@gmail.com>
Date: Wed, 25 Jun 2025 20:17:31 +0300
Subject: [PATCH 08/42] Github action for docs

---
 .github/workflows/build-docs.yml | 28 ++++++++++++++++++++++++++++
 1 file changed, 28 insertions(+)
 create mode 100644 .github/workflows/build-docs.yml

diff --git a/.github/workflows/build-docs.yml b/.github/workflows/build-docs.yml
new file mode 100644
index 00000000..50b0171d
--- /dev/null
+++ b/.github/workflows/build-docs.yml
@@ -0,0 +1,28 @@
+name: build-docs
+on:
+  push:
+    branches:
+      - main
+permissions:
+  contents: write
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Configure Git Credentials
+        run: |
+          git config user.name github-actions[bot]
+          git config user.email 41898282+github-actions[bot]@users.noreply.github.com
+      - uses: actions/setup-python@v5
+        with:
+          python-version: 3.x
+      - run: echo "cache_id=$(date --utc '+%V')" >> $GITHUB_ENV
+      - uses: actions/cache@v4
+        with:
+          key: mkdocs-material-${{ env.cache_id }}
+          path: .cache
+          restore-keys: |
+            mkdocs-material-
+      - run: pip install mkdocs-material
+      - run: mkdocs gh-deploy --force

From cf9ec04170c203e9286775aaa064c21c1cb4351b Mon Sep 17 00:00:00 2001
From: Yiorgis Gozadinos <ggozadinos@gmail.com>
Date: Thu, 26 Jun 2025 11:46:23 +0300
Subject: [PATCH 09/42] Fix docs

---
 README.md             | 225 +++++++++---------------------------------
 docs/configuration.md |  18 +++-
 2 files changed, 64 insertions(+), 179 deletions(-)

diff --git a/README.md b/README.md
index 866fdbca..7d1d6126 100644
--- a/README.md
+++ b/README.md
@@ -1,194 +1,65 @@
 # Haiku SQLite RAG
 
-A Retrieval-Augmented Generation (RAG) library on SQLite.
+Retrieval-Augmented Generation (RAG) library on SQLite.
 
 ## Features
-- **Local SQLite**: No need to run additional servers
-- **Support for various embedding providers**: You can use Ollama, VoyageAI, OpenAI or add your own
-- **Hybrid Search**: Vector search using `sqlite-vec` combined with full-text search `FTS5`, using Reciprocal Rank Fusion
-- **File monitoring** when run as a server automatically indexing your files
-- **Extended file format Support**: Parse 40+ file formats including PDF, DOCX, HTML, Markdown, audio and more. Or add a url!
-- **MCP server** Exposes functionality as MCP tools.
-- **CLI commands** Access all functionality from your terminal
-- **Python client** Call `haiku.rag` from your own python applications.
 
-## Installation
+- **Local SQLite**: No external servers required
+- **Multiple embedding providers**: Ollama, VoyageAI, OpenAI
+- **Hybrid search**: Vector + full-text search with Reciprocal Rank Fusion
+- **File monitoring**: Auto-index files when run as server
+- **40+ file formats**: PDF, DOCX, HTML, Markdown, audio, URLs
+- **MCP server**: Expose as tools for AI assistants
+- **CLI & Python API**: Use from command line or Python
+
+## Quick Start
 
 ```bash
+# Install
 uv pip install haiku.rag
+
+# Add documents
+haiku-rag add "Your content here"
+haiku-rag add-src document.pdf
+
+# Search
+haiku-rag search "query"
+
+# Start server with file monitoring
+export MONITOR_DIRECTORIES="/path/to/docs"
+haiku-rag serve
 ```
 
-By default Ollama (with the `mxbai-embed-large` model) is used for the embeddings.
-For other providers use:
-
-- **VoyageAI**: `uv pip install haiku.rag --extra voyageai`
-- **OpenAI**: `uv pip install haiku.rag --extra openai`
-
-## Configuration
-
-You can set the directories to monitor using the `MONITOR_DIRECTORIES` environment variable (as comma separated values) :
-
-```bash
-# Monitor single directory
-export MONITOR_DIRECTORIES="/path/to/documents,/another_path/to/documents"
-```
-
-If you want to use an alternative embeddings provider (Ollama being the default) you will need to set the provider details through environment variables:
-
-By default:
-
-```bash
-EMBEDDINGS_PROVIDER="ollama"
-EMBEDDINGS_MODEL="mxbai-embed-large" # or any other model
-EMBEDDINGS_VECTOR_DIM=1024
-```
-
-For VoyageAI:
-```bash
-EMBEDDINGS_PROVIDER="voyageai"
-EMBEDDINGS_MODEL="voyage-3.5" # or any other model
-EMBEDDINGS_VECTOR_DIM=1024
-VOYAGE_API_KEY="your-api-key"
-```
-
-For OpenAI:
-```bash
-EMBEDDINGS_PROVIDER="openai"
-EMBEDDINGS_MODEL="text-embedding-3-small" # or text-embedding-3-large
-EMBEDDINGS_VECTOR_DIM=1536
-OPENAI_API_KEY="your-api-key"
-```
-
-## Command Line Interface
-
-`haiku.rag` includes a CLI application for managing documents and performing searches from the command line:
-
-### Available Commands
-
-```bash
-# List all documents
-haiku-rag list
-
-# Add document from text
-haiku-rag add "Your document content here"
-
-# Add document from file or URL
-haiku-rag add-src /path/to/document.pdf
-haiku-rag add-src https://example.com/article.html
-
-# Get and display a specific document
-haiku-rag get 1
-
-# Delete a document by ID
-haiku-rag delete 1
-
-# Search documents
-haiku-rag search "machine learning"
-
-# Search with custom options
-haiku-rag search "python programming" --limit 10 --k 100
-
-# Start file monitoring & MCP server (default HTTP transport)
-haiku-rag serve # --stdio for stdio transport or --sse for SSE transport
-```
-
-All commands support the `--db` option to specify a custom database path. Run
-```bash
-haiku-rag command -h
-```
-to see additional parameters for a command.
-
-## File Monitoring & MCP server
-
-You can start the server (using Streamble HTTP, stdio or SSE transports) with:
-
-```bash
-# Start with default HTTP transport
-haiku-rag serve # --stdio for stdio transport or --sse for SSE transport
-```
-
-You need to have set the `MONITOR_DIRECTORIES` environment variable for monitoring to take place.
-
-### File monitoring
-
-`haiku.rag` can watch directories for changes and automatically update the document store:
-
-- **Startup**: Scan all monitored directories and add any new files
-- **File Added/Modified**: Automatically parse and add/update the document in the database
-- **File Deleted**: Remove the corresponding document from the database
-
-### MCP Server
-
-`haiku.rag` includes a Model Context Protocol (MCP) server that exposes RAG functionality as tools for AI assistants like Claude Desktop. The MCP server provides the following tools:
-
-- `add_document_from_file` - Add documents from local file paths
-- `add_document_from_url` - Add documents from URLs
-- `add_document_from_text` - Add documents from raw text content
-- `search_documents` - Search documents using hybrid search
-- `get_document` - Retrieve specific documents by ID
-- `list_documents` - List all documents with pagination
-- `delete_document` - Delete documents by ID
-
-## Using `haiku.rag` from python
-
-### Managing documents
+## Python Usage
 
 ```python
-from pathlib import Path
 from haiku.rag.client import HaikuRAG
 
-# Use as async context manager (recommended)
-async with HaikuRAG("path/to/database.db") as client:
-    # Create document from text
-    doc = await client.create_document(
-        content="Your document content here",
-        uri="doc://example",
-        metadata={"source": "manual", "topic": "example"}
-    )
-
-    # Create document from file (auto-parses content)
-    doc = await client.create_document_from_source("path/to/document.pdf")
-
-    # Create document from URL
-    doc = await client.create_document_from_source("https://example.com/article.html")
-
-    # Retrieve documents
-    doc = await client.get_document_by_id(1)
-    doc = await client.get_document_by_uri("file:///path/to/document.pdf")
-
-    # List all documents with pagination
-    docs = await client.list_documents(limit=10, offset=0)
-
-    # Update document content
-    doc.content = "Updated content"
-    await client.update_document(doc)
-
-    # Delete document
-    await client.delete_document(doc.id)
-
-    # Search documents using hybrid search (vector + full-text)
-    results = await client.search("machine learning algorithms", limit=5)
-    for chunk, score in results:
-        print(f"Score: {score:.3f}")
-        print(f"Content: {chunk.content}")
-        print(f"Document ID: {chunk.document_id}")
-        print("---")
-```
-
-## Searching documents
-
-```python
 async with HaikuRAG("database.db") as client:
+    # Add document
+    doc = await client.create_document("Your content")
 
-    results = await client.search(
-        query="machine learning",
-        limit=5,  # Maximum results to return, defaults to 5
-        k=60      # RRF parameter for reciprocal rank fusion, defaults to 60
-    )
-
-    # Process results
-    for chunk, relevance_score in results:
-        print(f"Relevance: {relevance_score:.3f}")
-        print(f"Content: {chunk.content}")
-        print(f"From document: {chunk.document_id}")
+    # Search
+    results = await client.search("query")
+    for chunk, score in results:
+        print(f"{score:.3f}: {chunk.content}")
 ```
+
+## MCP Server
+
+Use with AI assistants like Claude Desktop:
+
+```bash
+haiku-rag serve --stdio
+```
+
+Provides tools for document management and search directly in your AI assistant.
+
+## Documentation
+
+Full documentation at: https://ggozad.github.io/haiku.rag/
+
+- [Installation](https://ggozad.github.io/haiku.rag/installation/) - Provider setup
+- [Configuration](https://ggozad.github.io/haiku.rag/configuration/) - Environment variables
+- [CLI](https://ggozad.github.io/haiku.rag/cli/) - Command reference
+- [Python API](https://ggozad.github.io/haiku.rag/python/) - Complete API docs
diff --git a/docs/configuration.md b/docs/configuration.md
index 6e7c6610..d7cdd593 100644
--- a/docs/configuration.md
+++ b/docs/configuration.md
@@ -8,14 +8,16 @@ Set directories to monitor for automatic indexing:
 
 ```bash
 # Monitor single directory
-export MONITOR_DIRECTORIES="/path/to/documents"
+MONITOR_DIRECTORIES="/path/to/documents"
 
 # Monitor multiple directories
-export MONITOR_DIRECTORIES="/path/to/documents,/another_path/to/documents"
+MONITOR_DIRECTORIES="/path/to/documents,/another_path/to/documents"
 ```
 
 ## Embedding Providers
 
+If you use Ollama, you can use any pulled model that supports embeddings.
+
 ### Ollama (Default)
 
 ```bash
@@ -25,6 +27,11 @@ EMBEDDINGS_VECTOR_DIM=1024
 ```
 
 ### VoyageAI
+If you want to use VoyageAI embeddings you will need to install `haiku.rag` with the VoyageAI extras,
+
+```bash
+uv pip install haiku.rag --extra voyageai
+```
 
 ```bash
 EMBEDDINGS_PROVIDER="voyageai"
@@ -34,6 +41,13 @@ VOYAGE_API_KEY="your-api-key"
 ```
 
 ### OpenAI
+If you want to use OpenAI embeddings you will need to install `haiku.rag` with the VoyageAI extras,
+
+```bash
+uv pip install haiku.rag --extra openai
+```
+
+and set environment variables.
 
 ```bash
 EMBEDDINGS_PROVIDER="openai"

From e07c849d3c31fbba31591985de47b801c2afb660 Mon Sep 17 00:00:00 2001
From: Yiorgis Gozadinos <ggozadinos@gmail.com>
Date: Thu, 26 Jun 2025 11:49:19 +0300
Subject: [PATCH 10/42] mkdocs precommit hook

---
 .pre-commit-config.yaml | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 19c80c7c..57298115 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -20,3 +20,13 @@ repos:
     rev: v1.1.399
     hooks:
       - id: pyright
+
+  - repo: https://github.com/RodrigoGonzalez/check-mkdocs
+    rev: v1.2.0
+    hooks:
+      - id: check-mkdocs
+        name: check-mkdocs
+        args: ["--config", "mkdocs.yml"] # Optional, mkdocs.yml is the default
+        # If you have additional plugins or libraries that are not included in
+        # check-mkdocs, add them here
+        additional_dependencies: ["mkdocs-material"]

From 7a1b09d5e2ba25e5354fa586e8285996acc159a0 Mon Sep 17 00:00:00 2001
From: Yiorgis Gozadinos <ggozadinos@gmail.com>
Date: Fri, 27 Jun 2025 19:58:34 +0300
Subject: [PATCH 11/42] Minor doc fixes

---
 README.md             |  2 ++
 docs/configuration.md |  2 +-
 docs/index.md         |  3 ++-
 docs/mcp.md           | 11 ++---------
 docs/python.md        |  7 -------
 5 files changed, 7 insertions(+), 18 deletions(-)

diff --git a/README.md b/README.md
index 7d1d6126..8e20b3e1 100644
--- a/README.md
+++ b/README.md
@@ -2,6 +2,8 @@
 
 Retrieval-Augmented Generation (RAG) library on SQLite.
 
+`haiku.rag` is a Retrieval-Augmented Generation (RAG) library built to work on SQLite alone without the need for external vector databases. It uses [sqlite-vec](https://github.com/asg017/sqlite-vec) for storing the embeddings and performs semantic (vector) search as well as full-text search combined through Reciprocal Rank Fusion. Both open-source (Ollama) as well as commercial (OpenAI, VoyageAI) embedding providers are supported.
+
 ## Features
 
 - **Local SQLite**: No external servers required
diff --git a/docs/configuration.md b/docs/configuration.md
index d7cdd593..8ba6cc7e 100644
--- a/docs/configuration.md
+++ b/docs/configuration.md
@@ -1,6 +1,6 @@
 # Configuration
 
-Configuration is done through environment variables.
+Configuration is done through the use of environment variables.
 
 ## File Monitoring
 
diff --git a/docs/index.md b/docs/index.md
index c81ab0f2..404ae8b4 100644
--- a/docs/index.md
+++ b/docs/index.md
@@ -1,6 +1,7 @@
 # haiku.rag
 
-A Retrieval-Augmented Generation (RAG) library on SQLite.
+`haiku.rag` is a Retrieval-Augmented Generation (RAG) library built to work on SQLite alone without the need for external vector databases. It uses [sqlite-vec](https://github.com/asg017/sqlite-vec) for storing the embeddings and performs semantic (vector) search as well as full-text search combined through Reciprocal Rank Fusion. Both open-source (Ollama) as well as commercial (OpenAI, VoyageAI) embedding providers are supported.
+
 
 ## Features
 
diff --git a/docs/mcp.md b/docs/mcp.md
index 58adf7db..22f0f667 100644
--- a/docs/mcp.md
+++ b/docs/mcp.md
@@ -1,6 +1,6 @@
 # Model Context Protocol (MCP)
 
-The MCP server exposes RAG functionality as tools for AI assistants like Claude Desktop.
+The MCP server exposes `haiku.rag` as MCP tools for compatible MCP clients.
 
 ## Available Tools
 
@@ -19,7 +19,7 @@ The MCP server exposes RAG functionality as tools for AI assistants like Claude
 
 ## Starting MCP Server
 
-The MCP server starts automatically with the serve command:
+The MCP server starts automatically with the serve command and supports `Streamable HTTP`, `stdio` and `SSE` transports:
 
 ```bash
 # Default HTTP transport
@@ -31,10 +31,3 @@ haiku-rag serve --stdio
 # SSE transport
 haiku-rag serve --sse
 ```
-
-## Integration
-
-The MCP server follows the Model Context Protocol specification, making it compatible with:
-- Claude Desktop
-- Other MCP-compatible AI assistants
-- Custom MCP clients
diff --git a/docs/python.md b/docs/python.md
index 3b81c232..9908210e 100644
--- a/docs/python.md
+++ b/docs/python.md
@@ -92,10 +92,3 @@ for chunk, relevance_score in results:
     print(f"Content: {chunk.content}")
     print(f"From document: {chunk.document_id}")
 ```
-
-## Search Technology
-
-`haiku.rag` uses hybrid search combining:
-- **Vector search** using `sqlite-vec` for semantic similarity
-- **Full-text search** using SQLite's `FTS5` for keyword matching
-- **Reciprocal Rank Fusion** to combine and rank results

From f3ed88b97047d63611450e635874a87c189ffe03 Mon Sep 17 00:00:00 2001
From: Yiorgis Gozadinos <ggozadinos@gmail.com>
Date: Sat, 28 Jun 2025 09:10:23 +0300
Subject: [PATCH 12/42] Include document uri and meta in search results

---
 src/haiku/rag/store/models/chunk.py       |  4 +-
 src/haiku/rag/store/repositories/chunk.py | 45 +++++++++++++++++------
 tests/test_search.py                      | 30 +++++++++++++++
 3 files changed, 66 insertions(+), 13 deletions(-)

diff --git a/src/haiku/rag/store/models/chunk.py b/src/haiku/rag/store/models/chunk.py
index 5b17bea4..bd26aaed 100644
--- a/src/haiku/rag/store/models/chunk.py
+++ b/src/haiku/rag/store/models/chunk.py
@@ -3,10 +3,12 @@ from pydantic import BaseModel
 
 class Chunk(BaseModel):
     """
-    Represents a document with an ID, content, and metadata.
+    Represents a chunk with content, metadata, and optional document information.
     """
 
     id: int | None = None
     document_id: int
     content: str
     metadata: dict = {}
+    document_uri: str | None = None
+    document_meta: dict = {}
diff --git a/src/haiku/rag/store/repositories/chunk.py b/src/haiku/rag/store/repositories/chunk.py
index 1ffec47b..1f33ec68 100644
--- a/src/haiku/rag/store/repositories/chunk.py
+++ b/src/haiku/rag/store/repositories/chunk.py
@@ -240,9 +240,10 @@ class ChunkRepository(BaseRepository[Chunk]):
         # Search for similar chunks using sqlite-vec
         cursor.execute(
             """
-            SELECT c.id, c.document_id, c.content, c.metadata, distance
+            SELECT c.id, c.document_id, c.content, c.metadata, distance, d.uri, d.metadata as document_metadata
             FROM chunk_embeddings
             JOIN chunks c ON c.id = chunk_embeddings.chunk_id
+            JOIN documents d ON c.document_id = d.id
             WHERE embedding MATCH :embedding AND k = :k
             ORDER BY distance
             """,
@@ -257,10 +258,14 @@ class ChunkRepository(BaseRepository[Chunk]):
                     document_id=document_id,
                     content=content,
                     metadata=json.loads(metadata_json) if metadata_json else {},
+                    document_uri=document_uri,
+                    document_meta=json.loads(document_metadata_json)
+                    if document_metadata_json
+                    else {},
                 ),
                 1.0 / (1.0 + distance),
             )
-            for chunk_id, document_id, content, metadata_json, distance in results
+            for chunk_id, document_id, content, metadata_json, distance, document_uri, document_metadata_json in results
         ]
 
     async def search_chunks_fts(
@@ -281,9 +286,10 @@ class ChunkRepository(BaseRepository[Chunk]):
         # Search using FTS5
         cursor.execute(
             """
-            SELECT c.id, c.document_id, c.content, c.metadata, rank
+            SELECT c.id, c.document_id, c.content, c.metadata, rank, d.uri, d.metadata as document_metadata
             FROM chunks_fts
             JOIN chunks c ON c.id = chunks_fts.rowid
+            JOIN documents d ON c.document_id = d.id
             WHERE chunks_fts MATCH :query
             ORDER BY rank
             LIMIT :limit
@@ -300,10 +306,14 @@ class ChunkRepository(BaseRepository[Chunk]):
                     document_id=document_id,
                     content=content,
                     metadata=json.loads(metadata_json) if metadata_json else {},
+                    document_uri=document_uri,
+                    document_meta=json.loads(document_metadata_json)
+                    if document_metadata_json
+                    else {},
                 ),
                 -rank,
             )
-            for chunk_id, document_id, content, metadata_json, rank in results
+            for chunk_id, document_id, content, metadata_json, rank, document_uri, document_metadata_json in results
             # FTS5 rank is negative BM25 score
         ]
 
@@ -368,9 +378,10 @@ class ChunkRepository(BaseRepository[Chunk]):
                 LEFT JOIN vector_search v ON a.id = v.id
                 LEFT JOIN fts_search f ON a.id = f.id
             )
-            SELECT id, document_id, content, metadata, rrf_score
-            FROM rrf_scores
-            ORDER BY rrf_score DESC
+            SELECT r.id, r.document_id, r.content, r.metadata, r.rrf_score, d.uri, d.metadata as document_metadata
+            FROM rrf_scores r
+            JOIN documents d ON r.document_id = d.id
+            ORDER BY r.rrf_score DESC
             LIMIT :limit
             """,
             {
@@ -390,10 +401,14 @@ class ChunkRepository(BaseRepository[Chunk]):
                     document_id=document_id,
                     content=content,
                     metadata=json.loads(metadata_json) if metadata_json else {},
+                    document_uri=document_uri,
+                    document_meta=json.loads(document_metadata_json)
+                    if document_metadata_json
+                    else {},
                 ),
                 rrf_score,
             )
-            for chunk_id, document_id, content, metadata_json, rrf_score in results
+            for chunk_id, document_id, content, metadata_json, rrf_score, document_uri, document_metadata_json in results
         ]
 
     async def get_by_document_id(self, document_id: int) -> list[Chunk]:
@@ -404,9 +419,11 @@ class ChunkRepository(BaseRepository[Chunk]):
         cursor = self.store._connection.cursor()
         cursor.execute(
             """
-            SELECT id, document_id, content, metadata
-            FROM chunks WHERE document_id = :document_id
-            ORDER BY JSON_EXTRACT(metadata, '$.order')
+            SELECT c.id, c.document_id, c.content, c.metadata, d.uri, d.metadata as document_metadata
+            FROM chunks c
+            JOIN documents d ON c.document_id = d.id
+            WHERE c.document_id = :document_id
+            ORDER BY JSON_EXTRACT(c.metadata, '$.order')
             """,
             {"document_id": document_id},
         )
@@ -418,6 +435,10 @@ class ChunkRepository(BaseRepository[Chunk]):
                 document_id=document_id,
                 content=content,
                 metadata=json.loads(metadata_json) if metadata_json else {},
+                document_uri=document_uri,
+                document_meta=json.loads(document_metadata_json)
+                if document_metadata_json
+                else {},
             )
-            for chunk_id, document_id, content, metadata_json in rows
+            for chunk_id, document_id, content, metadata_json, document_uri, document_metadata_json in rows
         ]
diff --git a/tests/test_search.py b/tests/test_search.py
index 340a2525..57ffb74f 100644
--- a/tests/test_search.py
+++ b/tests/test_search.py
@@ -56,3 +56,33 @@ async def test_search_qa_corpus(qa_corpus: Dataset):
         assert target_document.id in target_document_ids
 
     store.close()
+
+
+@pytest.mark.asyncio
+async def test_chunks_include_document_info():
+    """Test that search results include document URI and metadata."""
+    store = Store(":memory:")
+    doc_repo = DocumentRepository(store)
+    chunk_repo = ChunkRepository(store)
+
+    # Create a document with URI and metadata
+    document = Document(
+        content="This is a test document with some content for searching.",
+        uri="https://example.com/test.html",
+        metadata={"title": "Test Document", "author": "Test Author"},
+    )
+
+    created_document = await doc_repo.create(document)
+
+    # Search for chunks
+    results = await chunk_repo.search_chunks_hybrid("test document", limit=1)
+
+    assert len(results) > 0
+    chunk, score = results[0]
+
+    # Verify the chunk includes document information
+    assert chunk.document_uri == "https://example.com/test.html"
+    assert chunk.document_meta == {"title": "Test Document", "author": "Test Author"}
+    assert chunk.document_id == created_document.id
+
+    store.close()

From 357460da577cf13ca9a1a9ffb02fa5f95e162ae3 Mon Sep 17 00:00:00 2001
From: Yiorgis Gozadinos <ggozadinos@gmail.com>
Date: Thu, 26 Jun 2025 11:56:49 +0300
Subject: [PATCH 13/42] Update benchmarks

---
 BENCHMARKS.md | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/BENCHMARKS.md b/BENCHMARKS.md
index bc66cab3..977e1927 100644
--- a/BENCHMARKS.md
+++ b/BENCHMARKS.md
@@ -7,3 +7,7 @@ We use [repliqa](https://huggingface.co/datasets/ServiceNow/repliqa) for the eva
 We load the `News Stories` from `repliqa_3` which is 1035 documents, using `tests/generate_benchmark_db.py`, using the `mxbai-embed-large` Ollama embeddings.
 
 Subsequently, we run a search over the `question` for each row of the dataset and check whether we match the document that answers the question. The recall obtained is ~0.75 for matching in the top result, raising to ~0.75 for the top 3 results.
+
+* Question/Answer evaluation
+
+We use the  `News Stories` from `repliqa_3` using the `mxbai-embed-large` Ollama embeddings, with a QA agent also using Ollama with the `qwen3` model (8b). For each story we ask the `question` and use an LLM judge (also `qwen3`) to evaluate whether the answer is correct or not. Thus we obtain accuracy of ~0.54.

From 85b106c4614b41b4508b2298dca7a8b317819fe1 Mon Sep 17 00:00:00 2001
From: Yiorgis Gozadinos <ggozadinos@gmail.com>
Date: Fri, 27 Jun 2025 09:14:16 +0300
Subject: [PATCH 14/42] OpenAI Question/Answer agent

---
 src/haiku/rag/config.py        |   8 +++
 src/haiku/rag/qa/__init__.py   |  26 +++++++++
 src/haiku/rag/qa/base.py       |  27 ++++++++-
 src/haiku/rag/qa/ollama.py     |  30 +---------
 src/haiku/rag/qa/openai.py     | 101 +++++++++++++++++++++++++++++++++
 tests/conftest.py              |   7 ---
 tests/generate_benchmark_db.py |   4 +-
 tests/test_qa.py               |  52 +++++++++++------
 8 files changed, 202 insertions(+), 53 deletions(-)
 create mode 100644 src/haiku/rag/qa/openai.py

diff --git a/src/haiku/rag/config.py b/src/haiku/rag/config.py
index 270c0aea..c388064f 100644
--- a/src/haiku/rag/config.py
+++ b/src/haiku/rag/config.py
@@ -27,6 +27,10 @@ class AppConfig(BaseModel):
 
     OLLAMA_BASE_URL: str = "http://localhost:11434"
 
+    # Provider keys
+    VOYAGE_API_KEY: str = ""
+    OPENAI_API_KEY: str = ""
+
     @field_validator("MONITOR_DIRECTORIES", mode="before")
     @classmethod
     def parse_monitor_directories(cls, v):
@@ -41,3 +45,7 @@ class AppConfig(BaseModel):
 
 # Expose Config object for app to import
 Config = AppConfig.model_validate(os.environ)
+if Config.OPENAI_API_KEY:
+    os.environ["OPENAI_API_KEY"] = Config.OPENAI_API_KEY
+if Config.VOYAGE_API_KEY:
+    os.environ["VOYAGE_API_KEY"] = Config.VOYAGE_API_KEY
diff --git a/src/haiku/rag/qa/__init__.py b/src/haiku/rag/qa/__init__.py
index e69de29b..69ff4523 100644
--- a/src/haiku/rag/qa/__init__.py
+++ b/src/haiku/rag/qa/__init__.py
@@ -0,0 +1,26 @@
+from haiku.rag.client import HaikuRAG
+from haiku.rag.config import Config
+from haiku.rag.qa.base import QuestionAnswerAgentBase
+from haiku.rag.qa.ollama import QuestionAnswerOllamaAgent
+
+
+def get_qa_agent(client: HaikuRAG, model: str = "") -> QuestionAnswerAgentBase:
+    """
+    Factory function to get the appropriate QA agent based on the configuration.
+    """
+
+    if Config.QA_PROVIDER == "ollama":
+        return QuestionAnswerOllamaAgent(client, model or Config.QA_MODEL)
+
+    if Config.QA_PROVIDER == "openai":
+        try:
+            from haiku.rag.qa.openai import QuestionAnswerOpenAIAgent
+        except ImportError:
+            raise ImportError(
+                "OpenAI QA agent requires the 'openai' package. "
+                "Please install haiku.rag with the 'openai' extra:"
+                "uv pip install haiku.rag --extra openai"
+            )
+        return QuestionAnswerOpenAIAgent(client, model or "gpt-4o-mini")
+
+    raise ValueError(f"Unsupported QA provider: {Config.QA_PROVIDER}")
diff --git a/src/haiku/rag/qa/base.py b/src/haiku/rag/qa/base.py
index 6c8f8359..0ff2a55b 100644
--- a/src/haiku/rag/qa/base.py
+++ b/src/haiku/rag/qa/base.py
@@ -2,7 +2,7 @@ from haiku.rag.client import HaikuRAG
 from haiku.rag.qa.prompts import SYSTEM_PROMPT
 
 
-class QABase:
+class QuestionAnswerAgentBase:
     _model: str = ""
     _system_prompt: str = SYSTEM_PROMPT
 
@@ -14,3 +14,28 @@ class QABase:
         raise NotImplementedError(
             "QABase is an abstract class. Please implement the answer method in a subclass."
         )
+
+    tools = [
+        {
+            "type": "function",
+            "function": {
+                "name": "search_documents",
+                "description": "Search the knowledge base for relevant documents",
+                "parameters": {
+                    "type": "object",
+                    "properties": {
+                        "query": {
+                            "type": "string",
+                            "description": "The search query to find relevant documents",
+                        },
+                        "limit": {
+                            "type": "integer",
+                            "description": "Maximum number of results to return",
+                            "default": 3,
+                        },
+                    },
+                    "required": ["query"],
+                },
+            },
+        }
+    ]
diff --git a/src/haiku/rag/qa/ollama.py b/src/haiku/rag/qa/ollama.py
index 021190ca..c8cac4ce 100644
--- a/src/haiku/rag/qa/ollama.py
+++ b/src/haiku/rag/qa/ollama.py
@@ -2,12 +2,12 @@ from ollama import AsyncClient
 
 from haiku.rag.client import HaikuRAG
 from haiku.rag.config import Config
-from haiku.rag.qa.base import QABase
+from haiku.rag.qa.base import QuestionAnswerAgentBase
 
 OLLAMA_OPTIONS = {"temperature": 0.0, "seed": 42, "num_ctx": 64000}
 
 
-class QA(QABase):
+class QuestionAnswerOllamaAgent(QuestionAnswerAgentBase):
     def __init__(self, client: HaikuRAG, model: str = Config.QA_MODEL):
         super().__init__(client, model or self._model)
 
@@ -15,30 +15,6 @@ class QA(QABase):
         ollama_client = AsyncClient(host=Config.OLLAMA_BASE_URL)
 
         # Define the search tool
-        tools = [
-            {
-                "type": "function",
-                "function": {
-                    "name": "search_documents",
-                    "description": "Search the knowledge base for relevant documents",
-                    "parameters": {
-                        "type": "object",
-                        "properties": {
-                            "query": {
-                                "type": "string",
-                                "description": "The search query to find relevant documents",
-                            },
-                            "limit": {
-                                "type": "integer",
-                                "description": "Maximum number of results to return",
-                                "default": 3,
-                            },
-                        },
-                        "required": ["query"],
-                    },
-                },
-            }
-        ]
 
         messages = [
             {"role": "system", "content": self._system_prompt},
@@ -49,7 +25,7 @@ class QA(QABase):
         response = await ollama_client.chat(
             model=self._model,
             messages=messages,
-            tools=tools,
+            tools=self.tools,
             options=OLLAMA_OPTIONS,
             think=False,
         )
diff --git a/src/haiku/rag/qa/openai.py b/src/haiku/rag/qa/openai.py
new file mode 100644
index 00000000..f75a7396
--- /dev/null
+++ b/src/haiku/rag/qa/openai.py
@@ -0,0 +1,101 @@
+from collections.abc import Sequence
+
+try:
+    from openai import AsyncOpenAI
+    from openai.types.chat import (
+        ChatCompletionAssistantMessageParam,
+        ChatCompletionMessageParam,
+        ChatCompletionSystemMessageParam,
+        ChatCompletionToolMessageParam,
+        ChatCompletionUserMessageParam,
+    )
+    from openai.types.chat.chat_completion_tool_param import ChatCompletionToolParam
+
+    from haiku.rag.client import HaikuRAG
+    from haiku.rag.qa.base import QuestionAnswerAgentBase
+
+    class QuestionAnswerOpenAIAgent(QuestionAnswerAgentBase):
+        def __init__(self, client: HaikuRAG, model: str = "gpt-4o-mini"):
+            super().__init__(client, model or self._model)
+            self.tools: Sequence[ChatCompletionToolParam] = [
+                ChatCompletionToolParam(tool) for tool in self.tools
+            ]
+
+        async def answer(self, question: str) -> str:
+            openai_client = AsyncOpenAI()
+
+            # Define the search tool
+
+            messages: list[ChatCompletionMessageParam] = [
+                ChatCompletionSystemMessageParam(
+                    role="system", content=self._system_prompt
+                ),
+                ChatCompletionUserMessageParam(role="user", content=question),
+            ]
+
+            # Initial response with tool calling
+            response = await openai_client.chat.completions.create(
+                model=self._model,
+                messages=messages,
+                tools=self.tools,
+                temperature=0.0,
+            )
+
+            response_message = response.choices[0].message
+
+            if response_message.tool_calls:
+                messages.append(
+                    ChatCompletionAssistantMessageParam(
+                        role="assistant",
+                        content=response_message.content,
+                        tool_calls=[
+                            {
+                                "id": tc.id,
+                                "type": "function",
+                                "function": {
+                                    "name": tc.function.name,
+                                    "arguments": tc.function.arguments,
+                                },
+                            }
+                            for tc in response_message.tool_calls
+                        ],
+                    )
+                )
+
+                for tool_call in response_message.tool_calls:
+                    if tool_call.function.name == "search_documents":
+                        import json
+
+                        args = json.loads(tool_call.function.arguments)
+                        query = args.get("query", question)
+                        limit = int(args.get("limit", 3))
+
+                        search_results = await self._client.search(query, limit=limit)
+
+                        context_chunks = []
+                        for chunk, score in search_results:
+                            context_chunks.append(
+                                f"Content: {chunk.content}\nScore: {score:.4f}"
+                            )
+
+                        context = "\n\n".join(context_chunks)
+
+                        messages.append(
+                            ChatCompletionToolMessageParam(
+                                role="tool",
+                                content=context,
+                                tool_call_id=tool_call.id,
+                            )
+                        )
+
+                final_response = await openai_client.chat.completions.create(
+                    model=self._model,
+                    messages=messages,
+                    temperature=0.0,
+                )
+                return final_response.choices[0].message.content or ""
+            else:
+                return response_message.content or ""
+
+except ImportError:
+    pass
diff --git a/tests/conftest.py b/tests/conftest.py
index 2dcea549..31ed812d 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -3,8 +3,6 @@ from pathlib import Path
 import pytest
 from datasets import Dataset, load_dataset, load_from_disk
 
-from .llm_judge import LLMJudge
-
 
 @pytest.fixture(scope="session")
 def qa_corpus() -> Dataset:
@@ -18,8 +16,3 @@ def qa_corpus() -> Dataset:
         corpus = ds.filter(lambda doc: doc["document_topic"] == "News Stories")
         corpus.save_to_disk(ds_path)
         return corpus
-
-
-@pytest.fixture(scope="session")
-def llm_judge() -> LLMJudge:
-    return LLMJudge()
diff --git a/tests/generate_benchmark_db.py b/tests/generate_benchmark_db.py
index bbebb5eb..a736317a 100644
--- a/tests/generate_benchmark_db.py
+++ b/tests/generate_benchmark_db.py
@@ -6,7 +6,7 @@ from llm_judge import LLMJudge
 from tqdm import tqdm
 
 from haiku.rag.client import HaikuRAG
-from haiku.rag.qa.ollama import QA
+from haiku.rag.qa.ollama import QuestionAnswerOllamaAgent
 
 db_path = Path(__file__).parent / "data" / "benchmark.sqlite"
 
@@ -88,7 +88,7 @@ async def run_qa_benchmark(k: int | None = None):
     total_questions = 0
 
     async with HaikuRAG(db_path) as rag:
-        qa = QA(rag)
+        qa = QuestionAnswerOllamaAgent(rag)
 
         for i, doc in enumerate(tqdm(corpus, desc="QA Benchmarking")):
             question = doc["question"]  # type: ignore
diff --git a/tests/test_qa.py b/tests/test_qa.py
index 18496df7..41312a75 100644
--- a/tests/test_qa.py
+++ b/tests/test_qa.py
@@ -1,29 +1,52 @@
-from typing import TYPE_CHECKING
-
 import pytest
 from datasets import Dataset
 
 from haiku.rag.client import HaikuRAG
-from haiku.rag.qa.ollama import QA
+from haiku.rag.qa.ollama import QuestionAnswerOllamaAgent
 
-if TYPE_CHECKING:
-    import sys
-    from pathlib import Path
+try:
+    from haiku.rag.qa.openai import QuestionAnswerOpenAIAgent
 
-    sys.path.append(str(Path(__file__).parent))
-    from llm_judge import LLMJudge
+    OPENAI_AVAILABLE = True
+except ImportError:
+    QuestionAnswerOpenAIAgent = None
+    OPENAI_AVAILABLE = False
+
+from .llm_judge import LLMJudge
 
 
 @pytest.mark.asyncio
-async def test_qa_with_dataset_question(qa_corpus: Dataset, llm_judge: "LLMJudge"):
+async def test_qa_ollama_with_dataset_question(qa_corpus: Dataset):
     """Test QA with actual question from the dataset using LLM judge."""
     client = HaikuRAG(":memory:")
-    qa = QA(client)
+    qa = QuestionAnswerOllamaAgent(client)
+    llm_judge = LLMJudge()
+
+    doc = qa_corpus[1]
+    await client.create_document(
+        content=doc["document_extracted"], uri=doc["document_id"]
+    )
+
+    question = doc["question"]
+    expected_answer = doc["answer"]
+
+    answer = await qa.answer(question)
+    is_equivalent = await llm_judge.judge_answers(question, answer, expected_answer)
+
+    assert is_equivalent, (
+        f"Generated answer not equivalent to expected answer.\nQuestion: {question}\nGenerated: {answer}\nExpected: {expected_answer}"
+    )
+
+
+@pytest.mark.asyncio
+@pytest.mark.skipif(not OPENAI_AVAILABLE, reason="OpenAI not available")
+async def test_qa_openai_basic(qa_corpus: Dataset):
+    """Test OpenAI QA basic functionality."""
+    client = HaikuRAG(":memory:")
+    qa = QuestionAnswerOpenAIAgent(client)  # type: ignore
+    llm_judge = LLMJudge()
 
-    # Use the first document from the corpus
     doc = qa_corpus[1]
-
-    # Add the document to database
     await client.create_document(
         content=doc["document_extracted"], uri=doc["document_id"]
     )
@@ -32,11 +55,8 @@ async def test_qa_with_dataset_question(qa_corpus: Dataset, llm_judge: "LLMJudge
     expected_answer = doc["answer"]
 
     answer = await qa.answer(question)
-    # Use LLM judge to evaluate answer equivalence
     is_equivalent = await llm_judge.judge_answers(question, answer, expected_answer)
 
-    assert isinstance(answer, str)
-    assert len(answer) > 0
     assert is_equivalent, (
         f"Generated answer not equivalent to expected answer.\nQuestion: {question}\nGenerated: {answer}\nExpected: {expected_answer}"
     )

From 4bbc23dbd40925bcf29b8c094546e43ea4a75c50 Mon Sep 17 00:00:00 2001
From: Yiorgis Gozadinos <ggozadinos@gmail.com>
Date: Sat, 28 Jun 2025 09:21:45 +0300
Subject: [PATCH 15/42] Add QA to client, cli

---
 src/haiku/rag/app.py    | 11 +++++++++++
 src/haiku/rag/cli.py    | 15 +++++++++++++++
 src/haiku/rag/client.py | 16 +++++++++++++++-
 3 files changed, 41 insertions(+), 1 deletion(-)

diff --git a/src/haiku/rag/app.py b/src/haiku/rag/app.py
index 6db14c83..7e33ba5b 100644
--- a/src/haiku/rag/app.py
+++ b/src/haiku/rag/app.py
@@ -61,6 +61,17 @@ class HaikuRAGApp:
             for chunk, score in results:
                 self._rich_print_search_result(chunk, score)
 
+    async def ask(self, question: str):
+        async with HaikuRAG(db_path=self.db_path) as self.client:
+            try:
+                answer = await self.client.ask(question)
+                self.console.print(f"[bold blue]Question:[/bold blue] {question}")
+                self.console.print()
+                self.console.print("[bold green]Answer:[/bold green]")
+                self.console.print(Markdown(answer))
+            except Exception as e:
+                self.console.print(f"[red]Error: {e}[/red]")
+
     def _rich_print_document(self, doc: Document, truncate: bool = False):
         """Format a document for display."""
         if truncate:
diff --git a/src/haiku/rag/cli.py b/src/haiku/rag/cli.py
index 71e2c8b9..2e012cf1 100644
--- a/src/haiku/rag/cli.py
+++ b/src/haiku/rag/cli.py
@@ -113,6 +113,21 @@ def search(
     event_loop.run_until_complete(app.search(query=query, limit=limit, k=k))
 
 
+@cli.command("ask", help="Ask a question using the QA agent")
+def ask(
+    question: str = typer.Argument(
+        help="The question to ask",
+    ),
+    db: Path = typer.Option(
+        get_default_data_dir() / "haiku.rag.sqlite",
+        "--db",
+        help="Path to the SQLite database file",
+    ),
+):
+    app = HaikuRAGApp(db_path=db)
+    event_loop.run_until_complete(app.ask(question=question))
+
+
 @cli.command(
     "serve", help="Start the haiku.rag MCP server (by default in streamable HTTP mode)"
 )
diff --git a/src/haiku/rag/client.py b/src/haiku/rag/client.py
index 920f262a..0f24b3b9 100644
--- a/src/haiku/rag/client.py
+++ b/src/haiku/rag/client.py
@@ -36,7 +36,7 @@ class HaikuRAG:
         """Async context manager entry."""
         return self
 
-    async def __aexit__(self, exc_type, exc_val, exc_tb):
+    async def __aexit__(self, exc_type, exc_val, exc_tb):  # noqa: ARG002
         """Async context manager exit."""
         self.close()
         return False
@@ -256,6 +256,20 @@ class HaikuRAG:
         """
         return await self.chunk_repository.search_chunks_hybrid(query, limit, k)
 
+    async def ask(self, question: str) -> str:
+        """Ask a question using the configured QA agent.
+
+        Args:
+            question: The question to ask
+
+        Returns:
+            The generated answer as a string
+        """
+        from haiku.rag.qa import get_qa_agent
+
+        qa_agent = get_qa_agent(self)
+        return await qa_agent.answer(question)
+
     def close(self):
         """Close the underlying store connection."""
         self.store.close()

From 273a3bda4f7899c73ebddb86ab2018b54b633cba Mon Sep 17 00:00:00 2001
From: Yiorgis Gozadinos <ggozadinos@gmail.com>
Date: Sat, 28 Jun 2025 09:37:59 +0300
Subject: [PATCH 16/42] Document QA

---
 README.md             |  8 ++++++++
 docs/cli.md           |  9 +++++++++
 docs/configuration.md | 47 +++++++++++++++++++++++++++++++++++++++++++
 docs/index.md         |  6 ++++++
 docs/python.md        | 15 ++++++++++++++
 5 files changed, 85 insertions(+)

diff --git a/README.md b/README.md
index 8e20b3e1..cf86ae99 100644
--- a/README.md
+++ b/README.md
@@ -9,6 +9,7 @@ Retrieval-Augmented Generation (RAG) library on SQLite.
 - **Local SQLite**: No external servers required
 - **Multiple embedding providers**: Ollama, VoyageAI, OpenAI
 - **Hybrid search**: Vector + full-text search with Reciprocal Rank Fusion
+- **Question answering**: Built-in QA agents on your documents
 - **File monitoring**: Auto-index files when run as server
 - **40+ file formats**: PDF, DOCX, HTML, Markdown, audio, URLs
 - **MCP server**: Expose as tools for AI assistants
@@ -27,6 +28,9 @@ haiku-rag add-src document.pdf
 # Search
 haiku-rag search "query"
 
+# Ask questions
+haiku-rag ask "Who is the author of haiku.rag?"
+
 # Start server with file monitoring
 export MONITOR_DIRECTORIES="/path/to/docs"
 haiku-rag serve
@@ -45,6 +49,10 @@ async with HaikuRAG("database.db") as client:
     results = await client.search("query")
     for chunk, score in results:
         print(f"{score:.3f}: {chunk.content}")
+
+    # Ask questions
+    answer = await client.ask("Who is the author of haiku.rag?")
+    print(answer)
 ```
 
 ## MCP Server
diff --git a/docs/cli.md b/docs/cli.md
index e5100062..fae3db8a 100644
--- a/docs/cli.md
+++ b/docs/cli.md
@@ -47,6 +47,15 @@ With options:
 haiku-rag search "python programming" --limit 10 --k 100
 ```
 
+## Question Answering
+
+Ask questions about your documents:
+```bash
+haiku-rag ask "Who is the author of haiku.rag?"
+```
+
+The QA agent will search your documents for relevant information and provide a comprehensive answer.
+
 ## Server
 
 Start the MCP server:
diff --git a/docs/configuration.md b/docs/configuration.md
index 8ba6cc7e..cae8a1dd 100644
--- a/docs/configuration.md
+++ b/docs/configuration.md
@@ -55,3 +55,50 @@ EMBEDDINGS_MODEL="text-embedding-3-small"  # or text-embedding-3-large
 EMBEDDINGS_VECTOR_DIM=1536
 OPENAI_API_KEY="your-api-key"
 ```
+
+## Question Answering Providers
+
+Configure which LLM provider to use for question answering.
+
+### Ollama (Default)
+
+```bash
+QA_PROVIDER="ollama"
+QA_MODEL="qwen3"
+OLLAMA_BASE_URL="http://localhost:11434"
+```
+
+### OpenAI
+
+For OpenAI QA, you need to install haiku.rag with OpenAI extras:
+
+```bash
+uv pip install haiku.rag --extra openai
+```
+
+Then configure:
+
+```bash
+QA_PROVIDER="openai"
+QA_MODEL="gpt-4o-mini"  # or gpt-4, gpt-3.5-turbo, etc.
+OPENAI_API_KEY="your-api-key"
+```
+
+## Other Settings
+
+### Database and Storage
+
+```bash
+# Default data directory (where SQLite database is stored)
+DEFAULT_DATA_DIR="/path/to/data"
+```
+
+### Document Processing
+
+```bash
+# Chunk size for document processing
+CHUNK_SIZE=256
+
+# Chunk overlap for better context
+CHUNK_OVERLAP=32
+```
diff --git a/docs/index.md b/docs/index.md
index 404ae8b4..66117c63 100644
--- a/docs/index.md
+++ b/docs/index.md
@@ -8,6 +8,7 @@
 - **Local SQLite**: No need to run additional servers
 - **Support for various embedding providers**: Ollama, VoyageAI, OpenAI or add your own
 - **Hybrid Search**: Vector search using `sqlite-vec` combined with full-text search `FTS5`, using Reciprocal Rank Fusion
+- **Question Answering**: Built-in QA agents using Ollama or OpenAI.
 - **File monitoring**: Automatically index files when run as a server
 - **Extended file format support**: Parse 40+ file formats including PDF, DOCX, HTML, Markdown, audio and more. Or add a URL!
 - **MCP server**: Exposes functionality as MCP tools
@@ -31,12 +32,16 @@ async with HaikuRAG("database.db") as client:
 
     # Search documents
     results = await client.search("query")
+
+    # Ask questions
+    answer = await client.ask("Who is the author of haiku.rag?")
 ```
 
 Or use the CLI:
 ```bash
 haiku-rag add "Your document content"
 haiku-rag search "query"
+haiku-rag ask "Who is the author of haiku.rag?"
 ```
 
 ## Documentation
@@ -44,6 +49,7 @@ haiku-rag search "query"
 - [Installation](installation.md) - Install haiku.rag with different providers
 - [Configuration](configuration.md) - Environment variables and settings
 - [CLI](cli.md) - Command line interface usage
+- [Question Answering](qa.md) - QA agents and natural language queries
 - [Server](server.md) - File monitoring and server mode
 - [MCP](mcp.md) - Model Context Protocol integration
 - [Python](python.md) - Python API reference
diff --git a/docs/python.md b/docs/python.md
index 9908210e..ebc87f4c 100644
--- a/docs/python.md
+++ b/docs/python.md
@@ -91,4 +91,19 @@ for chunk, relevance_score in results:
     print(f"Relevance: {relevance_score:.3f}")
     print(f"Content: {chunk.content}")
     print(f"From document: {chunk.document_id}")
+    print(f"Document URI: {chunk.document_uri}")
+    print(f"Document metadata: {chunk.document_meta}")
 ```
+
+## Question Answering
+
+Ask questions about your documents:
+
+```python
+answer = await client.ask("Who is the author of haiku.rag?")
+print(answer)
+```
+
+The QA agent will search your documents for relevant information and use the configured LLM to generate a comprehensive answer.
+
+The QA provider and model can be configured via environment variables (see [Configuration](configuration.md)).

From 8e77d6b9c128be13426bdd17f48cbda7fee322fe Mon Sep 17 00:00:00 2001
From: Yiorgis Gozadinos <ggozadinos@gmail.com>
Date: Sat, 28 Jun 2025 09:43:37 +0300
Subject: [PATCH 17/42] vb

---
 pyproject.toml | 2 +-
 uv.lock        | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/pyproject.toml b/pyproject.toml
index f9e5534c..3e33e1a8 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,6 +1,6 @@
 [project]
 name = "haiku.rag"
-version = "0.2.0"
+version = "0.3.0"
 description = "Retrieval Augmented Generation (RAG) with SQLite"
 authors = [{ name = "Yiorgis Gozadinos", email = "ggozadinos@gmail.com" }]
 license = { text = "MIT" }
diff --git a/uv.lock b/uv.lock
index 14f667c6..176dc658 100644
--- a/uv.lock
+++ b/uv.lock
@@ -798,7 +798,7 @@ wheels = [
 
 [[package]]
 name = "haiku-rag"
-version = "0.2.0"
+version = "0.3.0"
 source = { editable = "." }
 dependencies = [
     { name = "fastmcp" },

From 954e96370a4ffc337afdea7560cf49549a8fcc7c Mon Sep 17 00:00:00 2001
From: Yiorgis Gozadinos <ggozadinos@gmail.com>
Date: Sun, 29 Jun 2025 09:26:33 +0300
Subject: [PATCH 18/42] Log document uri, meta in search results

---
 src/haiku/rag/app.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/src/haiku/rag/app.py b/src/haiku/rag/app.py
index 7e33ba5b..1e81a14f 100644
--- a/src/haiku/rag/app.py
+++ b/src/haiku/rag/app.py
@@ -99,6 +99,12 @@ class HaikuRAGApp:
             f"[repr.attrib_name]document_id[/repr.attrib_name]: {chunk.document_id} "
             f"[repr.attrib_name]score[/repr.attrib_name]: {score:.4f}"
         )
+        if chunk.document_uri:
+            self.console.print("[repr.attrib_name]document uri[/repr.attrib_name]:")
+            self.console.print(chunk.document_uri)
+        if chunk.document_meta:
+            self.console.print("[repr.attrib_name]document meta[/repr.attrib_name]:")
+            self.console.print(chunk.document_meta)
         self.console.print("[repr.attrib_name]content[/repr.attrib_name]:")
         self.console.print(content)
         self.console.rule()

From 11d9f2701e3378112ddbe0c92436face57a5a8ed Mon Sep 17 00:00:00 2001
From: Yiorgis Gozadinos <ggozadinos@gmail.com>
Date: Wed, 2 Jul 2025 10:55:06 +0300
Subject: [PATCH 19/42] Rebuild database in client & cli

---
 src/haiku/rag/app.py                      |  8 ++++
 src/haiku/rag/cli.py                      | 15 ++++++++
 src/haiku/rag/client.py                   | 18 +++++++++
 src/haiku/rag/store/repositories/chunk.py | 16 ++++++++
 tests/test_rebuild.py                     | 46 +++++++++++++++++++++++
 5 files changed, 103 insertions(+)
 create mode 100644 tests/test_rebuild.py

diff --git a/src/haiku/rag/app.py b/src/haiku/rag/app.py
index 1e81a14f..dca36151 100644
--- a/src/haiku/rag/app.py
+++ b/src/haiku/rag/app.py
@@ -72,6 +72,14 @@ class HaikuRAGApp:
             except Exception as e:
                 self.console.print(f"[red]Error: {e}[/red]")
 
+    async def rebuild(self):
+        async with HaikuRAG(db_path=self.db_path) as client:
+            try:
+                await client.rebuild_database()
+                self.console.print("[b]Database rebuild completed successfully.[/b]")
+            except Exception as e:
+                self.console.print(f"[red]Error rebuilding database: {e}[/red]")
+
     def _rich_print_document(self, doc: Document, truncate: bool = False):
         """Format a document for display."""
         if truncate:
diff --git a/src/haiku/rag/cli.py b/src/haiku/rag/cli.py
index 2e012cf1..03f653a6 100644
--- a/src/haiku/rag/cli.py
+++ b/src/haiku/rag/cli.py
@@ -128,6 +128,21 @@ def ask(
     event_loop.run_until_complete(app.ask(question=question))
 
 
+@cli.command(
+    "rebuild",
+    help="Rebuild the database by deleting all chunks and re-indexing all documents",
+)
+def rebuild(
+    db: Path = typer.Option(
+        get_default_data_dir() / "haiku.rag.sqlite",
+        "--db",
+        help="Path to the SQLite database file",
+    ),
+):
+    app = HaikuRAGApp(db_path=db)
+    event_loop.run_until_complete(app.rebuild())
+
+
 @cli.command(
     "serve", help="Start the haiku.rag MCP server (by default in streamable HTTP mode)"
 )
diff --git a/src/haiku/rag/client.py b/src/haiku/rag/client.py
index 0f24b3b9..396a9fde 100644
--- a/src/haiku/rag/client.py
+++ b/src/haiku/rag/client.py
@@ -270,6 +270,24 @@ class HaikuRAG:
         qa_agent = get_qa_agent(self)
         return await qa_agent.answer(question)
 
+    async def rebuild_database(self) -> None:
+        """Rebuild the database by deleting all chunks and re-indexing all documents."""
+        documents = await self.list_documents()
+
+        if not documents:
+            return
+
+        await self.chunk_repository.delete_all()
+
+        for doc in documents:
+            if doc.id is not None:
+                await self.chunk_repository.create_chunks_for_document(
+                    doc.id, doc.content, commit=False
+                )
+
+        if self.store._connection:
+            self.store._connection.commit()
+
     def close(self):
         """Close the underlying store connection."""
         self.store.close()
diff --git a/src/haiku/rag/store/repositories/chunk.py b/src/haiku/rag/store/repositories/chunk.py
index 1f33ec68..4261cfeb 100644
--- a/src/haiku/rag/store/repositories/chunk.py
+++ b/src/haiku/rag/store/repositories/chunk.py
@@ -208,6 +208,22 @@ class ChunkRepository(BaseRepository[Chunk]):
 
         return created_chunks
 
+    async def delete_all(self, commit: bool = True) -> bool:
+        """Delete all chunks from the database."""
+        if self.store._connection is None:
+            raise ValueError("Store connection is not available")
+
+        cursor = self.store._connection.cursor()
+
+        cursor.execute("DELETE FROM chunks_fts")
+        cursor.execute("DELETE FROM chunk_embeddings")
+        cursor.execute("DELETE FROM chunks")
+
+        deleted = cursor.rowcount > 0
+        if commit:
+            self.store._connection.commit()
+        return deleted
+
     async def delete_by_document_id(
         self, document_id: int, commit: bool = True
     ) -> bool:
diff --git a/tests/test_rebuild.py b/tests/test_rebuild.py
new file mode 100644
index 00000000..1192e7d1
--- /dev/null
+++ b/tests/test_rebuild.py
@@ -0,0 +1,46 @@
+import pytest
+from datasets import Dataset
+
+from haiku.rag.client import HaikuRAG
+from haiku.rag.store.models.document import Document
+
+
+@pytest.mark.asyncio
+async def test_rebuild_database(qa_corpus: Dataset):
+    """Test rebuild functionality with existing documents."""
+    client = HaikuRAG(":memory:")
+
+    created_docs: list[Document] = []
+    for content in qa_corpus["document_extracted"][:3]:
+        doc = await client.create_document(
+            content=content,
+        )
+        created_docs.append(doc)
+
+    documents_before = await client.list_documents()
+    assert len(documents_before) == 3
+
+    chunks_before = []
+    for doc in created_docs:
+        assert doc.id is not None
+        doc_chunks = await client.chunk_repository.get_by_document_id(doc.id)
+        chunks_before.extend(doc_chunks)
+
+    assert len(chunks_before) > 0
+
+    # Perform rebuild
+    await client.rebuild_database()
+
+    documents_after = await client.list_documents()
+    assert len(documents_after) == 3
+
+    # Verify chunks were recreated
+    chunks_after = []
+    for doc in documents_after:
+        if doc.id is not None:
+            doc_chunks = await client.chunk_repository.get_by_document_id(doc.id)
+            chunks_after.extend(doc_chunks)
+
+    assert len(chunks_after) > 0
+
+    client.close()

From 85443ee963836654a388ed8c1a1f589d31ce93d4 Mon Sep 17 00:00:00 2001
From: Yiorgis Gozadinos <ggozadinos@gmail.com>
Date: Wed, 2 Jul 2025 11:10:31 +0300
Subject: [PATCH 20/42] Turn rebuild into a generator, track progress in
 command line

---
 src/haiku/rag/app.py    | 19 ++++++++++++++++++-
 src/haiku/rag/client.py | 10 ++++++++--
 tests/test_rebuild.py   |  8 +++++++-
 3 files changed, 33 insertions(+), 4 deletions(-)

diff --git a/src/haiku/rag/app.py b/src/haiku/rag/app.py
index dca36151..e7d483e1 100644
--- a/src/haiku/rag/app.py
+++ b/src/haiku/rag/app.py
@@ -3,6 +3,7 @@ from pathlib import Path
 
 from rich.console import Console
 from rich.markdown import Markdown
+from rich.progress import Progress
 
 from haiku.rag.client import HaikuRAG
 from haiku.rag.config import Config
@@ -75,7 +76,23 @@ class HaikuRAGApp:
     async def rebuild(self):
         async with HaikuRAG(db_path=self.db_path) as client:
             try:
-                await client.rebuild_database()
+                documents = await client.list_documents()
+                total_docs = len(documents)
+
+                if total_docs == 0:
+                    self.console.print(
+                        "[yellow]No documents found in database.[/yellow]"
+                    )
+                    return
+
+                self.console.print(
+                    f"[b]Rebuilding database with {total_docs} documents...[/b]"
+                )
+                with Progress() as progress:
+                    task = progress.add_task("Rebuilding...", total=total_docs)
+                    async for _ in client.rebuild_database():
+                        progress.update(task, advance=1)
+
                 self.console.print("[b]Database rebuild completed successfully.[/b]")
             except Exception as e:
                 self.console.print(f"[red]Error rebuilding database: {e}[/red]")
diff --git a/src/haiku/rag/client.py b/src/haiku/rag/client.py
index 396a9fde..74478654 100644
--- a/src/haiku/rag/client.py
+++ b/src/haiku/rag/client.py
@@ -1,6 +1,7 @@
 import hashlib
 import mimetypes
 import tempfile
+from collections.abc import AsyncGenerator
 from pathlib import Path
 from typing import Literal
 from urllib.parse import urlparse
@@ -270,8 +271,12 @@ class HaikuRAG:
         qa_agent = get_qa_agent(self)
         return await qa_agent.answer(question)
 
-    async def rebuild_database(self) -> None:
-        """Rebuild the database by deleting all chunks and re-indexing all documents."""
+    async def rebuild_database(self) -> AsyncGenerator[int, None]:
+        """Rebuild the database by deleting all chunks and re-indexing all documents.
+
+        Yields:
+            int: The ID of the document currently being processed
+        """
         documents = await self.list_documents()
 
         if not documents:
@@ -284,6 +289,7 @@ class HaikuRAG:
                 await self.chunk_repository.create_chunks_for_document(
                     doc.id, doc.content, commit=False
                 )
+                yield doc.id
 
         if self.store._connection:
             self.store._connection.commit()
diff --git a/tests/test_rebuild.py b/tests/test_rebuild.py
index 1192e7d1..3254ce1d 100644
--- a/tests/test_rebuild.py
+++ b/tests/test_rebuild.py
@@ -29,7 +29,13 @@ async def test_rebuild_database(qa_corpus: Dataset):
     assert len(chunks_before) > 0
 
     # Perform rebuild
-    await client.rebuild_database()
+    processed_doc_ids = []
+    async for doc_id in client.rebuild_database():
+        processed_doc_ids.append(doc_id)
+
+    # Verify all documents were processed
+    expected_doc_ids = [doc.id for doc in created_docs]
+    assert set(processed_doc_ids) == set(expected_doc_ids)
 
     documents_after = await client.list_documents()
     assert len(documents_after) == 3

From fb837ede699b1cab0281cb71f0e024ac1417fcbf Mon Sep 17 00:00:00 2001
From: Yiorgis Gozadinos <ggozadinos@gmail.com>
Date: Wed, 2 Jul 2025 11:17:16 +0300
Subject: [PATCH 21/42] Documentation

---
 README.md      |  3 +++
 docs/cli.md    | 10 ++++++++++
 docs/python.md |  7 +++++++
 3 files changed, 20 insertions(+)

diff --git a/README.md b/README.md
index cf86ae99..bc3a9567 100644
--- a/README.md
+++ b/README.md
@@ -31,6 +31,9 @@ haiku-rag search "query"
 # Ask questions
 haiku-rag ask "Who is the author of haiku.rag?"
 
+# Rebuild database (re-chunk and re-embed all documents)
+haiku-rag rebuild
+
 # Start server with file monitoring
 export MONITOR_DIRECTORIES="/path/to/docs"
 haiku-rag serve
diff --git a/docs/cli.md b/docs/cli.md
index fae3db8a..efedc679 100644
--- a/docs/cli.md
+++ b/docs/cli.md
@@ -35,6 +35,16 @@ haiku-rag get 1
 haiku-rag delete 1
 ```
 
+### Rebuild Database
+
+Rebuild the database by deleting all chunks & embeddings and re-indexing all documents:
+
+```bash
+haiku-rag rebuild
+```
+
+Use this when you want to change things like the embedding model or chunk size for example.
+
 ## Search
 
 Basic search:
diff --git a/docs/python.md b/docs/python.md
index ebc87f4c..8ad47f4e 100644
--- a/docs/python.md
+++ b/docs/python.md
@@ -67,6 +67,13 @@ await client.update_document(doc)
 await client.delete_document(doc.id)
 ```
 
+### Rebuilding the Database
+
+```python
+async for doc_id in client.rebuild_database():
+    print(f"Processed document {doc_id}")
+```
+
 ## Searching Documents
 
 Basic search:

From 27e1c9fd49baaf55c3a9977a2bd56354e5bb83ca Mon Sep 17 00:00:00 2001
From: Yiorgis Gozadinos <ggozadinos@gmail.com>
Date: Wed, 2 Jul 2025 13:18:29 +0300
Subject: [PATCH 22/42] Show settings command

---
 docs/cli.md          |  7 +++++++
 src/haiku/rag/app.py | 20 ++++++++++++++++++++
 src/haiku/rag/cli.py |  6 ++++++
 3 files changed, 33 insertions(+)

diff --git a/docs/cli.md b/docs/cli.md
index efedc679..3c857cd5 100644
--- a/docs/cli.md
+++ b/docs/cli.md
@@ -66,6 +66,13 @@ haiku-rag ask "Who is the author of haiku.rag?"
 
 The QA agent will search your documents for relevant information and provide a comprehensive answer.
 
+## Configuration
+
+View current configuration settings:
+```bash
+haiku-rag settings
+```
+
 ## Server
 
 Start the MCP server:
diff --git a/src/haiku/rag/app.py b/src/haiku/rag/app.py
index e7d483e1..dedfce4c 100644
--- a/src/haiku/rag/app.py
+++ b/src/haiku/rag/app.py
@@ -97,6 +97,26 @@ class HaikuRAGApp:
             except Exception as e:
                 self.console.print(f"[red]Error rebuilding database: {e}[/red]")
 
+    def show_settings(self):
+        """Display current configuration settings."""
+        self.console.print("[bold]haiku.rag configuration[/bold]")
+        self.console.print()
+
+        # Get all config fields dynamically
+        for field_name, field_value in Config.model_dump().items():
+            # Format the display value
+            if isinstance(field_value, str) and (
+                "key" in field_name.lower()
+                or "password" in field_name.lower()
+                or "token" in field_name.lower()
+            ):
+                # Hide sensitive values but show if they're set
+                display_value = "✓ Set" if field_value else "✗ Not set"
+            else:
+                display_value = field_value
+
+            self.console.print(f"  [cyan]{field_name}[/cyan]: {display_value}")
+
     def _rich_print_document(self, doc: Document, truncate: bool = False):
         """Format a document for display."""
         if truncate:
diff --git a/src/haiku/rag/cli.py b/src/haiku/rag/cli.py
index 03f653a6..426af784 100644
--- a/src/haiku/rag/cli.py
+++ b/src/haiku/rag/cli.py
@@ -128,6 +128,12 @@ def ask(
     event_loop.run_until_complete(app.ask(question=question))
 
 
+@cli.command("settings", help="Display current configuration settings")
+def settings():
+    app = HaikuRAGApp(db_path=Path())  # Don't need actual DB for settings
+    app.show_settings()
+
+
 @cli.command(
     "rebuild",
     help="Rebuild the database by deleting all chunks and re-indexing all documents",

From e66c160055065d74fd96378a011784485013b74d Mon Sep 17 00:00:00 2001
From: Yiorgis Gozadinos <ggozadinos@gmail.com>
Date: Fri, 4 Jul 2025 11:48:19 +0300
Subject: [PATCH 23/42] Support for anthropic in Question/Answering

---
 pyproject.toml                |   1 +
 src/haiku/rag/config.py       |   3 +
 src/haiku/rag/qa/__init__.py  |  13 ++++
 src/haiku/rag/qa/anthropic.py | 112 ++++++++++++++++++++++++++++++++++
 tests/test_qa.py              |  36 ++++++++++-
 uv.lock                       |  24 +++++++-
 6 files changed, 186 insertions(+), 3 deletions(-)
 create mode 100644 src/haiku/rag/qa/anthropic.py

diff --git a/pyproject.toml b/pyproject.toml
index 3e33e1a8..f0b0b457 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -38,6 +38,7 @@ dependencies = [
 [project.optional-dependencies]
 voyageai = ["voyageai>=0.3.2"]
 openai = ["openai>=1.0.0"]
+anthropic = ["anthropic>=0.56.0"]
 
 [project.scripts]
 haiku-rag = "haiku.rag.cli:cli"
diff --git a/src/haiku/rag/config.py b/src/haiku/rag/config.py
index c388064f..e1552873 100644
--- a/src/haiku/rag/config.py
+++ b/src/haiku/rag/config.py
@@ -30,6 +30,7 @@ class AppConfig(BaseModel):
     # Provider keys
     VOYAGE_API_KEY: str = ""
     OPENAI_API_KEY: str = ""
+    ANTHROPIC_API_KEY: str = ""
 
     @field_validator("MONITOR_DIRECTORIES", mode="before")
     @classmethod
@@ -49,3 +50,5 @@ if Config.OPENAI_API_KEY:
     os.environ["OPENAI_API_KEY"] = Config.OPENAI_API_KEY
 if Config.VOYAGE_API_KEY:
     os.environ["VOYAGE_API_KEY"] = Config.VOYAGE_API_KEY
+if Config.ANTHROPIC_API_KEY:
+    os.environ["ANTHROPIC_API_KEY"] = Config.ANTHROPIC_API_KEY
diff --git a/src/haiku/rag/qa/__init__.py b/src/haiku/rag/qa/__init__.py
index 69ff4523..bcf53380 100644
--- a/src/haiku/rag/qa/__init__.py
+++ b/src/haiku/rag/qa/__init__.py
@@ -23,4 +23,17 @@ def get_qa_agent(client: HaikuRAG, model: str = "") -> QuestionAnswerAgentBase:
             )
         return QuestionAnswerOpenAIAgent(client, model or "gpt-4o-mini")
 
+    if Config.QA_PROVIDER == "anthropic":
+        try:
+            from haiku.rag.qa.anthropic import QuestionAnswerAnthropicAgent
+        except ImportError:
+            raise ImportError(
+                "Anthropic QA agent requires the 'anthropic' package. "
+                "Please install haiku.rag with the 'anthropic' extra:"
+                "uv pip install haiku.rag --extra anthropic"
+            )
+        return QuestionAnswerAnthropicAgent(
+            client, model or "claude-3-5-haiku-20241022"
+        )
+
     raise ValueError(f"Unsupported QA provider: {Config.QA_PROVIDER}")
diff --git a/src/haiku/rag/qa/anthropic.py b/src/haiku/rag/qa/anthropic.py
new file mode 100644
index 00000000..5b4479b3
--- /dev/null
+++ b/src/haiku/rag/qa/anthropic.py
@@ -0,0 +1,112 @@
+from collections.abc import Sequence
+
+try:
+    from anthropic import AsyncAnthropic
+    from anthropic.types import MessageParam, TextBlock, ToolParam, ToolUseBlock
+
+    from haiku.rag.client import HaikuRAG
+    from haiku.rag.qa.base import QuestionAnswerAgentBase
+
+    class QuestionAnswerAnthropicAgent(QuestionAnswerAgentBase):
+        def __init__(self, client: HaikuRAG, model: str = "claude-3-5-haiku-20241022"):
+            super().__init__(client, model or self._model)
+            self.tools: Sequence[ToolParam] = [
+                ToolParam(
+                    name="search_documents",
+                    description="Search the knowledge base for relevant documents",
+                    input_schema={
+                        "type": "object",
+                        "properties": {
+                            "query": {
+                                "type": "string",
+                                "description": "The search query to find relevant documents",
+                            },
+                            "limit": {
+                                "type": "integer",
+                                "description": "Maximum number of results to return",
+                                "default": 3,
+                            },
+                        },
+                        "required": ["query"],
+                    },
+                )
+            ]
+
+        async def answer(self, question: str) -> str:
+            anthropic_client = AsyncAnthropic()
+
+            messages: list[MessageParam] = [{"role": "user", "content": question}]
+
+            response = await anthropic_client.messages.create(
+                model=self._model,
+                max_tokens=4096,
+                system=self._system_prompt,
+                messages=messages,
+                tools=self.tools,
+                temperature=0.0,
+            )
+
+            if response.stop_reason == "tool_use":
+                messages.append({"role": "assistant", "content": response.content})
+
+                # Process tool calls
+                tool_results = []
+                for content_block in response.content:
+                    if isinstance(content_block, ToolUseBlock):
+                        if content_block.name == "search_documents":
+                            args = content_block.input
+                            query = (
+                                args.get("query", question)
+                                if isinstance(args, dict)
+                                else question
+                            )
+                            limit = (
+                                int(args.get("limit", 3))
+                                if isinstance(args, dict)
+                                else 3
+                            )
+
+                            search_results = await self._client.search(
+                                query, limit=limit
+                            )
+
+                            context_chunks = []
+                            for chunk, score in search_results:
+                                context_chunks.append(
+                                    f"Content: {chunk.content}\nScore: {score:.4f}"
+                                )
+
+                            context = "\n\n".join(context_chunks)
+
+                            tool_results.append(
+                                {
+                                    "type": "tool_result",
+                                    "tool_use_id": content_block.id,
+                                    "content": context,
+                                }
+                            )
+
+                if tool_results:
+                    messages.append({"role": "user", "content": tool_results})
+
+                    final_response = await anthropic_client.messages.create(
+                        model=self._model,
+                        max_tokens=4096,
+                        system=self._system_prompt,
+                        messages=messages,
+                        temperature=0.0,
+                    )
+                    if final_response.content:
+                        first_content = final_response.content[0]
+                        if isinstance(first_content, TextBlock):
+                            return first_content.text
+                    return ""
+
+            if response.content:
+                first_content = response.content[0]
+                if isinstance(first_content, TextBlock):
+                    return first_content.text
+            return ""
+
+except ImportError:
+    pass
diff --git a/tests/test_qa.py b/tests/test_qa.py
index 41312a75..686fb53f 100644
--- a/tests/test_qa.py
+++ b/tests/test_qa.py
@@ -12,11 +12,19 @@ except ImportError:
     QuestionAnswerOpenAIAgent = None
     OPENAI_AVAILABLE = False
 
+try:
+    from haiku.rag.qa.anthropic import QuestionAnswerAnthropicAgent
+
+    ANTHROPIC_AVAILABLE = True
+except ImportError:
+    QuestionAnswerAnthropicAgent = None
+    ANTHROPIC_AVAILABLE = False
+
 from .llm_judge import LLMJudge
 
 
 @pytest.mark.asyncio
-async def test_qa_ollama_with_dataset_question(qa_corpus: Dataset):
+async def test_qa_ollama(qa_corpus: Dataset):
     """Test QA with actual question from the dataset using LLM judge."""
     client = HaikuRAG(":memory:")
     qa = QuestionAnswerOllamaAgent(client)
@@ -40,7 +48,7 @@ async def test_qa_ollama_with_dataset_question(qa_corpus: Dataset):
 
 @pytest.mark.asyncio
 @pytest.mark.skipif(not OPENAI_AVAILABLE, reason="OpenAI not available")
-async def test_qa_openai_basic(qa_corpus: Dataset):
+async def test_qa_openai(qa_corpus: Dataset):
     """Test OpenAI QA basic functionality."""
     client = HaikuRAG(":memory:")
     qa = QuestionAnswerOpenAIAgent(client)  # type: ignore
@@ -60,3 +68,27 @@ async def test_qa_openai_basic(qa_corpus: Dataset):
     assert is_equivalent, (
         f"Generated answer not equivalent to expected answer.\nQuestion: {question}\nGenerated: {answer}\nExpected: {expected_answer}"
     )
+
+
+@pytest.mark.asyncio
+@pytest.mark.skipif(not ANTHROPIC_AVAILABLE, reason="Anthropic not available")
+async def test_qa_anthropic(qa_corpus: Dataset):
+    """Test Anthropic QA basic functionality."""
+    client = HaikuRAG(":memory:")
+    qa = QuestionAnswerAnthropicAgent(client)  # type: ignore
+    llm_judge = LLMJudge()
+
+    doc = qa_corpus[1]
+    await client.create_document(
+        content=doc["document_extracted"], uri=doc["document_id"]
+    )
+
+    question = doc["question"]
+    expected_answer = doc["answer"]
+
+    answer = await qa.answer(question)
+    is_equivalent = await llm_judge.judge_answers(question, answer, expected_answer)
+
+    assert is_equivalent, (
+        f"Generated answer not equivalent to expected answer.\nQuestion: {question}\nGenerated: {answer}\nExpected: {expected_answer}"
+    )
diff --git a/uv.lock b/uv.lock
index 176dc658..d12ce5a2 100644
--- a/uv.lock
+++ b/uv.lock
@@ -133,6 +133,24 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl", hash = "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53", size = 13643, upload-time = "2024-05-20T21:33:24.1Z" },
 ]
 
+[[package]]
+name = "anthropic"
+version = "0.56.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "anyio" },
+    { name = "distro" },
+    { name = "httpx" },
+    { name = "jiter" },
+    { name = "pydantic" },
+    { name = "sniffio" },
+    { name = "typing-extensions" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/4d/40/0c4eb5728466849803782c8a86eb315af1a6eb0efea6a751de120ab845c9/anthropic-0.56.0.tar.gz", hash = "sha256:56fa9eb61afa004a1664bc85eed071e77b96c579b77395e9cc893097e599f72e", size = 421538, upload-time = "2025-07-01T19:39:10.805Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/e5/90/7f4d4084f9c35c3ea3e784646ec12f9b2c8cf8743b2bb5489252659b5bda/anthropic-0.56.0-py3-none-any.whl", hash = "sha256:91f1f74abdcf0958d3296b657304588cc244b1107b89f973ff6f511afdacfc56", size = 289603, upload-time = "2025-07-01T19:39:08.794Z" },
+]
+
 [[package]]
 name = "anyio"
 version = "4.9.0"
@@ -815,6 +833,9 @@ dependencies = [
 ]
 
 [package.optional-dependencies]
+anthropic = [
+    { name = "anthropic" },
+]
 openai = [
     { name = "openai" },
 ]
@@ -837,6 +858,7 @@ dev = [
 
 [package.metadata]
 requires-dist = [
+    { name = "anthropic", marker = "extra == 'anthropic'", specifier = ">=0.56.0" },
     { name = "fastmcp", specifier = ">=2.8.1" },
     { name = "httpx", specifier = ">=0.28.1" },
     { name = "markitdown", extras = ["audio-transcription", "docx", "pdf", "pptx", "xlsx"], specifier = ">=0.1.2" },
@@ -851,7 +873,7 @@ requires-dist = [
     { name = "voyageai", marker = "extra == 'voyageai'", specifier = ">=0.3.2" },
     { name = "watchfiles", specifier = ">=1.1.0" },
 ]
-provides-extras = ["voyageai", "openai"]
+provides-extras = ["voyageai", "openai", "anthropic"]
 
 [package.metadata.requires-dev]
 dev = [

From b28ed57cf520cb16948990a1facc5f94f228e47a Mon Sep 17 00:00:00 2001
From: Yiorgis Gozadinos <ggozadinos@gmail.com>
Date: Fri, 4 Jul 2025 11:50:39 +0300
Subject: [PATCH 24/42] Update docs

---
 README.md             |  1 +
 docs/configuration.md | 16 ++++++++++++++++
 docs/index.md         |  2 +-
 docs/installation.md  |  6 ++++++
 4 files changed, 24 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index bc3a9567..873364f4 100644
--- a/README.md
+++ b/README.md
@@ -8,6 +8,7 @@ Retrieval-Augmented Generation (RAG) library on SQLite.
 
 - **Local SQLite**: No external servers required
 - **Multiple embedding providers**: Ollama, VoyageAI, OpenAI
+- **Multiple QA providers**: Ollama, OpenAI, Anthropic
 - **Hybrid search**: Vector + full-text search with Reciprocal Rank Fusion
 - **Question answering**: Built-in QA agents on your documents
 - **File monitoring**: Auto-index files when run as server
diff --git a/docs/configuration.md b/docs/configuration.md
index cae8a1dd..a9846508 100644
--- a/docs/configuration.md
+++ b/docs/configuration.md
@@ -84,6 +84,22 @@ QA_MODEL="gpt-4o-mini"  # or gpt-4, gpt-3.5-turbo, etc.
 OPENAI_API_KEY="your-api-key"
 ```
 
+### Anthropic
+
+For Anthropic QA, you need to install haiku.rag with Anthropic extras:
+
+```bash
+uv pip install haiku.rag --extra anthropic
+```
+
+Then configure:
+
+```bash
+QA_PROVIDER="anthropic"
+QA_MODEL="claude-3-5-haiku-20241022"  # or claude-3-5-sonnet-20241022, etc.
+ANTHROPIC_API_KEY="your-api-key"
+```
+
 ## Other Settings
 
 ### Database and Storage
diff --git a/docs/index.md b/docs/index.md
index 66117c63..19da8d6d 100644
--- a/docs/index.md
+++ b/docs/index.md
@@ -8,7 +8,7 @@
 - **Local SQLite**: No need to run additional servers
 - **Support for various embedding providers**: Ollama, VoyageAI, OpenAI or add your own
 - **Hybrid Search**: Vector search using `sqlite-vec` combined with full-text search `FTS5`, using Reciprocal Rank Fusion
-- **Question Answering**: Built-in QA agents using Ollama or OpenAI.
+- **Question Answering**: Built-in QA agents using Ollama, OpenAI, or Anthropic.
 - **File monitoring**: Automatically index files when run as a server
 - **Extended file format support**: Parse 40+ file formats including PDF, DOCX, HTML, Markdown, audio and more. Or add a URL!
 - **MCP server**: Exposes functionality as MCP tools
diff --git a/docs/installation.md b/docs/installation.md
index b3da3847..fd5eb509 100644
--- a/docs/installation.md
+++ b/docs/installation.md
@@ -24,6 +24,12 @@ uv pip install haiku.rag --extra voyageai
 uv pip install haiku.rag --extra openai
 ```
 
+### Anthropic
+
+```bash
+uv pip install haiku.rag --extra anthropic
+```
+
 ## Requirements
 
 - Python 3.10+

From ae1188da6b929a106944e36f0ebf58f84096ae98 Mon Sep 17 00:00:00 2001
From: Yiorgis Gozadinos <ggozadinos@gmail.com>
Date: Fri, 4 Jul 2025 11:58:23 +0300
Subject: [PATCH 25/42] vb

---
 pyproject.toml | 2 +-
 uv.lock        | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/pyproject.toml b/pyproject.toml
index f0b0b457..25c1fa2f 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,6 +1,6 @@
 [project]
 name = "haiku.rag"
-version = "0.3.0"
+version = "0.3.1"
 description = "Retrieval Augmented Generation (RAG) with SQLite"
 authors = [{ name = "Yiorgis Gozadinos", email = "ggozadinos@gmail.com" }]
 license = { text = "MIT" }
diff --git a/uv.lock b/uv.lock
index d12ce5a2..71155922 100644
--- a/uv.lock
+++ b/uv.lock
@@ -816,7 +816,7 @@ wheels = [
 
 [[package]]
 name = "haiku-rag"
-version = "0.3.0"
+version = "0.3.1"
 source = { editable = "." }
 dependencies = [
     { name = "fastmcp" },

From 1705e7a21cff15b93179f1287c5fa116359f6170 Mon Sep 17 00:00:00 2001
From: Yiorgis Gozadinos <ggozadinos@gmail.com>
Date: Fri, 4 Jul 2025 12:14:13 +0300
Subject: [PATCH 26/42] Honour QA_MODEL for anthropic and OpenAI

---
 src/haiku/rag/qa/__init__.py   | 7 ++-----
 tests/generate_benchmark_db.py | 4 ++--
 2 files changed, 4 insertions(+), 7 deletions(-)

diff --git a/src/haiku/rag/qa/__init__.py b/src/haiku/rag/qa/__init__.py
index bcf53380..f9047a91 100644
--- a/src/haiku/rag/qa/__init__.py
+++ b/src/haiku/rag/qa/__init__.py
@@ -8,7 +8,6 @@ def get_qa_agent(client: HaikuRAG, model: str = "") -> QuestionAnswerAgentBase:
     """
     Factory function to get the appropriate QA agent based on the configuration.
     """
-
     if Config.QA_PROVIDER == "ollama":
         return QuestionAnswerOllamaAgent(client, model or Config.QA_MODEL)
 
@@ -21,7 +20,7 @@ def get_qa_agent(client: HaikuRAG, model: str = "") -> QuestionAnswerAgentBase:
                 "Please install haiku.rag with the 'openai' extra:"
                 "uv pip install haiku.rag --extra openai"
             )
-        return QuestionAnswerOpenAIAgent(client, model or "gpt-4o-mini")
+        return QuestionAnswerOpenAIAgent(client, model or Config.QA_MODEL)
 
     if Config.QA_PROVIDER == "anthropic":
         try:
@@ -32,8 +31,6 @@ def get_qa_agent(client: HaikuRAG, model: str = "") -> QuestionAnswerAgentBase:
                 "Please install haiku.rag with the 'anthropic' extra:"
                 "uv pip install haiku.rag --extra anthropic"
             )
-        return QuestionAnswerAnthropicAgent(
-            client, model or "claude-3-5-haiku-20241022"
-        )
+        return QuestionAnswerAnthropicAgent(client, model or Config.QA_MODEL)
 
     raise ValueError(f"Unsupported QA provider: {Config.QA_PROVIDER}")
diff --git a/tests/generate_benchmark_db.py b/tests/generate_benchmark_db.py
index a736317a..b1ade523 100644
--- a/tests/generate_benchmark_db.py
+++ b/tests/generate_benchmark_db.py
@@ -6,7 +6,7 @@ from llm_judge import LLMJudge
 from tqdm import tqdm
 
 from haiku.rag.client import HaikuRAG
-from haiku.rag.qa.ollama import QuestionAnswerOllamaAgent
+from haiku.rag.qa import get_qa_agent
 
 db_path = Path(__file__).parent / "data" / "benchmark.sqlite"
 
@@ -88,7 +88,7 @@ async def run_qa_benchmark(k: int | None = None):
     total_questions = 0
 
     async with HaikuRAG(db_path) as rag:
-        qa = QuestionAnswerOllamaAgent(rag)
+        qa = get_qa_agent(rag)
 
         for i, doc in enumerate(tqdm(corpus, desc="QA Benchmarking")):
             question = doc["question"]  # type: ignore

From dd39cf59271b302451513160879099ab44ae62f1 Mon Sep 17 00:00:00 2001
From: Yiorgis Gozadinos <ggozadinos@gmail.com>
Date: Fri, 4 Jul 2025 12:15:31 +0300
Subject: [PATCH 27/42] vb

---
 pyproject.toml | 2 +-
 uv.lock        | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/pyproject.toml b/pyproject.toml
index 25c1fa2f..91801092 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,6 +1,6 @@
 [project]
 name = "haiku.rag"
-version = "0.3.1"
+version = "0.3.2"
 description = "Retrieval Augmented Generation (RAG) with SQLite"
 authors = [{ name = "Yiorgis Gozadinos", email = "ggozadinos@gmail.com" }]
 license = { text = "MIT" }
diff --git a/uv.lock b/uv.lock
index 71155922..7c8a8615 100644
--- a/uv.lock
+++ b/uv.lock
@@ -816,7 +816,7 @@ wheels = [
 
 [[package]]
 name = "haiku-rag"
-version = "0.3.1"
+version = "0.3.2"
 source = { editable = "." }
 dependencies = [
     { name = "fastmcp" },

From bea96f1b1b9acd1f86d365cdd35a6d7c523d6816 Mon Sep 17 00:00:00 2001
From: Yiorgis Gozadinos <ggozadinos@gmail.com>
Date: Fri, 4 Jul 2025 13:29:51 +0300
Subject: [PATCH 28/42] Better prompts

---
 src/haiku/rag/qa/prompts.py | 22 +++++++++++++++++-----
 tests/llm_judge.py          |  1 +
 2 files changed, 18 insertions(+), 5 deletions(-)

diff --git a/src/haiku/rag/qa/prompts.py b/src/haiku/rag/qa/prompts.py
index fc8f2c9b..6a63ccb3 100644
--- a/src/haiku/rag/qa/prompts.py
+++ b/src/haiku/rag/qa/prompts.py
@@ -1,7 +1,19 @@
 SYSTEM_PROMPT = """
-You are a helpful assistant that uses a RAG library to answer the user's prompt.
-Your task is to provide a concise and accurate answer based on the provided context.
-You should ask the provided tools to find relevant documents and then use the content of those documents to answer the question.
-Never make up information, always use the context to answer the question.
-If the context does not contain enough information to answer the question, respond with "I cannot answer that based on the provided context."
+You are a knowledgeable assistant that helps users find information from a document knowledge base.
+
+Your process:
+1. When a user asks a question, use the search_documents tool to find relevant information
+2. Search with specific keywords and phrases from the user's question
+3. Review the search results and their relevance scores
+4. Provide a comprehensive answer based only on the retrieved documents
+
+Guidelines:
+- Base your answers strictly on the provided document content
+- Quote or reference specific information when possible
+- If multiple documents contain relevant information, synthesize them coherently
+- Indicate when information is incomplete or when you need to search for additional context
+- If the retrieved documents don't contain sufficient information, clearly state: "I cannot find enough information in the knowledge base to answer this question."
+- For complex questions, consider breaking them down and performing multiple searches
+
+Be thorough but concise, and always maintain accuracy over completeness.
 """
diff --git a/tests/llm_judge.py b/tests/llm_judge.py
index 66bfd2cb..5af4cf0e 100644
--- a/tests/llm_judge.py
+++ b/tests/llm_judge.py
@@ -49,6 +49,7 @@ class LLMJudge:
         1. Do both answers provide the same answer?
         2. Do both answers directly address the question asked?
         3. Minor differences in wording or style are acceptable if the meaning of the answer is the same.
+        4. If one answer is more detailed but the other is correct, they can still be considered equivalent.
 
         Be strict but fair in your evaluation. Focus on factual correctness and whether both answers would satisfy someone asking the question."""
 

From 2f261a33f5f45ee7e3dfd33dcabdfb6493ddb74a Mon Sep 17 00:00:00 2001
From: Yiorgis Gozadinos <ggozadinos@gmail.com>
Date: Tue, 8 Jul 2025 12:51:58 +0300
Subject: [PATCH 29/42] Allow multiple tool calling rounds in QA agent

---
 BENCHMARKS.md => docs/benchmarks.md |   0
 src/haiku/rag/qa/anthropic.py       | 108 ++++++++++++-------------
 src/haiku/rag/qa/ollama.py          |  79 +++++++++---------
 src/haiku/rag/qa/openai.py          | 119 ++++++++++++++--------------
 src/haiku/rag/qa/prompts.py         |   5 +-
 5 files changed, 151 insertions(+), 160 deletions(-)
 rename BENCHMARKS.md => docs/benchmarks.md (100%)

diff --git a/BENCHMARKS.md b/docs/benchmarks.md
similarity index 100%
rename from BENCHMARKS.md
rename to docs/benchmarks.md
diff --git a/src/haiku/rag/qa/anthropic.py b/src/haiku/rag/qa/anthropic.py
index 5b4479b3..8827c5cb 100644
--- a/src/haiku/rag/qa/anthropic.py
+++ b/src/haiku/rag/qa/anthropic.py
@@ -37,75 +37,69 @@ try:
 
             messages: list[MessageParam] = [{"role": "user", "content": question}]
 
-            response = await anthropic_client.messages.create(
-                model=self._model,
-                max_tokens=4096,
-                system=self._system_prompt,
-                messages=messages,
-                tools=self.tools,
-                temperature=0.0,
-            )
+            max_rounds = 5  # Prevent infinite loops
 
-            if response.stop_reason == "tool_use":
-                messages.append({"role": "assistant", "content": response.content})
+            for _ in range(max_rounds):
+                response = await anthropic_client.messages.create(
+                    model=self._model,
+                    max_tokens=4096,
+                    system=self._system_prompt,
+                    messages=messages,
+                    tools=self.tools,
+                    temperature=0.0,
+                )
 
-                # Process tool calls
-                tool_results = []
-                for content_block in response.content:
-                    if isinstance(content_block, ToolUseBlock):
-                        if content_block.name == "search_documents":
-                            args = content_block.input
-                            query = (
-                                args.get("query", question)
-                                if isinstance(args, dict)
-                                else question
-                            )
-                            limit = (
-                                int(args.get("limit", 3))
-                                if isinstance(args, dict)
-                                else 3
-                            )
+                if response.stop_reason == "tool_use":
+                    messages.append({"role": "assistant", "content": response.content})
 
-                            search_results = await self._client.search(
-                                query, limit=limit
-                            )
-
-                            context_chunks = []
-                            for chunk, score in search_results:
-                                context_chunks.append(
-                                    f"Content: {chunk.content}\nScore: {score:.4f}"
+                    # Process tool calls
+                    tool_results = []
+                    for content_block in response.content:
+                        if isinstance(content_block, ToolUseBlock):
+                            if content_block.name == "search_documents":
+                                args = content_block.input
+                                query = (
+                                    args.get("query", question)
+                                    if isinstance(args, dict)
+                                    else question
+                                )
+                                limit = (
+                                    int(args.get("limit", 3))
+                                    if isinstance(args, dict)
+                                    else 3
                                 )
 
-                            context = "\n\n".join(context_chunks)
+                                search_results = await self._client.search(
+                                    query, limit=limit
+                                )
 
-                            tool_results.append(
-                                {
-                                    "type": "tool_result",
-                                    "tool_use_id": content_block.id,
-                                    "content": context,
-                                }
-                            )
+                                context_chunks = []
+                                for chunk, score in search_results:
+                                    context_chunks.append(
+                                        f"Content: {chunk.content}\nScore: {score:.4f}"
+                                    )
 
-                if tool_results:
-                    messages.append({"role": "user", "content": tool_results})
+                                context = "\n\n".join(context_chunks)
 
-                    final_response = await anthropic_client.messages.create(
-                        model=self._model,
-                        max_tokens=4096,
-                        system=self._system_prompt,
-                        messages=messages,
-                        temperature=0.0,
-                    )
-                    if final_response.content:
-                        first_content = final_response.content[0]
+                                tool_results.append(
+                                    {
+                                        "type": "tool_result",
+                                        "tool_use_id": content_block.id,
+                                        "content": context,
+                                    }
+                                )
+
+                    if tool_results:
+                        messages.append({"role": "user", "content": tool_results})
+                else:
+                    # No tool use, return the response
+                    if response.content:
+                        first_content = response.content[0]
                         if isinstance(first_content, TextBlock):
                             return first_content.text
                     return ""
 
-            if response.content:
-                first_content = response.content[0]
-                if isinstance(first_content, TextBlock):
-                    return first_content.text
+            # If we've exhausted max rounds, return empty string
             return ""
 
 except ImportError:
diff --git a/src/haiku/rag/qa/ollama.py b/src/haiku/rag/qa/ollama.py
index c8cac4ce..9c4ee01a 100644
--- a/src/haiku/rag/qa/ollama.py
+++ b/src/haiku/rag/qa/ollama.py
@@ -14,54 +14,51 @@ class QuestionAnswerOllamaAgent(QuestionAnswerAgentBase):
     async def answer(self, question: str) -> str:
         ollama_client = AsyncClient(host=Config.OLLAMA_BASE_URL)
 
-        # Define the search tool
-
         messages = [
             {"role": "system", "content": self._system_prompt},
             {"role": "user", "content": question},
         ]
 
-        # Initial response with tool calling
-        response = await ollama_client.chat(
-            model=self._model,
-            messages=messages,
-            tools=self.tools,
-            options=OLLAMA_OPTIONS,
-            think=False,
-        )
+        max_rounds = 5  # Prevent infinite loops
 
-        if response.get("message", {}).get("tool_calls"):
-            for tool_call in response["message"]["tool_calls"]:
-                if tool_call["function"]["name"] == "search_documents":
-                    args = tool_call["function"]["arguments"]
-                    query = args.get("query", question)
-                    limit = int(args.get("limit", 3))
-
-                    search_results = await self._client.search(query, limit=limit)
-
-                    context_chunks = []
-                    for chunk, score in search_results:
-                        context_chunks.append(
-                            f"Content: {chunk.content}\nScore: {score:.4f}"
-                        )
-
-                    context = "\n\n".join(context_chunks)
-
-                    messages.append(response["message"])
-                    messages.append(
-                        {
-                            "role": "tool",
-                            "content": context,
-                            "tool_call_id": tool_call.get("id", "search_tool"),
-                        }
-                    )
-
-            final_response = await ollama_client.chat(
+        for _ in range(max_rounds):
+            response = await ollama_client.chat(
                 model=self._model,
                 messages=messages,
-                think=False,
+                tools=self.tools,
                 options=OLLAMA_OPTIONS,
+                think=False,
             )
-            return final_response["message"]["content"]
-        else:
-            return response["message"]["content"]
+
+            if response.get("message", {}).get("tool_calls"):
+                messages.append(response["message"])
+
+                for tool_call in response["message"]["tool_calls"]:
+                    if tool_call["function"]["name"] == "search_documents":
+                        args = tool_call["function"]["arguments"]
+                        query = args.get("query", question)
+                        limit = int(args.get("limit", 3))
+
+                        search_results = await self._client.search(query, limit=limit)
+
+                        context_chunks = []
+                        for chunk, score in search_results:
+                            context_chunks.append(
+                                f"Content: {chunk.content}\nScore: {score:.4f}"
+                            )
+
+                        context = "\n\n".join(context_chunks)
+
+                        messages.append(
+                            {
+                                "role": "tool",
+                                "content": context,
+                                "tool_call_id": tool_call.get("id", "search_tool"),
+                            }
+                        )
+            else:
+                # No tool calls, return the response
+                return response["message"]["content"]
+
+        # If we've exhausted max rounds, return empty string
+        return ""
diff --git a/src/haiku/rag/qa/openai.py b/src/haiku/rag/qa/openai.py
index f75a7396..24f58cf9 100644
--- a/src/haiku/rag/qa/openai.py
+++ b/src/haiku/rag/qa/openai.py
@@ -24,8 +24,6 @@ try:
         async def answer(self, question: str) -> str:
             openai_client = AsyncOpenAI()
 
-            # Define the search tool
-
             messages: list[ChatCompletionMessageParam] = [
                 ChatCompletionSystemMessageParam(
                     role="system", content=self._system_prompt
@@ -33,69 +31,70 @@ try:
                 ChatCompletionUserMessageParam(role="user", content=question),
             ]
 
-            # Initial response with tool calling
-            response = await openai_client.chat.completions.create(
-                model=self._model,
-                messages=messages,
-                tools=self.tools,
-                temperature=0.0,
-            )
+            max_rounds = 5  # Prevent infinite loops
 
-            response_message = response.choices[0].message
-
-            if response_message.tool_calls:
-                messages.append(
-                    ChatCompletionAssistantMessageParam(
-                        role="assistant",
-                        content=response_message.content,
-                        tool_calls=[
-                            {
-                                "id": tc.id,
-                                "type": "function",
-                                "function": {
-                                    "name": tc.function.name,
-                                    "arguments": tc.function.arguments,
-                                },
-                            }
-                            for tc in response_message.tool_calls
-                        ],
-                    )
-                )
-
-                for tool_call in response_message.tool_calls:
-                    if tool_call.function.name == "search_documents":
-                        import json
-
-                        args = json.loads(tool_call.function.arguments)
-                        query = args.get("query", question)
-                        limit = int(args.get("limit", 3))
-
-                        search_results = await self._client.search(query, limit=limit)
-
-                        context_chunks = []
-                        for chunk, score in search_results:
-                            context_chunks.append(
-                                f"Content: {chunk.content}\nScore: {score:.4f}"
-                            )
-
-                        context = "\n\n".join(context_chunks)
-
-                        messages.append(
-                            ChatCompletionToolMessageParam(
-                                role="tool",
-                                content=context,
-                                tool_call_id=tool_call.id,
-                            )
-                        )
-
-                final_response = await openai_client.chat.completions.create(
+            for _ in range(max_rounds):
+                response = await openai_client.chat.completions.create(
                     model=self._model,
                     messages=messages,
+                    tools=self.tools,
                     temperature=0.0,
                 )
-                return final_response.choices[0].message.content or ""
-            else:
-                return response_message.content or ""
+
+                response_message = response.choices[0].message
+
+                if response_message.tool_calls:
+                    messages.append(
+                        ChatCompletionAssistantMessageParam(
+                            role="assistant",
+                            content=response_message.content,
+                            tool_calls=[
+                                {
+                                    "id": tc.id,
+                                    "type": "function",
+                                    "function": {
+                                        "name": tc.function.name,
+                                        "arguments": tc.function.arguments,
+                                    },
+                                }
+                                for tc in response_message.tool_calls
+                            ],
+                        )
+                    )
+
+                    for tool_call in response_message.tool_calls:
+                        if tool_call.function.name == "search_documents":
+                            import json
+
+                            args = json.loads(tool_call.function.arguments)
+                            query = args.get("query", question)
+                            limit = int(args.get("limit", 3))
+
+                            search_results = await self._client.search(
+                                query, limit=limit
+                            )
+
+                            context_chunks = []
+                            for chunk, score in search_results:
+                                context_chunks.append(
+                                    f"Content: {chunk.content}\nScore: {score:.4f}"
+                                )
+
+                            context = "\n\n".join(context_chunks)
+
+                            messages.append(
+                                ChatCompletionToolMessageParam(
+                                    role="tool",
+                                    content=context,
+                                    tool_call_id=tool_call.id,
+                                )
+                            )
+                else:
+                    # No tool calls, return the response
+                    return response_message.content or ""
+
+            # If we've exhausted max rounds, return empty string
+            return ""
 
 except ImportError:
     pass
diff --git a/src/haiku/rag/qa/prompts.py b/src/haiku/rag/qa/prompts.py
index 6a63ccb3..283c40e2 100644
--- a/src/haiku/rag/qa/prompts.py
+++ b/src/haiku/rag/qa/prompts.py
@@ -5,7 +5,8 @@ Your process:
 1. When a user asks a question, use the search_documents tool to find relevant information
 2. Search with specific keywords and phrases from the user's question
 3. Review the search results and their relevance scores
-4. Provide a comprehensive answer based only on the retrieved documents
+4. If you need additional context, perform follow-up searches with different keywords
+5. Provide a comprehensive answer based only on the retrieved documents
 
 Guidelines:
 - Base your answers strictly on the provided document content
@@ -15,5 +16,5 @@ Guidelines:
 - If the retrieved documents don't contain sufficient information, clearly state: "I cannot find enough information in the knowledge base to answer this question."
 - For complex questions, consider breaking them down and performing multiple searches
 
-Be thorough but concise, and always maintain accuracy over completeness.
+Be concise, and always maintain accuracy over completeness. Prefer short, direct answers that are well-supported by the documents.
 """

From 9d72a158a6a5a8fd23bf0afa67842b1346c2b1c8 Mon Sep 17 00:00:00 2001
From: Yiorgis Gozadinos <ggozadinos@gmail.com>
Date: Tue, 8 Jul 2025 19:01:54 +0300
Subject: [PATCH 30/42] Document benchmarks

---
 docs/benchmarks.md             | 28 +++++++++++++++++++++-------
 mkdocs.yml                     |  1 +
 tests/generate_benchmark_db.py |  1 +
 3 files changed, 23 insertions(+), 7 deletions(-)

diff --git a/docs/benchmarks.md b/docs/benchmarks.md
index 977e1927..e1fc2bf7 100644
--- a/docs/benchmarks.md
+++ b/docs/benchmarks.md
@@ -1,13 +1,27 @@
-# `haiku.rag` benchmarks
+# Benchmarks
 
-We use [repliqa](https://huggingface.co/datasets/ServiceNow/repliqa) for the evaluation of `haiku.rag`
+We use the [repliqa](https://huggingface.co/datasets/ServiceNow/repliqa) dataset for the evaluation of `haiku.rag`.
 
-* Recall
+You can perform your own evaluations using as example the script found at
+`tests/generate_benchmark_db.py`.
 
-We load the `News Stories` from `repliqa_3` which is 1035 documents, using `tests/generate_benchmark_db.py`, using the `mxbai-embed-large` Ollama embeddings.
+## Recall
 
-Subsequently, we run a search over the `question` for each row of the dataset and check whether we match the document that answers the question. The recall obtained is ~0.75 for matching in the top result, raising to ~0.75 for the top 3 results.
+In order to calculate recall, we load the `News Stories` from `repliqa_3` which is 1035 documents and index them in a sqlite db. Subsequently, we run a search over the `question` field for each row of the dataset and check whether we match the document that answers the question.
 
-* Question/Answer evaluation
 
-We use the  `News Stories` from `repliqa_3` using the `mxbai-embed-large` Ollama embeddings, with a QA agent also using Ollama with the `qwen3` model (8b). For each story we ask the `question` and use an LLM judge (also `qwen3`) to evaluate whether the answer is correct or not. Thus we obtain accuracy of ~0.54.
+The recall obtained is ~0.73 for matching in the top result, raising to ~0.75 for the top 3 results.
+
+| Model                                 | Document in top 1 | Document in top 3 |
+|---------------------------------------|-------------------|-------------------|
+| Ollama / `mxbai-embed-large`          | 0.73              | 0.75              |
+| OpenAI / `text-embeddings-3-small`    |                   |                   |
+
+## Question/Answer evaluation
+
+Again using the same dataset, we use a QA agent to answer the question. In addition we use an LLM judge (using the Ollama `qwen3`) to evaluate whether the answer is correct or not. The obtained accuracy is as follows:
+
+| Embedding Model              | QA Model                          | Accuracy  |
+|------------------------------|-----------------------------------|-----------|
+| Ollama / `mxbai-embed-large` | Ollama / `qwen3`                  | 0.64      |
+| Ollama / `mxbai-embed-large` | Anthropic / `Claude Sonnet 3.7`   | 0.79      |
diff --git a/mkdocs.yml b/mkdocs.yml
index e9767caa..07bcb523 100644
--- a/mkdocs.yml
+++ b/mkdocs.yml
@@ -63,6 +63,7 @@ nav:
       - Server: server.md
       - MCP: mcp.md
       - Python: python.md
+      - Benchmarks: benchmarks.md
 markdown_extensions:
   - admonition
   - attr_list
diff --git a/tests/generate_benchmark_db.py b/tests/generate_benchmark_db.py
index b1ade523..20bb01b2 100644
--- a/tests/generate_benchmark_db.py
+++ b/tests/generate_benchmark_db.py
@@ -106,6 +106,7 @@ async def run_qa_benchmark(k: int | None = None):
             if is_equivalent:
                 correct_answers += 1
             total_questions += 1
+            print("Current score:", correct_answers, "/", total_questions)
 
     accuracy = correct_answers / total_questions if total_questions > 0 else 0
 

From 42e62f92a1b44237f6f4d0c5e715bcd92c2fa35d Mon Sep 17 00:00:00 2001
From: Yiorgis Gozadinos <ggozadinos@gmail.com>
Date: Wed, 9 Jul 2025 10:27:32 +0300
Subject: [PATCH 31/42] Better formatting of benchmark script

---
 docs/benchmarks.md             |   2 +-
 tests/generate_benchmark_db.py | 143 +++++++++++++++++++--------------
 2 files changed, 83 insertions(+), 62 deletions(-)

diff --git a/docs/benchmarks.md b/docs/benchmarks.md
index e1fc2bf7..948cba93 100644
--- a/docs/benchmarks.md
+++ b/docs/benchmarks.md
@@ -15,7 +15,7 @@ The recall obtained is ~0.73 for matching in the top result, raising to ~0.75 fo
 | Model                                 | Document in top 1 | Document in top 3 |
 |---------------------------------------|-------------------|-------------------|
 | Ollama / `mxbai-embed-large`          | 0.73              | 0.75              |
-| OpenAI / `text-embeddings-3-small`    |                   |                   |
+| OpenAI / `text-embeddings-3-small`    | 0.75              | 0.88              |
 
 ## Question/Answer evaluation
 
diff --git a/tests/generate_benchmark_db.py b/tests/generate_benchmark_db.py
index 20bb01b2..70b9c468 100644
--- a/tests/generate_benchmark_db.py
+++ b/tests/generate_benchmark_db.py
@@ -3,28 +3,37 @@ from pathlib import Path
 
 from datasets import Dataset, load_dataset
 from llm_judge import LLMJudge
-from tqdm import tqdm
+from rich.console import Console
+from rich.progress import Progress
 
 from haiku.rag.client import HaikuRAG
 from haiku.rag.qa import get_qa_agent
 
+console = Console()
+
 db_path = Path(__file__).parent / "data" / "benchmark.sqlite"
 
 
 async def populate_db():
-    if (db_path).exists():
-        print("Benchmark database already exists. Skipping creation.")
-        return
-
     ds: Dataset = load_dataset("ServiceNow/repliqa")["repliqa_3"]  # type: ignore
     corpus = ds.filter(lambda doc: doc["document_topic"] == "News Stories")
 
-    async with HaikuRAG(db_path) as rag:
-        for i, doc in enumerate(tqdm(corpus)):
-            await rag.create_document(
-                content=doc["document_extracted"],  # type: ignore
-                uri=doc["document_id"],  # type: ignore
-            )
+    with Progress() as progress:
+        task = progress.add_task("[green]Populating database...", total=len(corpus))
+
+        async with HaikuRAG(db_path) as rag:
+            for doc in corpus:
+                uri = doc["document_id"]  # type: ignore
+                existing_doc = await rag.get_document_by_uri(uri)
+                if existing_doc is not None:
+                    progress.advance(task)
+                    continue
+
+                await rag.create_document(
+                    content=doc["document_extracted"],  # type: ignore
+                    uri=uri,
+                )
+                progress.advance(task)
 
 
 async def run_match_benchmark():
@@ -36,41 +45,48 @@ async def run_match_benchmark():
     correct_at_3 = 0
     total_queries = 0
 
-    async with HaikuRAG(db_path) as rag:
-        for i, doc in enumerate(tqdm(corpus)):
-            doc_id = doc["document_id"]  # type: ignore
-            matches = await rag.search(
-                query=doc["question"],  # type: ignore
-                limit=3,
-            )
+    with Progress() as progress:
+        task = progress.add_task(
+            "[blue]Running retrieval benchmark...", total=len(corpus)
+        )
 
-            total_queries += 1
+        async with HaikuRAG(db_path) as rag:
+            for doc in corpus:
+                doc_id = doc["document_id"]  # type: ignore
+                matches = await rag.search(
+                    query=doc["question"],  # type: ignore
+                    limit=3,
+                )
 
-            # Check position of correct document in results
-            for position, (chunk, _) in enumerate(matches):
-                retrieved = await rag.get_document_by_id(chunk.document_id)
-                if retrieved and retrieved.uri == doc_id:
-                    if position == 0:  # First position
-                        correct_at_1 += 1
-                        correct_at_2 += 1
-                        correct_at_3 += 1
-                    elif position == 1:  # Second position
-                        correct_at_2 += 1
-                        correct_at_3 += 1
-                    elif position == 2:  # Third position
-                        correct_at_3 += 1
-                    break
+                total_queries += 1
+
+                # Check position of correct document in results
+                for position, (chunk, _) in enumerate(matches):
+                    retrieved = await rag.get_document_by_id(chunk.document_id)
+                    if retrieved and retrieved.uri == doc_id:
+                        if position == 0:  # First position
+                            correct_at_1 += 1
+                            correct_at_2 += 1
+                            correct_at_3 += 1
+                        elif position == 1:  # Second position
+                            correct_at_2 += 1
+                            correct_at_3 += 1
+                        elif position == 2:  # Third position
+                            correct_at_3 += 1
+                        break
+
+                progress.advance(task)
 
     # Calculate recall metrics
     recall_at_1 = correct_at_1 / total_queries
     recall_at_2 = correct_at_2 / total_queries
     recall_at_3 = correct_at_3 / total_queries
 
-    print("\n=== Retrieval Benchmark Results ===")
-    print(f"Total queries: {total_queries}")
-    print(f"Recall@1: {recall_at_1:.4f}")
-    print(f"Recall@2: {recall_at_2:.4f}")
-    print(f"Recall@3: {recall_at_3:.4f}")
+    console.print("\n=== Retrieval Benchmark Results ===", style="bold cyan")
+    console.print(f"Total queries: {total_queries}")
+    console.print(f"Recall@1: {recall_at_1:.4f}")
+    console.print(f"Recall@2: {recall_at_2:.4f}")
+    console.print(f"Recall@3: {recall_at_3:.4f}")
 
     return {"recall@1": recall_at_1, "recall@2": recall_at_2, "recall@3": recall_at_3}
 
@@ -87,42 +103,47 @@ async def run_qa_benchmark(k: int | None = None):
     correct_answers = 0
     total_questions = 0
 
-    async with HaikuRAG(db_path) as rag:
-        qa = get_qa_agent(rag)
+    with Progress() as progress:
+        task = progress.add_task("[yellow]Running QA benchmark...", total=len(corpus))
 
-        for i, doc in enumerate(tqdm(corpus, desc="QA Benchmarking")):
-            question = doc["question"]  # type: ignore
-            expected_answer = doc["answer"]  # type: ignore
+        async with HaikuRAG(db_path) as rag:
+            qa = get_qa_agent(rag)
 
-            generated_answer = await qa.answer(question)
-            is_equivalent = await judge.judge_answers(
-                question, generated_answer, expected_answer
-            )
-            print(f"Question: {question}")
-            print(f"Expected: {expected_answer}")
-            print(f"Generated: {generated_answer}")
-            print(f"Equivalent: {is_equivalent}\n")
+            for doc in corpus:
+                question = doc["question"]  # type: ignore
+                expected_answer = doc["answer"]  # type: ignore
 
-            if is_equivalent:
-                correct_answers += 1
-            total_questions += 1
-            print("Current score:", correct_answers, "/", total_questions)
+                generated_answer = await qa.answer(question)
+                is_equivalent = await judge.judge_answers(
+                    question, generated_answer, expected_answer
+                )
+                console.print(f"Question: {question}")
+                console.print(f"Expected: {expected_answer}")
+                console.print(f"Generated: {generated_answer}")
+                console.print(f"Equivalent: {is_equivalent}\n")
+
+                if is_equivalent:
+                    correct_answers += 1
+                total_questions += 1
+                console.print("Current score:", correct_answers, "/", total_questions)
+
+                progress.advance(task)
 
     accuracy = correct_answers / total_questions if total_questions > 0 else 0
 
-    print("\n=== QA Benchmark Results ===")
-    print(f"Total questions: {total_questions}")
-    print(f"Correct answers: {correct_answers}")
-    print(f"QA Accuracy: {accuracy:.4f} ({accuracy * 100:.2f}%)")
+    console.print("\n=== QA Benchmark Results ===", style="bold cyan")
+    console.print(f"Total questions: {total_questions}")
+    console.print(f"Correct answers: {correct_answers}")
+    console.print(f"QA Accuracy: {accuracy:.4f} ({accuracy * 100:.2f}%)")
 
 
 async def main():
     await populate_db()
 
-    print("Running retrieval benchmarks...")
+    console.print("Running retrieval benchmarks...", style="bold blue")
     await run_match_benchmark()
 
-    print("\nRunning QA benchmarks...")
+    console.print("\nRunning QA benchmarks...", style="bold yellow")
     await run_qa_benchmark()
 
 

From 556a5d2b58a0584b48bee152afe23278aa9f401a Mon Sep 17 00:00:00 2001
From: Yiorgis Gozadinos <ggozadinos@gmail.com>
Date: Wed, 9 Jul 2025 10:49:41 +0300
Subject: [PATCH 32/42] vb

---
 README.md      | 1 +
 pyproject.toml | 2 +-
 uv.lock        | 2 +-
 3 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 873364f4..968c6fa1 100644
--- a/README.md
+++ b/README.md
@@ -77,3 +77,4 @@ Full documentation at: https://ggozad.github.io/haiku.rag/
 - [Configuration](https://ggozad.github.io/haiku.rag/configuration/) - Environment variables
 - [CLI](https://ggozad.github.io/haiku.rag/cli/) - Command reference
 - [Python API](https://ggozad.github.io/haiku.rag/python/) - Complete API docs
+- [Benchmarks](https://ggozad.github.io/haiku.rag/benchmarks/) - Performance Benchmarks
diff --git a/pyproject.toml b/pyproject.toml
index 91801092..49237c9a 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,6 +1,6 @@
 [project]
 name = "haiku.rag"
-version = "0.3.2"
+version = "0.3.3"
 description = "Retrieval Augmented Generation (RAG) with SQLite"
 authors = [{ name = "Yiorgis Gozadinos", email = "ggozadinos@gmail.com" }]
 license = { text = "MIT" }
diff --git a/uv.lock b/uv.lock
index 7c8a8615..088ad51f 100644
--- a/uv.lock
+++ b/uv.lock
@@ -816,7 +816,7 @@ wheels = [
 
 [[package]]
 name = "haiku-rag"
-version = "0.3.2"
+version = "0.3.3"
 source = { editable = "." }
 dependencies = [
     { name = "fastmcp" },

From eefd51ead600619952e6244241a9385f83126787 Mon Sep 17 00:00:00 2001
From: Yiorgis Gozadinos <ggozadinos@gmail.com>
Date: Wed, 9 Jul 2025 12:46:11 +0300
Subject: [PATCH 33/42] Update mxbai recall stats

---
 docs/benchmarks.md | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/docs/benchmarks.md b/docs/benchmarks.md
index 948cba93..d5a85a04 100644
--- a/docs/benchmarks.md
+++ b/docs/benchmarks.md
@@ -14,7 +14,8 @@ The recall obtained is ~0.73 for matching in the top result, raising to ~0.75 fo
 
 | Model                                 | Document in top 1 | Document in top 3 |
 |---------------------------------------|-------------------|-------------------|
-| Ollama / `mxbai-embed-large`          | 0.73              | 0.75              |
+| Ollama / `mxbai-embed-large`          | 0.77              | 0.89              |
+| Ollama / `nomic-embed-text`           | 0.74              | 0.88              |
 | OpenAI / `text-embeddings-3-small`    | 0.75              | 0.88              |
 
 ## Question/Answer evaluation

From a59b38158d64be0a4118aa3c2800f797488e7aa5 Mon Sep 17 00:00:00 2001
From: Yiorgis Gozadinos <ggozadinos@gmail.com>
Date: Sat, 12 Jul 2025 20:21:34 +0300
Subject: [PATCH 34/42] Settings table & repo, tests

---
 pyproject.toml                               |  2 +-
 src/haiku/rag/store/engine.py                | 17 +++++++++
 src/haiku/rag/store/repositories/settings.py | 36 ++++++++++++++++++++
 tests/test_settings.py                       | 33 ++++++++++++++++++
 uv.lock                                      |  8 ++---
 5 files changed, 91 insertions(+), 5 deletions(-)
 create mode 100644 src/haiku/rag/store/repositories/settings.py
 create mode 100644 tests/test_settings.py

diff --git a/pyproject.toml b/pyproject.toml
index 49237c9a..30dbb443 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -56,7 +56,7 @@ dev = [
     "mkdocs>=1.6.1",
     "mkdocs-material>=9.6.14",
     "pre-commit>=4.2.0",
-    "pyright>=1.1.402",
+    "pyright>=1.1.403",
     "pytest>=8.4.0",
     "pytest-asyncio>=1.0.0",
     "pytest-cov>=6.2.1",
diff --git a/src/haiku/rag/store/engine.py b/src/haiku/rag/store/engine.py
index cdf1d2ca..4133b6dc 100644
--- a/src/haiku/rag/store/engine.py
+++ b/src/haiku/rag/store/engine.py
@@ -60,11 +60,28 @@ class Store:
             )
         """)
 
+        # Create settings table for storing current configuration
+        db.execute("""
+            CREATE TABLE IF NOT EXISTS settings (
+                id INTEGER PRIMARY KEY DEFAULT 1,
+                settings TEXT NOT NULL DEFAULT '{}'
+            )
+        """)
+
         # Create indexes for better performance
         db.execute(
             "CREATE INDEX IF NOT EXISTS idx_chunks_document_id ON chunks(document_id)"
         )
 
+        # Save current settings to the new database
+        from haiku.rag.config import Config
+
+        settings_json = Config.model_dump_json()
+        db.execute(
+            "INSERT OR IGNORE INTO settings (id, settings) VALUES (1, ?)",
+            (settings_json,),
+        )
+
         db.commit()
         return db
 
diff --git a/src/haiku/rag/store/repositories/settings.py b/src/haiku/rag/store/repositories/settings.py
new file mode 100644
index 00000000..d51cb0d5
--- /dev/null
+++ b/src/haiku/rag/store/repositories/settings.py
@@ -0,0 +1,36 @@
+import json
+from typing import Any
+
+from haiku.rag.store.engine import Store
+
+
+class SettingsRepository:
+    def __init__(self, store: Store):
+        self.store = store
+
+    def get(self) -> dict[str, Any]:
+        """Get all settings from the database."""
+        if self.store._connection is None:
+            raise ValueError("Store connection is not available")
+
+        cursor = self.store._connection.execute("SELECT settings FROM settings LIMIT 1")
+        row = cursor.fetchone()
+        if row:
+            return json.loads(row[0])
+        return {}
+
+    def save(self) -> None:
+        """Sync settings from the current AppConfig to database."""
+        if self.store._connection is None:
+            raise ValueError("Store connection is not available")
+
+        from haiku.rag.config import Config
+
+        settings_json = Config.model_dump_json()
+
+        self.store._connection.execute(
+            "INSERT INTO settings (id, settings) VALUES (1, ?) ON CONFLICT(id) DO UPDATE SET settings = excluded.settings",
+            (settings_json,),
+        )
+
+        self.store._connection.commit()
diff --git a/tests/test_settings.py b/tests/test_settings.py
new file mode 100644
index 00000000..aaec6ac5
--- /dev/null
+++ b/tests/test_settings.py
@@ -0,0 +1,33 @@
+from haiku.rag.config import Config
+from haiku.rag.store.engine import Store
+from haiku.rag.store.repositories.settings import SettingsRepository
+
+
+def test_settings_table_populated_on_store_init():
+    """Test that settings table is populated with current config when store is initialized."""
+
+    store = Store(":memory:")
+    settings_repo = SettingsRepository(store)
+
+    db_settings = settings_repo.get()
+    config_dict = Config.model_dump(mode="json")
+
+    assert db_settings == config_dict
+
+    store.close()
+
+
+def test_settings_save_and_retrieve():
+    """Test saving and retrieving settings after config change."""
+    store = Store(":memory:")
+    settings_repo = SettingsRepository(store)
+
+    original_chunk_size = Config.CHUNK_SIZE
+    Config.CHUNK_SIZE = 2 * original_chunk_size
+
+    settings_repo.save()
+    retrieved_settings = settings_repo.get()
+    assert retrieved_settings["CHUNK_SIZE"] == 2 * original_chunk_size
+
+    Config.CHUNK_SIZE = original_chunk_size
+    store.close()
diff --git a/uv.lock b/uv.lock
index 088ad51f..fd6b3649 100644
--- a/uv.lock
+++ b/uv.lock
@@ -881,7 +881,7 @@ dev = [
     { name = "mkdocs", specifier = ">=1.6.1" },
     { name = "mkdocs-material", specifier = ">=9.6.14" },
     { name = "pre-commit", specifier = ">=4.2.0" },
-    { name = "pyright", specifier = ">=1.1.402" },
+    { name = "pyright", specifier = ">=1.1.403" },
     { name = "pytest", specifier = ">=8.4.0" },
     { name = "pytest-asyncio", specifier = ">=1.0.0" },
     { name = "pytest-cov", specifier = ">=6.2.1" },
@@ -2305,15 +2305,15 @@ wheels = [
 
 [[package]]
 name = "pyright"
-version = "1.1.402"
+version = "1.1.403"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "nodeenv" },
     { name = "typing-extensions" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/aa/04/ce0c132d00e20f2d2fb3b3e7c125264ca8b909e693841210534b1ea1752f/pyright-1.1.402.tar.gz", hash = "sha256:85a33c2d40cd4439c66aa946fd4ce71ab2f3f5b8c22ce36a623f59ac22937683", size = 3888207, upload-time = "2025-06-11T08:48:35.759Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/fe/f6/35f885264ff08c960b23d1542038d8da86971c5d8c955cfab195a4f672d7/pyright-1.1.403.tar.gz", hash = "sha256:3ab69b9f41c67fb5bbb4d7a36243256f0d549ed3608678d381d5f51863921104", size = 3913526, upload-time = "2025-07-09T07:15:52.882Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/fe/37/1a1c62d955e82adae588be8e374c7f77b165b6cb4203f7d581269959abbc/pyright-1.1.402-py3-none-any.whl", hash = "sha256:2c721f11869baac1884e846232800fe021c33f1b4acb3929cff321f7ea4e2982", size = 5624004, upload-time = "2025-06-11T08:48:33.998Z" },
+    { url = "https://files.pythonhosted.org/packages/49/b6/b04e5c2f41a5ccad74a1a4759da41adb20b4bc9d59a5e08d29ba60084d07/pyright-1.1.403-py3-none-any.whl", hash = "sha256:c0eeca5aa76cbef3fcc271259bbd785753c7ad7bcac99a9162b4c4c7daed23b3", size = 5684504, upload-time = "2025-07-09T07:15:50.958Z" },
 ]
 
 [[package]]

From 4444ec50da96e236935c7784d731631aabf113ed Mon Sep 17 00:00:00 2001
From: Yiorgis Gozadinos <ggozadinos@gmail.com>
Date: Sat, 12 Jul 2025 20:56:49 +0300
Subject: [PATCH 35/42] Check if settings are compatible when loading a db

---
 src/haiku/rag/store/engine.py                |  6 +++
 src/haiku/rag/store/repositories/settings.py | 42 ++++++++++++++++++++
 tests/test_settings.py                       | 42 +++++++++++++++++++-
 3 files changed, 89 insertions(+), 1 deletion(-)

diff --git a/src/haiku/rag/store/engine.py b/src/haiku/rag/store/engine.py
index 4133b6dc..af7b27c4 100644
--- a/src/haiku/rag/store/engine.py
+++ b/src/haiku/rag/store/engine.py
@@ -13,6 +13,12 @@ class Store:
         self.db_path: Path | Literal[":memory:"] = db_path
         self._connection = self.create_db()
 
+        # Validate config compatibility after connection is established
+        from haiku.rag.store.repositories.settings import SettingsRepository
+
+        settings_repo = SettingsRepository(self)
+        settings_repo.validate_config_compatibility()
+
     def create_db(self) -> sqlite3.Connection:
         """Create the database and tables with sqlite-vec support for embeddings."""
         db = sqlite3.connect(self.db_path)
diff --git a/src/haiku/rag/store/repositories/settings.py b/src/haiku/rag/store/repositories/settings.py
index d51cb0d5..add87fa5 100644
--- a/src/haiku/rag/store/repositories/settings.py
+++ b/src/haiku/rag/store/repositories/settings.py
@@ -4,6 +4,12 @@ from typing import Any
 from haiku.rag.store.engine import Store
 
 
+class ConfigMismatchError(Exception):
+    """Raised when current config doesn't match stored settings."""
+
+    pass
+
+
 class SettingsRepository:
     def __init__(self, store: Store):
         self.store = store
@@ -34,3 +40,39 @@ class SettingsRepository:
         )
 
         self.store._connection.commit()
+
+    def validate_config_compatibility(self) -> None:
+        """Check if current config is compatible with stored settings.
+
+        Raises ConfigMismatchError if there are incompatible differences.
+        If no settings exist, saves current config.
+        """
+        db_settings = self.get()
+        if not db_settings:
+            # No settings in DB, save current config
+            self.save()
+            return
+
+        from haiku.rag.config import Config
+
+        current_config = Config.model_dump(mode="json")
+
+        # Critical settings that must match
+        critical_settings = [
+            "EMBEDDINGS_PROVIDER",
+            "EMBEDDINGS_MODEL",
+            "EMBEDDINGS_VECTOR_DIM",
+            "CHUNK_SIZE",
+            "CHUNK_OVERLAP",
+        ]
+
+        errors = []
+        for setting in critical_settings:
+            if db_settings.get(setting) != current_config.get(setting):
+                errors.append(
+                    f"{setting}: current={current_config.get(setting)}, stored={db_settings.get(setting)}"
+                )
+
+        if errors:
+            error_msg = f"Config mismatch detected: {'; '.join(errors)}. Consider rebuilding the database with the current configuration."
+            raise ConfigMismatchError(error_msg)
diff --git a/tests/test_settings.py b/tests/test_settings.py
index aaec6ac5..8f6f49d7 100644
--- a/tests/test_settings.py
+++ b/tests/test_settings.py
@@ -1,6 +1,14 @@
+import tempfile
+from pathlib import Path
+
+import pytest
+
 from haiku.rag.config import Config
 from haiku.rag.store.engine import Store
-from haiku.rag.store.repositories.settings import SettingsRepository
+from haiku.rag.store.repositories.settings import (
+    ConfigMismatchError,
+    SettingsRepository,
+)
 
 
 def test_settings_table_populated_on_store_init():
@@ -31,3 +39,35 @@ def test_settings_save_and_retrieve():
 
     Config.CHUNK_SIZE = original_chunk_size
     store.close()
+
+
+def test_config_validation_on_db_load():
+    """Test that config validation fails when loading db with mismatched settings."""
+    # Create a temporary database file
+    with tempfile.NamedTemporaryFile(suffix=".sqlite", delete=False) as tmp:
+        db_path = Path(tmp.name)
+
+    try:
+        # Create store and save settings
+        store1 = Store(db_path)
+        SettingsRepository(store1)
+        store1.close()
+
+        # Change config
+        original_chunk_size = Config.CHUNK_SIZE
+        Config.CHUNK_SIZE = 999
+
+        # Loading the database should raise ConfigMismatchError
+        with pytest.raises(ConfigMismatchError) as exc_info:
+            Store(db_path)
+
+        assert "CHUNK_SIZE" in str(exc_info.value)
+        assert "Consider rebuilding" in str(exc_info.value)
+
+        # Restore original config
+        Config.CHUNK_SIZE = original_chunk_size
+
+    finally:
+        # Cleanup
+        if db_path.exists():
+            db_path.unlink()

From 62a519b094cb5425e9aa558a139f461999db24df Mon Sep 17 00:00:00 2001
From: Yiorgis Gozadinos <ggozadinos@gmail.com>
Date: Sat, 12 Jul 2025 21:21:14 +0300
Subject: [PATCH 36/42] Prevent rebuild from throwing and recreate the
 embeddings table

---
 src/haiku/rag/app.py          |  2 +-
 src/haiku/rag/client.py       | 17 +++++++++++------
 src/haiku/rag/store/engine.py | 30 ++++++++++++++++++++++++++----
 tests/test_settings.py        | 16 ++++++++++++++--
 4 files changed, 52 insertions(+), 13 deletions(-)

diff --git a/src/haiku/rag/app.py b/src/haiku/rag/app.py
index dedfce4c..0e2d1822 100644
--- a/src/haiku/rag/app.py
+++ b/src/haiku/rag/app.py
@@ -74,7 +74,7 @@ class HaikuRAGApp:
                 self.console.print(f"[red]Error: {e}[/red]")
 
     async def rebuild(self):
-        async with HaikuRAG(db_path=self.db_path) as client:
+        async with HaikuRAG(db_path=self.db_path, skip_validation=True) as client:
             try:
                 documents = await client.list_documents()
                 total_docs = len(documents)
diff --git a/src/haiku/rag/client.py b/src/haiku/rag/client.py
index 74478654..72e5b0a6 100644
--- a/src/haiku/rag/client.py
+++ b/src/haiku/rag/client.py
@@ -24,12 +24,13 @@ class HaikuRAG:
         self,
         db_path: Path | Literal[":memory:"] = Config.DEFAULT_DATA_DIR
         / "haiku.rag.sqlite",
+        skip_validation: bool = False,
     ):
         """Initialize the RAG client with a database path."""
         if isinstance(db_path, Path):
             if not db_path.parent.exists():
                 Path.mkdir(db_path.parent, parents=True)
-        self.store = Store(db_path)
+        self.store = Store(db_path, skip_validation=skip_validation)
         self.document_repository = DocumentRepository(self.store)
         self.chunk_repository = ChunkRepository(self.store)
 
@@ -277,12 +278,16 @@ class HaikuRAG:
         Yields:
             int: The ID of the document currently being processed
         """
-        documents = await self.list_documents()
-
-        if not documents:
-            return
-
         await self.chunk_repository.delete_all()
+        self.store.recreate_embeddings_table()
+
+        # Update settings to current config
+        from haiku.rag.store.repositories.settings import SettingsRepository
+
+        settings_repo = SettingsRepository(self.store)
+        settings_repo.save()
+
+        documents = await self.list_documents()
 
         for doc in documents:
             if doc.id is not None:
diff --git a/src/haiku/rag/store/engine.py b/src/haiku/rag/store/engine.py
index af7b27c4..a2701ddf 100644
--- a/src/haiku/rag/store/engine.py
+++ b/src/haiku/rag/store/engine.py
@@ -9,15 +9,18 @@ from haiku.rag.embeddings import get_embedder
 
 
 class Store:
-    def __init__(self, db_path: Path | Literal[":memory:"]):
+    def __init__(
+        self, db_path: Path | Literal[":memory:"], skip_validation: bool = False
+    ):
         self.db_path: Path | Literal[":memory:"] = db_path
         self._connection = self.create_db()
 
         # Validate config compatibility after connection is established
-        from haiku.rag.store.repositories.settings import SettingsRepository
+        if not skip_validation:
+            from haiku.rag.store.repositories.settings import SettingsRepository
 
-        settings_repo = SettingsRepository(self)
-        settings_repo.validate_config_compatibility()
+            settings_repo = SettingsRepository(self)
+            settings_repo.validate_config_compatibility()
 
     def create_db(self) -> sqlite3.Connection:
         """Create the database and tables with sqlite-vec support for embeddings."""
@@ -91,6 +94,25 @@ class Store:
         db.commit()
         return db
 
+    def recreate_embeddings_table(self) -> None:
+        """Recreate the embeddings table with current vector dimensions."""
+        if self._connection is None:
+            raise ValueError("Store connection is not available")
+
+        # Drop existing embeddings table
+        self._connection.execute("DROP TABLE IF EXISTS chunk_embeddings")
+
+        # Recreate with current dimensions
+        embedder = get_embedder()
+        self._connection.execute(f"""
+            CREATE VIRTUAL TABLE chunk_embeddings USING vec0(
+                chunk_id INTEGER PRIMARY KEY,
+                embedding FLOAT[{embedder._vector_dim}]
+            )
+        """)
+
+        self._connection.commit()
+
     @staticmethod
     def serialize_embedding(embedding: list[float]) -> bytes:
         """Serialize a list of floats to bytes for sqlite-vec storage."""
diff --git a/tests/test_settings.py b/tests/test_settings.py
index 8f6f49d7..0db16cc8 100644
--- a/tests/test_settings.py
+++ b/tests/test_settings.py
@@ -3,6 +3,7 @@ from pathlib import Path
 
 import pytest
 
+from haiku.rag.client import HaikuRAG
 from haiku.rag.config import Config
 from haiku.rag.store.engine import Store
 from haiku.rag.store.repositories.settings import (
@@ -41,7 +42,7 @@ def test_settings_save_and_retrieve():
     store.close()
 
 
-def test_config_validation_on_db_load():
+async def test_config_validation_on_db_load():
     """Test that config validation fails when loading db with mismatched settings."""
     # Create a temporary database file
     with tempfile.NamedTemporaryFile(suffix=".sqlite", delete=False) as tmp:
@@ -64,7 +65,18 @@ def test_config_validation_on_db_load():
         assert "CHUNK_SIZE" in str(exc_info.value)
         assert "Consider rebuilding" in str(exc_info.value)
 
-        # Restore original config
+        # Rebuild
+        async with HaikuRAG(db_path=db_path, skip_validation=True) as client:
+            async for _ in client.rebuild_database():
+                pass  # Process all documents
+
+        # Verify we can now load the database without exception (settings were updated)
+        store2 = Store(db_path)
+        settings_repo2 = SettingsRepository(store2)
+        db_settings = settings_repo2.get()
+        assert db_settings["CHUNK_SIZE"] == 999
+        store2.close()
+
         Config.CHUNK_SIZE = original_chunk_size
 
     finally:

From a2e0f2cefd50fabb9f3beac535913feea927ce3f Mon Sep 17 00:00:00 2001
From: Yiorgis Gozadinos <ggozadinos@gmail.com>
Date: Sat, 12 Jul 2025 23:00:49 +0300
Subject: [PATCH 37/42] Remove Claude nonsense, use context manager

---
 src/haiku/rag/client.py |  27 +-
 tests/test_client.py    | 638 +++++++++++++++++++---------------------
 tests/test_monitor.py   |  20 +-
 tests/test_rebuild.py   |  71 +++--
 tests/test_settings.py  |  42 ++-
 5 files changed, 366 insertions(+), 432 deletions(-)

diff --git a/src/haiku/rag/client.py b/src/haiku/rag/client.py
index 72e5b0a6..ca30098c 100644
--- a/src/haiku/rag/client.py
+++ b/src/haiku/rag/client.py
@@ -166,29 +166,26 @@ class HaikuRAG:
 
             # Create a temporary file with the appropriate extension
             with tempfile.NamedTemporaryFile(
-                mode="wb", suffix=file_extension, delete=False
+                mode="wb", suffix=file_extension
             ) as temp_file:
                 temp_file.write(response.content)
+                temp_file.flush()  # Ensure content is written to disk
                 temp_path = Path(temp_file.name)
 
-            try:
                 # Parse the content using FileReader
                 content = FileReader.parse_file(temp_path)
 
-                # Merge metadata with contentType and md5
-                metadata.update({"contentType": content_type, "md5": md5_hash})
+            # Merge metadata with contentType and md5
+            metadata.update({"contentType": content_type, "md5": md5_hash})
 
-                if existing_doc:
-                    existing_doc.content = content
-                    existing_doc.metadata = metadata
-                    return await self.update_document(existing_doc)
-                else:
-                    return await self.create_document(
-                        content=content, uri=url, metadata=metadata
-                    )
-            finally:
-                # Clean up temporary file
-                temp_path.unlink(missing_ok=True)
+            if existing_doc:
+                existing_doc.content = content
+                existing_doc.metadata = metadata
+                return await self.update_document(existing_doc)
+            else:
+                return await self.create_document(
+                    content=content, uri=url, metadata=metadata
+                )
 
     def _get_extension_from_content_type_or_url(
         self, url: str, content_type: str
diff --git a/tests/test_client.py b/tests/test_client.py
index 086facc0..c5aaeb1c 100644
--- a/tests/test_client.py
+++ b/tests/test_client.py
@@ -12,300 +12,270 @@ from haiku.rag.client import HaikuRAG
 @pytest.mark.asyncio
 async def test_client_document_crud(qa_corpus: Dataset):
     """Test HaikuRAG CRUD operations for documents."""
-    # Create client with in-memory database
-    client = HaikuRAG(":memory:")
+    async with HaikuRAG(":memory:") as client:
+        # Get test data
+        first_doc = qa_corpus[0]
+        document_text = first_doc["document_extracted"]
+        test_uri = "file:///path/to/test.txt"
+        test_metadata = {"source": "test", "topic": "testing"}
 
-    # Get test data
-    first_doc = qa_corpus[0]
-    document_text = first_doc["document_extracted"]
-    test_uri = "file:///path/to/test.txt"
-    test_metadata = {"source": "test", "topic": "testing"}
+        # Test create_document
+        created_doc = await client.create_document(
+            content=document_text, uri=test_uri, metadata=test_metadata
+        )
 
-    # Test create_document
-    created_doc = await client.create_document(
-        content=document_text, uri=test_uri, metadata=test_metadata
-    )
+        assert created_doc.id is not None
+        assert created_doc.content == document_text
+        assert created_doc.uri == test_uri
+        assert created_doc.metadata == test_metadata
 
-    assert created_doc.id is not None
-    assert created_doc.content == document_text
-    assert created_doc.uri == test_uri
-    assert created_doc.metadata == test_metadata
+        # Test get_document_by_id
+        retrieved_doc = await client.get_document_by_id(created_doc.id)
+        assert retrieved_doc is not None
+        assert retrieved_doc.id == created_doc.id
+        assert retrieved_doc.content == document_text
+        assert retrieved_doc.uri == test_uri
 
-    # Test get_document_by_id
-    retrieved_doc = await client.get_document_by_id(created_doc.id)
-    assert retrieved_doc is not None
-    assert retrieved_doc.id == created_doc.id
-    assert retrieved_doc.content == document_text
-    assert retrieved_doc.uri == test_uri
+        # Test get_document_by_uri
+        retrieved_by_uri = await client.get_document_by_uri(test_uri)
+        assert retrieved_by_uri is not None
+        assert retrieved_by_uri.id == created_doc.id
+        assert retrieved_by_uri.content == document_text
 
-    # Test get_document_by_uri
-    retrieved_by_uri = await client.get_document_by_uri(test_uri)
-    assert retrieved_by_uri is not None
-    assert retrieved_by_uri.id == created_doc.id
-    assert retrieved_by_uri.content == document_text
+        # Test get_document_by_uri with non-existent URI
+        non_existent = await client.get_document_by_uri("file:///non/existent.txt")
+        assert non_existent is None
 
-    # Test get_document_by_uri with non-existent URI
-    non_existent = await client.get_document_by_uri("file:///non/existent.txt")
-    assert non_existent is None
+        # Test update_document
+        retrieved_doc.content = "Updated content"
+        retrieved_doc.uri = "file:///updated/path.txt"
+        updated_doc = await client.update_document(retrieved_doc)
+        assert updated_doc.content == "Updated content"
+        assert updated_doc.uri == "file:///updated/path.txt"
 
-    # Test update_document
-    retrieved_doc.content = "Updated content"
-    retrieved_doc.uri = "file:///updated/path.txt"
-    updated_doc = await client.update_document(retrieved_doc)
-    assert updated_doc.content == "Updated content"
-    assert updated_doc.uri == "file:///updated/path.txt"
+        # Test list_documents
+        all_docs = await client.list_documents()
+        assert len(all_docs) == 1
+        assert all_docs[0].id == created_doc.id
 
-    # Test list_documents
-    all_docs = await client.list_documents()
-    assert len(all_docs) == 1
-    assert all_docs[0].id == created_doc.id
+        # Test list_documents with pagination
+        limited_docs = await client.list_documents(limit=10, offset=0)
+        assert len(limited_docs) == 1
 
-    # Test list_documents with pagination
-    limited_docs = await client.list_documents(limit=10, offset=0)
-    assert len(limited_docs) == 1
+        # Test delete_document
+        deleted = await client.delete_document(created_doc.id)
+        assert deleted is True
 
-    # Test delete_document
-    deleted = await client.delete_document(created_doc.id)
-    assert deleted is True
+        # Verify document is gone
+        retrieved_doc = await client.get_document_by_id(created_doc.id)
+        assert retrieved_doc is None
 
-    # Verify document is gone
-    retrieved_doc = await client.get_document_by_id(created_doc.id)
-    assert retrieved_doc is None
-
-    # Test delete non-existent document
-    deleted_again = await client.delete_document(created_doc.id)
-    assert deleted_again is False
-
-    client.close()
+        # Test delete non-existent document
+        deleted_again = await client.delete_document(created_doc.id)
+        assert deleted_again is False
 
 
 @pytest.mark.asyncio
 async def test_client_create_document_from_source():
     """Test creating a document from a file source."""
-    client = HaikuRAG(":memory:")
+    async with HaikuRAG(":memory:") as client:
+        with tempfile.TemporaryDirectory() as temp_dir:
+            test_content = "This is test content from a file."
+            temp_path = Path(temp_dir) / "test.txt"
+            temp_path.write_text(test_content)
 
-    # Create a temporary text file
-    with tempfile.NamedTemporaryFile(mode="w", suffix=".txt", delete=False) as f:
-        test_content = "This is test content from a file."
-        f.write(test_content)
-        temp_path = Path(f.name)
+            # Test create_document_from_source with Path
+            doc = await client.create_document_from_source(source=temp_path)
 
-    try:
-        # Test create_document_from_source with Path
-        doc = await client.create_document_from_source(
-            source=temp_path, metadata={"source_type": "file"}
-        )
+            assert doc.id is not None
+            assert doc.content == test_content
+            assert doc.uri == temp_path.as_uri()
+            assert "contentType" in doc.metadata
+            assert "md5" in doc.metadata
+            assert doc.metadata["contentType"] == "text/plain"
 
-        assert doc.id is not None
-        assert doc.content == test_content
-        assert doc.uri == temp_path.as_uri()
-        assert doc.metadata["source_type"] == "file"
-        assert "contentType" in doc.metadata
-        assert "md5" in doc.metadata
-        assert doc.metadata["contentType"] == "text/plain"
+            # Test create_document_from_source with string path
+            doc2 = await client.create_document_from_source(source=str(temp_path))
 
-        # Test create_document_from_source with string path
-        doc2 = await client.create_document_from_source(source=str(temp_path))
-
-        assert doc2.id is not None
-        assert doc2.content == test_content
-        assert doc2.uri == temp_path.as_uri()
-        assert "contentType" in doc2.metadata
-        assert "md5" in doc2.metadata
-
-    finally:
-        # Clean up
-        temp_path.unlink()
-        client.close()
+            assert doc2.id is not None
+            assert doc2.content == test_content
+            assert doc2.uri == temp_path.as_uri()
+            assert "contentType" in doc2.metadata
+            assert "md5" in doc2.metadata
 
 
 @pytest.mark.asyncio
 async def test_client_create_document_from_source_unsupported():
     """Test creating a document from an unsupported file type."""
-    client = HaikuRAG(":memory:")
+    async with HaikuRAG(":memory:") as client:
+        # Create a temporary file with unsupported extension
+        with tempfile.NamedTemporaryFile(
+            mode="w", suffix=".unsupported", delete=False
+        ) as f:
+            f.write("content")
+            temp_path = Path(f.name)
 
-    # Create a temporary file with unsupported extension
-    with tempfile.NamedTemporaryFile(
-        mode="w", suffix=".unsupported", delete=False
-    ) as f:
-        f.write("content")
-        temp_path = Path(f.name)
-
-    try:
-        # Should raise ValueError for unsupported extension
-        with pytest.raises(ValueError, match="Unsupported file extension"):
-            await client.create_document_from_source(temp_path)
-
-    finally:
-        temp_path.unlink()
-        client.close()
+            # Should raise ValueError for unsupported extension
+            with pytest.raises(ValueError, match="Unsupported file extension"):
+                await client.create_document_from_source(temp_path)
 
 
 @pytest.mark.asyncio
 async def test_client_create_document_from_source_nonexistent():
     """Test creating a document from a non-existent file."""
-    client = HaikuRAG(":memory:")
+    async with HaikuRAG(":memory:") as client:
+        non_existent_path = Path("/non/existent/file.txt")
 
-    non_existent_path = Path("/non/existent/file.txt")
-
-    # Should raise ValueError when file doesn't exist
-    with pytest.raises(ValueError, match="File does not exist"):
-        await client.create_document_from_source(non_existent_path)
-
-    client.close()
+        # Should raise ValueError when file doesn't exist
+        with pytest.raises(ValueError, match="File does not exist"):
+            await client.create_document_from_source(non_existent_path)
 
 
 @pytest.mark.asyncio
 async def test_client_create_document_from_url():
     """Test creating a document from a URL."""
-    client = HaikuRAG(":memory:")
+    async with HaikuRAG(":memory:") as client:
+        # Mock the HTTP response
+        mock_response = AsyncMock()
+        mock_response.content = b"<html><body><h1>Test Page</h1><p>This is test content from a webpage.</p></body></html>"
+        mock_response.headers = {"content-type": "text/html"}
+        mock_response.raise_for_status = AsyncMock()
 
-    # Mock the HTTP response
-    mock_response = AsyncMock()
-    mock_response.content = b"<html><body><h1>Test Page</h1><p>This is test content from a webpage.</p></body></html>"
-    mock_response.headers = {"content-type": "text/html"}
-    mock_response.raise_for_status = AsyncMock()
+        with patch("httpx.AsyncClient.get", return_value=mock_response):
+            doc = await client.create_document_from_source(
+                source="https://example.com/test.html", metadata={"source_type": "web"}
+            )
 
-    with patch("httpx.AsyncClient.get", return_value=mock_response):
-        doc = await client.create_document_from_source(
-            source="https://example.com/test.html", metadata={"source_type": "web"}
-        )
-
-        assert doc.id is not None
-        assert "Test Page" in doc.content
-        assert "test content" in doc.content
-        assert doc.uri == "https://example.com/test.html"
-        assert doc.metadata["source_type"] == "web"
-        assert "contentType" in doc.metadata
-        assert "md5" in doc.metadata
-        assert doc.metadata["contentType"] == "text/html"
-
-    client.close()
+            assert doc.id is not None
+            assert "Test Page" in doc.content
+            assert "test content" in doc.content
+            assert doc.uri == "https://example.com/test.html"
+            assert doc.metadata["source_type"] == "web"
+            assert "contentType" in doc.metadata
+            assert "md5" in doc.metadata
+            assert doc.metadata["contentType"] == "text/html"
 
 
 @pytest.mark.asyncio
 async def test_client_create_document_from_url_with_different_content_types():
     """Test creating documents from URLs with different content types."""
-    client = HaikuRAG(":memory:")
-
-    # Test JSON content
-    mock_json_response = AsyncMock()
-    mock_json_response.content = (
-        b'{"title": "Test JSON", "content": "This is JSON content"}'
-    )
-    mock_json_response.headers = {"content-type": "application/json"}
-    mock_json_response.raise_for_status = AsyncMock()
-
-    with patch("httpx.AsyncClient.get", return_value=mock_json_response):
-        doc = await client.create_document_from_source(
-            "https://api.example.com/data.json"
+    async with HaikuRAG(":memory:") as client:
+        # Test JSON content
+        mock_json_response = AsyncMock()
+        mock_json_response.content = (
+            b'{"title": "Test JSON", "content": "This is JSON content"}'
         )
+        mock_json_response.headers = {"content-type": "application/json"}
+        mock_json_response.raise_for_status = AsyncMock()
 
-        assert doc.id is not None
-        assert "Test JSON" in doc.content
-        assert doc.uri == "https://api.example.com/data.json"
-        assert "contentType" in doc.metadata
-        assert "md5" in doc.metadata
-        assert doc.metadata["contentType"] == "application/json"
+        with patch("httpx.AsyncClient.get", return_value=mock_json_response):
+            doc = await client.create_document_from_source(
+                "https://api.example.com/data.json"
+            )
 
-    # Test plain text content
-    mock_text_response = AsyncMock()
-    mock_text_response.content = b"This is plain text content from a URL."
-    mock_text_response.headers = {"content-type": "text/plain"}
-    mock_text_response.raise_for_status = AsyncMock()
+            assert doc.id is not None
+            assert "Test JSON" in doc.content
+            assert doc.uri == "https://api.example.com/data.json"
+            assert "contentType" in doc.metadata
+            assert "md5" in doc.metadata
+            assert doc.metadata["contentType"] == "application/json"
 
-    with patch("httpx.AsyncClient.get", return_value=mock_text_response):
-        doc = await client.create_document_from_source("https://example.com/readme.txt")
+        # Test plain text content
+        mock_text_response = AsyncMock()
+        mock_text_response.content = b"This is plain text content from a URL."
+        mock_text_response.headers = {"content-type": "text/plain"}
+        mock_text_response.raise_for_status = AsyncMock()
 
-        assert doc.id is not None
-        assert doc.content == "This is plain text content from a URL."
-        assert doc.uri == "https://example.com/readme.txt"
-        assert "contentType" in doc.metadata
-        assert "md5" in doc.metadata
-        assert doc.metadata["contentType"] == "text/plain"
+        with patch("httpx.AsyncClient.get", return_value=mock_text_response):
+            doc = await client.create_document_from_source(
+                "https://example.com/readme.txt"
+            )
 
-    client.close()
+            assert doc.id is not None
+            assert doc.content == "This is plain text content from a URL."
+            assert doc.uri == "https://example.com/readme.txt"
+            assert "contentType" in doc.metadata
+            assert "md5" in doc.metadata
+            assert doc.metadata["contentType"] == "text/plain"
 
 
 @pytest.mark.asyncio
 async def test_client_create_document_from_url_unsupported_content():
     """Test creating a document from URL with unsupported content type."""
-    client = HaikuRAG(":memory:")
+    async with HaikuRAG(":memory:") as client:
+        # Mock response with unsupported content type
+        mock_response = AsyncMock()
+        mock_response.content = b"binary content"
+        mock_response.headers = {"content-type": "application/octet-stream"}
+        mock_response.raise_for_status = AsyncMock()
 
-    # Mock response with unsupported content type
-    mock_response = AsyncMock()
-    mock_response.content = b"binary content"
-    mock_response.headers = {"content-type": "application/octet-stream"}
-    mock_response.raise_for_status = AsyncMock()
-
-    with patch("httpx.AsyncClient.get", return_value=mock_response):
-        with pytest.raises(ValueError, match="Unsupported content type"):
-            await client.create_document_from_source("https://example.com/binary.bin")
-
-    client.close()
+        with patch("httpx.AsyncClient.get", return_value=mock_response):
+            with pytest.raises(ValueError, match="Unsupported content type"):
+                await client.create_document_from_source(
+                    "https://example.com/binary.bin"
+                )
 
 
 @pytest.mark.asyncio
 async def test_client_create_document_from_url_http_error():
     """Test handling HTTP errors when creating document from URL."""
-    client = HaikuRAG(":memory:")
-
-    with patch("httpx.AsyncClient.get") as mock_get:
-        mock_get.side_effect = httpx.HTTPStatusError(
-            "404 Not Found",
-            request=httpx.Request("GET", "https://example.com/notfound.html"),
-            response=httpx.Response(404),
-        )
-
-        with pytest.raises(httpx.HTTPStatusError):
-            await client.create_document_from_source(
-                "https://example.com/notfound.html"
+    async with HaikuRAG(":memory:") as client:
+        with patch("httpx.AsyncClient.get") as mock_get:
+            mock_get.side_effect = httpx.HTTPStatusError(
+                "404 Not Found",
+                request=httpx.Request("GET", "https://example.com/notfound.html"),
+                response=httpx.Response(404),
             )
 
-    client.close()
+            with pytest.raises(httpx.HTTPStatusError):
+                await client.create_document_from_source(
+                    "https://example.com/notfound.html"
+                )
 
 
 @pytest.mark.asyncio
 async def test_get_extension_from_content_type_or_url():
     """Test the helper method for determining file extensions."""
-    client = HaikuRAG(":memory:")
-
-    # Test content type mappings
-    assert client._get_extension_from_content_type_or_url("", "text/html") == ".html"
-    assert (
-        client._get_extension_from_content_type_or_url("", "application/pdf") == ".pdf"
-    )
-    assert client._get_extension_from_content_type_or_url("", "text/plain") == ".txt"
-
-    # Test URL extension detection
-    assert (
-        client._get_extension_from_content_type_or_url(
-            "https://example.com/doc.pdf", ""
+    async with HaikuRAG(":memory:") as client:
+        # Test content type mappings
+        assert (
+            client._get_extension_from_content_type_or_url("", "text/html") == ".html"
         )
-        == ".pdf"
-    )
-    assert (
-        client._get_extension_from_content_type_or_url(
-            "https://example.com/data.json", ""
+        assert (
+            client._get_extension_from_content_type_or_url("", "application/pdf")
+            == ".pdf"
         )
-        == ".json"
-    )
-
-    # Test default fallback
-    assert (
-        client._get_extension_from_content_type_or_url("https://example.com/", "")
-        == ".html"
-    )
-
-    # Test content type priority over URL extension
-    assert (
-        client._get_extension_from_content_type_or_url(
-            "https://example.com/file.txt", "application/pdf"
+        assert (
+            client._get_extension_from_content_type_or_url("", "text/plain") == ".txt"
         )
-        == ".pdf"
-    )
 
-    client.close()
+        # Test URL extension detection
+        assert (
+            client._get_extension_from_content_type_or_url(
+                "https://example.com/doc.pdf", ""
+            )
+            == ".pdf"
+        )
+        assert (
+            client._get_extension_from_content_type_or_url(
+                "https://example.com/data.json", ""
+            )
+            == ".json"
+        )
+
+        # Test default fallback
+        assert (
+            client._get_extension_from_content_type_or_url("https://example.com/", "")
+            == ".html"
+        )
+
+        # Test content type priority over URL extension
+        assert (
+            client._get_extension_from_content_type_or_url(
+                "https://example.com/file.txt", "application/pdf"
+            )
+            == ".pdf"
+        )
 
 
 @pytest.mark.asyncio
@@ -313,165 +283,147 @@ async def test_client_metadata_content_type_and_md5():
     """Test that contentType and md5 metadata are correctly set."""
     import hashlib
 
-    client = HaikuRAG(":memory:")
+    async with HaikuRAG(":memory:") as client:
+        # Create a temporary file with known content
+        test_content = "Test content for MD5 calculation."
+        expected_md5 = hashlib.md5(test_content.encode()).hexdigest()
 
-    # Create a temporary file with known content
-    test_content = "Test content for MD5 calculation."
-    expected_md5 = hashlib.md5(test_content.encode()).hexdigest()
+        with tempfile.TemporaryDirectory() as temp_dir:
+            temp_path = Path(temp_dir) / "test.txt"
+            temp_path.write_text(test_content)
 
-    with tempfile.NamedTemporaryFile(mode="w", suffix=".txt", delete=False) as f:
-        f.write(test_content)
-        temp_path = Path(f.name)
+            doc = await client.create_document_from_source(temp_path)
 
-    try:
-        doc = await client.create_document_from_source(temp_path)
+            assert doc.metadata["contentType"] == "text/plain"
+            assert doc.metadata["md5"] == expected_md5
 
-        assert doc.metadata["contentType"] == "text/plain"
-        assert doc.metadata["md5"] == expected_md5
+            mock_response = AsyncMock()
+            mock_response.content = test_content.encode()
+            mock_response.headers = {"content-type": "text/plain"}
+            mock_response.raise_for_status = AsyncMock()
 
-        mock_response = AsyncMock()
-        mock_response.content = test_content.encode()
-        mock_response.headers = {"content-type": "text/plain"}
-        mock_response.raise_for_status = AsyncMock()
+            with patch("httpx.AsyncClient.get", return_value=mock_response):
+                url_doc = await client.create_document_from_source(
+                    "https://example.com/test.txt"
+                )
 
-        with patch("httpx.AsyncClient.get", return_value=mock_response):
-            url_doc = await client.create_document_from_source(
-                "https://example.com/test.txt"
-            )
-
-            assert url_doc.metadata["contentType"] == "text/plain"
-            assert url_doc.metadata["md5"] == expected_md5
-
-    finally:
-        temp_path.unlink()
-        client.close()
+                assert url_doc.metadata["contentType"] == "text/plain"
+                assert url_doc.metadata["md5"] == expected_md5
 
 
 @pytest.mark.asyncio
 async def test_client_create_update_no_op_behavior():
     """Test create/update/no-op behavior based on MD5 changes."""
-    client = HaikuRAG(":memory:")
+    async with HaikuRAG(":memory:") as client:
+        # Create a temporary file
+        test_content = "Original content for testing."
+        with tempfile.TemporaryDirectory() as temp_dir:
+            temp_path = Path(temp_dir) / "test.txt"
+            temp_path.write_text(test_content)
 
-    # Create a temporary file
-    test_content = "Original content for testing."
-    with tempfile.NamedTemporaryFile(mode="w", suffix=".txt", delete=False) as f:
-        f.write(test_content)
-        temp_path = Path(f.name)
+            # First call - should create new document
+            doc1 = await client.create_document_from_source(temp_path)
+            assert doc1.id is not None
+            assert doc1.content == test_content
+            original_id = doc1.id
 
-    try:
-        # First call - should create new document
-        doc1 = await client.create_document_from_source(temp_path)
-        assert doc1.id is not None
-        assert doc1.content == test_content
-        original_id = doc1.id
+            # Second call with same content - should return existing document (no-op)
+            doc2 = await client.create_document_from_source(temp_path)
+            assert doc2.id == original_id  # Same document
+            assert doc2.content == test_content
 
-        # Second call with same content - should return existing document (no-op)
-        doc2 = await client.create_document_from_source(temp_path)
-        assert doc2.id == original_id  # Same document
-        assert doc2.content == test_content
+            # Modify file content
+            updated_content = "Updated content for testing."
+            temp_path.write_text(updated_content)
 
-        # Modify file content
-        updated_content = "Updated content for testing."
-        temp_path.write_text(updated_content)
+            # Third call with changed content - should update existing document
+            doc3 = await client.create_document_from_source(temp_path)
+            assert doc3.id == original_id  # Same document ID
+            assert doc3.content == updated_content  # Updated content
 
-        # Third call with changed content - should update existing document
-        doc3 = await client.create_document_from_source(temp_path)
-        assert doc3.id == original_id  # Same document ID
-        assert doc3.content == updated_content  # Updated content
-
-        # Verify the document was actually updated in database
-        retrieved_doc = await client.get_document_by_id(original_id)
-        assert retrieved_doc is not None
-        assert retrieved_doc.content == updated_content
-
-    finally:
-        temp_path.unlink()
-        client.close()
+            # Verify the document was actually updated in database
+            retrieved_doc = await client.get_document_by_id(original_id)
+            assert retrieved_doc is not None
+            assert retrieved_doc.content == updated_content
 
 
 @pytest.mark.asyncio
 async def test_client_url_create_update_no_op_behavior():
     """Test create/update/no-op behavior for URLs based on MD5 changes."""
-    client = HaikuRAG(":memory:")
+    async with HaikuRAG(":memory:") as client:
+        url = "https://example.com/test.txt"
+        original_content = b"Original URL content"
+        updated_content = b"Updated URL content"
 
-    url = "https://example.com/test.txt"
-    original_content = b"Original URL content"
-    updated_content = b"Updated URL content"
+        # Mock first response
+        mock_response1 = AsyncMock()
+        mock_response1.content = original_content
+        mock_response1.headers = {"content-type": "text/plain"}
+        mock_response1.raise_for_status = AsyncMock()
 
-    # Mock first response
-    mock_response1 = AsyncMock()
-    mock_response1.content = original_content
-    mock_response1.headers = {"content-type": "text/plain"}
-    mock_response1.raise_for_status = AsyncMock()
+        with patch("httpx.AsyncClient.get", return_value=mock_response1):
+            # First call - should create new document
+            doc1 = await client.create_document_from_source(url)
+            assert doc1.id is not None
+            original_id = doc1.id
 
-    with patch("httpx.AsyncClient.get", return_value=mock_response1):
-        # First call - should create new document
-        doc1 = await client.create_document_from_source(url)
-        assert doc1.id is not None
-        original_id = doc1.id
+            # Second call with same content - should return existing document (no-op)
+            doc2 = await client.create_document_from_source(url)
+            assert doc2.id == original_id  # Same document
 
-        # Second call with same content - should return existing document (no-op)
-        doc2 = await client.create_document_from_source(url)
-        assert doc2.id == original_id  # Same document
+        mock_response2 = AsyncMock()
+        mock_response2.content = updated_content
+        mock_response2.headers = {"content-type": "text/plain"}
+        mock_response2.raise_for_status = AsyncMock()
 
-    mock_response2 = AsyncMock()
-    mock_response2.content = updated_content
-    mock_response2.headers = {"content-type": "text/plain"}
-    mock_response2.raise_for_status = AsyncMock()
-
-    with patch("httpx.AsyncClient.get", return_value=mock_response2):
-        # Third call with changed content - should update existing document
-        doc3 = await client.create_document_from_source(url)
-        assert doc3.id == original_id  # Same document ID
-        assert doc3.content == updated_content.decode()  # Updated content
-
-    client.close()
+        with patch("httpx.AsyncClient.get", return_value=mock_response2):
+            # Third call with changed content - should update existing document
+            doc3 = await client.create_document_from_source(url)
+            assert doc3.id == original_id  # Same document ID
+            assert doc3.content == updated_content.decode()  # Updated content
 
 
 @pytest.mark.asyncio
 async def test_client_search():
     """Test HaikuRAG search functionality."""
-    client = HaikuRAG(":memory:")
+    async with HaikuRAG(":memory:") as client:
+        # Add multiple documents to search from
+        doc1_text = "Python is a high-level programming language known for its simplicity and readability."
+        doc2_text = "Machine learning algorithms help computers learn patterns from data without explicit programming."
+        doc3_text = "Data science combines statistics, programming, and domain expertise to extract insights."
 
-    # Add multiple documents to search from
-    doc1_text = "Python is a high-level programming language known for its simplicity and readability."
-    doc2_text = "Machine learning algorithms help computers learn patterns from data without explicit programming."
-    doc3_text = "Data science combines statistics, programming, and domain expertise to extract insights."
+        # Create documents
+        doc1 = await client.create_document(
+            content=doc1_text, uri="doc1.txt", metadata={"topic": "python"}
+        )
+        doc2 = await client.create_document(
+            content=doc2_text, uri="doc2.txt", metadata={"topic": "ml"}
+        )
+        await client.create_document(
+            content=doc3_text, uri="doc3.txt", metadata={"topic": "data_science"}
+        )
 
-    # Create documents
-    doc1 = await client.create_document(
-        content=doc1_text, uri="doc1.txt", metadata={"topic": "python"}
-    )
-    doc2 = await client.create_document(
-        content=doc2_text, uri="doc2.txt", metadata={"topic": "ml"}
-    )
-    await client.create_document(
-        content=doc3_text, uri="doc3.txt", metadata={"topic": "data_science"}
-    )
+        # Test search with keyword that should match doc1
+        results = await client.search("Python programming", limit=3)
 
-    # Test search with keyword that should match doc1
-    results = await client.search("Python programming", limit=3)
+        assert len(results) > 0
+        assert all(len(result) == 2 for result in results)
 
-    assert len(results) > 0
-    assert all(len(result) == 2 for result in results)
+        # Verify first result is from the Python document (doc1)
+        first_chunk, _ = results[0]
+        assert first_chunk.document_id == doc1.id
 
-    # Verify first result is from the Python document (doc1)
-    first_chunk, _ = results[0]
-    assert first_chunk.document_id == doc1.id
+        # Test search with different query
+        ml_results = await client.search("machine learning data", limit=2)
+        assert len(ml_results) > 0
 
-    # Test search with different query
-    ml_results = await client.search("machine learning data", limit=2)
-    assert len(ml_results) > 0
+        # Verify first result is from the machine learning document (doc2)
+        first_ml_chunk, _ = ml_results[0]
+        assert first_ml_chunk.document_id == doc2.id
 
-    # Verify first result is from the machine learning document (doc2)
-    first_ml_chunk, _ = ml_results[0]
-    assert first_ml_chunk.document_id == doc2.id
-
-    # Test search with limit parameter
-    limited_results = await client.search("programming", limit=1)
-    assert len(limited_results) <= 1
-
-    client.close()
+        # Test search with limit parameter
+        limited_results = await client.search("programming", limit=1)
+        assert len(limited_results) <= 1
 
 
 @pytest.mark.asyncio
diff --git a/tests/test_monitor.py b/tests/test_monitor.py
index ac909631..73992416 100644
--- a/tests/test_monitor.py
+++ b/tests/test_monitor.py
@@ -13,11 +13,10 @@ from haiku.rag.store.models.document import Document
 async def test_file_watcher_upsert_document():
     """Test FileWatcher._upsert_document method."""
 
-    with tempfile.NamedTemporaryFile(mode="w", suffix=".txt", delete=False) as f:
-        f.write("Test content for file watcher")
-        temp_path = Path(f.name)
+    with tempfile.TemporaryDirectory() as temp_dir:
+        temp_path = Path(temp_dir) / "test.txt"
+        temp_path.write_text("Test content for file watcher")
 
-    try:
         mock_client = AsyncMock(spec=HaikuRAG)
         mock_doc = Document(id=1, content="Test content", uri=temp_path.as_uri())
         mock_client.create_document_from_source.return_value = mock_doc
@@ -32,19 +31,15 @@ async def test_file_watcher_upsert_document():
         mock_client.get_document_by_uri.assert_called_once_with(temp_path.as_uri())
         mock_client.create_document_from_source.assert_called_once_with(str(temp_path))
 
-    finally:
-        temp_path.unlink(missing_ok=True)
-
 
 @pytest.mark.asyncio
 async def test_file_watcher_upsert_existing_document():
     """Test FileWatcher._upsert_document with existing document."""
 
-    with tempfile.NamedTemporaryFile(mode="w", suffix=".txt", delete=False) as f:
-        f.write("Test content for file watcher")
-        temp_path = Path(f.name)
+    with tempfile.TemporaryDirectory() as temp_dir:
+        temp_path = Path(temp_dir) / "test.txt"
+        temp_path.write_text("Test content for file watcher")
 
-    try:
         mock_client = AsyncMock(spec=HaikuRAG)
         existing_doc = Document(id=1, content="Old content", uri=temp_path.as_uri())
         updated_doc = Document(id=1, content="Updated content", uri=temp_path.as_uri())
@@ -61,9 +56,6 @@ async def test_file_watcher_upsert_existing_document():
         mock_client.get_document_by_uri.assert_called_once_with(temp_path.as_uri())
         mock_client.create_document_from_source.assert_called_once_with(str(temp_path))
 
-    finally:
-        temp_path.unlink(missing_ok=True)
-
 
 @pytest.mark.asyncio
 async def test_file_watcher_delete_document():
diff --git a/tests/test_rebuild.py b/tests/test_rebuild.py
index 3254ce1d..04fd8533 100644
--- a/tests/test_rebuild.py
+++ b/tests/test_rebuild.py
@@ -8,45 +8,42 @@ from haiku.rag.store.models.document import Document
 @pytest.mark.asyncio
 async def test_rebuild_database(qa_corpus: Dataset):
     """Test rebuild functionality with existing documents."""
-    client = HaikuRAG(":memory:")
+    async with HaikuRAG(":memory:") as client:
+        created_docs: list[Document] = []
+        for content in qa_corpus["document_extracted"][:3]:
+            doc = await client.create_document(
+                content=content,
+            )
+            created_docs.append(doc)
 
-    created_docs: list[Document] = []
-    for content in qa_corpus["document_extracted"][:3]:
-        doc = await client.create_document(
-            content=content,
-        )
-        created_docs.append(doc)
+        documents_before = await client.list_documents()
+        assert len(documents_before) == 3
 
-    documents_before = await client.list_documents()
-    assert len(documents_before) == 3
-
-    chunks_before = []
-    for doc in created_docs:
-        assert doc.id is not None
-        doc_chunks = await client.chunk_repository.get_by_document_id(doc.id)
-        chunks_before.extend(doc_chunks)
-
-    assert len(chunks_before) > 0
-
-    # Perform rebuild
-    processed_doc_ids = []
-    async for doc_id in client.rebuild_database():
-        processed_doc_ids.append(doc_id)
-
-    # Verify all documents were processed
-    expected_doc_ids = [doc.id for doc in created_docs]
-    assert set(processed_doc_ids) == set(expected_doc_ids)
-
-    documents_after = await client.list_documents()
-    assert len(documents_after) == 3
-
-    # Verify chunks were recreated
-    chunks_after = []
-    for doc in documents_after:
-        if doc.id is not None:
+        chunks_before = []
+        for doc in created_docs:
+            assert doc.id is not None
             doc_chunks = await client.chunk_repository.get_by_document_id(doc.id)
-            chunks_after.extend(doc_chunks)
+            chunks_before.extend(doc_chunks)
 
-    assert len(chunks_after) > 0
+        assert len(chunks_before) > 0
 
-    client.close()
+        # Perform rebuild
+        processed_doc_ids = []
+        async for doc_id in client.rebuild_database():
+            processed_doc_ids.append(doc_id)
+
+        # Verify all documents were processed
+        expected_doc_ids = [doc.id for doc in created_docs]
+        assert set(processed_doc_ids) == set(expected_doc_ids)
+
+        documents_after = await client.list_documents()
+        assert len(documents_after) == 3
+
+        # Verify chunks were recreated
+        chunks_after = []
+        for doc in documents_after:
+            if doc.id is not None:
+                doc_chunks = await client.chunk_repository.get_by_document_id(doc.id)
+                chunks_after.extend(doc_chunks)
+
+        assert len(chunks_after) > 0
diff --git a/tests/test_settings.py b/tests/test_settings.py
index 0db16cc8..6a531685 100644
--- a/tests/test_settings.py
+++ b/tests/test_settings.py
@@ -45,10 +45,9 @@ def test_settings_save_and_retrieve():
 async def test_config_validation_on_db_load():
     """Test that config validation fails when loading db with mismatched settings."""
     # Create a temporary database file
-    with tempfile.NamedTemporaryFile(suffix=".sqlite", delete=False) as tmp:
+    with tempfile.NamedTemporaryFile(suffix=".sqlite") as tmp:
         db_path = Path(tmp.name)
 
-    try:
         # Create store and save settings
         store1 = Store(db_path)
         SettingsRepository(store1)
@@ -58,28 +57,25 @@ async def test_config_validation_on_db_load():
         original_chunk_size = Config.CHUNK_SIZE
         Config.CHUNK_SIZE = 999
 
-        # Loading the database should raise ConfigMismatchError
-        with pytest.raises(ConfigMismatchError) as exc_info:
-            Store(db_path)
+        try:
+            # Loading the database should raise ConfigMismatchError
+            with pytest.raises(ConfigMismatchError) as exc_info:
+                Store(db_path)
 
-        assert "CHUNK_SIZE" in str(exc_info.value)
-        assert "Consider rebuilding" in str(exc_info.value)
+            assert "CHUNK_SIZE" in str(exc_info.value)
+            assert "Consider rebuilding" in str(exc_info.value)
 
-        # Rebuild
-        async with HaikuRAG(db_path=db_path, skip_validation=True) as client:
-            async for _ in client.rebuild_database():
-                pass  # Process all documents
+            # Rebuild
+            async with HaikuRAG(db_path=db_path, skip_validation=True) as client:
+                async for _ in client.rebuild_database():
+                    pass  # Process all documents
 
-        # Verify we can now load the database without exception (settings were updated)
-        store2 = Store(db_path)
-        settings_repo2 = SettingsRepository(store2)
-        db_settings = settings_repo2.get()
-        assert db_settings["CHUNK_SIZE"] == 999
-        store2.close()
+            # Verify we can now load the database without exception (settings were updated)
+            store2 = Store(db_path)
+            settings_repo2 = SettingsRepository(store2)
+            db_settings = settings_repo2.get()
+            assert db_settings["CHUNK_SIZE"] == 999
+            store2.close()
 
-        Config.CHUNK_SIZE = original_chunk_size
-
-    finally:
-        # Cleanup
-        if db_path.exists():
-            db_path.unlink()
+        finally:
+            Config.CHUNK_SIZE = original_chunk_size

From a2d7e527d193ea4677d2fe813d87ed7fda8307a8 Mon Sep 17 00:00:00 2001
From: Yiorgis Gozadinos <ggozadinos@gmail.com>
Date: Mon, 14 Jul 2025 12:00:03 +0300
Subject: [PATCH 38/42] Update docs on configuration changes

---
 docs/configuration.md | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/docs/configuration.md b/docs/configuration.md
index a9846508..5dbba71b 100644
--- a/docs/configuration.md
+++ b/docs/configuration.md
@@ -2,6 +2,9 @@
 
 Configuration is done through the use of environment variables.
 
+!!! note
+    If you create a db with certain settings and later change them, `haiku.rag` will detect incompatibilities (for example, if you change embedding provider) and will exit. You can **rebuild** the database to apply the new settings, see [Rebuild Database](./cli.md#rebuild-database).
+
 ## File Monitoring
 
 Set directories to monitor for automatic indexing:

From f4aef4eb9c55d4ad6fb7e9c7732f201a64c4077e Mon Sep 17 00:00:00 2001
From: Yiorgis Gozadinos <ggozadinos@gmail.com>
Date: Mon, 14 Jul 2025 12:13:01 +0300
Subject: [PATCH 39/42] Check pypi version on startup

---
 src/haiku/rag/cli.py   | 19 ++++++++++++++++++-
 src/haiku/rag/utils.py | 24 ++++++++++++++++++++++++
 2 files changed, 42 insertions(+), 1 deletion(-)

diff --git a/src/haiku/rag/cli.py b/src/haiku/rag/cli.py
index 426af784..80ee047e 100644
--- a/src/haiku/rag/cli.py
+++ b/src/haiku/rag/cli.py
@@ -5,7 +5,7 @@ import typer
 from rich.console import Console
 
 from haiku.rag.app import HaikuRAGApp
-from haiku.rag.utils import get_default_data_dir
+from haiku.rag.utils import get_default_data_dir, is_up_to_date
 
 cli = typer.Typer(
     context_settings={"help_option_names": ["-h", "--help"]}, no_args_is_help=True
@@ -15,6 +15,23 @@ console = Console()
 event_loop = asyncio.get_event_loop()
 
 
+async def check_version():
+    """Check if haiku.rag is up to date and show warning if not."""
+    up_to_date, current_version, latest_version = await is_up_to_date()
+    if not up_to_date:
+        console.print(
+            f"[yellow]Warning: haiku.rag is outdated. Current: {current_version}, Latest: {latest_version}[/yellow]"
+        )
+        console.print("[yellow]Please update.[/yellow]")
+
+
+@cli.callback()
+def main():
+    """haiku.rag CLI - SQLite-based RAG system"""
+    # Run version check before any command
+    event_loop.run_until_complete(check_version())
+
+
 @cli.command("list", help="List all stored documents")
 def list_documents(
     db: Path = typer.Option(
diff --git a/src/haiku/rag/utils.py b/src/haiku/rag/utils.py
index 03c160bf..8b78f008 100644
--- a/src/haiku/rag/utils.py
+++ b/src/haiku/rag/utils.py
@@ -1,6 +1,10 @@
 import sys
+from importlib import metadata
 from pathlib import Path
 
+import httpx
+from packaging.version import Version, parse
+
 
 def get_default_data_dir() -> Path:
     """
@@ -23,3 +27,23 @@ def get_default_data_dir() -> Path:
 
     data_path = system_paths[sys.platform]
     return data_path
+
+
+async def is_up_to_date() -> tuple[bool, Version, Version]:
+    """
+    Checks whether haiku.rag is current.
+
+    :return: A tuple containing a boolean indicating whether haiku.rag is current, the running version and the latest version
+    :rtype: tuple[bool, Version, Version]
+    """
+
+    async with httpx.AsyncClient() as client:
+        running_version = parse(metadata.version("haiku.rag"))
+        try:
+            response = await client.get("https://pypi.org/pypi/haiku.rag/json")
+            data = response.json()
+            pypi_version = parse(data["info"]["version"])
+        except Exception:
+            # If no network connection, do not raise alarms.
+            pypi_version = running_version
+    return running_version >= pypi_version, running_version, pypi_version

From 0bf5606117776c5e5dafb1f898fc1d5e1d6a0ba9 Mon Sep 17 00:00:00 2001
From: Yiorgis Gozadinos <ggozadinos@gmail.com>
Date: Wed, 16 Jul 2025 15:32:24 +0300
Subject: [PATCH 40/42] Store haiku.rag version, enabling upgrades

---
 pyproject.toml                           |  2 +-
 src/haiku/rag/store/engine.py            | 68 ++++++++++++++++++------
 src/haiku/rag/store/upgrades/__init__.py |  3 ++
 src/haiku/rag/store/upgrades/v0_3_4.py   | 26 +++++++++
 src/haiku/rag/utils.py                   | 31 +++++++++++
 tests/test_settings.py                   |  1 -
 tests/test_utils.py                      | 15 ++++++
 uv.lock                                  |  2 +-
 8 files changed, 129 insertions(+), 19 deletions(-)
 create mode 100644 src/haiku/rag/store/upgrades/__init__.py
 create mode 100644 src/haiku/rag/store/upgrades/v0_3_4.py
 create mode 100644 tests/test_utils.py

diff --git a/pyproject.toml b/pyproject.toml
index 30dbb443..3bfd9784 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,6 +1,6 @@
 [project]
 name = "haiku.rag"
-version = "0.3.3"
+version = "0.3.4"
 description = "Retrieval Augmented Generation (RAG) with SQLite"
 authors = [{ name = "Yiorgis Gozadinos", email = "ggozadinos@gmail.com" }]
 license = { text = "MIT" }
diff --git a/src/haiku/rag/store/engine.py b/src/haiku/rag/store/engine.py
index a2701ddf..72f17402 100644
--- a/src/haiku/rag/store/engine.py
+++ b/src/haiku/rag/store/engine.py
@@ -1,11 +1,16 @@
 import sqlite3
 import struct
+from importlib import metadata
 from pathlib import Path
 from typing import Literal
 
 import sqlite_vec
+from packaging.version import parse
 
+from haiku.rag.config import Config
 from haiku.rag.embeddings import get_embedder
+from haiku.rag.store.upgrades import upgrades
+from haiku.rag.utils import int_to_semantic_version, semantic_version_to_int
 
 
 class Store:
@@ -13,7 +18,7 @@ class Store:
         self, db_path: Path | Literal[":memory:"], skip_validation: bool = False
     ):
         self.db_path: Path | Literal[":memory:"] = db_path
-        self._connection = self.create_db()
+        self.create_or_update_db()
 
         # Validate config compatibility after connection is established
         if not skip_validation:
@@ -21,12 +26,35 @@ class Store:
 
             settings_repo = SettingsRepository(self)
             settings_repo.validate_config_compatibility()
+        current_version = metadata.version("haiku.rag")
+        self.set_user_version(current_version)
 
-    def create_db(self) -> sqlite3.Connection:
+    def create_or_update_db(self):
         """Create the database and tables with sqlite-vec support for embeddings."""
+        current_version = metadata.version("haiku.rag")
+
         db = sqlite3.connect(self.db_path)
         db.enable_load_extension(True)
         sqlite_vec.load(db)
+        self._connection = db
+        existing_tables = [
+            row[0]
+            for row in db.execute(
+                "SELECT name FROM sqlite_master WHERE type='table';"
+            ).fetchall()
+        ]
+
+        # If we have a db already, perform upgrades and return
+        if self.db_path != ":memory:" and "documents" in existing_tables:
+            # Upgrade database
+            db_version = self.get_user_version()
+            for version, steps in upgrades:
+                if parse(current_version) >= parse(version) and parse(version) > parse(
+                    db_version
+                ):
+                    for step in steps:
+                        step(db)
+            return
 
         # Create documents table
         db.execute("""
@@ -39,7 +67,6 @@ class Store:
                 updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
             )
         """)
-
         # Create chunks table
         db.execute("""
             CREATE TABLE IF NOT EXISTS chunks (
@@ -50,7 +77,6 @@ class Store:
                 FOREIGN KEY (document_id) REFERENCES documents (id) ON DELETE CASCADE
             )
         """)
-
         # Create vector table for chunk embeddings
         embedder = get_embedder()
         db.execute(f"""
@@ -59,7 +85,6 @@ class Store:
                 embedding FLOAT[{embedder._vector_dim}]
             )
         """)
-
         # Create FTS5 table for full-text search
         db.execute("""
             CREATE VIRTUAL TABLE IF NOT EXISTS chunks_fts USING fts5(
@@ -68,7 +93,6 @@ class Store:
                 content_rowid='id'
             )
         """)
-
         # Create settings table for storing current configuration
         db.execute("""
             CREATE TABLE IF NOT EXISTS settings (
@@ -76,23 +100,35 @@ class Store:
                 settings TEXT NOT NULL DEFAULT '{}'
             )
         """)
-
-        # Create indexes for better performance
-        db.execute(
-            "CREATE INDEX IF NOT EXISTS idx_chunks_document_id ON chunks(document_id)"
-        )
-
         # Save current settings to the new database
-        from haiku.rag.config import Config
-
         settings_json = Config.model_dump_json()
         db.execute(
             "INSERT OR IGNORE INTO settings (id, settings) VALUES (1, ?)",
             (settings_json,),
         )
-
+        # Create indexes for better performance
+        db.execute(
+            "CREATE INDEX IF NOT EXISTS idx_chunks_document_id ON chunks(document_id)"
+        )
         db.commit()
-        return db
+
+    def get_user_version(self) -> str:
+        """Returns the SQLite user version"""
+        if self._connection is None:
+            raise ValueError("Store connection is not available")
+
+        cursor = self._connection.execute("PRAGMA user_version;")
+        version = cursor.fetchone()
+        return int_to_semantic_version(version[0])
+
+    def set_user_version(self, version: str) -> None:
+        """Updates the SQLite user version"""
+        if self._connection is None:
+            raise ValueError("Store connection is not available")
+
+        self._connection.execute(
+            f"PRAGMA user_version = {semantic_version_to_int(version)};"
+        )
 
     def recreate_embeddings_table(self) -> None:
         """Recreate the embeddings table with current vector dimensions."""
diff --git a/src/haiku/rag/store/upgrades/__init__.py b/src/haiku/rag/store/upgrades/__init__.py
new file mode 100644
index 00000000..3954309e
--- /dev/null
+++ b/src/haiku/rag/store/upgrades/__init__.py
@@ -0,0 +1,3 @@
+from haiku.rag.store.upgrades.v0_3_4 import upgrades as v0_3_4_upgrades
+
+upgrades = v0_3_4_upgrades
diff --git a/src/haiku/rag/store/upgrades/v0_3_4.py b/src/haiku/rag/store/upgrades/v0_3_4.py
new file mode 100644
index 00000000..9ba912a1
--- /dev/null
+++ b/src/haiku/rag/store/upgrades/v0_3_4.py
@@ -0,0 +1,26 @@
+from collections.abc import Callable
+from sqlite3 import Connection
+
+from haiku.rag.config import Config
+
+
+def add_settings_table(db: Connection) -> None:
+    # Create settings table for storing current configuration
+    db.execute("""
+        CREATE TABLE settings (
+            id INTEGER PRIMARY KEY DEFAULT 1,
+            settings TEXT NOT NULL DEFAULT '{}'
+        )
+    """)
+
+    settings_json = Config.model_dump_json()
+    db.execute(
+        "INSERT INTO settings (id, settings) VALUES (1, ?)",
+        (settings_json,),
+    )
+    db.commit()
+
+
+upgrades: list[tuple[str, list[Callable[[Connection], None]]]] = [
+    ("0.3.4", [add_settings_table])
+]
diff --git a/src/haiku/rag/utils.py b/src/haiku/rag/utils.py
index 8b78f008..91ead27b 100644
--- a/src/haiku/rag/utils.py
+++ b/src/haiku/rag/utils.py
@@ -29,6 +29,37 @@ def get_default_data_dir() -> Path:
     return data_path
 
 
+def semantic_version_to_int(version: str) -> int:
+    """
+    Convert a semantic version string to an integer.
+
+    :param version: Semantic version string
+    :type version: str
+    :return: Integer representation of semantic version
+    :rtype: int
+    """
+    major, minor, patch = version.split(".")
+    major = int(major) << 16
+    minor = int(minor) << 8
+    patch = int(patch)
+    return major + minor + patch
+
+
+def int_to_semantic_version(version: int) -> str:
+    """
+    Convert an integer to a semantic version string.
+
+    :param version: Integer representation of semantic version
+    :type version: int
+    :return: Semantic version string
+    :rtype: str
+    """
+    major = version >> 16
+    minor = (version >> 8) & 255
+    patch = version & 255
+    return f"{major}.{minor}.{patch}"
+
+
 async def is_up_to_date() -> tuple[bool, Version, Version]:
     """
     Checks whether haiku.rag is current.
diff --git a/tests/test_settings.py b/tests/test_settings.py
index 6a531685..08c21b07 100644
--- a/tests/test_settings.py
+++ b/tests/test_settings.py
@@ -50,7 +50,6 @@ async def test_config_validation_on_db_load():
 
         # Create store and save settings
         store1 = Store(db_path)
-        SettingsRepository(store1)
         store1.close()
 
         # Change config
diff --git a/tests/test_utils.py b/tests/test_utils.py
new file mode 100644
index 00000000..cd5b742c
--- /dev/null
+++ b/tests/test_utils.py
@@ -0,0 +1,15 @@
+from haiku.rag.utils import int_to_semantic_version, semantic_version_to_int
+
+
+def test_sqlite_user_version():
+    version = "0.1.5"
+    assert semantic_version_to_int(version) == 261
+    assert int_to_semantic_version(261) == version
+
+    version = "0.0.0"
+    assert semantic_version_to_int(version) == 0
+    assert int_to_semantic_version(0) == version
+
+    version = "255.255.255"
+    assert semantic_version_to_int(version) == 16777215
+    assert int_to_semantic_version(16777215) == version
diff --git a/uv.lock b/uv.lock
index fd6b3649..d0ac66e2 100644
--- a/uv.lock
+++ b/uv.lock
@@ -816,7 +816,7 @@ wheels = [
 
 [[package]]
 name = "haiku-rag"
-version = "0.3.3"
+version = "0.3.4"
 source = { editable = "." }
 dependencies = [
     { name = "fastmcp" },

From 4d8088efa8fb4024f47c168f196d31a1d8892c90 Mon Sep 17 00:00:00 2001
From: Yiorgis Gozadinos <ggozadinos@gmail.com>
Date: Wed, 16 Jul 2025 18:32:58 +0300
Subject: [PATCH 41/42] Log db upgrades

---
 src/haiku/rag/store/engine.py          | 5 +++++
 src/haiku/rag/store/upgrades/v0_3_4.py | 2 +-
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/src/haiku/rag/store/engine.py b/src/haiku/rag/store/engine.py
index 72f17402..7e746bc7 100644
--- a/src/haiku/rag/store/engine.py
+++ b/src/haiku/rag/store/engine.py
@@ -6,6 +6,7 @@ from typing import Literal
 
 import sqlite_vec
 from packaging.version import parse
+from rich.console import Console
 
 from haiku.rag.config import Config
 from haiku.rag.embeddings import get_embedder
@@ -47,6 +48,7 @@ class Store:
         # If we have a db already, perform upgrades and return
         if self.db_path != ":memory:" and "documents" in existing_tables:
             # Upgrade database
+            console = Console()
             db_version = self.get_user_version()
             for version, steps in upgrades:
                 if parse(current_version) >= parse(version) and parse(version) > parse(
@@ -54,6 +56,9 @@ class Store:
                 ):
                     for step in steps:
                         step(db)
+                        console.print(
+                            f"[green][b]DB Upgrade: [/b]{step.__doc__}[/green]"
+                        )
             return
 
         # Create documents table
diff --git a/src/haiku/rag/store/upgrades/v0_3_4.py b/src/haiku/rag/store/upgrades/v0_3_4.py
index 9ba912a1..56e73f00 100644
--- a/src/haiku/rag/store/upgrades/v0_3_4.py
+++ b/src/haiku/rag/store/upgrades/v0_3_4.py
@@ -5,7 +5,7 @@ from haiku.rag.config import Config
 
 
 def add_settings_table(db: Connection) -> None:
-    # Create settings table for storing current configuration
+    """Create settings table for storing current configuration"""
     db.execute("""
         CREATE TABLE settings (
             id INTEGER PRIMARY KEY DEFAULT 1,

From 20beb23a389778461e8280f1442ea196e52182e3 Mon Sep 17 00:00:00 2001
From: Yiorgis Gozadinos <ggozadinos@gmail.com>
Date: Wed, 16 Jul 2025 19:06:54 +0300
Subject: [PATCH 42/42] Fix docstrings

---
 src/haiku/rag/chunker.py | 27 +++++++------------
 src/haiku/rag/client.py  | 58 ++++++++++++++++++++++++++++++++--------
 src/haiku/rag/utils.py   | 39 +++++++++++++--------------
 3 files changed, 75 insertions(+), 49 deletions(-)

diff --git a/src/haiku/rag/chunker.py b/src/haiku/rag/chunker.py
index 3cb5808b..3d7db951 100644
--- a/src/haiku/rag/chunker.py
+++ b/src/haiku/rag/chunker.py
@@ -6,15 +6,11 @@ from haiku.rag.config import Config
 
 
 class Chunker:
-    """
-    A class that chunks text into smaller pieces for embedding and retrieval.
+    """A class that chunks text into smaller pieces for embedding and retrieval.
 
-    Parameters
-    ----------
-    chunk_size : int
-        The maximum size of a chunk in characters.
-    chunk_overlap : int
-        The number of characters of overlap between chunks.
+    Args:
+        chunk_size: The maximum size of a chunk in tokens.
+        chunk_overlap: The number of tokens of overlap between chunks.
     """
 
     encoder: ClassVar[tiktoken.Encoding] = tiktoken.encoding_for_model("gpt-4o")
@@ -28,18 +24,13 @@ class Chunker:
         self.chunk_overlap = chunk_overlap
 
     async def chunk(self, text: str) -> list[str]:
-        """
-        Split the text into chunks.
+        """Split the text into chunks based on token boundaries.
 
-        Parameters
-        ----------
-        text : str
-            The text to be split into chunks.
+        Args:
+            text: The text to be split into chunks.
 
-        Returns
-        -------
-        list
-            A list of text chunks.
+        Returns:
+            A list of text chunks with token-based boundaries and overlap.
         """
         if not text:
             return []
diff --git a/src/haiku/rag/client.py b/src/haiku/rag/client.py
index ca30098c..7f375155 100644
--- a/src/haiku/rag/client.py
+++ b/src/haiku/rag/client.py
@@ -26,7 +26,12 @@ class HaikuRAG:
         / "haiku.rag.sqlite",
         skip_validation: bool = False,
     ):
-        """Initialize the RAG client with a database path."""
+        """Initialize the RAG client with a database path.
+
+        Args:
+            db_path: Path to the SQLite database file or ":memory:" for in-memory database.
+            skip_validation: Whether to skip configuration validation on database load.
+        """
         if isinstance(db_path, Path):
             if not db_path.parent.exists():
                 Path.mkdir(db_path.parent, parents=True)
@@ -46,7 +51,16 @@ class HaikuRAG:
     async def create_document(
         self, content: str, uri: str | None = None, metadata: dict | None = None
     ) -> Document:
-        """Create a new document with optional URI and metadata."""
+        """Create a new document with optional URI and metadata.
+
+        Args:
+            content: The text content of the document.
+            uri: Optional URI identifier for the document.
+            metadata: Optional metadata dictionary.
+
+        Returns:
+            The created Document instance.
+        """
         document = Document(
             content=content,
             uri=uri,
@@ -219,11 +233,25 @@ class HaikuRAG:
         return ".html"
 
     async def get_document_by_id(self, document_id: int) -> Document | None:
-        """Get a document by its ID."""
+        """Get a document by its ID.
+
+        Args:
+            document_id: The unique identifier of the document.
+
+        Returns:
+            The Document instance if found, None otherwise.
+        """
         return await self.document_repository.get_by_id(document_id)
 
     async def get_document_by_uri(self, uri: str) -> Document | None:
-        """Get a document by its URI."""
+        """Get a document by its URI.
+
+        Args:
+            uri: The URI identifier of the document.
+
+        Returns:
+            The Document instance if found, None otherwise.
+        """
         return await self.document_repository.get_by_uri(uri)
 
     async def update_document(self, document: Document) -> Document:
@@ -237,7 +265,15 @@ class HaikuRAG:
     async def list_documents(
         self, limit: int | None = None, offset: int | None = None
     ) -> list[Document]:
-        """List all documents with optional pagination."""
+        """List all documents with optional pagination.
+
+        Args:
+            limit: Maximum number of documents to return.
+            offset: Number of documents to skip.
+
+        Returns:
+            List of Document instances.
+        """
         return await self.document_repository.list_all(limit=limit, offset=offset)
 
     async def search(
@@ -246,12 +282,12 @@ class HaikuRAG:
         """Search for relevant chunks using hybrid search (vector similarity + full-text search).
 
         Args:
-            query: The search query string
-            limit: Maximum number of results to return
-            k: Parameter for Reciprocal Rank Fusion (default: 60)
+            query: The search query string.
+            limit: Maximum number of results to return.
+            k: Parameter for Reciprocal Rank Fusion (default: 60).
 
         Returns:
-            List of (chunk, score) tuples ordered by relevance
+            List of (chunk, score) tuples ordered by relevance.
         """
         return await self.chunk_repository.search_chunks_hybrid(query, limit, k)
 
@@ -259,10 +295,10 @@ class HaikuRAG:
         """Ask a question using the configured QA agent.
 
         Args:
-            question: The question to ask
+            question: The question to ask.
 
         Returns:
-            The generated answer as a string
+            The generated answer as a string.
         """
         from haiku.rag.qa import get_qa_agent
 
diff --git a/src/haiku/rag/utils.py b/src/haiku/rag/utils.py
index 91ead27b..f5f2b5eb 100644
--- a/src/haiku/rag/utils.py
+++ b/src/haiku/rag/utils.py
@@ -7,15 +7,14 @@ from packaging.version import Version, parse
 
 
 def get_default_data_dir() -> Path:
-    """
-    Get the user data directory for the current system platform.
+    """Get the user data directory for the current system platform.
 
     Linux: ~/.local/share/haiku.rag
     macOS: ~/Library/Application Support/haiku.rag
     Windows: C:/Users/<USER>/AppData/Roaming/haiku.rag
 
-    :return: User Data Path
-    :rtype: Path
+    Returns:
+        User Data Path.
     """
     home = Path.home()
 
@@ -30,13 +29,13 @@ def get_default_data_dir() -> Path:
 
 
 def semantic_version_to_int(version: str) -> int:
-    """
-    Convert a semantic version string to an integer.
+    """Convert a semantic version string to an integer.
 
-    :param version: Semantic version string
-    :type version: str
-    :return: Integer representation of semantic version
-    :rtype: int
+    Args:
+        version: Semantic version string.
+
+    Returns:
+        Integer representation of semantic version.
     """
     major, minor, patch = version.split(".")
     major = int(major) << 16
@@ -46,13 +45,13 @@ def semantic_version_to_int(version: str) -> int:
 
 
 def int_to_semantic_version(version: int) -> str:
-    """
-    Convert an integer to a semantic version string.
+    """Convert an integer to a semantic version string.
 
-    :param version: Integer representation of semantic version
-    :type version: int
-    :return: Semantic version string
-    :rtype: str
+    Args:
+        version: Integer representation of semantic version.
+
+    Returns:
+        Semantic version string.
     """
     major = version >> 16
     minor = (version >> 8) & 255
@@ -61,11 +60,11 @@ def int_to_semantic_version(version: int) -> str:
 
 
 async def is_up_to_date() -> tuple[bool, Version, Version]:
-    """
-    Checks whether haiku.rag is current.
+    """Check whether haiku.rag is current.
 
-    :return: A tuple containing a boolean indicating whether haiku.rag is current, the running version and the latest version
-    :rtype: tuple[bool, Version, Version]
+    Returns:
+        A tuple containing a boolean indicating whether haiku.rag is current,
+        the running version and the latest version.
     """
 
     async with httpx.AsyncClient() as client: