DB migration

CHANGELOG.md

@@ -6,7 +6,7 @@
 - **Download Models Progress**: `haiku-rag download-models` now shows real-time progress with Rich progress bars for Ollama model downloads
 - **Refactored Download Models**: Moved core download logic to `HaikuRAG.download_models()` async generator that yields `DownloadProgress` events, separating business logic from UI
 
-## [0.19.6] - 2025-12-03
+## [0.20.0] - 2025-11-28
 
 ## [0.19.6] - 2025-12-03
 

evaluations/pyproject.toml

@@ -2,7 +2,7 @@
 
 name = "haiku.rag-evals"
 description = "Benchmarking and evaluation scripts for haiku.rag"
-version = "0.19.6"
+version = "0.20.0"
 authors = [{ name = "Yiorgis Gozadinos", email = "ggozadinos@gmail.com" }]
 license = { text = "MIT" }
 requires-python = ">=3.12"

haiku_rag_slim/haiku/rag/store/upgrades/__init__.py

@@ -6,6 +6,16 @@ from packaging.version import Version, parse
 
 from haiku.rag.store.engine import Store
 
+from .v0_9_3 import upgrade_fts_phrase as upgrade_0_9_3_fts  # noqa: E402
+from .v0_9_3 import upgrade_order as upgrade_0_9_3_order  # noqa: E402
+from .v0_10_1 import upgrade_add_title as upgrade_0_10_1_add_title  # noqa: E402
+from .v0_19_6 import (  # noqa: E402
+    upgrade_embeddings_model_config as upgrade_0_19_6_embeddings,
+)
+from .v0_20_0 import (
+    upgrade_add_docling_document as upgrade_0_20_0_docling,  # noqa: E402
+)
+
 logger = logging.getLogger(__name__)
 
 
@@ -53,14 +63,8 @@ def run_pending_upgrades(store: Store, from_version: str, to_version: str) -> No
         logger.info("Completed upgrade %s", step.version)
 
 
-from .v0_9_3 import upgrade_fts_phrase as upgrade_0_9_3_fts  # noqa: E402
-from .v0_9_3 import upgrade_order as upgrade_0_9_3_order  # noqa: E402
-from .v0_10_1 import upgrade_add_title as upgrade_0_10_1_add_title  # noqa: E402
-from .v0_19_6 import (  # noqa: E402
-    upgrade_embeddings_model_config as upgrade_0_19_6_embeddings,
-)
-
 upgrades.append(upgrade_0_9_3_order)
 upgrades.append(upgrade_0_9_3_fts)
 upgrades.append(upgrade_0_10_1_add_title)
 upgrades.append(upgrade_0_19_6_embeddings)
+upgrades.append(upgrade_0_20_0_docling)

haiku_rag_slim/haiku/rag/store/upgrades/v0_20_0.py

@@ -0,0 +1,68 @@
+import json
+
+from lancedb.pydantic import LanceModel
+from pydantic import Field
+
+from haiku.rag.store.engine import Store
+from haiku.rag.store.upgrades import Upgrade
+
+
+def _apply_add_docling_document_columns(store: Store) -> None:
+    """Add 'docling_document_json' and 'docling_version' columns to documents table."""
+
+    # Read existing rows using Arrow for schema-agnostic access
+    try:
+        docs_arrow = store.documents_table.search().to_arrow()
+        rows = docs_arrow.to_pylist()
+    except Exception:
+        rows = []
+
+    class DocumentRecordV3(LanceModel):
+        id: str
+        content: str
+        uri: str | None = None
+        title: str | None = None
+        metadata: str = Field(default="{}")
+        docling_document_json: str | None = None
+        docling_version: str | None = None
+        created_at: str = Field(default_factory=lambda: "")
+        updated_at: str = Field(default_factory=lambda: "")
+
+    # Drop and recreate documents table with the new schema
+    try:
+        store.db.drop_table("documents")
+    except Exception:
+        pass
+
+    store.documents_table = store.db.create_table("documents", schema=DocumentRecordV3)
+
+    # Reinsert previous rows with new columns as None
+    if rows:
+        backfilled = []
+        for row in rows:
+            backfilled.append(
+                DocumentRecordV3(
+                    id=row.get("id"),
+                    content=row.get("content", ""),
+                    uri=row.get("uri"),
+                    title=row.get("title"),
+                    metadata=(
+                        row.get("metadata")
+                        if isinstance(row.get("metadata"), str)
+                        else json.dumps(row.get("metadata") or {})
+                    ),
+                    docling_document_json=None,
+                    docling_version=None,
+                    created_at=row.get("created_at", ""),
+                    updated_at=row.get("updated_at", ""),
+                )
+            )
+
+        store.documents_table.add(backfilled)
+
+
+upgrade_add_docling_document = Upgrade(
+    version="0.20.0",
+    apply=_apply_add_docling_document_columns,
+    description="Add 'docling_document_json' and 'docling_version' columns to documents table",
+)

haiku_rag_slim/pyproject.toml

@@ -2,7 +2,7 @@
 
 name = "haiku.rag-slim"
 description = "Opinionated agentic RAG powered by LanceDB, Pydantic AI, and Docling - Minimal dependencies"
-version = "0.19.6"
+version = "0.20.0"
 authors = [{ name = "Yiorgis Gozadinos", email = "ggozadinos@gmail.com" }]
 license = { text = "MIT" }
 readme = { file = "README.md", content-type = "text/markdown" }

pyproject.toml

@@ -2,7 +2,7 @@
 
 name = "haiku.rag"
 description = "Opinionated agentic RAG powered by LanceDB, Pydantic AI, and Docling"
-version = "0.19.6"
+version = "0.20.0"
 authors = [{ name = "Yiorgis Gozadinos", email = "ggozadinos@gmail.com" }]
 license = { text = "MIT" }
 readme = { file = "README.md", content-type = "text/markdown" }
@@ -30,7 +30,11 @@ classifiers = [
 ]
 
 dependencies = [
+<<<<<<< HEAD
     "haiku.rag-slim[docling,voyageai,mxbai,cohere,zeroentropy,inspector]==0.19.6",
+=======
+    "haiku.rag-slim[docling,voyageai,mxbai,cohere,zeroentropy,inspector]==0.20.0",
+>>>>>>> 4dba94cc93ba (DB migration)
 ]
 
 [project.scripts]

uv.lock

@@ -1264,7 +1264,7 @@ wheels = [
 
 [[package]]
 name = "haiku-rag"
-version = "0.19.6"
+version = "0.20.0"
 source = { editable = "." }
 dependencies = [
     { name = "haiku-rag-slim", extra = ["cohere", "docling", "inspector", "mxbai", "voyageai", "zeroentropy"] },
@@ -1312,7 +1312,7 @@ dev = [
 
 [[package]]
 name = "haiku-rag-evals"
-version = "0.19.6"
+version = "0.20.0"
 source = { editable = "evaluations" }
 dependencies = [
     { name = "datasets" },
@@ -1333,7 +1333,7 @@ requires-dist = [
 
 [[package]]
 name = "haiku-rag-slim"
-version = "0.19.6"
+version = "0.20.0"
 source = { editable = "haiku_rag_slim" }
 dependencies = [
     { name = "docling-core" },