From fe4ac530be8ed648fda3425f5e5d2c94605a3b80 Mon Sep 17 00:00:00 2001 From: Yiorgis Gozadinos Date: Fri, 28 Nov 2025 15:54:48 +0200 Subject: [PATCH] DB migration --- CHANGELOG.md | 2 +- evaluations/pyproject.toml | 2 +- .../haiku/rag/store/upgrades/__init__.py | 18 +++-- .../haiku/rag/store/upgrades/v0_20_0.py | 68 +++++++++++++++++++ haiku_rag_slim/pyproject.toml | 2 +- pyproject.toml | 6 +- uv.lock | 6 +- 7 files changed, 90 insertions(+), 14 deletions(-) create mode 100644 haiku_rag_slim/haiku/rag/store/upgrades/v0_20_0.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 947e6c51..1b255e2c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ - **Download Models Progress**: `haiku-rag download-models` now shows real-time progress with Rich progress bars for Ollama model downloads - **Refactored Download Models**: Moved core download logic to `HaikuRAG.download_models()` async generator that yields `DownloadProgress` events, separating business logic from UI -## [0.19.6] - 2025-12-03 +## [0.20.0] - 2025-11-28 ## [0.19.6] - 2025-12-03 diff --git a/evaluations/pyproject.toml b/evaluations/pyproject.toml index 0977c9eb..c84b95a3 100644 --- a/evaluations/pyproject.toml +++ b/evaluations/pyproject.toml @@ -2,7 +2,7 @@ name = "haiku.rag-evals" description = "Benchmarking and evaluation scripts for haiku.rag" -version = "0.19.6" +version = "0.20.0" authors = [{ name = "Yiorgis Gozadinos", email = "ggozadinos@gmail.com" }] license = { text = "MIT" } requires-python = ">=3.12" diff --git a/haiku_rag_slim/haiku/rag/store/upgrades/__init__.py b/haiku_rag_slim/haiku/rag/store/upgrades/__init__.py index 3c8c0143..0ece23d1 100644 --- a/haiku_rag_slim/haiku/rag/store/upgrades/__init__.py +++ b/haiku_rag_slim/haiku/rag/store/upgrades/__init__.py @@ -6,6 +6,16 @@ from packaging.version import Version, parse from haiku.rag.store.engine import Store +from .v0_9_3 import upgrade_fts_phrase as upgrade_0_9_3_fts # noqa: E402 +from .v0_9_3 import upgrade_order as upgrade_0_9_3_order # noqa: E402 +from .v0_10_1 import upgrade_add_title as upgrade_0_10_1_add_title # noqa: E402 +from .v0_19_6 import ( # noqa: E402 + upgrade_embeddings_model_config as upgrade_0_19_6_embeddings, +) +from .v0_20_0 import ( + upgrade_add_docling_document as upgrade_0_20_0_docling, # noqa: E402 +) + logger = logging.getLogger(__name__) @@ -53,14 +63,8 @@ def run_pending_upgrades(store: Store, from_version: str, to_version: str) -> No logger.info("Completed upgrade %s", step.version) -from .v0_9_3 import upgrade_fts_phrase as upgrade_0_9_3_fts # noqa: E402 -from .v0_9_3 import upgrade_order as upgrade_0_9_3_order # noqa: E402 -from .v0_10_1 import upgrade_add_title as upgrade_0_10_1_add_title # noqa: E402 -from .v0_19_6 import ( # noqa: E402 - upgrade_embeddings_model_config as upgrade_0_19_6_embeddings, -) - upgrades.append(upgrade_0_9_3_order) upgrades.append(upgrade_0_9_3_fts) upgrades.append(upgrade_0_10_1_add_title) upgrades.append(upgrade_0_19_6_embeddings) +upgrades.append(upgrade_0_20_0_docling) diff --git a/haiku_rag_slim/haiku/rag/store/upgrades/v0_20_0.py b/haiku_rag_slim/haiku/rag/store/upgrades/v0_20_0.py new file mode 100644 index 00000000..e7a2a0b0 --- /dev/null +++ b/haiku_rag_slim/haiku/rag/store/upgrades/v0_20_0.py @@ -0,0 +1,68 @@ +import json + +from lancedb.pydantic import LanceModel +from pydantic import Field + +from haiku.rag.store.engine import Store +from haiku.rag.store.upgrades import Upgrade + + +def _apply_add_docling_document_columns(store: Store) -> None: + """Add 'docling_document_json' and 'docling_version' columns to documents table.""" + + # Read existing rows using Arrow for schema-agnostic access + try: + docs_arrow = store.documents_table.search().to_arrow() + rows = docs_arrow.to_pylist() + except Exception: + rows = [] + + class DocumentRecordV3(LanceModel): + id: str + content: str + uri: str | None = None + title: str | None = None + metadata: str = Field(default="{}") + docling_document_json: str | None = None + docling_version: str | None = None + created_at: str = Field(default_factory=lambda: "") + updated_at: str = Field(default_factory=lambda: "") + + # Drop and recreate documents table with the new schema + try: + store.db.drop_table("documents") + except Exception: + pass + + store.documents_table = store.db.create_table("documents", schema=DocumentRecordV3) + + # Reinsert previous rows with new columns as None + if rows: + backfilled = [] + for row in rows: + backfilled.append( + DocumentRecordV3( + id=row.get("id"), + content=row.get("content", ""), + uri=row.get("uri"), + title=row.get("title"), + metadata=( + row.get("metadata") + if isinstance(row.get("metadata"), str) + else json.dumps(row.get("metadata") or {}) + ), + docling_document_json=None, + docling_version=None, + created_at=row.get("created_at", ""), + updated_at=row.get("updated_at", ""), + ) + ) + + store.documents_table.add(backfilled) + + +upgrade_add_docling_document = Upgrade( + version="0.20.0", + apply=_apply_add_docling_document_columns, + description="Add 'docling_document_json' and 'docling_version' columns to documents table", +) diff --git a/haiku_rag_slim/pyproject.toml b/haiku_rag_slim/pyproject.toml index 90a40878..6b304761 100644 --- a/haiku_rag_slim/pyproject.toml +++ b/haiku_rag_slim/pyproject.toml @@ -2,7 +2,7 @@ name = "haiku.rag-slim" description = "Opinionated agentic RAG powered by LanceDB, Pydantic AI, and Docling - Minimal dependencies" -version = "0.19.6" +version = "0.20.0" authors = [{ name = "Yiorgis Gozadinos", email = "ggozadinos@gmail.com" }] license = { text = "MIT" } readme = { file = "README.md", content-type = "text/markdown" } diff --git a/pyproject.toml b/pyproject.toml index 106aab5b..b41d36e7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,7 +2,7 @@ name = "haiku.rag" description = "Opinionated agentic RAG powered by LanceDB, Pydantic AI, and Docling" -version = "0.19.6" +version = "0.20.0" authors = [{ name = "Yiorgis Gozadinos", email = "ggozadinos@gmail.com" }] license = { text = "MIT" } readme = { file = "README.md", content-type = "text/markdown" } @@ -30,7 +30,11 @@ classifiers = [ ] dependencies = [ +<<<<<<< HEAD "haiku.rag-slim[docling,voyageai,mxbai,cohere,zeroentropy,inspector]==0.19.6", +======= + "haiku.rag-slim[docling,voyageai,mxbai,cohere,zeroentropy,inspector]==0.20.0", +>>>>>>> 4dba94cc93ba (DB migration) ] [project.scripts] diff --git a/uv.lock b/uv.lock index a9c1b931..ae96a4b5 100644 --- a/uv.lock +++ b/uv.lock @@ -1264,7 +1264,7 @@ wheels = [ [[package]] name = "haiku-rag" -version = "0.19.6" +version = "0.20.0" source = { editable = "." } dependencies = [ { name = "haiku-rag-slim", extra = ["cohere", "docling", "inspector", "mxbai", "voyageai", "zeroentropy"] }, @@ -1312,7 +1312,7 @@ dev = [ [[package]] name = "haiku-rag-evals" -version = "0.19.6" +version = "0.20.0" source = { editable = "evaluations" } dependencies = [ { name = "datasets" }, @@ -1333,7 +1333,7 @@ requires-dist = [ [[package]] name = "haiku-rag-slim" -version = "0.19.6" +version = "0.20.0" source = { editable = "haiku_rag_slim" } dependencies = [ { name = "docling-core" },