Remove obsolete migrations
This commit is contained in:
Yiorgis Gozadinos
parent
119ca84da8
commit
061095e855
5 changed files with 4 additions and 253 deletions
|
|
@ -14,6 +14,10 @@
|
|||
|
||||
- **Dependencies**: Updated lancedb 0.26.0 → 0.26.1, docling 2.65.0 → 2.67.0
|
||||
|
||||
### Removed
|
||||
|
||||
- **Legacy Migrations**: Removed obsolete database migration files (`v0_9_3.py`, `v0_10_1.py`, `v0_19_6.py`). These migrations were for versions prior to 0.20.0 and are no longer needed since the current release requires a database rebuild anyway.
|
||||
|
||||
## [0.24.2] - 2026-01-08
|
||||
|
||||
### Fixed
|
||||
|
|
|
|||
|
|
@ -55,14 +55,6 @@ def run_pending_upgrades(store: Store, from_version: str, to_version: str) -> No
|
|||
|
||||
# Import upgrade modules AFTER Upgrade class is defined to avoid circular imports
|
||||
# ruff: noqa: E402, I001
|
||||
from haiku.rag.store.upgrades.v0_9_3 import upgrade_fts_phrase as upgrade_0_9_3_fts
|
||||
from haiku.rag.store.upgrades.v0_9_3 import upgrade_order as upgrade_0_9_3_order
|
||||
from haiku.rag.store.upgrades.v0_10_1 import (
|
||||
upgrade_add_title as upgrade_0_10_1_add_title,
|
||||
)
|
||||
from haiku.rag.store.upgrades.v0_19_6 import (
|
||||
upgrade_embeddings_model_config as upgrade_0_19_6_embeddings,
|
||||
)
|
||||
from haiku.rag.store.upgrades.v0_20_0 import (
|
||||
upgrade_add_docling_document as upgrade_0_20_0_docling,
|
||||
)
|
||||
|
|
@ -73,10 +65,6 @@ from haiku.rag.store.upgrades.v0_25_0 import (
|
|||
upgrade_compress_docling_document as upgrade_0_25_0_compress,
|
||||
)
|
||||
|
||||
upgrades.append(upgrade_0_9_3_order)
|
||||
upgrades.append(upgrade_0_9_3_fts)
|
||||
upgrades.append(upgrade_0_10_1_add_title)
|
||||
upgrades.append(upgrade_0_19_6_embeddings)
|
||||
upgrades.append(upgrade_0_20_0_docling)
|
||||
upgrades.append(upgrade_0_23_1_contextualize)
|
||||
upgrades.append(upgrade_0_25_0_compress)
|
||||
|
|
|
|||
|
|
@ -1,64 +0,0 @@
|
|||
import json
|
||||
|
||||
from lancedb.pydantic import LanceModel
|
||||
from pydantic import Field
|
||||
|
||||
from haiku.rag.store.engine import Store
|
||||
from haiku.rag.store.upgrades import Upgrade
|
||||
|
||||
|
||||
def _apply_add_document_title(store: Store) -> None: # pragma: no cover
|
||||
"""Add a nullable 'title' column to the documents table."""
|
||||
|
||||
# Read existing rows using Arrow for schema-agnostic access
|
||||
try:
|
||||
docs_arrow = store.documents_table.search().to_arrow()
|
||||
rows = docs_arrow.to_pylist()
|
||||
except Exception:
|
||||
rows = []
|
||||
|
||||
class DocumentRecordV2(LanceModel):
|
||||
id: str
|
||||
content: str
|
||||
uri: str | None = None
|
||||
title: str | None = None
|
||||
metadata: str = Field(default="{}")
|
||||
created_at: str = Field(default_factory=lambda: "")
|
||||
updated_at: str = Field(default_factory=lambda: "")
|
||||
|
||||
# Drop and recreate documents table with the new schema
|
||||
try:
|
||||
store.db.drop_table("documents")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
store.documents_table = store.db.create_table("documents", schema=DocumentRecordV2)
|
||||
|
||||
# Reinsert previous rows with title=None
|
||||
if rows:
|
||||
backfilled = []
|
||||
for row in rows:
|
||||
backfilled.append(
|
||||
DocumentRecordV2(
|
||||
id=row.get("id"),
|
||||
content=row.get("content", ""),
|
||||
uri=row.get("uri"),
|
||||
title=None,
|
||||
metadata=(
|
||||
row.get("metadata")
|
||||
if isinstance(row.get("metadata"), str)
|
||||
else json.dumps(row.get("metadata") or {})
|
||||
),
|
||||
created_at=row.get("created_at", ""),
|
||||
updated_at=row.get("updated_at", ""),
|
||||
)
|
||||
)
|
||||
|
||||
store.documents_table.add(backfilled)
|
||||
|
||||
|
||||
upgrade_add_title = Upgrade(
|
||||
version="0.10.1",
|
||||
apply=_apply_add_document_title,
|
||||
description="Add nullable 'title' column to documents table",
|
||||
)
|
||||
|
|
@ -1,65 +0,0 @@
|
|||
import json
|
||||
import logging
|
||||
|
||||
from haiku.rag.store.engine import SettingsRecord, Store
|
||||
from haiku.rag.store.upgrades import Upgrade
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _apply_embeddings_model_config(store: Store) -> None: # pragma: no cover
|
||||
"""Migrate embeddings config from flat to nested EmbeddingModelConfig structure."""
|
||||
results = list(
|
||||
store.settings_table.search()
|
||||
.where("id = 'settings'")
|
||||
.limit(1)
|
||||
.to_pydantic(SettingsRecord)
|
||||
)
|
||||
|
||||
if not results or not results[0].settings:
|
||||
return
|
||||
|
||||
settings = json.loads(results[0].settings)
|
||||
embeddings = settings.get("embeddings", {})
|
||||
|
||||
# Check if already migrated (model is a dict with nested structure)
|
||||
if isinstance(embeddings.get("model"), dict):
|
||||
return
|
||||
|
||||
# Migrate from flat structure to nested EmbeddingModelConfig
|
||||
old_provider = embeddings.get("provider", "ollama")
|
||||
old_model = embeddings.get("model", "qwen3-embedding:4b")
|
||||
old_vector_dim = embeddings.get("vector_dim", 2560)
|
||||
|
||||
logger.info(
|
||||
"Migrating embeddings config to new nested structure: "
|
||||
"embeddings.{provider,model,vector_dim} -> embeddings.model.{provider,name,vector_dim}"
|
||||
)
|
||||
|
||||
# Create new nested structure
|
||||
settings["embeddings"] = {
|
||||
"model": {
|
||||
"provider": old_provider,
|
||||
"name": old_model,
|
||||
"vector_dim": old_vector_dim,
|
||||
}
|
||||
}
|
||||
|
||||
store.settings_table.update(
|
||||
where="id = 'settings'",
|
||||
values={"settings": json.dumps(settings)},
|
||||
)
|
||||
|
||||
logger.info(
|
||||
"Embeddings config migrated: provider=%s, name=%s, vector_dim=%d",
|
||||
old_provider,
|
||||
old_model,
|
||||
old_vector_dim,
|
||||
)
|
||||
|
||||
|
||||
upgrade_embeddings_model_config = Upgrade(
|
||||
version="0.19.6",
|
||||
apply=_apply_embeddings_model_config,
|
||||
description="Migrate embeddings config to nested EmbeddingModelConfig structure",
|
||||
)
|
||||
|
|
@ -1,112 +0,0 @@
|
|||
import json
|
||||
|
||||
from lancedb.pydantic import LanceModel, Vector
|
||||
from pydantic import Field
|
||||
|
||||
from haiku.rag.store.engine import Store
|
||||
from haiku.rag.store.upgrades import Upgrade
|
||||
|
||||
|
||||
def _infer_vector_dim(store: Store) -> int: # pragma: no cover
|
||||
"""Infer vector dimension from existing data; fallback to embedder config."""
|
||||
try:
|
||||
arrow = store.chunks_table.search().limit(1).to_arrow()
|
||||
rows = arrow.to_pylist()
|
||||
if rows:
|
||||
vec = rows[0].get("vector")
|
||||
if isinstance(vec, list) and vec:
|
||||
return len(vec)
|
||||
except Exception:
|
||||
pass
|
||||
# Fallback to configured embedder vector dim
|
||||
return getattr(store.embedder, "_vector_dim", 1024)
|
||||
|
||||
|
||||
def _apply_chunk_order(store: Store) -> None: # pragma: no cover
|
||||
"""Add integer 'order' column to chunks and backfill from metadata."""
|
||||
|
||||
vector_dim = _infer_vector_dim(store)
|
||||
|
||||
class ChunkRecordV2(LanceModel):
|
||||
id: str
|
||||
document_id: str
|
||||
content: str
|
||||
metadata: str = Field(default="{}")
|
||||
order: int = Field(default=0)
|
||||
vector: Vector(vector_dim) = Field( # type: ignore
|
||||
default_factory=lambda: [0.0] * vector_dim
|
||||
)
|
||||
|
||||
# Read existing chunks
|
||||
try:
|
||||
chunks_arrow = store.chunks_table.search().to_arrow()
|
||||
rows = chunks_arrow.to_pylist()
|
||||
except Exception:
|
||||
rows = []
|
||||
|
||||
new_chunk_records: list[ChunkRecordV2] = []
|
||||
for row in rows:
|
||||
md_raw = row.get("metadata") or "{}"
|
||||
try:
|
||||
md = json.loads(md_raw) if isinstance(md_raw, str) else md_raw
|
||||
except Exception:
|
||||
md = {}
|
||||
# Extract and normalize order
|
||||
order_val = 0
|
||||
try:
|
||||
if isinstance(md, dict) and "order" in md:
|
||||
order_val = int(md["order"]) # type: ignore[arg-type]
|
||||
except Exception:
|
||||
order_val = 0
|
||||
|
||||
if isinstance(md, dict) and "order" in md:
|
||||
md = {k: v for k, v in md.items() if k != "order"}
|
||||
|
||||
vec = row.get("vector") or [0.0] * vector_dim
|
||||
|
||||
new_chunk_records.append(
|
||||
ChunkRecordV2(
|
||||
id=row.get("id"),
|
||||
document_id=row.get("document_id"),
|
||||
content=row.get("content", ""),
|
||||
metadata=json.dumps(md),
|
||||
order=order_val,
|
||||
vector=vec,
|
||||
)
|
||||
)
|
||||
|
||||
# Recreate chunks table with new schema
|
||||
try:
|
||||
store.db.drop_table("chunks")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
store.chunks_table = store.db.create_table("chunks", schema=ChunkRecordV2)
|
||||
store.chunks_table.create_fts_index("content", replace=True)
|
||||
|
||||
if new_chunk_records:
|
||||
store.chunks_table.add(new_chunk_records)
|
||||
|
||||
|
||||
upgrade_order = Upgrade(
|
||||
version="0.9.3",
|
||||
apply=_apply_chunk_order,
|
||||
description="Add 'order' column to chunks and backfill from metadata",
|
||||
)
|
||||
|
||||
|
||||
def _apply_fts_phrase_support(store: Store) -> None: # pragma: no cover
|
||||
"""Recreate FTS index with phrase query support and no stop-word removal."""
|
||||
try:
|
||||
store.chunks_table.create_fts_index(
|
||||
"content", replace=True, with_position=True, remove_stop_words=False
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
upgrade_fts_phrase = Upgrade(
|
||||
version="0.9.3",
|
||||
apply=_apply_fts_phrase_support,
|
||||
description="Enable FTS phrase queries (with positions) and keep stop-words",
|
||||
)
|
||||
Loading…
Reference in a new issue