diff --git a/README.md b/README.md index 8f0c2da3..8109b390 100644 --- a/README.md +++ b/README.md @@ -4,8 +4,6 @@ Retrieval-Augmented Generation (RAG) library built on LanceDB. `haiku.rag` is a Retrieval-Augmented Generation (RAG) library built to work with LanceDB as a local vector database. It uses LanceDB for storing embeddings and performs semantic (vector) search as well as full-text search combined through native hybrid search with Reciprocal Rank Fusion. Both open-source (Ollama) as well as commercial (OpenAI, VoyageAI) embedding providers are supported. -> **Note**: Starting with version 0.7.0, haiku.rag uses LanceDB instead of SQLite. If you have an existing SQLite database, use `haiku-rag migrate old_database.sqlite` to migrate your data safely. - ## Features - **Local LanceDB**: No external servers required, supports also LanceDB cloud storage, S3, Google Cloud & Azure @@ -59,9 +57,6 @@ haiku-rag research \ # Rebuild database (re-chunk and re-embed all documents) haiku-rag rebuild -# Migrate from SQLite to LanceDB -haiku-rag migrate old_database.sqlite - # Start server with file monitoring export MONITOR_DIRECTORIES="/path/to/docs" haiku-rag serve diff --git a/docs/cli.md b/docs/cli.md index 3663ea32..2b2e88ec 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -227,20 +227,3 @@ haiku-rag download-models This command: - Downloads Docling OCR/conversion models (no-op if already present). - Pulls Ollama models referenced in your configuration (embeddings, QA, research, rerank). - -## Migration - -### Migrate from SQLite to LanceDB - -Migrate an existing SQLite database to LanceDB: - -```bash -haiku-rag migrate /path/to/old_database.sqlite -``` - -This will: -- Read all documents, chunks, embeddings, and settings from the SQLite database -- Create a new LanceDB database with the same data in the same directory -- Optimize the new database for best performance - -The original SQLite database remains unchanged, so you can safely migrate without risk of data loss. diff --git a/docs/index.md b/docs/index.md index fc3cde59..0f7ab515 100644 --- a/docs/index.md +++ b/docs/index.md @@ -2,8 +2,6 @@ `haiku.rag` is a Retrieval-Augmented Generation (RAG) library built to work with LanceDB as a local vector database. It uses LanceDB for storing embeddings and performs semantic (vector) search as well as full-text search combined through native hybrid search with Reciprocal Rank Fusion. Both open-source (Ollama, MixedBread AI) as well as commercial (OpenAI, VoyageAI) embedding providers are supported. -> **Note**: Starting with version 0.7.0, haiku.rag uses LanceDB instead of SQLite. If you have an existing SQLite database, use `haiku-rag migrate old_database.sqlite` to migrate your data safely. - ## Features - **Local LanceDB**: No need to run additional servers @@ -51,7 +49,6 @@ haiku-rag add "Your document content" --meta author=alice haiku-rag add-src /path/to/document.pdf --title "Q3 Financial Report" --meta source=manual haiku-rag search "query" haiku-rag ask "Who is the author of haiku.rag?" -haiku-rag migrate old_database.sqlite # Migrate from SQLite ``` ## Documentation diff --git a/src/haiku/rag/cli.py b/src/haiku/rag/cli.py index 7394585a..31742e10 100644 --- a/src/haiku/rag/cli.py +++ b/src/haiku/rag/cli.py @@ -442,24 +442,6 @@ def serve( ) -@cli.command("migrate", help="Migrate an SQLite database to LanceDB") -def migrate( - sqlite_path: Path = typer.Argument( - help="Path to the SQLite database file to migrate", - ), -): - # Generate LanceDB path in same parent directory - lancedb_path = sqlite_path.parent / (sqlite_path.stem + ".lancedb") - - # Lazy import to avoid heavy deps on simple invocations - from haiku.rag.migration import migrate_sqlite_to_lancedb - - success = asyncio.run(migrate_sqlite_to_lancedb(sqlite_path, lancedb_path)) - - if not success: - raise typer.Exit(1) - - @cli.command( "a2aclient", help="Run interactive client to chat with haiku.rag's A2A server" ) diff --git a/src/haiku/rag/migration.py b/src/haiku/rag/migration.py deleted file mode 100644 index da2d21e6..00000000 --- a/src/haiku/rag/migration.py +++ /dev/null @@ -1,316 +0,0 @@ -import json -import sqlite3 -import struct -from pathlib import Path -from uuid import uuid4 - -from rich.console import Console -from rich.progress import Progress, TaskID - -from haiku.rag.store.engine import Store - - -def deserialize_sqlite_embedding(data: bytes) -> list[float]: - """Deserialize sqlite-vec embedding from bytes.""" - if not data: - return [] - # sqlite-vec stores embeddings as float32 arrays - num_floats = len(data) // 4 - return list(struct.unpack(f"{num_floats}f", data)) - - -class SQLiteToLanceDBMigrator: - """Migrates data from SQLite to LanceDB.""" - - def __init__(self, sqlite_path: Path, lancedb_path: Path): - self.sqlite_path = sqlite_path - self.lancedb_path = lancedb_path - self.console = Console() - - async def migrate(self) -> bool: - """Perform the migration.""" - try: - self.console.print( - f"[blue]Starting migration from {self.sqlite_path} to {self.lancedb_path}[/blue]" - ) - - # Check if SQLite database exists - if not self.sqlite_path.exists(): - self.console.print( - f"[red]SQLite database not found: {self.sqlite_path}[/red]" - ) - return False - - # Connect to SQLite database - sqlite_conn = sqlite3.connect(self.sqlite_path) - sqlite_conn.row_factory = sqlite3.Row - - # Load the sqlite-vec extension - try: - import sqlite_vec # type: ignore - - sqlite_conn.enable_load_extension(True) - sqlite_vec.load(sqlite_conn) - self.console.print("[cyan]Loaded sqlite-vec extension[/cyan]") - except Exception as e: - self.console.print( - f"[yellow]Warning: Could not load sqlite-vec extension: {e}[/yellow]" - ) - self.console.print( - "[yellow]Install sqlite-vec with[/yellow]\n[green]uv pip install sqlite-vec [/green]" - ) - exit(1) - - # Create LanceDB store - lance_store = Store(self.lancedb_path, skip_validation=True) - - with Progress() as progress: - # Migrate documents - doc_task = progress.add_task( - "[green]Migrating documents...", total=None - ) - document_id_mapping = self._migrate_documents( - sqlite_conn, lance_store, progress, doc_task - ) - - # Migrate chunks and embeddings - chunk_task = progress.add_task( - "[yellow]Migrating chunks and embeddings...", total=None - ) - self._migrate_chunks( - sqlite_conn, lance_store, progress, chunk_task, document_id_mapping - ) - - # Migrate settings - settings_task = progress.add_task( - "[blue]Migrating settings...", total=None - ) - self._migrate_settings( - sqlite_conn, lance_store, progress, settings_task - ) - - sqlite_conn.close() - - # Optimize and cleanup using centralized vacuum - self.console.print("[cyan]Optimizing LanceDB...[/cyan]") - try: - await lance_store.vacuum() - self.console.print("[green]✅ Optimization completed[/green]") - except Exception as e: - self.console.print( - f"[yellow]Warning: Optimization failed: {e}[/yellow]" - ) - - lance_store.close() - - self.console.print("[green]✅ Migration completed successfully![/green]") - self.console.print( - f"[green]✅ Migrated {len(document_id_mapping)} documents[/green]" - ) - return True - - except Exception as e: - self.console.print(f"[red]❌ Migration failed: {e}[/red]") - import traceback - - self.console.print(f"[red]{traceback.format_exc()}[/red]") - return False - - def _migrate_documents( - self, - sqlite_conn: sqlite3.Connection, - lance_store: Store, - progress: Progress, - task: TaskID, - ) -> dict[int, str]: - """Migrate documents from SQLite to LanceDB and return ID mapping.""" - cursor = sqlite_conn.cursor() - cursor.execute( - "SELECT id, content, uri, metadata, created_at, updated_at FROM documents ORDER BY id" - ) - - documents = [] - id_mapping = {} # Maps old integer ID to new UUID - - for row in cursor.fetchall(): - new_uuid = str(uuid4()) - id_mapping[row["id"]] = new_uuid - - doc_data = { - "id": new_uuid, - "content": row["content"], - "uri": row["uri"], - "metadata": json.loads(row["metadata"]) if row["metadata"] else {}, - "created_at": row["created_at"], - "updated_at": row["updated_at"], - } - documents.append(doc_data) - - # Batch insert documents to LanceDB - if documents: - from haiku.rag.store.engine import DocumentRecord - - doc_records = [ - DocumentRecord( - id=doc["id"], - content=doc["content"], - uri=doc["uri"], - metadata=json.dumps(doc["metadata"]), - created_at=doc["created_at"], - updated_at=doc["updated_at"], - ) - for doc in documents - ] - lance_store.documents_table.add(doc_records) - - progress.update(task, completed=len(documents), total=len(documents)) - return id_mapping - - def _migrate_chunks( - self, - sqlite_conn: sqlite3.Connection, - lance_store: Store, - progress: Progress, - task: TaskID, - document_id_mapping: dict[int, str], - ): - """Migrate chunks and embeddings from SQLite to LanceDB.""" - cursor = sqlite_conn.cursor() - - # Get chunks first - cursor.execute(""" - SELECT id, document_id, content, metadata - FROM chunks - ORDER BY id - """) - - chunks_data = cursor.fetchall() - - # Get embeddings using the sqlite-vec virtual table - embeddings_map = {} - try: - # Use the virtual table to get embeddings properly - cursor.execute(""" - SELECT chunk_id, embedding - FROM chunk_embeddings - """) - - for row in cursor.fetchall(): - chunk_id = row[0] - embedding_blob = row[1] - if embedding_blob and chunk_id not in embeddings_map: - embeddings_map[chunk_id] = embedding_blob - - except sqlite3.OperationalError as e: - self.console.print( - f"[yellow]Warning: Could not extract embeddings from virtual table: {e}[/yellow]" - ) - - chunks = [] - for row in chunks_data: - # Generate new UUID for chunk - chunk_uuid = str(uuid4()) - - # Map the old document_id to new UUID - document_uuid = document_id_mapping.get(row["document_id"]) - if not document_uuid: - self.console.print( - f"[yellow]Warning: Document ID {row['document_id']} not found in mapping for chunk {row['id']}[/yellow]" - ) - continue - - # Get embedding for this chunk - embedding = [] - embedding_blob = embeddings_map.get(row["id"]) - if embedding_blob: - try: - embedding = deserialize_sqlite_embedding(embedding_blob) - except Exception as e: - self.console.print( - f"[yellow]Warning: Failed to deserialize embedding for chunk {row['id']}: {e}[/yellow]" - ) - # Generate a zero vector of the expected dimension - embedding = [0.0] * lance_store.embedder._vector_dim - else: - # No embedding found, generate zero vector - embedding = [0.0] * lance_store.embedder._vector_dim - - chunk_data = { - "id": chunk_uuid, - "document_id": document_uuid, - "content": row["content"], - "metadata": json.loads(row["metadata"]) if row["metadata"] else {}, - "vector": embedding, - } - chunks.append(chunk_data) - - # Batch insert chunks to LanceDB - if chunks: - chunk_records = [ - lance_store.ChunkRecord( - id=chunk["id"], - document_id=chunk["document_id"], - content=chunk["content"], - metadata=json.dumps(chunk["metadata"]), - vector=chunk["vector"], - ) - for chunk in chunks - ] - lance_store.chunks_table.add(chunk_records) - - progress.update(task, completed=len(chunks), total=len(chunks)) - - def _migrate_settings( - self, - sqlite_conn: sqlite3.Connection, - lance_store: Store, - progress: Progress, - task: TaskID, - ): - """Migrate settings from SQLite to LanceDB.""" - cursor = sqlite_conn.cursor() - - try: - cursor.execute("SELECT id, settings FROM settings WHERE id = 1") - row = cursor.fetchone() - - if row: - settings_data = json.loads(row["settings"]) if row["settings"] else {} - - # Update the existing settings in LanceDB (use string ID) - lance_store.settings_table.update( - where="id = 'settings'", - values={"settings": json.dumps(settings_data)}, - ) - - progress.update(task, completed=1, total=1) - else: - progress.update(task, completed=0, total=0) - - except sqlite3.OperationalError: - # Settings table doesn't exist in old SQLite database - self.console.print( - "[yellow]No settings table found in SQLite database[/yellow]" - ) - progress.update(task, completed=0, total=0) - - -async def migrate_sqlite_to_lancedb( - sqlite_path: Path, lancedb_path: Path | None = None -) -> bool: - """ - Migrate an existing SQLite database to LanceDB. - - Args: - sqlite_path: Path to the existing SQLite database - lancedb_path: Path for the new LanceDB database (optional, will auto-generate if not provided) - - Returns: - True if migration was successful, False otherwise - """ - if lancedb_path is None: - # Auto-generate LanceDB path - lancedb_path = sqlite_path.parent / (sqlite_path.stem + ".lancedb") - - migrator = SQLiteToLanceDBMigrator(sqlite_path, lancedb_path) - return await migrator.migrate()