From d233788400961947e8282bca9abe0d8693ce8f5b Mon Sep 17 00:00:00 2001
From: Yiorgis Gozadinos <ggozadinos@gmail.com>
Date: Mon, 1 Sep 2025 14:46:22 +0300
Subject: [PATCH] Optimize table after insertion, update instead of recreating
 the index

---
 src/haiku/rag/store/engine.py             |  5 +++++
 src/haiku/rag/store/repositories/chunk.py | 14 ++++++++------
 2 files changed, 13 insertions(+), 6 deletions(-)

diff --git a/src/haiku/rag/store/engine.py b/src/haiku/rag/store/engine.py
index 78f3d7d0..cf0ed233 100644
--- a/src/haiku/rag/store/engine.py
+++ b/src/haiku/rag/store/engine.py
@@ -82,6 +82,8 @@ class Store:
             self.chunks_table = self.db.open_table("chunks")
         else:
             self.chunks_table = self.db.create_table("chunks", schema=self.ChunkRecord)
+            # Create FTS index on the new table
+            self.chunks_table.create_fts_index("content", replace=True)
 
         # Create or get settings table
         if "settings" in existing_tables:
@@ -167,6 +169,9 @@ class Store:
         self.ChunkRecord = create_chunk_model(self.embedder._vector_dim)
         self.chunks_table = self.db.create_table("chunks", schema=self.ChunkRecord)
 
+        # Create FTS index on the new table
+        self.chunks_table.create_fts_index("content", replace=True)
+
     def close(self):
         """Close the database connection."""
         # LanceDB connections are automatically managed
diff --git a/src/haiku/rag/store/repositories/chunk.py b/src/haiku/rag/store/repositories/chunk.py
index acc7384e..23bee6fd 100644
--- a/src/haiku/rag/store/repositories/chunk.py
+++ b/src/haiku/rag/store/repositories/chunk.py
@@ -47,6 +47,9 @@ class ChunkRepository:
         self.store.chunks_table.add([chunk_record])
 
         entity.id = chunk_id
+
+        # Optimize table after insert to update indexes
+        self.store.chunks_table.optimize()
         return entity
 
     async def get_by_id(self, entity_id: str) -> Chunk | None:
@@ -84,6 +87,8 @@ class ChunkRepository:
                 "vector": embedding,
             },
         )
+        # Optimize table after update to refresh indexes
+        self.store.chunks_table.optimize()
 
         return entity
 
@@ -151,6 +156,9 @@ class ChunkRepository:
                 self.store.chunks_table = self.store.db.create_table(
                     "chunks", schema=self.store.ChunkRecord
                 )
+                # Create FTS index on the new table
+                self.store.chunks_table.create_fts_index("content", replace=True)
+
                 return True
             return False
         except Exception:
@@ -192,18 +200,12 @@ class ChunkRepository:
             return await self._process_search_results(results)
 
         elif search_type == "fts":
-            # Ensure FTS index exists
-            self._ensure_fts_index()
-
             results = self.store.chunks_table.search(query, query_type="fts").limit(
                 limit
             )
             return await self._process_search_results(results)
 
         else:  # hybrid (default)
-            # Ensure FTS index exists for hybrid search
-            self._ensure_fts_index()
-
             query_embedding = await self.embedder.embed(query)
 
             # Create RRF reranker