diff --git a/CHANGELOG.md b/CHANGELOG.md index 004bf3bb..f47ae492 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,12 +1,26 @@ # Changelog ## [Unreleased] +### Added + +- **Manual Vector Indexing**: New `create-index` CLI command for explicit vector index creation + - Creates IVF_PQ indexes + - Requires minimum 256 chunks (LanceDB training data requirement) + - New `search.vector_index_metric` config option: `cosine` (default), `l2`, or `dot` + - New `search.vector_refine_factor` config option (default: 10) for accuracy/speed tradeoff + - Indexes not created automatically during ingestion to avoid performance degradation + - Manual rebuilding required after adding significant new data + ### Changed -- **Evaluations**: Improved evaluation dataset naming and simplified evaluator configuration +- **Evaluations**: Improved evaluation dataset naming and simplified evaluator +- configuration - `EvalDataset` now accepts dataset name for better organization in Logfire - Added `--name` CLI parameter to override evaluation run names - Removed `IsInstance` evaluator, using only `LLMJudge` for QA evaluation +- **Search Accuracy**: Applied `refine_factor` to vector and hybrid searches for improved accuracy + - Retrieves `refine_factor * limit` candidates and re-ranks in memory + - Higher values increase accuracy but slow down queries ### Fixed @@ -16,6 +30,7 @@ - `get_model()` utility function accepts `config` parameter (defaults to global Config) - Allows creating multiple graphs with different configurations in the same application + ## [0.17.2] - 2025-11-19 ### Added diff --git a/docs/configuration.md b/docs/configuration.md index 9e50ee4d..d7463d0e 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -86,6 +86,10 @@ research: confidence_threshold: 0.8 max_concurrency: 1 +search: + vector_index_metric: cosine # cosine, l2, or dot + vector_refine_factor: 10 + agui: host: "0.0.0.0" port: 8000 @@ -721,6 +725,47 @@ haiku.rag intelligently handles database creation based on operation type: This prevents the common mistake where a search query accidentally creates an empty database. To initialize your database, simply add your first document using `haiku-rag add` or `haiku-rag add-src`. +### Vector Indexing + +Configure vector indexing behavior for efficient similarity search: + +```yaml +search: + vector_index_metric: cosine # cosine, l2, or dot + vector_refine_factor: 10 # Re-ranking factor for accuracy +``` + +- **vector_index_metric**: Distance metric for vector similarity: + - `cosine`: Cosine similarity (default, best for most embeddings) + - `l2`: Euclidean distance + - `dot`: Dot product similarity +- **vector_refine_factor**: Retrieve `refine_factor * limit` candidates and re-rank in memory for better accuracy. Higher values increase accuracy but slow down queries. Default: 10 + +**Index creation:** + +Vector indexes are **not created automatically** during document ingestion to avoid slowing down the process. After you've added documents (at least 256 chunks required), create the index manually: + +```bash +haiku-rag create-index +``` + +This command: +- Checks if you have enough data (minimum 256 chunks) +- Creates an IVF_PQ index for fast approximate nearest neighbor search +- Uses LanceDB's automatic parameter calculation based on your dataset size and vector dimensions + +**Re-indexing:** + +Indexes are not automatically updated when you add new documents. After adding a significant amount of new data: + +```bash +haiku-rag create-index # Rebuilds the index with all data +``` + +Searches still work with stale indexes - LanceDB uses the index for old data and brute-force for new unindexed rows, then combines the results. However, performance degrades as more unindexed data accumulates. + +For datasets with fewer than 256 chunks, searches use brute-force scans which are slower but still functional. + ### Document Processing ```yaml diff --git a/haiku_rag_slim/haiku/rag/app.py b/haiku_rag_slim/haiku/rag/app.py index 01bf2401..af2d15d6 100644 --- a/haiku_rag_slim/haiku/rag/app.py +++ b/haiku_rag_slim/haiku/rag/app.py @@ -397,6 +397,39 @@ class HaikuRAGApp: except Exception as e: self.console.print(f"[red]Error during vacuum: {e}[/red]") + async def create_index(self): + """Create vector index on the chunks table.""" + try: + async with HaikuRAG( + db_path=self.db_path, config=self.config, skip_validation=True + ) as client: + row_count = client.store.chunks_table.count_rows() + self.console.print(f"Chunks in database: {row_count}") + + if row_count < 256: + self.console.print( + f"[yellow]Warning: Need at least 256 chunks to create an index (have {row_count})[/yellow]" + ) + return + + # Check if index already exists + indices = client.store.chunks_table.list_indices() + has_vector_index = any("vector" in str(idx).lower() for idx in indices) + + if has_vector_index: + self.console.print( + "[yellow]Rebuilding existing vector index...[/yellow]" + ) + else: + self.console.print("[bold]Creating vector index...[/bold]") + + client.store._ensure_vector_index() + self.console.print( + "[bold green]Vector index created successfully.[/bold green]" + ) + except Exception as e: + self.console.print(f"[red]Error creating index: {e}[/red]") + def show_settings(self): """Display current configuration settings.""" self.console.print("[bold]haiku.rag configuration[/bold]") diff --git a/haiku_rag_slim/haiku/rag/cli.py b/haiku_rag_slim/haiku/rag/cli.py index dc99ec2a..e086179d 100644 --- a/haiku_rag_slim/haiku/rag/cli.py +++ b/haiku_rag_slim/haiku/rag/cli.py @@ -375,6 +375,18 @@ def vacuum( asyncio.run(app.vacuum()) +@cli.command("create-index", help="Create vector index for efficient similarity search") +def create_index( + db: Path | None = typer.Option( + None, + "--db", + help="Path to the LanceDB database file", + ), +): + app = create_app(db) + asyncio.run(app.create_index()) + + @cli.command("info", help="Show read-only database info (no upgrades or writes)") def info( db: Path | None = typer.Option( diff --git a/haiku_rag_slim/haiku/rag/config/models.py b/haiku_rag_slim/haiku/rag/config/models.py index f71fc03b..fdd1a09f 100644 --- a/haiku_rag_slim/haiku/rag/config/models.py +++ b/haiku_rag_slim/haiku/rag/config/models.py @@ -81,6 +81,11 @@ class ProcessingConfig(BaseModel): conversion_options: ConversionOptions = Field(default_factory=ConversionOptions) +class SearchConfig(BaseModel): + vector_index_metric: Literal["cosine", "l2", "dot"] = "cosine" + vector_refine_factor: int = 10 + + class OllamaConfig(BaseModel): base_url: str = Field( default_factory=lambda: __import__("os").environ.get( @@ -127,5 +132,6 @@ class AppConfig(BaseModel): qa: QAConfig = Field(default_factory=QAConfig) research: ResearchConfig = Field(default_factory=ResearchConfig) processing: ProcessingConfig = Field(default_factory=ProcessingConfig) + search: SearchConfig = Field(default_factory=SearchConfig) providers: ProvidersConfig = Field(default_factory=ProvidersConfig) agui: AGUIConfig = Field(default_factory=AGUIConfig) diff --git a/haiku_rag_slim/haiku/rag/store/engine.py b/haiku_rag_slim/haiku/rag/store/engine.py index 4c2a54e3..820b25ee 100644 --- a/haiku_rag_slim/haiku/rag/store/engine.py +++ b/haiku_rag_slim/haiku/rag/store/engine.py @@ -145,6 +145,42 @@ class Store: and self._config.lancedb.region ) + def _ensure_vector_index(self) -> None: + """Create or rebuild vector index on chunks table. + + Cloud deployments auto-create indexes, so we skip for those. + For self-hosted, creates an IVF_PQ index. If an index exists, + it will be replaced (using replace=True parameter). + Note: Index creation requires sufficient training data. + """ + if self._has_cloud_config(): + return + + try: + # Check if table has enough data (indexes require training data) + row_count = self.chunks_table.count_rows() + if row_count < 256: + logger.debug( + f"Skipping vector index creation: need at least 256 rows, have {row_count}" + ) + return + + # Create or replace index (replace=True is the default) + logger.info("Creating vector index on chunks table...") + self.chunks_table.create_index( + metric=self._config.search.vector_index_metric, + index_type="IVF_PQ", + replace=True, # Explicit: replace existing index + ) + + # Wait for index creation to complete + # Index name is column_name + "_idx" + self.chunks_table.wait_for_index(["vector_idx"], timeout=timedelta(hours=1)) + + logger.info("Vector index created successfully") + except Exception as e: + logger.warning(f"Could not create vector index: {e}") + def _validate_configuration(self) -> None: """Validate that the configuration is compatible with the database.""" from haiku.rag.store.repositories.settings import SettingsRepository diff --git a/haiku_rag_slim/haiku/rag/store/repositories/chunk.py b/haiku_rag_slim/haiku/rag/store/repositories/chunk.py index d5083808..dd251e6a 100644 --- a/haiku_rag_slim/haiku/rag/store/repositories/chunk.py +++ b/haiku_rag_slim/haiku/rag/store/repositories/chunk.py @@ -1,12 +1,16 @@ import inspect import json import logging -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, cast from uuid import uuid4 if TYPE_CHECKING: import pandas as pd - from lancedb.query import LanceQueryBuilder + from lancedb.query import ( + LanceHybridQueryBuilder, + LanceQueryBuilder, + LanceVectorQueryBuilder, + ) from lancedb.rerankers import RRFReranker @@ -319,8 +323,14 @@ class ChunkRepository: # Prepare search query based on search type if search_type == "vector": query_embedding = await self.embedder.embed(query) - results = self.store.chunks_table.search( - query_embedding, query_type="vector", vector_column_name="vector" + vector_query = cast( + "LanceVectorQueryBuilder", + self.store.chunks_table.search( + query_embedding, query_type="vector", vector_column_name="vector" + ), + ) + results = vector_query.refine_factor( + self.store._config.search.vector_refine_factor ) elif search_type == "fts": @@ -331,12 +341,15 @@ class ChunkRepository: # Create RRF reranker reranker = RRFReranker() # Perform native hybrid search with RRF reranking - results = ( + hybrid_query = cast( + "LanceHybridQueryBuilder", self.store.chunks_table.search(query_type="hybrid") .vector(query_embedding) - .text(query) - .rerank(reranker) + .text(query), ) + results = hybrid_query.refine_factor( + self.store._config.search.vector_refine_factor + ).rerank(reranker) # Apply filtering if needed (common for all search types) if filtered_doc_ids is not None: