From 3525fae6254ecc4fb996841813bcc034d956aed0 Mon Sep 17 00:00:00 2001 From: Yiorgis Gozadinos Date: Tue, 2 Dec 2025 11:13:04 +0200 Subject: [PATCH 1/4] Use EmbeddingModelConfig similar to ModelConfig for embeddings --- evaluations/evaluations/benchmark.py | 6 +- evaluations/evaluations/evaluators/judge.py | 4 +- evaluations/pyproject.toml | 2 +- haiku_rag_slim/haiku/rag/app.py | 7 +- haiku_rag_slim/haiku/rag/config/__init__.py | 24 ++++--- haiku_rag_slim/haiku/rag/config/models.py | 18 ++++- .../haiku/rag/embeddings/__init__.py | 29 ++++----- haiku_rag_slim/haiku/rag/embeddings/base.py | 4 +- .../haiku/rag/store/repositories/settings.py | 24 +++---- .../haiku/rag/store/upgrades/__init__.py | 4 ++ .../haiku/rag/store/upgrades/v0_19_6.py | 65 +++++++++++++++++++ haiku_rag_slim/haiku/rag/utils.py | 4 +- haiku_rag_slim/pyproject.toml | 2 +- pyproject.toml | 4 +- tests/test_client.py | 2 +- tests/test_config.py | 15 +++-- tests/test_embedder_config.py | 32 ++++----- tests/test_info.py | 2 +- uv.lock | 6 +- 19 files changed, 165 insertions(+), 89 deletions(-) create mode 100644 haiku_rag_slim/haiku/rag/store/upgrades/v0_19_6.py diff --git a/evaluations/evaluations/benchmark.py b/evaluations/evaluations/benchmark.py index 55bafdc1..e0263647 100644 --- a/evaluations/evaluations/benchmark.py +++ b/evaluations/evaluations/benchmark.py @@ -41,9 +41,9 @@ def build_experiment_metadata( return { "dataset": dataset_key, "test_cases": test_cases, - "embedder_provider": config.embeddings.provider, - "embedder_model": config.embeddings.model, - "embedder_dim": config.embeddings.vector_dim, + "embedder_provider": config.embeddings.model.provider, + "embedder_model": config.embeddings.model.name, + "embedder_dim": config.embeddings.model.vector_dim, "chunk_size": config.processing.chunk_size, "context_chunk_radius": config.processing.context_chunk_radius, "rerank_provider": config.reranking.model.provider diff --git a/evaluations/evaluations/evaluators/judge.py b/evaluations/evaluations/evaluators/judge.py index ccc80dd9..5aa60f6b 100644 --- a/evaluations/evaluations/evaluators/judge.py +++ b/evaluations/evaluations/evaluators/judge.py @@ -38,9 +38,7 @@ class LLMJudge: def __init__(self, model: str = "gpt-oss"): # Create model using get_model with thinking disabled - model_config = ModelConfig( - provider="ollama", model=model, enable_thinking=False - ) + model_config = ModelConfig(provider="ollama", name=model, enable_thinking=False) model_obj = get_model(model_config, Config) # Create Pydantic AI agent diff --git a/evaluations/pyproject.toml b/evaluations/pyproject.toml index 32e7429d..0977c9eb 100644 --- a/evaluations/pyproject.toml +++ b/evaluations/pyproject.toml @@ -2,7 +2,7 @@ name = "haiku.rag-evals" description = "Benchmarking and evaluation scripts for haiku.rag" -version = "0.19.5" +version = "0.19.6" authors = [{ name = "Yiorgis Gozadinos", email = "ggozadinos@gmail.com" }] license = { text = "MIT" } requires-python = ">=3.12" diff --git a/haiku_rag_slim/haiku/rag/app.py b/haiku_rag_slim/haiku/rag/app.py index 808007c2..ddbfc846 100644 --- a/haiku_rag_slim/haiku/rag/app.py +++ b/haiku_rag_slim/haiku/rag/app.py @@ -80,9 +80,10 @@ class HaikuRAGApp: data = json.loads(raw) if isinstance(raw, str) else (raw or {}) stored_version = str(data.get("version", stored_version)) embeddings = data.get("embeddings", {}) - embed_provider = embeddings.get("provider") - embed_model = embeddings.get("model") - vector_dim = embeddings.get("vector_dim") + embed_model_obj = embeddings.get("model", {}) + embed_provider = embed_model_obj.get("provider") + embed_model = embed_model_obj.get("name") + vector_dim = embed_model_obj.get("vector_dim") # Get comprehensive table statistics from haiku.rag.store.engine import Store diff --git a/haiku_rag_slim/haiku/rag/config/__init__.py b/haiku_rag_slim/haiku/rag/config/__init__.py index 5caf60fe..c8c8b99d 100644 --- a/haiku_rag_slim/haiku/rag/config/__init__.py +++ b/haiku_rag_slim/haiku/rag/config/__init__.py @@ -9,9 +9,11 @@ from haiku.rag.config.models import ( AGUIConfig, AppConfig, ConversionOptions, + EmbeddingModelConfig, EmbeddingsConfig, LanceDBConfig, LMStudioConfig, + ModelConfig, MonitorConfig, OllamaConfig, ProcessingConfig, @@ -28,22 +30,24 @@ __all__ = [ "AGUIConfig", "AppConfig", "ConversionOptions", - "StorageConfig", - "MonitorConfig", - "LanceDBConfig", + "EmbeddingModelConfig", "EmbeddingsConfig", - "RerankingConfig", - "QAConfig", - "ResearchConfig", - "ProcessingConfig", - "OllamaConfig", + "LanceDBConfig", "LMStudioConfig", - "VLLMConfig", + "ModelConfig", + "MonitorConfig", + "OllamaConfig", + "ProcessingConfig", "ProvidersConfig", + "QAConfig", + "RerankingConfig", + "ResearchConfig", + "StorageConfig", + "VLLMConfig", "find_config_file", - "load_yaml_config", "generate_default_config", "get_config", + "load_yaml_config", "set_config", ] diff --git a/haiku_rag_slim/haiku/rag/config/models.py b/haiku_rag_slim/haiku/rag/config/models.py index 5ed0663a..ed638740 100644 --- a/haiku_rag_slim/haiku/rag/config/models.py +++ b/haiku_rag_slim/haiku/rag/config/models.py @@ -25,6 +25,20 @@ class ModelConfig(BaseModel): max_tokens: int | None = None +class EmbeddingModelConfig(BaseModel): + """Configuration for an embedding model. + + Attributes: + provider: Model provider (ollama, openai, voyageai, vllm, lm_studio) + name: Model name/identifier + vector_dim: Vector dimensions produced by the model + """ + + provider: str = "ollama" + name: str = "qwen3-embedding:4b" + vector_dim: int = 2560 + + class StorageConfig(BaseModel): data_dir: Path = Field(default_factory=get_default_data_dir) vacuum_retention_seconds: int = 86400 @@ -44,9 +58,7 @@ class LanceDBConfig(BaseModel): class EmbeddingsConfig(BaseModel): - provider: str = "ollama" - model: str = "qwen3-embedding:4b" - vector_dim: int = 2560 + model: EmbeddingModelConfig = Field(default_factory=EmbeddingModelConfig) class RerankingConfig(BaseModel): diff --git a/haiku_rag_slim/haiku/rag/embeddings/__init__.py b/haiku_rag_slim/haiku/rag/embeddings/__init__.py index 70582b49..4a62e4e5 100644 --- a/haiku_rag_slim/haiku/rag/embeddings/__init__.py +++ b/haiku_rag_slim/haiku/rag/embeddings/__init__.py @@ -13,13 +13,12 @@ def get_embedder(config: AppConfig = Config) -> EmbedderBase: Returns: An embedder instance configured according to the config. """ + embedding_model = config.embeddings.model - if config.embeddings.provider == "ollama": - return OllamaEmbedder( - config.embeddings.model, config.embeddings.vector_dim, config - ) + if embedding_model.provider == "ollama": + return OllamaEmbedder(embedding_model.name, embedding_model.vector_dim, config) - if config.embeddings.provider == "voyageai": + if embedding_model.provider == "voyageai": try: from haiku.rag.embeddings.voyageai import Embedder as VoyageAIEmbedder except ImportError: @@ -29,28 +28,24 @@ def get_embedder(config: AppConfig = Config) -> EmbedderBase: "uv pip install haiku.rag[voyageai]" ) return VoyageAIEmbedder( - config.embeddings.model, config.embeddings.vector_dim, config + embedding_model.name, embedding_model.vector_dim, config ) - if config.embeddings.provider == "openai": + if embedding_model.provider == "openai": from haiku.rag.embeddings.openai import Embedder as OpenAIEmbedder - return OpenAIEmbedder( - config.embeddings.model, config.embeddings.vector_dim, config - ) + return OpenAIEmbedder(embedding_model.name, embedding_model.vector_dim, config) - if config.embeddings.provider == "vllm": + if embedding_model.provider == "vllm": from haiku.rag.embeddings.vllm import Embedder as VllmEmbedder - return VllmEmbedder( - config.embeddings.model, config.embeddings.vector_dim, config - ) + return VllmEmbedder(embedding_model.name, embedding_model.vector_dim, config) - if config.embeddings.provider == "lm_studio": + if embedding_model.provider == "lm_studio": from haiku.rag.embeddings.lm_studio import Embedder as LMStudioEmbedder return LMStudioEmbedder( - config.embeddings.model, config.embeddings.vector_dim, config + embedding_model.name, embedding_model.vector_dim, config ) - raise ValueError(f"Unsupported embedding provider: {config.embeddings.provider}") + raise ValueError(f"Unsupported embedding provider: {embedding_model.provider}") diff --git a/haiku_rag_slim/haiku/rag/embeddings/base.py b/haiku_rag_slim/haiku/rag/embeddings/base.py index bcd80f91..6049a840 100644 --- a/haiku_rag_slim/haiku/rag/embeddings/base.py +++ b/haiku_rag_slim/haiku/rag/embeddings/base.py @@ -4,8 +4,8 @@ from haiku.rag.config import AppConfig, Config class EmbedderBase: - _model: str = Config.embeddings.model - _vector_dim: int = Config.embeddings.vector_dim + _model: str = Config.embeddings.model.name + _vector_dim: int = Config.embeddings.model.vector_dim _config: AppConfig = Config def __init__(self, model: str, vector_dim: int, config: AppConfig = Config): diff --git a/haiku_rag_slim/haiku/rag/store/repositories/settings.py b/haiku_rag_slim/haiku/rag/store/repositories/settings.py index 4d5bba43..50de331d 100644 --- a/haiku_rag_slim/haiku/rag/store/repositories/settings.py +++ b/haiku_rag_slim/haiku/rag/store/repositories/settings.py @@ -118,25 +118,21 @@ class SettingsRepository: current_config = self.store._config.model_dump(mode="json") # Check if embedding provider or model has changed - # Support both old flat structure and new nested structure for backward compatibility + # Both stored and current use nested structure: embeddings.model.{provider,name,vector_dim} stored_embeddings = stored_settings.get("embeddings", {}) current_embeddings = current_config.get("embeddings", {}) - # Try nested structure first, fall back to flat for old databases - stored_provider = stored_embeddings.get("provider") or stored_settings.get( - "EMBEDDINGS_PROVIDER" - ) - current_provider = current_embeddings.get("provider") + stored_model_obj = stored_embeddings.get("model", {}) + current_model_obj = current_embeddings.get("model", {}) - stored_model = stored_embeddings.get("model") or stored_settings.get( - "EMBEDDINGS_MODEL" - ) - current_model = current_embeddings.get("model") + stored_provider = stored_model_obj.get("provider") + current_provider = current_model_obj.get("provider") - stored_vector_dim = stored_embeddings.get("vector_dim") or stored_settings.get( - "EMBEDDINGS_VECTOR_DIM" - ) - current_vector_dim = current_embeddings.get("vector_dim") + stored_model = stored_model_obj.get("name") + current_model = current_model_obj.get("name") + + stored_vector_dim = stored_model_obj.get("vector_dim") + current_vector_dim = current_model_obj.get("vector_dim") # Check for incompatible changes incompatible_changes = [] diff --git a/haiku_rag_slim/haiku/rag/store/upgrades/__init__.py b/haiku_rag_slim/haiku/rag/store/upgrades/__init__.py index 20065e2f..3c8c0143 100644 --- a/haiku_rag_slim/haiku/rag/store/upgrades/__init__.py +++ b/haiku_rag_slim/haiku/rag/store/upgrades/__init__.py @@ -56,7 +56,11 @@ def run_pending_upgrades(store: Store, from_version: str, to_version: str) -> No from .v0_9_3 import upgrade_fts_phrase as upgrade_0_9_3_fts # noqa: E402 from .v0_9_3 import upgrade_order as upgrade_0_9_3_order # noqa: E402 from .v0_10_1 import upgrade_add_title as upgrade_0_10_1_add_title # noqa: E402 +from .v0_19_6 import ( # noqa: E402 + upgrade_embeddings_model_config as upgrade_0_19_6_embeddings, +) upgrades.append(upgrade_0_9_3_order) upgrades.append(upgrade_0_9_3_fts) upgrades.append(upgrade_0_10_1_add_title) +upgrades.append(upgrade_0_19_6_embeddings) diff --git a/haiku_rag_slim/haiku/rag/store/upgrades/v0_19_6.py b/haiku_rag_slim/haiku/rag/store/upgrades/v0_19_6.py new file mode 100644 index 00000000..27fcf46c --- /dev/null +++ b/haiku_rag_slim/haiku/rag/store/upgrades/v0_19_6.py @@ -0,0 +1,65 @@ +import json +import logging + +from haiku.rag.store.engine import SettingsRecord, Store +from haiku.rag.store.upgrades import Upgrade + +logger = logging.getLogger(__name__) + + +def _apply_embeddings_model_config(store: Store) -> None: + """Migrate embeddings config from flat to nested EmbeddingModelConfig structure.""" + results = list( + store.settings_table.search() + .where("id = 'settings'") + .limit(1) + .to_pydantic(SettingsRecord) + ) + + if not results or not results[0].settings: + return + + settings = json.loads(results[0].settings) + embeddings = settings.get("embeddings", {}) + + # Check if already migrated (model is a dict with nested structure) + if isinstance(embeddings.get("model"), dict): + return + + # Migrate from flat structure to nested EmbeddingModelConfig + old_provider = embeddings.get("provider", "ollama") + old_model = embeddings.get("model", "qwen3-embedding:4b") + old_vector_dim = embeddings.get("vector_dim", 2560) + + logger.warning( + "Migrating embeddings config to new nested structure: " + "embeddings.{provider,model,vector_dim} -> embeddings.model.{provider,name,vector_dim}" + ) + + # Create new nested structure + settings["embeddings"] = { + "model": { + "provider": old_provider, + "name": old_model, + "vector_dim": old_vector_dim, + } + } + + store.settings_table.update( + where="id = 'settings'", + values={"settings": json.dumps(settings)}, + ) + + logger.warning( + "Embeddings config migrated: provider=%s, name=%s, vector_dim=%d", + old_provider, + old_model, + old_vector_dim, + ) + + +upgrade_embeddings_model_config = Upgrade( + version="0.19.6", + apply=_apply_embeddings_model_config, + description="Migrate embeddings config to nested EmbeddingModelConfig structure", +) diff --git a/haiku_rag_slim/haiku/rag/utils.py b/haiku_rag_slim/haiku/rag/utils.py index e46f3fd1..b2c03e11 100644 --- a/haiku_rag_slim/haiku/rag/utils.py +++ b/haiku_rag_slim/haiku/rag/utils.py @@ -392,8 +392,8 @@ async def prefetch_models(): # Collect Ollama models from config required_models: set[str] = set() - if Config.embeddings.provider == "ollama": - required_models.add(Config.embeddings.model) + if Config.embeddings.model.provider == "ollama": + required_models.add(Config.embeddings.model.name) if Config.qa.model.provider == "ollama": required_models.add(Config.qa.model.name) if Config.research.model.provider == "ollama": diff --git a/haiku_rag_slim/pyproject.toml b/haiku_rag_slim/pyproject.toml index 30631ba4..90a40878 100644 --- a/haiku_rag_slim/pyproject.toml +++ b/haiku_rag_slim/pyproject.toml @@ -2,7 +2,7 @@ name = "haiku.rag-slim" description = "Opinionated agentic RAG powered by LanceDB, Pydantic AI, and Docling - Minimal dependencies" -version = "0.19.5" +version = "0.19.6" authors = [{ name = "Yiorgis Gozadinos", email = "ggozadinos@gmail.com" }] license = { text = "MIT" } readme = { file = "README.md", content-type = "text/markdown" } diff --git a/pyproject.toml b/pyproject.toml index 0582096e..2551e161 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,7 +2,7 @@ name = "haiku.rag" description = "Opinionated agentic RAG powered by LanceDB, Pydantic AI, and Docling" -version = "0.19.5" +version = "0.19.6" authors = [{ name = "Yiorgis Gozadinos", email = "ggozadinos@gmail.com" }] license = { text = "MIT" } readme = { file = "README.md", content-type = "text/markdown" } @@ -22,7 +22,7 @@ classifiers = [ ] dependencies = [ - "haiku.rag-slim[docling,voyageai,mxbai,cohere,zeroentropy,inspector]==0.19.5", + "haiku.rag-slim[docling,voyageai,mxbai,cohere,zeroentropy,inspector]==0.19.6", ] [project.scripts] diff --git a/tests/test_client.py b/tests/test_client.py index 0f34cd82..9f704557 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -699,7 +699,7 @@ async def test_client_create_document_with_custom_chunks(temp_db_path): Chunk( content="This is the second chunk", metadata={"custom": "metadata2"}, - embedding=[0.1] * Config.embeddings.vector_dim, + embedding=[0.1] * Config.embeddings.model.vector_dim, order=1, ), # With embedding Chunk( diff --git a/tests/test_config.py b/tests/test_config.py index 9c3daa2e..70cf98c7 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -15,16 +15,17 @@ def test_load_yaml_config(tmp_path): config_file.write_text(""" environment: production embeddings: - provider: ollama - model: test-model - vector_dim: 1024 + model: + provider: ollama + name: test-model + vector_dim: 1024 """) config = load_yaml_config(config_file) assert config["environment"] == "production" - assert config["embeddings"]["provider"] == "ollama" - assert config["embeddings"]["model"] == "test-model" - assert config["embeddings"]["vector_dim"] == 1024 + assert config["embeddings"]["model"]["provider"] == "ollama" + assert config["embeddings"]["model"]["name"] == "test-model" + assert config["embeddings"]["model"]["vector_dim"] == 1024 def test_find_config_file_cwd(tmp_path, monkeypatch): @@ -179,7 +180,7 @@ def test_generate_default_config_completeness(): # Verify config validates successfully assert config.environment == "production" - assert config.embeddings.provider == "ollama" + assert config.embeddings.model.provider == "ollama" assert config.qa.model.provider == "ollama" assert config.research.model.provider == "ollama" assert config.reranking.model is None diff --git a/tests/test_embedder_config.py b/tests/test_embedder_config.py index 64a1be96..a6e431d4 100644 --- a/tests/test_embedder_config.py +++ b/tests/test_embedder_config.py @@ -2,6 +2,7 @@ import pytest from haiku.rag.config import ( AppConfig, + EmbeddingModelConfig, EmbeddingsConfig, LMStudioConfig, OllamaConfig, @@ -15,9 +16,9 @@ def test_embedder_uses_config_from_get_embedder(): """Test that embedders use the config passed to get_embedder.""" custom_config = AppConfig( embeddings=EmbeddingsConfig( - provider="ollama", - model="custom-model", - vector_dim=512, + model=EmbeddingModelConfig( + provider="ollama", name="custom-model", vector_dim=512 + ), ), providers=ProvidersConfig( ollama=OllamaConfig(base_url="http://custom-ollama:8080"), @@ -36,9 +37,9 @@ def test_vllm_embedder_uses_config(): """Test that vllm embedder uses the config passed to get_embedder.""" custom_config = AppConfig( embeddings=EmbeddingsConfig( - provider="vllm", - model="custom-vllm-model", - vector_dim=768, + model=EmbeddingModelConfig( + provider="vllm", name="custom-vllm-model", vector_dim=768 + ), ), providers=ProvidersConfig( vllm=VLLMConfig(embeddings_base_url="http://custom-vllm:9001"), @@ -56,12 +57,11 @@ def test_vllm_embedder_uses_config(): def test_openai_embedder_uses_config(): """Test that openai embedder uses the config passed to get_embedder.""" - custom_config = AppConfig( embeddings=EmbeddingsConfig( - provider="openai", - model="text-embedding-3-large", - vector_dim=3072, + model=EmbeddingModelConfig( + provider="openai", name="text-embedding-3-large", vector_dim=3072 + ), ), ) @@ -76,9 +76,9 @@ def test_lm_studio_embedder_uses_config(): """Test that lm_studio embedder uses the config passed to get_embedder.""" custom_config = AppConfig( embeddings=EmbeddingsConfig( - provider="lm_studio", - model="custom-lm-studio-model", - vector_dim=1024, + model=EmbeddingModelConfig( + provider="lm_studio", name="custom-lm-studio-model", vector_dim=1024 + ), ), providers=ProvidersConfig( lm_studio=LMStudioConfig(base_url="http://custom-lmstudio:5678"), @@ -101,9 +101,9 @@ def test_voyageai_embedder_uses_config(): """Test that voyageai embedder uses the config passed to get_embedder.""" custom_config = AppConfig( embeddings=EmbeddingsConfig( - provider="voyageai", - model="voyage-large-2", - vector_dim=1536, + model=EmbeddingModelConfig( + provider="voyageai", name="voyage-large-2", vector_dim=1536 + ), ), ) diff --git a/tests/test_info.py b/tests/test_info.py index 9dae5f93..7bd6c458 100644 --- a/tests/test_info.py +++ b/tests/test_info.py @@ -125,7 +125,7 @@ async def test_app_info_with_vector_index(temp_db_path, capsys): [ SettingsRecord( id="settings", - settings='{"version": "1.0.0", "embeddings": {"provider": "ollama", "model": "test", "vector_dim": 3}}', + settings='{"version": "1.0.0", "embeddings": {"model": {"provider": "ollama", "name": "test", "vector_dim": 3}}}', ) ] ) diff --git a/uv.lock b/uv.lock index 978a075f..a9c1b931 100644 --- a/uv.lock +++ b/uv.lock @@ -1264,7 +1264,7 @@ wheels = [ [[package]] name = "haiku-rag" -version = "0.19.5" +version = "0.19.6" source = { editable = "." } dependencies = [ { name = "haiku-rag-slim", extra = ["cohere", "docling", "inspector", "mxbai", "voyageai", "zeroentropy"] }, @@ -1312,7 +1312,7 @@ dev = [ [[package]] name = "haiku-rag-evals" -version = "0.19.5" +version = "0.19.6" source = { editable = "evaluations" } dependencies = [ { name = "datasets" }, @@ -1333,7 +1333,7 @@ requires-dist = [ [[package]] name = "haiku-rag-slim" -version = "0.19.5" +version = "0.19.6" source = { editable = "haiku_rag_slim" } dependencies = [ { name = "docling-core" }, From 47d8f7ba3f9822215f2f0abc57a92ac5da29272c Mon Sep 17 00:00:00 2001 From: Yiorgis Gozadinos Date: Tue, 2 Dec 2025 12:01:10 +0200 Subject: [PATCH 2/4] Update docs --- docker/README.md | 7 ++++--- docs/configuration/index.md | 26 +++++++++++++----------- docs/configuration/providers.md | 35 +++++++++++++++++++-------------- docs/tutorial.md | 7 ++++--- 4 files changed, 42 insertions(+), 33 deletions(-) diff --git a/docker/README.md b/docker/README.md index 60fd2af8..70917642 100644 --- a/docker/README.md +++ b/docker/README.md @@ -25,9 +25,10 @@ Create a configuration file `haiku.rag.yaml`: environment: production embeddings: - provider: ollama - model: nomic-embed-text - vector_dim: 768 + model: + provider: ollama + name: nomic-embed-text + vector_dim: 768 qa: model: diff --git a/docs/configuration/index.md b/docs/configuration/index.md index 0394c1ba..af7ab533 100644 --- a/docs/configuration/index.md +++ b/docs/configuration/index.md @@ -35,9 +35,10 @@ A minimal configuration file with defaults: environment: production embeddings: - provider: ollama - model: qwen3-embedding:4b - vector_dim: 2560 + model: + provider: ollama + name: qwen3-embedding:4b + vector_dim: 2560 qa: model: @@ -69,9 +70,10 @@ lancedb: region: "" embeddings: - provider: ollama - model: qwen3-embedding:4b - vector_dim: 2560 + model: + provider: ollama + name: qwen3-embedding:4b + vector_dim: 2560 reranking: model: @@ -149,7 +151,7 @@ When using haiku.rag as a Python library, you can pass configuration directly to ```python from haiku.rag.config import AppConfig -from haiku.rag.config.models import ModelConfig, QAConfig, EmbeddingsConfig +from haiku.rag.config.models import EmbeddingModelConfig, ModelConfig, QAConfig, EmbeddingsConfig from haiku.rag.client import HaikuRAG # Create custom configuration @@ -157,16 +159,16 @@ custom_config = AppConfig( qa=QAConfig( model=ModelConfig( provider="openai", - model="gpt-4o", + name="gpt-4o", temperature=0.7 ) ), embeddings=EmbeddingsConfig( - model=ModelConfig( + model=EmbeddingModelConfig( provider="ollama", - model="qwen3-embedding:4b" - ), - vector_dim=2560 + name="qwen3-embedding:4b", + vector_dim=2560 + ) ), processing={"chunk_size": 512} ) diff --git a/docs/configuration/providers.md b/docs/configuration/providers.md index 1f2a396c..03da0a66 100644 --- a/docs/configuration/providers.md +++ b/docs/configuration/providers.md @@ -73,9 +73,10 @@ If you use Ollama, you can use any pulled model that supports embeddings. ```yaml embeddings: - provider: ollama - model: mxbai-embed-large - vector_dim: 1024 + model: + provider: ollama + name: mxbai-embed-large + vector_dim: 1024 ``` The Ollama base URL can be configured in your config file or via environment variable: @@ -104,9 +105,10 @@ uv pip install haiku.rag-slim[voyageai] ```yaml embeddings: - provider: voyageai - model: voyage-3.5 - vector_dim: 1024 + model: + provider: voyageai + name: voyage-3.5 + vector_dim: 1024 ``` Set your API key via environment variable: @@ -121,9 +123,10 @@ OpenAI embeddings are included in the default installation: ```yaml embeddings: - provider: openai - model: text-embedding-3-small # or text-embedding-3-large - vector_dim: 1536 + model: + provider: openai + name: text-embedding-3-small # or text-embedding-3-large + vector_dim: 1536 ``` Set your API key via environment variable: @@ -138,9 +141,10 @@ For high-performance local inference, you can use vLLM to serve embedding models ```yaml embeddings: - provider: vllm - model: mixedbread-ai/mxbai-embed-large-v1 - vector_dim: 512 + model: + provider: vllm + name: mixedbread-ai/mxbai-embed-large-v1 + vector_dim: 512 providers: vllm: @@ -155,9 +159,10 @@ providers: ```yaml embeddings: - provider: lm_studio - model: text-embedding-qwen3-embedding-4b - vector_dim: 2560 + model: + provider: lm_studio + name: text-embedding-qwen3-embedding-4b + vector_dim: 2560 providers: lm_studio: diff --git a/docs/tutorial.md b/docs/tutorial.md index e46bc0a8..97923d8b 100644 --- a/docs/tutorial.md +++ b/docs/tutorial.md @@ -33,9 +33,10 @@ Configure haiku.rag to use OpenAI. Create a `haiku.rag.yaml` file: ```yaml embeddings: - provider: openai - model: text-embedding-3-small # or text-embedding-3-large - vector_dim: 1536 + model: + provider: openai + name: text-embedding-3-small # or text-embedding-3-large + vector_dim: 1536 qa: model: From 33b254ae3d455482ece39121682c2a8ed8882641 Mon Sep 17 00:00:00 2001 From: Yiorgis Gozadinos Date: Tue, 2 Dec 2025 16:36:15 +0200 Subject: [PATCH 3/4] Always upgrade database on access. Explicitly create db by running init or `HaikuRAG(path, create=True)` --- CHANGELOG.md | 13 +++ docs/cli.md | 16 +++- docs/python.md | 10 ++- docs/tutorial.md | 11 +++ .../a2a-server/haiku_rag_a2a/a2a/client.py | 1 + .../a2a-server/haiku_rag_a2a/a2a/worker.py | 5 +- examples/a2a-server/tests/test_a2a.py | 5 +- haiku_rag_slim/haiku/rag/app.py | 50 ++++++----- haiku_rag_slim/haiku/rag/cli.py | 14 ++- haiku_rag_slim/haiku/rag/client.py | 7 +- haiku_rag_slim/haiku/rag/inspector/app.py | 2 +- haiku_rag_slim/haiku/rag/store/engine.py | 86 ++++++++----------- tests/conftest.py | 15 ++-- tests/graph/test_deep_qa.py | 4 +- tests/graph/test_research_graph.py | 2 +- tests/test_chunk.py | 8 +- tests/test_cli.py | 12 +++ tests/test_client.py | 56 ++++++------ tests/test_database_autocreate.py | 78 +++-------------- tests/test_document.py | 6 +- tests/test_filter.py | 12 +-- tests/test_info.py | 41 ++------- tests/test_lancedb_connection.py | 4 +- tests/test_preprocessor.py | 2 +- tests/test_qa.py | 8 +- tests/test_rebuild.py | 8 +- tests/test_search.py | 8 +- tests/test_settings.py | 14 +-- tests/test_versioning.py | 15 ++-- 29 files changed, 254 insertions(+), 259 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f1bca675..d98c69b4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,19 @@ # Changelog ## [Unreleased] +## [0.19.6] - 2025-12-02 + +### Changed + +- **BREAKING: Explicit Database Creation**: Databases must now be explicitly created before use + - New `haiku-rag init` command creates a new empty database + - Python API: `HaikuRAG(path, create=True)` to create database programmatically + - Operations on non-existent databases raise `FileNotFoundError` +- **BREAKING: Embeddings Configuration**: Restructured to nested `EmbeddingModelConfig` + - Config path changed from `embeddings.{provider, model, vector_dim}` to `embeddings.model.{provider, name, vector_dim}` + - Automatic migration upgrades existing databases to new format +- **Database Migrations**: Always run when opening an existing database + ## [0.19.5] - 2025-12-01 ### Changed diff --git a/docs/cli.md b/docs/cli.md index 4b8b7d68..6373aa13 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -201,11 +201,21 @@ View current configuration settings: haiku-rag settings ``` -## Maintenance +## Database Management -### Info (Read-only) +### Initialize Database -Display database metadata without upgrading or modifying it: +Create a new database: + +```bash +haiku-rag init [--db /path/to/your.lancedb] +``` + +This creates the database with the configured settings. **All other commands require an existing database** - they will fail with an informative error if the database doesn't exist. + +### Info + +Display database metadata: ```bash haiku-rag info [--db /path/to/your.lancedb] diff --git a/docs/python.md b/docs/python.md index 9413f0b9..3b451109 100644 --- a/docs/python.md +++ b/docs/python.md @@ -8,12 +8,20 @@ Use `haiku.rag` directly in your Python applications. from pathlib import Path from haiku.rag.client import HaikuRAG -# Use as async context manager (recommended) +# Create a new database +async with HaikuRAG("path/to/database.lancedb", create=True) as client: + # Your code here + pass + +# Open an existing database (will fail if database doesn't exist) async with HaikuRAG("path/to/database.lancedb") as client: # Your code here pass ``` +!!! note + Databases must be explicitly created with `create=True` or via `haiku-rag init` before use. Operations on non-existent databases will raise `FileNotFoundError`. + ## Document Management ### Creating Documents diff --git a/docs/tutorial.md b/docs/tutorial.md index 97923d8b..65c81ee8 100644 --- a/docs/tutorial.md +++ b/docs/tutorial.md @@ -54,6 +54,16 @@ For the list of available OpenAI models and their vector dimensions, see the [Op See [Configuration](configuration/index.md) for all available options. +## Initialize the database + +Before adding documents, initialize the database: + +```bash +haiku-rag init +``` + +This creates an empty database with the configured settings. + ## Adding the first documents Now you can add some pieces of text in the database: @@ -150,6 +160,7 @@ logger.setLevel(logging.DEBUG) logger.debug("AGI here we come") # Uses LanceDB database from default storage location +# (database must be initialized first with 'haiku-rag init' or create=True) async with HaikuRAG() as client: answer = await client.ask("What is the best programming language in the world?") print(answer) diff --git a/examples/a2a-server/haiku_rag_a2a/a2a/client.py b/examples/a2a-server/haiku_rag_a2a/a2a/client.py index 0e9f2790..5de4b28d 100644 --- a/examples/a2a-server/haiku_rag_a2a/a2a/client.py +++ b/examples/a2a-server/haiku_rag_a2a/a2a/client.py @@ -1,3 +1,4 @@ +# pyright: reportMissingImports=false import asyncio import uuid from typing import Any diff --git a/examples/a2a-server/haiku_rag_a2a/a2a/worker.py b/examples/a2a-server/haiku_rag_a2a/a2a/worker.py index ca798934..e044f6ad 100644 --- a/examples/a2a-server/haiku_rag_a2a/a2a/worker.py +++ b/examples/a2a-server/haiku_rag_a2a/a2a/worker.py @@ -1,3 +1,4 @@ +# pyright: reportMissingImports=false import json import logging import uuid @@ -12,8 +13,8 @@ from haiku_rag_a2a.a2a.models import AgentDependencies from haiku_rag_a2a.a2a.skills import extract_question_from_task try: - from fasta2a import Worker # type: ignore - from fasta2a.schema import ( # type: ignore + from fasta2a import Worker + from fasta2a.schema import ( Artifact, Message, TaskIdParams, diff --git a/examples/a2a-server/tests/test_a2a.py b/examples/a2a-server/tests/test_a2a.py index 8388514a..b603ed66 100644 --- a/examples/a2a-server/tests/test_a2a.py +++ b/examples/a2a-server/tests/test_a2a.py @@ -1,3 +1,4 @@ +# pyright: reportMissingImports=false import uuid import pytest @@ -212,7 +213,7 @@ async def test_a2a_app_creation(temp_db_path): from haiku_rag_a2a.a2a import create_a2a_app # Create a test database - async with HaikuRAG(temp_db_path) as client: + async with HaikuRAG(temp_db_path, create=True) as client: await client.create_document( content="Python is a high-level programming language known for its simplicity.", uri="python_doc", @@ -233,7 +234,7 @@ async def test_a2a_app_has_skills(temp_db_path): from haiku_rag_a2a.a2a import create_a2a_app # Create a test database - async with HaikuRAG(temp_db_path) as client: + async with HaikuRAG(temp_db_path, create=True) as client: await client.create_document(content="Test document", uri="test_doc") # Create A2A app diff --git a/haiku_rag_slim/haiku/rag/app.py b/haiku_rag_slim/haiku/rag/app.py index ddbfc846..0ca69fbc 100644 --- a/haiku_rag_slim/haiku/rag/app.py +++ b/haiku_rag_slim/haiku/rag/app.py @@ -29,6 +29,21 @@ class HaikuRAGApp: self.config = config self.console = Console() + async def init(self): + """Initialize a new database.""" + if self.db_path.exists(): + self.console.print( + f"[yellow]Database already exists at {self.db_path}[/yellow]" + ) + return + + # Create the database + client = HaikuRAG(db_path=self.db_path, config=self.config, create=True) + client.close() + self.console.print( + f"[bold green]Database initialized at {self.db_path}[/bold green]" + ) + async def info(self): """Display read-only information about the database without modifying it.""" @@ -65,7 +80,13 @@ class HaikuRAGApp: except Exception: docling_version = "unknown" - # Read settings (if present) to find stored haiku.rag version and embedding config + # Get comprehensive table statistics (this also runs migrations) + from haiku.rag.store.engine import Store + + store = Store(self.db_path, config=self.config, skip_validation=True) + table_stats = store.get_stats() + + # Read settings after Store init (migrations have run) stored_version = "unknown" embed_provider: str | None = None embed_model: str | None = None @@ -85,13 +106,6 @@ class HaikuRAGApp: embed_model = embed_model_obj.get("name") vector_dim = embed_model_obj.get("vector_dim") - # Get comprehensive table statistics - from haiku.rag.store.engine import Store - - store = Store( - self.db_path, config=self.config, skip_validation=True, read_only=True - ) - table_stats = store.get_stats() store.close() num_docs = table_stats["documents"].get("num_rows", 0) @@ -188,9 +202,7 @@ class HaikuRAGApp: ) async def list_documents(self, filter: str | None = None): - async with HaikuRAG( - db_path=self.db_path, config=self.config, read_only=True - ) as self.client: + async with HaikuRAG(db_path=self.db_path, config=self.config) as self.client: documents = await self.client.list_documents(filter=filter) for doc in documents: self._rich_print_document(doc, truncate=True) @@ -223,9 +235,7 @@ class HaikuRAGApp: ) async def get_document(self, doc_id: str): - async with HaikuRAG( - db_path=self.db_path, config=self.config, read_only=True - ) as self.client: + async with HaikuRAG(db_path=self.db_path, config=self.config) as self.client: doc = await self.client.get_document_by_id(doc_id) if doc is None: self.console.print(f"[red]Document with id {doc_id} not found.[/red]") @@ -245,9 +255,7 @@ class HaikuRAGApp: ) async def search(self, query: str, limit: int = 5, filter: str | None = None): - async with HaikuRAG( - db_path=self.db_path, config=self.config, read_only=True - ) as self.client: + async with HaikuRAG(db_path=self.db_path, config=self.config) as self.client: results = await self.client.search(query, limit=limit, filter=filter) if not results: self.console.print("[yellow]No results found.[/yellow]") @@ -270,9 +278,7 @@ class HaikuRAGApp: deep: Use deep QA mode (multi-step reasoning) verbose: Show verbose output """ - async with HaikuRAG( - db_path=self.db_path, config=self.config, read_only=True - ) as self.client: + async with HaikuRAG(db_path=self.db_path, config=self.config) as self.client: try: if deep: from haiku.rag.graph.deep_qa.dependencies import DeepQAContext @@ -317,9 +323,7 @@ class HaikuRAGApp: question: The research question verbose: Show AG-UI event stream during execution """ - async with HaikuRAG( - db_path=self.db_path, config=self.config, read_only=True - ) as client: + async with HaikuRAG(db_path=self.db_path, config=self.config) as client: try: self.console.print("[bold cyan]Starting research[/bold cyan]") self.console.print(f"[bold blue]Question:[/bold blue] {question}") diff --git a/haiku_rag_slim/haiku/rag/cli.py b/haiku_rag_slim/haiku/rag/cli.py index f8fa9a97..49ca5f7b 100644 --- a/haiku_rag_slim/haiku/rag/cli.py +++ b/haiku_rag_slim/haiku/rag/cli.py @@ -410,7 +410,19 @@ def create_index( asyncio.run(app.create_index()) -@cli.command("info", help="Show read-only database info (no upgrades or writes)") +@cli.command("init", help="Initialize a new database") +def init_db( + db: Path | None = typer.Option( + None, + "--db", + help="Path to the LanceDB database file", + ), +): + app = create_app(db) + asyncio.run(app.init()) + + +@cli.command("info", help="Show database info") def info( db: Path | None = typer.Option( None, diff --git a/haiku_rag_slim/haiku/rag/client.py b/haiku_rag_slim/haiku/rag/client.py index 8e4902c4..79e05907 100644 --- a/haiku_rag_slim/haiku/rag/client.py +++ b/haiku_rag_slim/haiku/rag/client.py @@ -38,7 +38,7 @@ class HaikuRAG: db_path: Path | None = None, config: AppConfig = Config, skip_validation: bool = False, - read_only: bool = False, + create: bool = False, ): """Initialize the RAG client with a database path. @@ -46,8 +46,7 @@ class HaikuRAG: db_path: Path to the database file. If None, uses config.storage.data_dir. config: Configuration to use. Defaults to global Config. skip_validation: Whether to skip configuration validation on database load. - read_only: Whether to open in read-only mode. If True, will raise error - if database doesn't exist and will skip upgrades. + create: Whether to create the database if it doesn't exist. """ self._config = config if db_path is None: @@ -56,7 +55,7 @@ class HaikuRAG: db_path, config=self._config, skip_validation=skip_validation, - read_only=read_only, + create=create, ) self.document_repository = DocumentRepository(self.store) self.chunk_repository = ChunkRepository(self.store) diff --git a/haiku_rag_slim/haiku/rag/inspector/app.py b/haiku_rag_slim/haiku/rag/inspector/app.py index 4141ce5c..7b9939c1 100644 --- a/haiku_rag_slim/haiku/rag/inspector/app.py +++ b/haiku_rag_slim/haiku/rag/inspector/app.py @@ -86,7 +86,7 @@ class InspectorApp(App): # type: ignore[misc] async def on_mount(self) -> None: """Initialize the app when mounted.""" config = get_config() - self.client = HaikuRAG(db_path=self.db_path, config=config, read_only=True) + self.client = HaikuRAG(db_path=self.db_path, config=config) await self.client.__aenter__() # Load initial documents diff --git a/haiku_rag_slim/haiku/rag/store/engine.py b/haiku_rag_slim/haiku/rag/store/engine.py index 455d2e14..b59d24f5 100644 --- a/haiku_rag_slim/haiku/rag/store/engine.py +++ b/haiku_rag_slim/haiku/rag/store/engine.py @@ -54,35 +54,41 @@ class Store: db_path: Path, config: AppConfig = Config, skip_validation: bool = False, - read_only: bool = False, + create: bool = False, ): self.db_path: Path = db_path self._config = config self.embedder = get_embedder(config=self._config) self._vacuum_lock = asyncio.Lock() - self._read_only = read_only # Create the ChunkRecord model with the correct vector dimension self.ChunkRecord = create_chunk_model(self.embedder._vector_dim) - # Local filesystem handling for DB directory + # Check if database exists (for local filesystem only) + is_new_db = False if not self._has_cloud_config(): - if read_only: - # Read operations should not create the database - if not db_path.exists(): + if not db_path.exists(): + if not create: raise FileNotFoundError( - f"Database does not exist: {db_path}. Use a write operation (add, add-src) to create it." + f"Database does not exist at {db_path}. " + "Use 'haiku-rag init' to create a new database." ) - else: - # Write operations - ensure parent directories exist + is_new_db = True + # Ensure parent directories exist for new databases if not db_path.parent.exists(): Path.mkdir(db_path.parent, parents=True) # Connect to LanceDB self.db = self._connect_to_lancedb(db_path) - # Initialize tables - self.create_or_update_db() + # Initialize tables (creates them if they don't exist) + self._init_tables() + + # Run upgrades only on existing databases, set version for new ones + if is_new_db: + self._set_initial_version() + else: + self._run_upgrades() # Validate config compatibility after connection is established if not skip_validation: @@ -234,9 +240,8 @@ class Store: settings_repo = SettingsRepository(self) settings_repo.validate_config_compatibility() - def create_or_update_db(self): - """Create the database tables.""" - + def _init_tables(self): + """Initialize database tables (create if they don't exist).""" # Get list of existing tables existing_tables = self.db.table_names() @@ -271,44 +276,29 @@ class Store: [SettingsRecord(id="settings", settings=json.dumps(settings_data))] ) - # Run pending upgrades based on stored version and package version - # Skip in read-only mode to avoid modifying the database - if not self._read_only: - try: - from haiku.rag.store.upgrades import run_pending_upgrades + def _set_initial_version(self): + """Set the initial version for a new database.""" + self.set_haiku_version(metadata.version("haiku.rag-slim")) - current_version = metadata.version("haiku.rag-slim") - db_version = self.get_haiku_version() + def _run_upgrades(self): + """Run pending database upgrades.""" + try: + from haiku.rag.store.upgrades import run_pending_upgrades - if db_version != "0.0.0": - run_pending_upgrades(self, db_version, current_version) + current_version = metadata.version("haiku.rag-slim") + db_version = self.get_haiku_version() - # After upgrades complete (or if none), set stored version - # to the greater of the installed package version and the - # highest available upgrade step version in code. - try: - from packaging.version import parse as _v + run_pending_upgrades(self, db_version, current_version) - from haiku.rag.store.upgrades import upgrades as _steps - - highest_step = max((_v(u.version) for u in _steps), default=None) - effective_version = ( - str(max(_v(current_version), highest_step)) - if highest_step is not None - else current_version - ) - except Exception: - effective_version = current_version - - self.set_haiku_version(effective_version) - except Exception as e: - # Avoid hard failure on initial connection; log and continue so CLI remains usable. - logger.warning( - "Skipping upgrade due to error (db=%s -> pkg=%s): %s", - self.get_haiku_version(), - metadata.version("haiku.rag-slim"), - e, - ) + self.set_haiku_version(current_version) + except Exception as e: + # Avoid hard failure on initial connection; log and continue so CLI remains usable. + logger.warning( + "Skipping upgrade due to error (db=%s -> pkg=%s): %s", + self.get_haiku_version(), + metadata.version("haiku.rag-slim"), + e, + ) def get_haiku_version(self) -> str: """Returns the user version stored in settings.""" diff --git a/tests/conftest.py b/tests/conftest.py index 81c9295e..87bf19b8 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -22,7 +22,10 @@ def qa_corpus() -> Dataset: @pytest.fixture def temp_db_path(): - """Create a temporary database path for testing.""" + """Create a temporary database path for testing. + + Note: Tests that need a database should use HaikuRAG with create=True. + """ with tempfile.TemporaryDirectory() as temp_dir: yield Path(temp_dir) / "test.lancedb" @@ -43,11 +46,13 @@ def temp_yaml_config(tmp_path, monkeypatch): "vacuum_retention_seconds": 60, }, "embeddings": { - "provider": "ollama", - "model": "qwen3-embedding:4b", - "vector_dim": 2560, + "model": { + "provider": "ollama", + "name": "qwen3-embedding:4b", + "vector_dim": 2560, + } }, - "qa": {"provider": "ollama", "model": "gpt-oss"}, + "qa": {"model": {"provider": "ollama", "name": "gpt-oss"}}, } with open(config_file, "w") as f: diff --git a/tests/graph/test_deep_qa.py b/tests/graph/test_deep_qa.py index c96cda33..2bcaa221 100644 --- a/tests/graph/test_deep_qa.py +++ b/tests/graph/test_deep_qa.py @@ -32,7 +32,7 @@ async def test_deep_qa_graph_end_to_end(monkeypatch, temp_db_path): ) # Use real client but with TestModel for LLM calls - client = HaikuRAG(temp_db_path) + client = HaikuRAG(temp_db_path, create=True) deps = DeepQADeps(client=client) result = await graph.run(state=state, deps=deps) @@ -67,7 +67,7 @@ async def test_deep_qa_with_citations(monkeypatch, temp_db_path): ) # Use real client but with TestModel for LLM calls - client = HaikuRAG(temp_db_path) + client = HaikuRAG(temp_db_path, create=True) deps = DeepQADeps(client=client) result = await graph.run(state=state, deps=deps) diff --git a/tests/graph/test_research_graph.py b/tests/graph/test_research_graph.py index 6311ba4e..fcca2967 100644 --- a/tests/graph/test_research_graph.py +++ b/tests/graph/test_research_graph.py @@ -47,7 +47,7 @@ async def test_graph_end_to_end_with_test_model(monkeypatch, temp_db_path): ) # Use real client but with TestModel for LLM calls - client = HaikuRAG(temp_db_path) + client = HaikuRAG(temp_db_path, create=True) deps = ResearchDeps(client=client) events = [] diff --git a/tests/test_chunk.py b/tests/test_chunk.py index 08cf5984..cbb341fa 100644 --- a/tests/test_chunk.py +++ b/tests/test_chunk.py @@ -15,7 +15,7 @@ from haiku.rag.store.repositories.document import DocumentRepository async def test_chunk_repository_operations(qa_corpus: Dataset, temp_db_path): """Test ChunkRepository operations.""" # Create client - client = HaikuRAG(db_path=temp_db_path, config=Config) + client = HaikuRAG(db_path=temp_db_path, config=Config, create=True) # Get the first document from the corpus first_doc = qa_corpus[0] @@ -56,7 +56,7 @@ async def test_chunk_repository_operations(qa_corpus: Dataset, temp_db_path): async def test_create_chunks_for_document(qa_corpus: Dataset, temp_db_path): """Test creating chunks for a document.""" # Create a store and repositories - store = Store(temp_db_path) + store = Store(temp_db_path, create=True) chunk_repo = ChunkRepository(store) doc_repo = DocumentRepository(store) @@ -98,7 +98,7 @@ async def test_create_chunks_for_document(qa_corpus: Dataset, temp_db_path): async def test_chunk_repository_crud(temp_db_path): """Test basic CRUD operations in ChunkRepository.""" # Create a store - store = Store(temp_db_path) + store = Store(temp_db_path, create=True) chunk_repo = ChunkRepository(store) doc_repo = DocumentRepository(store) @@ -151,7 +151,7 @@ async def test_chunk_repository_crud(temp_db_path): @pytest.mark.asyncio async def test_adjacent_chunks(temp_db_path): """Test the get_adjacent_chunks repository method.""" - store = Store(temp_db_path) + store = Store(temp_db_path, create=True) doc_repo = DocumentRepository(store) chunk_repo = ChunkRepository(store) diff --git a/tests/test_cli.py b/tests/test_cli.py index 17b22bb1..4ef0521f 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -339,6 +339,18 @@ def test_ask_with_deep_and_verbose(): ) +def test_init(): + with patch("haiku.rag.cli.HaikuRAGApp") as mock_app: + mock_app_instance = MagicMock() + mock_app_instance.init = AsyncMock() + mock_app.return_value = mock_app_instance + + result = runner.invoke(cli, ["init"]) + + assert result.exit_code == 0 + mock_app_instance.init.assert_called_once() + + def test_info(): with patch("haiku.rag.cli.HaikuRAGApp") as mock_app: mock_app_instance = MagicMock() diff --git a/tests/test_client.py b/tests/test_client.py index 9f704557..8d648f70 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -15,7 +15,7 @@ from haiku.rag.store.models.document import Document @pytest.mark.asyncio async def test_client_document_crud(qa_corpus: Dataset, temp_db_path): """Test HaikuRAG CRUD operations for documents.""" - async with HaikuRAG(temp_db_path) as client: + async with HaikuRAG(temp_db_path, create=True) as client: # Get test data first_doc = qa_corpus[0] document_text = first_doc["document_extracted"] @@ -81,7 +81,7 @@ async def test_client_document_crud(qa_corpus: Dataset, temp_db_path): @pytest.mark.asyncio async def test_client_update_document_fields(qa_corpus: Dataset, temp_db_path): """Test updating document with individual parameters.""" - async with HaikuRAG(temp_db_path) as client: + async with HaikuRAG(temp_db_path, create=True) as client: # Get test data first_doc = qa_corpus[0] document_text = first_doc["document_extracted"] @@ -152,7 +152,7 @@ async def test_client_update_document_fields(qa_corpus: Dataset, temp_db_path): @pytest.mark.asyncio async def test_client_create_document_from_source(temp_db_path): """Test creating a document from a file source.""" - async with HaikuRAG(temp_db_path) as client: + async with HaikuRAG(temp_db_path, create=True) as client: with tempfile.TemporaryDirectory() as temp_dir: test_content = "This is test content from a file." temp_path = Path(temp_dir) / "test.txt" @@ -183,7 +183,7 @@ async def test_client_create_document_from_source(temp_db_path): @pytest.mark.asyncio async def test_client_create_document_from_source_with_title(temp_db_path): """Test creating a document from a file source with a title.""" - async with HaikuRAG(temp_db_path) as client: + async with HaikuRAG(temp_db_path, create=True) as client: with tempfile.TemporaryDirectory() as temp_dir: test_content = "This is test content from a file." temp_path = Path(temp_dir) / "test_title.txt" @@ -200,7 +200,7 @@ async def test_client_create_document_from_source_with_title(temp_db_path): @pytest.mark.asyncio async def test_client_update_title_noop_behavior(temp_db_path): """When content is unchanged, updating title should update document without re-chunking.""" - async with HaikuRAG(temp_db_path) as client: + async with HaikuRAG(temp_db_path, create=True) as client: with tempfile.TemporaryDirectory() as temp_dir: temp_path = Path(temp_dir) / "test_update_title.txt" temp_path.write_text("Original content") @@ -222,7 +222,7 @@ async def test_client_update_title_noop_behavior(temp_db_path): @pytest.mark.asyncio async def test_client_create_document_from_source_unsupported(temp_db_path): """Test creating a document from an unsupported file type.""" - async with HaikuRAG(temp_db_path) as client: + async with HaikuRAG(temp_db_path, create=True) as client: # Create a temporary file with unsupported extension with tempfile.NamedTemporaryFile( mode="w", suffix=".unsupported", delete=False @@ -238,7 +238,7 @@ async def test_client_create_document_from_source_unsupported(temp_db_path): @pytest.mark.asyncio async def test_client_create_document_from_source_nonexistent(temp_db_path): """Test creating a document from a non-existent file.""" - async with HaikuRAG(temp_db_path) as client: + async with HaikuRAG(temp_db_path, create=True) as client: non_existent_path = Path("/non/existent/file.txt") # Should raise ValueError when file doesn't exist @@ -249,7 +249,7 @@ async def test_client_create_document_from_source_nonexistent(temp_db_path): @pytest.mark.asyncio async def test_client_create_document_from_directory(temp_db_path): """Test creating documents from a directory recursively.""" - async with HaikuRAG(temp_db_path) as client: + async with HaikuRAG(temp_db_path, create=True) as client: with tempfile.TemporaryDirectory() as temp_dir: test_dir = Path(temp_dir) / "test_docs" test_dir.mkdir() @@ -294,7 +294,7 @@ async def test_client_create_document_from_directory_with_filters( "haiku.rag.client.Config.monitor.include_patterns", ["**/include/**/*.txt"] ) - async with HaikuRAG(temp_db_path) as client: + async with HaikuRAG(temp_db_path, create=True) as client: with tempfile.TemporaryDirectory() as temp_dir: test_dir = Path(temp_dir) / "test_docs" test_dir.mkdir() @@ -333,7 +333,7 @@ async def test_client_create_document_from_directory_with_filters( @pytest.mark.asyncio async def test_client_create_document_from_url(temp_db_path): """Test creating a document from a URL.""" - async with HaikuRAG(temp_db_path) as client: + async with HaikuRAG(temp_db_path, create=True) as client: # Mock the HTTP response mock_response = AsyncMock() mock_response.content = b"

Test Page

This is test content from a webpage.

" @@ -361,7 +361,7 @@ async def test_client_create_document_from_url_with_different_content_types( temp_db_path, ): """Test creating documents from URLs with different content types.""" - async with HaikuRAG(temp_db_path) as client: + async with HaikuRAG(temp_db_path, create=True) as client: # Test JSON content mock_json_response = AsyncMock() mock_json_response.content = ( @@ -406,7 +406,7 @@ async def test_client_create_document_from_url_with_different_content_types( @pytest.mark.asyncio async def test_client_create_document_from_url_unsupported_content(temp_db_path): """Test creating a document from URL with unsupported content type.""" - async with HaikuRAG(temp_db_path) as client: + async with HaikuRAG(temp_db_path, create=True) as client: # Mock response with unsupported content type mock_response = AsyncMock() mock_response.content = b"binary content" @@ -423,7 +423,7 @@ async def test_client_create_document_from_url_unsupported_content(temp_db_path) @pytest.mark.asyncio async def test_client_create_document_from_url_http_error(temp_db_path): """Test handling HTTP errors when creating document from URL.""" - async with HaikuRAG(temp_db_path) as client: + async with HaikuRAG(temp_db_path, create=True) as client: with patch("httpx.AsyncClient.get") as mock_get: mock_get.side_effect = httpx.HTTPStatusError( "404 Not Found", @@ -440,7 +440,7 @@ async def test_client_create_document_from_url_http_error(temp_db_path): @pytest.mark.asyncio async def test_get_extension_from_content_type_or_url(temp_db_path): """Test the helper method for determining file extensions.""" - async with HaikuRAG(temp_db_path) as client: + async with HaikuRAG(temp_db_path, create=True) as client: # Test content type mappings assert ( client._get_extension_from_content_type_or_url("", "text/html") == ".html" @@ -487,7 +487,7 @@ async def test_client_metadata_content_type_and_md5(temp_db_path): """Test that contentType and md5 metadata are correctly set.""" import hashlib - async with HaikuRAG(temp_db_path) as client: + async with HaikuRAG(temp_db_path, create=True) as client: # Create a temporary file with known content test_content = "Test content for MD5 calculation." expected_md5 = hashlib.md5(test_content.encode()).hexdigest() @@ -520,7 +520,7 @@ async def test_client_metadata_content_type_and_md5(temp_db_path): @pytest.mark.asyncio async def test_client_create_update_no_op_behavior(temp_db_path): """Test create/update/no-op behavior based on MD5 changes.""" - async with HaikuRAG(temp_db_path) as client: + async with HaikuRAG(temp_db_path, create=True) as client: # Create a temporary file test_content = "Original content for testing." with tempfile.TemporaryDirectory() as temp_dir: @@ -559,7 +559,7 @@ async def test_client_create_update_no_op_behavior(temp_db_path): @pytest.mark.asyncio async def test_client_unchanged_file_keeps_timestamp(temp_db_path): """Test that unchanged files don't update the updated_at timestamp.""" - async with HaikuRAG(temp_db_path) as client: + async with HaikuRAG(temp_db_path, create=True) as client: # Create a temporary file test_content = "Test content for timestamp check." with tempfile.TemporaryDirectory() as temp_dir: @@ -581,7 +581,7 @@ async def test_client_unchanged_file_keeps_timestamp(temp_db_path): @pytest.mark.asyncio async def test_client_url_create_update_no_op_behavior(temp_db_path): """Test create/update/no-op behavior for URLs based on MD5 changes.""" - async with HaikuRAG(temp_db_path) as client: + async with HaikuRAG(temp_db_path, create=True) as client: url = "https://example.com/test.txt" original_content = b"Original URL content" updated_content = b"Updated URL content" @@ -620,7 +620,7 @@ async def test_client_url_create_update_no_op_behavior(temp_db_path): @pytest.mark.asyncio async def test_client_search(temp_db_path): """Test HaikuRAG search functionality.""" - async with HaikuRAG(temp_db_path) as client: + async with HaikuRAG(temp_db_path, create=True) as client: # Add multiple documents to search from doc1_text = "Python is a high-level programming language known for its simplicity and readability." doc2_text = "Machine learning algorithms help computers learn patterns from data without explicit programming." @@ -665,7 +665,7 @@ async def test_client_async_context_manager(temp_db_path): """Test HaikuRAG as async context manager.""" # Test that context manager works and auto-closes - async with HaikuRAG(temp_db_path) as client: + async with HaikuRAG(temp_db_path, create=True) as client: # Create a document to ensure the client works doc = await client.create_document( content="Test content for context manager", @@ -688,7 +688,7 @@ async def test_client_async_context_manager(temp_db_path): @pytest.mark.asyncio async def test_client_create_document_with_custom_chunks(temp_db_path): """Test creating a document with pre-created chunks.""" - async with HaikuRAG(temp_db_path) as client: + async with HaikuRAG(temp_db_path, create=True) as client: # Create some custom chunks with and without embeddings chunks = [ Chunk( @@ -741,7 +741,7 @@ async def test_client_ask_without_cite(monkeypatch, temp_db_path): "haiku.rag.utils.get_model", lambda *args, **kwargs: TestModel() ) - async with HaikuRAG(temp_db_path) as client: + async with HaikuRAG(temp_db_path, create=True) as client: # Create a test document for the agent to search await client.create_document( content="Python is a high-level programming language.", uri="test.txt" @@ -765,7 +765,7 @@ async def test_client_ask_with_cite(monkeypatch, temp_db_path): "haiku.rag.utils.get_model", lambda *args, **kwargs: TestModel() ) - async with HaikuRAG(temp_db_path) as client: + async with HaikuRAG(temp_db_path, create=True) as client: # Create a test document await client.create_document( content="Python is a high-level programming language.", uri="test.txt" @@ -784,7 +784,7 @@ async def test_client_expand_context(temp_db_path): """Test expanding search results with adjacent chunks.""" # Mock Config to have CONTEXT_CHUNK_RADIUS = 2 with patch("haiku.rag.client.Config.processing.context_chunk_radius", 2): - async with HaikuRAG(temp_db_path) as client: + async with HaikuRAG(temp_db_path, create=True) as client: # Create chunks manually with precomputed embeddings to avoid network dim = client.chunk_repository.embedder._vector_dim z = [0.0] * dim @@ -836,7 +836,7 @@ async def test_client_expand_context(temp_db_path): @pytest.mark.asyncio async def test_client_expand_context_radius_zero(temp_db_path): """Test expand_context with radius 0 returns original results.""" - async with HaikuRAG(temp_db_path) as client: + async with HaikuRAG(temp_db_path, create=True) as client: # Create a simple document doc = await client.create_document(content="Simple test content") assert doc.id is not None @@ -853,7 +853,7 @@ async def test_client_expand_context_radius_zero(temp_db_path): async def test_client_expand_context_multiple_chunks(temp_db_path): """Test expand_context with multiple search results.""" with patch("haiku.rag.client.Config.processing.context_chunk_radius", 1): - async with HaikuRAG(temp_db_path) as client: + async with HaikuRAG(temp_db_path, create=True) as client: # Create first document with manual chunks doc1_chunks = [ Chunk(content="Doc1 Part A", order=0), @@ -906,7 +906,7 @@ async def test_client_expand_context_multiple_chunks(temp_db_path): @pytest.mark.asyncio async def test_client_expand_context_merges_overlapping_chunks(temp_db_path): """Test that overlapping expanded chunks are merged into one.""" - async with HaikuRAG(temp_db_path) as client: + async with HaikuRAG(temp_db_path, create=True) as client: # Create document with 5 chunks manual_chunks = [ Chunk(content="Chunk 0", order=0), @@ -953,7 +953,7 @@ async def test_client_expand_context_merges_overlapping_chunks(temp_db_path): @pytest.mark.asyncio async def test_client_expand_context_keeps_separate_non_overlapping(temp_db_path): """Test that non-overlapping expanded chunks remain separate.""" - async with HaikuRAG(temp_db_path) as client: + async with HaikuRAG(temp_db_path, create=True) as client: # Create document with chunks far apart manual_chunks = [ Chunk(content="Chunk 0", order=0), diff --git a/tests/test_database_autocreate.py b/tests/test_database_autocreate.py index 77ef7c15..aabb6306 100644 --- a/tests/test_database_autocreate.py +++ b/tests/test_database_autocreate.py @@ -7,93 +7,43 @@ from haiku.rag.client import HaikuRAG from haiku.rag.config import AppConfig -def test_read_operations_do_not_create_database(): - """Test that read operations fail if database doesn't exist.""" +def test_database_not_created_without_create_flag(): + """Test that database is not created without create=True.""" with tempfile.TemporaryDirectory() as tmpdir: db_path = Path(tmpdir) / "test.lancedb" config = AppConfig() - # Read operation with read_only=True should fail - with pytest.raises( - FileNotFoundError, - match="Database does not exist.*Use a write operation", - ): - HaikuRAG(db_path=db_path, config=config, read_only=True) + with pytest.raises(FileNotFoundError, match="Database does not exist"): + HaikuRAG(db_path=db_path, config=config) -def test_write_operations_create_database(): - """Test that write operations create the database.""" +def test_database_created_with_create_flag(): + """Test that database is created with create=True.""" with tempfile.TemporaryDirectory() as tmpdir: db_path = Path(tmpdir) / "test.lancedb" config = AppConfig() - # Write operation with read_only=False (default) should succeed - client = HaikuRAG(db_path=db_path, config=config, read_only=False) + client = HaikuRAG(db_path=db_path, config=config, create=True) assert db_path.exists() client.close() -async def test_add_document_creates_database(): - """Test that add operations create the database.""" +@pytest.mark.asyncio +async def test_operations_work_after_database_created(): + """Test that operations work after DB is created.""" with tempfile.TemporaryDirectory() as tmpdir: db_path = Path(tmpdir) / "test.lancedb" config = AppConfig() - # Create a document (write operation) should work and create DB - async with HaikuRAG(db_path=db_path, config=config, read_only=False) as client: - doc = await client.create_document("Test content") - assert doc.id is not None - assert doc.content == "Test content" - assert db_path.exists() - - -async def test_search_fails_if_database_does_not_exist(): - """Test that search operations fail if DB doesn't exist.""" - with tempfile.TemporaryDirectory() as tmpdir: - db_path = Path(tmpdir) / "test.lancedb" - - config = AppConfig() - - # Read operation (search) should fail if DB doesn't exist - with pytest.raises( - FileNotFoundError, - match="Database does not exist.*Use a write operation", - ): - async with HaikuRAG( - db_path=db_path, config=config, read_only=True - ) as client: - await client.search("test query") - - -async def test_read_operations_work_after_database_created(): - """Test that read operations work after DB is created via write operation.""" - with tempfile.TemporaryDirectory() as tmpdir: - db_path = Path(tmpdir) / "test.lancedb" - - config = AppConfig() - - # First, create DB via write operation - async with HaikuRAG(db_path=db_path, config=config, read_only=False) as client: + # First, create DB with create=True and add document + async with HaikuRAG(db_path=db_path, config=config, create=True) as client: await client.create_document("Test content", uri="test://doc1") - # Now read operations should work since DB exists - async with HaikuRAG(db_path=db_path, config=config, read_only=True) as client: + # Re-open without create flag and verify we can read the document + async with HaikuRAG(db_path=db_path, config=config) as client: docs = await client.list_documents() assert len(docs) == 1 assert docs[0].content == "Test content" - - -def test_default_read_only_is_false(): - """Test that read_only defaults to False for backward compatibility.""" - with tempfile.TemporaryDirectory() as tmpdir: - db_path = Path(tmpdir) / "test.lancedb" - - config = AppConfig() - - # Without specifying read_only, it should default to False (allow creation) - client = HaikuRAG(db_path=db_path, config=config) - assert db_path.exists() - client.close() diff --git a/tests/test_document.py b/tests/test_document.py index 61e3ae84..97855a39 100644 --- a/tests/test_document.py +++ b/tests/test_document.py @@ -12,7 +12,7 @@ from haiku.rag.store.repositories.document import DocumentRepository async def test_create_document_with_chunks(qa_corpus: Dataset, temp_db_path): """Test creating a document with chunks from the qa_corpus using repository.""" # Create client - client = HaikuRAG(db_path=temp_db_path, config=Config) + client = HaikuRAG(db_path=temp_db_path, config=Config, create=True) # Get the first document from the corpus first_doc = qa_corpus[0] @@ -44,7 +44,7 @@ async def test_create_document_with_chunks(qa_corpus: Dataset, temp_db_path): async def test_document_repository_crud(qa_corpus: Dataset, temp_db_path): """Test CRUD operations in DocumentRepository.""" # Create a store and repository - store = Store(temp_db_path) + store = Store(temp_db_path, create=True) doc_repo = DocumentRepository(store) # Get the first document from the corpus @@ -100,7 +100,7 @@ async def test_document_repository_crud(qa_corpus: Dataset, temp_db_path): @pytest.mark.asyncio async def test_document_list_with_filter(qa_corpus: Dataset, temp_db_path): """Test listing documents with filter clause.""" - store = Store(temp_db_path) + store = Store(temp_db_path, create=True) doc_repo = DocumentRepository(store) first_doc = qa_corpus[0] diff --git a/tests/test_filter.py b/tests/test_filter.py index 8e8664a8..cc9c4918 100644 --- a/tests/test_filter.py +++ b/tests/test_filter.py @@ -6,7 +6,7 @@ from haiku.rag.client import HaikuRAG @pytest.mark.asyncio async def test_search_with_uri_filter(temp_db_path): """Test filtering by document URI.""" - async with HaikuRAG(db_path=temp_db_path) as client: + async with HaikuRAG(db_path=temp_db_path, create=True) as client: # Add multiple test documents await client.create_document( content="Python tutorial content", @@ -40,7 +40,7 @@ async def test_search_with_uri_filter(temp_db_path): @pytest.mark.asyncio async def test_search_with_title_filter(temp_db_path): """Test filtering by document title.""" - async with HaikuRAG(db_path=temp_db_path) as client: + async with HaikuRAG(db_path=temp_db_path, create=True) as client: # Add test documents await client.create_document( content="Programming content", @@ -66,7 +66,7 @@ async def test_search_with_title_filter(temp_db_path): @pytest.mark.asyncio async def test_search_with_combined_filters(temp_db_path): """Test filtering with AND/OR conditions.""" - async with HaikuRAG(db_path=temp_db_path) as client: + async with HaikuRAG(db_path=temp_db_path, create=True) as client: # Add test documents await client.create_document( content="Content about AI", @@ -105,7 +105,7 @@ async def test_search_with_combined_filters(temp_db_path): @pytest.mark.asyncio async def test_search_with_no_matching_filter(temp_db_path): """Test that search returns empty results when filter matches no documents.""" - async with HaikuRAG(db_path=temp_db_path) as client: + async with HaikuRAG(db_path=temp_db_path, create=True) as client: # Add a test document await client.create_document( content="Test content", @@ -123,7 +123,7 @@ async def test_search_with_no_matching_filter(temp_db_path): @pytest.mark.asyncio async def test_search_with_invalid_filter(temp_db_path): """Test that invalid filter syntax raises an appropriate error.""" - async with HaikuRAG(db_path=temp_db_path) as client: + async with HaikuRAG(db_path=temp_db_path, create=True) as client: # Add a test document await client.create_document( content="Test content", @@ -139,7 +139,7 @@ async def test_search_with_invalid_filter(temp_db_path): @pytest.mark.asyncio async def test_search_filter_with_all_search_types(temp_db_path): """Test that filtering works with all search types (vector, fts, hybrid).""" - async with HaikuRAG(db_path=temp_db_path) as client: + async with HaikuRAG(db_path=temp_db_path, create=True) as client: await client.create_document( content="Machine learning is a subset of artificial intelligence", uri="https://ai.example.com/ml.html", diff --git a/tests/test_info.py b/tests/test_info.py index 7bd6c458..ba909a96 100644 --- a/tests/test_info.py +++ b/tests/test_info.py @@ -6,7 +6,7 @@ from haiku.rag.app import HaikuRAGApp @pytest.mark.asyncio -async def test_app_info_outputs_and_read_only(temp_db_path, capsys): +async def test_app_info_outputs(temp_db_path, capsys): # Build a minimal LanceDB with settings, documents, and chunks without using Store import lancedb from lancedb.pydantic import LanceModel, Vector @@ -32,7 +32,7 @@ async def test_app_info_outputs_and_read_only(temp_db_path, capsys): docs_tbl = db.create_table("documents", schema=DocumentRecord) chunks_tbl = db.create_table("chunks", schema=ChunkRecord) - # Insert one of each + # Insert one of each - using the new config format settings_tbl.add( [ SettingsRecord( @@ -41,9 +41,11 @@ async def test_app_info_outputs_and_read_only(temp_db_path, capsys): { "version": "1.2.3", "embeddings": { - "provider": "openai", - "model": "text-embedding-3-small", - "vector_dim": 3, + "model": { + "provider": "openai", + "name": "text-embedding-3-small", + "vector_dim": 3, + } }, } ), @@ -55,20 +57,13 @@ async def test_app_info_outputs_and_read_only(temp_db_path, capsys): [ChunkRecord(id="c1", document_id="doc-1", content="c", vector=[0.1, 0.2, 0.3])] ) - # Capture versions before - before_versions = { - "settings": int(settings_tbl.version), - "documents": int(docs_tbl.version), - "chunks": int(chunks_tbl.version), - } - app = HaikuRAGApp(db_path=temp_db_path) await app.info() out = capsys.readouterr().out # Validate expected content substrings assert f"path: \n{temp_db_path}" in out - assert "haiku.rag version (db): 1.2.3" in out + assert "haiku.rag version (db):" in out assert "embeddings: openai/text-embedding-3-small (dim: 3)" in out assert "documents: 1" in out assert "chunks: 1" in out @@ -85,13 +80,6 @@ async def test_app_info_outputs_and_read_only(temp_db_path, capsys): assert "lancedb:" in out assert "haiku.rag:" in out - # Verify no versions changed (read-only) - # Re-open to ensure fresh view - db2 = lancedb.connect(temp_db_path) - assert int(db2.open_table("settings").version) == before_versions["settings"] - assert int(db2.open_table("documents").version) == before_versions["documents"] - assert int(db2.open_table("chunks").version) == before_versions["chunks"] - @pytest.mark.asyncio async def test_app_info_with_vector_index(temp_db_path, capsys): @@ -148,13 +136,6 @@ async def test_app_info_with_vector_index(temp_db_path, capsys): # Create vector index chunks_tbl.create_index(metric="cosine", index_type="IVF_PQ") - # Capture versions before - before_versions = { - "settings": int(settings_tbl.version), - "documents": int(docs_tbl.version), - "chunks": int(chunks_tbl.version), - } - app = HaikuRAGApp(db_path=temp_db_path) await app.info() @@ -168,9 +149,3 @@ async def test_app_info_with_vector_index(temp_db_path, capsys): # Check basic info still present assert "documents: 1" in out assert "chunks: 512" in out - - # Verify no versions changed (read-only) - db2 = lancedb.connect(temp_db_path) - assert int(db2.open_table("settings").version) == before_versions["settings"] - assert int(db2.open_table("documents").version) == before_versions["documents"] - assert int(db2.open_table("chunks").version) == before_versions["chunks"] diff --git a/tests/test_lancedb_connection.py b/tests/test_lancedb_connection.py index d1de0877..e9b21ad5 100644 --- a/tests/test_lancedb_connection.py +++ b/tests/test_lancedb_connection.py @@ -10,7 +10,7 @@ from haiku.rag.store.engine import Store async def test_lancedb_cloud_skips_optimization(temp_db_path): """Test that vacuum is skipped when using LanceDB Cloud (db:// URI).""" # Create a store - store = Store(temp_db_path) + store = Store(temp_db_path, create=True) # Mock all cloud config to simulate LanceDB Cloud usage with ( @@ -33,7 +33,7 @@ async def test_lancedb_cloud_skips_optimization(temp_db_path): async def test_local_storage_calls_optimization(temp_db_path): """Test that vacuum calls optimization for local storage.""" # Create a store - store = Store(temp_db_path) + store = Store(temp_db_path, create=True) # Ensure uri is empty (local storage) with patch.object(Config.lancedb, "uri", ""): diff --git a/tests/test_preprocessor.py b/tests/test_preprocessor.py index 9f7d9b07..34339c50 100644 --- a/tests/test_preprocessor.py +++ b/tests/test_preprocessor.py @@ -45,7 +45,7 @@ def add_marker(text: str) -> str: try: Config.processing.markdown_preprocessor = f"{pre_file}:add_marker" - store = Store(temp_db_path) + store = Store(temp_db_path, create=True) chunk_repo = ChunkRepository(store) doc_repo = DocumentRepository(store) diff --git a/tests/test_qa.py b/tests/test_qa.py index 08f41521..113f5219 100644 --- a/tests/test_qa.py +++ b/tests/test_qa.py @@ -17,7 +17,7 @@ VLLM_QA_AVAILABLE = bool(Config.providers.vllm.qa_base_url) @pytest.mark.asyncio async def test_qa_ollama(qa_corpus: Dataset, temp_db_path): """Test Ollama QA with LLM judge.""" - client = HaikuRAG(temp_db_path) + client = HaikuRAG(temp_db_path, create=True) qa = QuestionAnswerAgent( client, ModelConfig(provider="ollama", name="gpt-oss", enable_thinking=False) ) @@ -43,7 +43,7 @@ async def test_qa_ollama(qa_corpus: Dataset, temp_db_path): @pytest.mark.skipif(not OPENAI_AVAILABLE, reason="OpenAI not available") async def test_qa_openai(qa_corpus: Dataset, temp_db_path): """Test OpenAI QA with LLM judge.""" - client = HaikuRAG(temp_db_path) + client = HaikuRAG(temp_db_path, create=True) qa = QuestionAnswerAgent(client, ModelConfig(provider="openai", name="gpt-4o-mini")) llm_judge = LLMJudge() @@ -67,7 +67,7 @@ async def test_qa_openai(qa_corpus: Dataset, temp_db_path): @pytest.mark.skipif(not ANTHROPIC_AVAILABLE, reason="Anthropic not available") async def test_qa_anthropic(qa_corpus: Dataset, temp_db_path): """Test Anthropic QA with LLM judge.""" - client = HaikuRAG(temp_db_path) + client = HaikuRAG(temp_db_path, create=True) qa = QuestionAnswerAgent( client, ModelConfig(provider="anthropic", name="claude-3-5-haiku-20241022") ) @@ -93,7 +93,7 @@ async def test_qa_anthropic(qa_corpus: Dataset, temp_db_path): @pytest.mark.skipif(not VLLM_QA_AVAILABLE, reason="vLLM QA server not configured") async def test_qa_vllm(qa_corpus: Dataset, temp_db_path): """Test vLLM QA with LLM judge.""" - client = HaikuRAG(temp_db_path) + client = HaikuRAG(temp_db_path, create=True) qa = QuestionAnswerAgent(client, ModelConfig(provider="vllm", name="Qwen/Qwen3-4B")) llm_judge = LLMJudge() diff --git a/tests/test_rebuild.py b/tests/test_rebuild.py index d145fd4a..2f749faf 100644 --- a/tests/test_rebuild.py +++ b/tests/test_rebuild.py @@ -7,7 +7,7 @@ from haiku.rag.client import HaikuRAG, RebuildMode @pytest.mark.asyncio async def test_rebuild_full(qa_corpus: Dataset, temp_db_path): """Test full rebuild: converts, chunks, and embeds all documents.""" - async with HaikuRAG(temp_db_path) as client: + async with HaikuRAG(temp_db_path, create=True) as client: doc = await client.create_document(content=qa_corpus["document_extracted"][0]) assert doc.id is not None @@ -30,7 +30,7 @@ async def test_rebuild_full(qa_corpus: Dataset, temp_db_path): @pytest.mark.asyncio async def test_rebuild_embed_only(qa_corpus: Dataset, temp_db_path): """Test embed-only rebuild: keeps chunks, only regenerates embeddings.""" - async with HaikuRAG(temp_db_path) as client: + async with HaikuRAG(temp_db_path, create=True) as client: doc = await client.create_document(content=qa_corpus["document_extracted"][0]) assert doc.id is not None @@ -60,7 +60,7 @@ async def test_rebuild_embed_only(qa_corpus: Dataset, temp_db_path): @pytest.mark.asyncio async def test_rebuild_embed_only_skips_unchanged(qa_corpus: Dataset, temp_db_path): """Test embed-only rebuild skips chunks with unchanged embeddings.""" - async with HaikuRAG(temp_db_path) as client: + async with HaikuRAG(temp_db_path, create=True) as client: doc = await client.create_document(content=qa_corpus["document_extracted"][0]) assert doc.id is not None @@ -96,7 +96,7 @@ async def test_rebuild_embed_only_skips_unchanged(qa_corpus: Dataset, temp_db_pa @pytest.mark.asyncio async def test_rebuild_rechunk(qa_corpus: Dataset, temp_db_path): """Test rechunk rebuild: re-chunks from content without accessing source files.""" - async with HaikuRAG(temp_db_path) as client: + async with HaikuRAG(temp_db_path, create=True) as client: doc = await client.create_document(content=qa_corpus["document_extracted"][0]) assert doc.id is not None diff --git a/tests/test_search.py b/tests/test_search.py index 1783aa90..6436d88f 100644 --- a/tests/test_search.py +++ b/tests/test_search.py @@ -9,7 +9,7 @@ from haiku.rag.config import Config async def test_search_qa_corpus(qa_corpus: Dataset, temp_db_path): """Test that documents can be found by searching with their associated questions.""" # Create client - client = HaikuRAG(db_path=temp_db_path, config=Config) + client = HaikuRAG(db_path=temp_db_path, config=Config, create=True) # Load unique documents (limited to 10) seen_documents = set() @@ -61,7 +61,7 @@ async def test_search_qa_corpus(qa_corpus: Dataset, temp_db_path): @pytest.mark.asyncio async def test_chunks_include_document_info(temp_db_path): """Test that search results include document URI and metadata.""" - client = HaikuRAG(db_path=temp_db_path, config=Config) + client = HaikuRAG(db_path=temp_db_path, config=Config, create=True) # Create a document with URI and metadata created_document = await client.create_document( @@ -93,7 +93,7 @@ async def test_chunks_include_document_info(temp_db_path): @pytest.mark.asyncio async def test_chunks_include_document_title(temp_db_path): """Test that search results include the parent document title when present.""" - client = HaikuRAG(db_path=temp_db_path, config=Config) + client = HaikuRAG(db_path=temp_db_path, config=Config, create=True) # Create a document with URI and title await client.create_document( @@ -119,7 +119,7 @@ async def test_chunks_include_document_title(temp_db_path): @pytest.mark.asyncio async def test_search_score_types(temp_db_path): """Test that different search types return appropriate score ranges.""" - client = HaikuRAG(db_path=temp_db_path, config=Config) + client = HaikuRAG(db_path=temp_db_path, config=Config, create=True) # Create multiple documents with different content documents_content = [ diff --git a/tests/test_settings.py b/tests/test_settings.py index d6e72d29..8e0f2007 100644 --- a/tests/test_settings.py +++ b/tests/test_settings.py @@ -12,7 +12,7 @@ def test_settings_table_populated_on_store_init(temp_db_path): from haiku.rag.store.engine import Store from haiku.rag.store.repositories.settings import SettingsRepository - store = Store(temp_db_path) + store = Store(temp_db_path, create=True) settings_repo = SettingsRepository(store) db_settings = settings_repo.get_current_settings() @@ -32,7 +32,7 @@ def test_settings_save_and_retrieve(temp_db_path): from haiku.rag.store.engine import Store from haiku.rag.store.repositories.settings import SettingsRepository - store = Store(temp_db_path) + store = Store(temp_db_path, create=True) settings_repo = SettingsRepository(store) original_chunk_size = Config.processing.chunk_size @@ -53,7 +53,7 @@ async def test_config_validation_on_db_load(temp_db_path): from haiku.rag.store.repositories.settings import SettingsRepository # Create store and save settings - store1 = Store(temp_db_path) + store1 = Store(temp_db_path, create=True) store1.close() # Change config @@ -63,18 +63,20 @@ async def test_config_validation_on_db_load(temp_db_path): try: # Loading the database should raise ConfigMismatchError with pytest.raises(ConfigMismatchError) as exc_info: - Store(temp_db_path) + Store(temp_db_path, create=True) assert "chunk_size" in str(exc_info.value) assert "rebuild" in str(exc_info.value).lower() # Rebuild - async with HaikuRAG(db_path=temp_db_path, skip_validation=True) as client: + async with HaikuRAG( + db_path=temp_db_path, skip_validation=True, create=True + ) as client: async for _ in client.rebuild_database(): pass # Process all documents # Verify we can now load the database without exception (settings were updated) - store2 = Store(temp_db_path) + store2 = Store(temp_db_path, create=True) settings_repo2 = SettingsRepository(store2) db_settings = settings_repo2.get_current_settings() assert db_settings["processing"]["chunk_size"] == 999 diff --git a/tests/test_versioning.py b/tests/test_versioning.py index e38986b6..2c8a5789 100644 --- a/tests/test_versioning.py +++ b/tests/test_versioning.py @@ -10,7 +10,7 @@ from haiku.rag.store.repositories.document import DocumentRepository @pytest.mark.asyncio async def test_version_rollback_on_create_failure(temp_db_path): - store = Store(temp_db_path) + store = Store(temp_db_path, create=True) repo = DocumentRepository(store) # Ensure chunk repository is instantiated and stub embeddings to avoid network @@ -51,7 +51,7 @@ async def test_version_rollback_on_create_failure(temp_db_path): @pytest.mark.asyncio async def test_version_rollback_on_update_failure(temp_db_path): - store = Store(temp_db_path) + store = Store(temp_db_path, create=True) repo = DocumentRepository(store) # Stub embeddings to avoid network @@ -106,11 +106,11 @@ def test_new_database_does_not_run_upgrades(monkeypatch, temp_db_path): fail_if_called, ) - Store(temp_db_path) + Store(temp_db_path, create=True) def test_existing_database_runs_upgrades(monkeypatch, temp_db_path): - Store(temp_db_path) + Store(temp_db_path, create=True) called = {"value": False} @@ -122,6 +122,7 @@ def test_existing_database_runs_upgrades(monkeypatch, temp_db_path): mark_called, ) + # Opening an existing database should trigger upgrades Store(temp_db_path) assert called["value"] @@ -129,7 +130,7 @@ def test_existing_database_runs_upgrades(monkeypatch, temp_db_path): @pytest.mark.asyncio async def test_vacuum_with_retention_threshold(temp_db_path): - store = Store(temp_db_path) + store = Store(temp_db_path, create=True) repo = DocumentRepository(store) # Stub embeddings to avoid network @@ -209,14 +210,14 @@ async def test_vacuum_completes_before_context_exit(temp_db_path, monkeypatch): # Set aggressive vacuum retention for this test monkeypatch.setattr(Config.storage, "vacuum_retention_seconds", 0) - async with HaikuRAG(db_path=temp_db_path) as client: + async with HaikuRAG(db_path=temp_db_path, create=True) as client: # Create multiple documents - each creation triggers automatic vacuum with retention=0 # This aggressively cleans up old versions between operations for i in range(3): await client.create_document(content=f"Test document {i}") # After context exit, automatic vacuum should have kept versions minimal - store = Store(temp_db_path) + store = Store(temp_db_path, create=True) final_versions = len(list(store.documents_table.list_versions())) # With retention_seconds=0, vacuum aggressively cleans up between operations From 78f1d0cef511383df137270e3fb707ee02c73d37 Mon Sep 17 00:00:00 2001 From: Yiorgis Gozadinos Date: Wed, 3 Dec 2025 08:51:25 +0200 Subject: [PATCH 4/4] No warnings, just info on db upgrade --- haiku_rag_slim/haiku/rag/store/engine.py | 2 +- haiku_rag_slim/haiku/rag/store/upgrades/v0_19_6.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/haiku_rag_slim/haiku/rag/store/engine.py b/haiku_rag_slim/haiku/rag/store/engine.py index b59d24f5..060bba7a 100644 --- a/haiku_rag_slim/haiku/rag/store/engine.py +++ b/haiku_rag_slim/haiku/rag/store/engine.py @@ -70,7 +70,7 @@ class Store: if not db_path.exists(): if not create: raise FileNotFoundError( - f"Database does not exist at {db_path}. " + f"Database does not exist at {self.db_path.absolute()}. " "Use 'haiku-rag init' to create a new database." ) is_new_db = True diff --git a/haiku_rag_slim/haiku/rag/store/upgrades/v0_19_6.py b/haiku_rag_slim/haiku/rag/store/upgrades/v0_19_6.py index 27fcf46c..b4213314 100644 --- a/haiku_rag_slim/haiku/rag/store/upgrades/v0_19_6.py +++ b/haiku_rag_slim/haiku/rag/store/upgrades/v0_19_6.py @@ -31,7 +31,7 @@ def _apply_embeddings_model_config(store: Store) -> None: old_model = embeddings.get("model", "qwen3-embedding:4b") old_vector_dim = embeddings.get("vector_dim", 2560) - logger.warning( + logger.info( "Migrating embeddings config to new nested structure: " "embeddings.{provider,model,vector_dim} -> embeddings.model.{provider,name,vector_dim}" ) @@ -50,7 +50,7 @@ def _apply_embeddings_model_config(store: Store) -> None: values={"settings": json.dumps(settings)}, ) - logger.warning( + logger.info( "Embeddings config migrated: provider=%s, name=%s, vector_dim=%d", old_provider, old_model,