From 33b254ae3d455482ece39121682c2a8ed8882641 Mon Sep 17 00:00:00 2001 From: Yiorgis Gozadinos Date: Tue, 2 Dec 2025 16:36:15 +0200 Subject: [PATCH] Always upgrade database on access. Explicitly create db by running init or `HaikuRAG(path, create=True)` --- CHANGELOG.md | 13 +++ docs/cli.md | 16 +++- docs/python.md | 10 ++- docs/tutorial.md | 11 +++ .../a2a-server/haiku_rag_a2a/a2a/client.py | 1 + .../a2a-server/haiku_rag_a2a/a2a/worker.py | 5 +- examples/a2a-server/tests/test_a2a.py | 5 +- haiku_rag_slim/haiku/rag/app.py | 50 ++++++----- haiku_rag_slim/haiku/rag/cli.py | 14 ++- haiku_rag_slim/haiku/rag/client.py | 7 +- haiku_rag_slim/haiku/rag/inspector/app.py | 2 +- haiku_rag_slim/haiku/rag/store/engine.py | 86 ++++++++----------- tests/conftest.py | 15 ++-- tests/graph/test_deep_qa.py | 4 +- tests/graph/test_research_graph.py | 2 +- tests/test_chunk.py | 8 +- tests/test_cli.py | 12 +++ tests/test_client.py | 56 ++++++------ tests/test_database_autocreate.py | 78 +++-------------- tests/test_document.py | 6 +- tests/test_filter.py | 12 +-- tests/test_info.py | 41 ++------- tests/test_lancedb_connection.py | 4 +- tests/test_preprocessor.py | 2 +- tests/test_qa.py | 8 +- tests/test_rebuild.py | 8 +- tests/test_search.py | 8 +- tests/test_settings.py | 14 +-- tests/test_versioning.py | 15 ++-- 29 files changed, 254 insertions(+), 259 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f1bca675..d98c69b4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,19 @@ # Changelog ## [Unreleased] +## [0.19.6] - 2025-12-02 + +### Changed + +- **BREAKING: Explicit Database Creation**: Databases must now be explicitly created before use + - New `haiku-rag init` command creates a new empty database + - Python API: `HaikuRAG(path, create=True)` to create database programmatically + - Operations on non-existent databases raise `FileNotFoundError` +- **BREAKING: Embeddings Configuration**: Restructured to nested `EmbeddingModelConfig` + - Config path changed from `embeddings.{provider, model, vector_dim}` to `embeddings.model.{provider, name, vector_dim}` + - Automatic migration upgrades existing databases to new format +- **Database Migrations**: Always run when opening an existing database + ## [0.19.5] - 2025-12-01 ### Changed diff --git a/docs/cli.md b/docs/cli.md index 4b8b7d68..6373aa13 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -201,11 +201,21 @@ View current configuration settings: haiku-rag settings ``` -## Maintenance +## Database Management -### Info (Read-only) +### Initialize Database -Display database metadata without upgrading or modifying it: +Create a new database: + +```bash +haiku-rag init [--db /path/to/your.lancedb] +``` + +This creates the database with the configured settings. **All other commands require an existing database** - they will fail with an informative error if the database doesn't exist. + +### Info + +Display database metadata: ```bash haiku-rag info [--db /path/to/your.lancedb] diff --git a/docs/python.md b/docs/python.md index 9413f0b9..3b451109 100644 --- a/docs/python.md +++ b/docs/python.md @@ -8,12 +8,20 @@ Use `haiku.rag` directly in your Python applications. from pathlib import Path from haiku.rag.client import HaikuRAG -# Use as async context manager (recommended) +# Create a new database +async with HaikuRAG("path/to/database.lancedb", create=True) as client: + # Your code here + pass + +# Open an existing database (will fail if database doesn't exist) async with HaikuRAG("path/to/database.lancedb") as client: # Your code here pass ``` +!!! note + Databases must be explicitly created with `create=True` or via `haiku-rag init` before use. Operations on non-existent databases will raise `FileNotFoundError`. + ## Document Management ### Creating Documents diff --git a/docs/tutorial.md b/docs/tutorial.md index 97923d8b..65c81ee8 100644 --- a/docs/tutorial.md +++ b/docs/tutorial.md @@ -54,6 +54,16 @@ For the list of available OpenAI models and their vector dimensions, see the [Op See [Configuration](configuration/index.md) for all available options. +## Initialize the database + +Before adding documents, initialize the database: + +```bash +haiku-rag init +``` + +This creates an empty database with the configured settings. + ## Adding the first documents Now you can add some pieces of text in the database: @@ -150,6 +160,7 @@ logger.setLevel(logging.DEBUG) logger.debug("AGI here we come") # Uses LanceDB database from default storage location +# (database must be initialized first with 'haiku-rag init' or create=True) async with HaikuRAG() as client: answer = await client.ask("What is the best programming language in the world?") print(answer) diff --git a/examples/a2a-server/haiku_rag_a2a/a2a/client.py b/examples/a2a-server/haiku_rag_a2a/a2a/client.py index 0e9f2790..5de4b28d 100644 --- a/examples/a2a-server/haiku_rag_a2a/a2a/client.py +++ b/examples/a2a-server/haiku_rag_a2a/a2a/client.py @@ -1,3 +1,4 @@ +# pyright: reportMissingImports=false import asyncio import uuid from typing import Any diff --git a/examples/a2a-server/haiku_rag_a2a/a2a/worker.py b/examples/a2a-server/haiku_rag_a2a/a2a/worker.py index ca798934..e044f6ad 100644 --- a/examples/a2a-server/haiku_rag_a2a/a2a/worker.py +++ b/examples/a2a-server/haiku_rag_a2a/a2a/worker.py @@ -1,3 +1,4 @@ +# pyright: reportMissingImports=false import json import logging import uuid @@ -12,8 +13,8 @@ from haiku_rag_a2a.a2a.models import AgentDependencies from haiku_rag_a2a.a2a.skills import extract_question_from_task try: - from fasta2a import Worker # type: ignore - from fasta2a.schema import ( # type: ignore + from fasta2a import Worker + from fasta2a.schema import ( Artifact, Message, TaskIdParams, diff --git a/examples/a2a-server/tests/test_a2a.py b/examples/a2a-server/tests/test_a2a.py index 8388514a..b603ed66 100644 --- a/examples/a2a-server/tests/test_a2a.py +++ b/examples/a2a-server/tests/test_a2a.py @@ -1,3 +1,4 @@ +# pyright: reportMissingImports=false import uuid import pytest @@ -212,7 +213,7 @@ async def test_a2a_app_creation(temp_db_path): from haiku_rag_a2a.a2a import create_a2a_app # Create a test database - async with HaikuRAG(temp_db_path) as client: + async with HaikuRAG(temp_db_path, create=True) as client: await client.create_document( content="Python is a high-level programming language known for its simplicity.", uri="python_doc", @@ -233,7 +234,7 @@ async def test_a2a_app_has_skills(temp_db_path): from haiku_rag_a2a.a2a import create_a2a_app # Create a test database - async with HaikuRAG(temp_db_path) as client: + async with HaikuRAG(temp_db_path, create=True) as client: await client.create_document(content="Test document", uri="test_doc") # Create A2A app diff --git a/haiku_rag_slim/haiku/rag/app.py b/haiku_rag_slim/haiku/rag/app.py index ddbfc846..0ca69fbc 100644 --- a/haiku_rag_slim/haiku/rag/app.py +++ b/haiku_rag_slim/haiku/rag/app.py @@ -29,6 +29,21 @@ class HaikuRAGApp: self.config = config self.console = Console() + async def init(self): + """Initialize a new database.""" + if self.db_path.exists(): + self.console.print( + f"[yellow]Database already exists at {self.db_path}[/yellow]" + ) + return + + # Create the database + client = HaikuRAG(db_path=self.db_path, config=self.config, create=True) + client.close() + self.console.print( + f"[bold green]Database initialized at {self.db_path}[/bold green]" + ) + async def info(self): """Display read-only information about the database without modifying it.""" @@ -65,7 +80,13 @@ class HaikuRAGApp: except Exception: docling_version = "unknown" - # Read settings (if present) to find stored haiku.rag version and embedding config + # Get comprehensive table statistics (this also runs migrations) + from haiku.rag.store.engine import Store + + store = Store(self.db_path, config=self.config, skip_validation=True) + table_stats = store.get_stats() + + # Read settings after Store init (migrations have run) stored_version = "unknown" embed_provider: str | None = None embed_model: str | None = None @@ -85,13 +106,6 @@ class HaikuRAGApp: embed_model = embed_model_obj.get("name") vector_dim = embed_model_obj.get("vector_dim") - # Get comprehensive table statistics - from haiku.rag.store.engine import Store - - store = Store( - self.db_path, config=self.config, skip_validation=True, read_only=True - ) - table_stats = store.get_stats() store.close() num_docs = table_stats["documents"].get("num_rows", 0) @@ -188,9 +202,7 @@ class HaikuRAGApp: ) async def list_documents(self, filter: str | None = None): - async with HaikuRAG( - db_path=self.db_path, config=self.config, read_only=True - ) as self.client: + async with HaikuRAG(db_path=self.db_path, config=self.config) as self.client: documents = await self.client.list_documents(filter=filter) for doc in documents: self._rich_print_document(doc, truncate=True) @@ -223,9 +235,7 @@ class HaikuRAGApp: ) async def get_document(self, doc_id: str): - async with HaikuRAG( - db_path=self.db_path, config=self.config, read_only=True - ) as self.client: + async with HaikuRAG(db_path=self.db_path, config=self.config) as self.client: doc = await self.client.get_document_by_id(doc_id) if doc is None: self.console.print(f"[red]Document with id {doc_id} not found.[/red]") @@ -245,9 +255,7 @@ class HaikuRAGApp: ) async def search(self, query: str, limit: int = 5, filter: str | None = None): - async with HaikuRAG( - db_path=self.db_path, config=self.config, read_only=True - ) as self.client: + async with HaikuRAG(db_path=self.db_path, config=self.config) as self.client: results = await self.client.search(query, limit=limit, filter=filter) if not results: self.console.print("[yellow]No results found.[/yellow]") @@ -270,9 +278,7 @@ class HaikuRAGApp: deep: Use deep QA mode (multi-step reasoning) verbose: Show verbose output """ - async with HaikuRAG( - db_path=self.db_path, config=self.config, read_only=True - ) as self.client: + async with HaikuRAG(db_path=self.db_path, config=self.config) as self.client: try: if deep: from haiku.rag.graph.deep_qa.dependencies import DeepQAContext @@ -317,9 +323,7 @@ class HaikuRAGApp: question: The research question verbose: Show AG-UI event stream during execution """ - async with HaikuRAG( - db_path=self.db_path, config=self.config, read_only=True - ) as client: + async with HaikuRAG(db_path=self.db_path, config=self.config) as client: try: self.console.print("[bold cyan]Starting research[/bold cyan]") self.console.print(f"[bold blue]Question:[/bold blue] {question}") diff --git a/haiku_rag_slim/haiku/rag/cli.py b/haiku_rag_slim/haiku/rag/cli.py index f8fa9a97..49ca5f7b 100644 --- a/haiku_rag_slim/haiku/rag/cli.py +++ b/haiku_rag_slim/haiku/rag/cli.py @@ -410,7 +410,19 @@ def create_index( asyncio.run(app.create_index()) -@cli.command("info", help="Show read-only database info (no upgrades or writes)") +@cli.command("init", help="Initialize a new database") +def init_db( + db: Path | None = typer.Option( + None, + "--db", + help="Path to the LanceDB database file", + ), +): + app = create_app(db) + asyncio.run(app.init()) + + +@cli.command("info", help="Show database info") def info( db: Path | None = typer.Option( None, diff --git a/haiku_rag_slim/haiku/rag/client.py b/haiku_rag_slim/haiku/rag/client.py index 8e4902c4..79e05907 100644 --- a/haiku_rag_slim/haiku/rag/client.py +++ b/haiku_rag_slim/haiku/rag/client.py @@ -38,7 +38,7 @@ class HaikuRAG: db_path: Path | None = None, config: AppConfig = Config, skip_validation: bool = False, - read_only: bool = False, + create: bool = False, ): """Initialize the RAG client with a database path. @@ -46,8 +46,7 @@ class HaikuRAG: db_path: Path to the database file. If None, uses config.storage.data_dir. config: Configuration to use. Defaults to global Config. skip_validation: Whether to skip configuration validation on database load. - read_only: Whether to open in read-only mode. If True, will raise error - if database doesn't exist and will skip upgrades. + create: Whether to create the database if it doesn't exist. """ self._config = config if db_path is None: @@ -56,7 +55,7 @@ class HaikuRAG: db_path, config=self._config, skip_validation=skip_validation, - read_only=read_only, + create=create, ) self.document_repository = DocumentRepository(self.store) self.chunk_repository = ChunkRepository(self.store) diff --git a/haiku_rag_slim/haiku/rag/inspector/app.py b/haiku_rag_slim/haiku/rag/inspector/app.py index 4141ce5c..7b9939c1 100644 --- a/haiku_rag_slim/haiku/rag/inspector/app.py +++ b/haiku_rag_slim/haiku/rag/inspector/app.py @@ -86,7 +86,7 @@ class InspectorApp(App): # type: ignore[misc] async def on_mount(self) -> None: """Initialize the app when mounted.""" config = get_config() - self.client = HaikuRAG(db_path=self.db_path, config=config, read_only=True) + self.client = HaikuRAG(db_path=self.db_path, config=config) await self.client.__aenter__() # Load initial documents diff --git a/haiku_rag_slim/haiku/rag/store/engine.py b/haiku_rag_slim/haiku/rag/store/engine.py index 455d2e14..b59d24f5 100644 --- a/haiku_rag_slim/haiku/rag/store/engine.py +++ b/haiku_rag_slim/haiku/rag/store/engine.py @@ -54,35 +54,41 @@ class Store: db_path: Path, config: AppConfig = Config, skip_validation: bool = False, - read_only: bool = False, + create: bool = False, ): self.db_path: Path = db_path self._config = config self.embedder = get_embedder(config=self._config) self._vacuum_lock = asyncio.Lock() - self._read_only = read_only # Create the ChunkRecord model with the correct vector dimension self.ChunkRecord = create_chunk_model(self.embedder._vector_dim) - # Local filesystem handling for DB directory + # Check if database exists (for local filesystem only) + is_new_db = False if not self._has_cloud_config(): - if read_only: - # Read operations should not create the database - if not db_path.exists(): + if not db_path.exists(): + if not create: raise FileNotFoundError( - f"Database does not exist: {db_path}. Use a write operation (add, add-src) to create it." + f"Database does not exist at {db_path}. " + "Use 'haiku-rag init' to create a new database." ) - else: - # Write operations - ensure parent directories exist + is_new_db = True + # Ensure parent directories exist for new databases if not db_path.parent.exists(): Path.mkdir(db_path.parent, parents=True) # Connect to LanceDB self.db = self._connect_to_lancedb(db_path) - # Initialize tables - self.create_or_update_db() + # Initialize tables (creates them if they don't exist) + self._init_tables() + + # Run upgrades only on existing databases, set version for new ones + if is_new_db: + self._set_initial_version() + else: + self._run_upgrades() # Validate config compatibility after connection is established if not skip_validation: @@ -234,9 +240,8 @@ class Store: settings_repo = SettingsRepository(self) settings_repo.validate_config_compatibility() - def create_or_update_db(self): - """Create the database tables.""" - + def _init_tables(self): + """Initialize database tables (create if they don't exist).""" # Get list of existing tables existing_tables = self.db.table_names() @@ -271,44 +276,29 @@ class Store: [SettingsRecord(id="settings", settings=json.dumps(settings_data))] ) - # Run pending upgrades based on stored version and package version - # Skip in read-only mode to avoid modifying the database - if not self._read_only: - try: - from haiku.rag.store.upgrades import run_pending_upgrades + def _set_initial_version(self): + """Set the initial version for a new database.""" + self.set_haiku_version(metadata.version("haiku.rag-slim")) - current_version = metadata.version("haiku.rag-slim") - db_version = self.get_haiku_version() + def _run_upgrades(self): + """Run pending database upgrades.""" + try: + from haiku.rag.store.upgrades import run_pending_upgrades - if db_version != "0.0.0": - run_pending_upgrades(self, db_version, current_version) + current_version = metadata.version("haiku.rag-slim") + db_version = self.get_haiku_version() - # After upgrades complete (or if none), set stored version - # to the greater of the installed package version and the - # highest available upgrade step version in code. - try: - from packaging.version import parse as _v + run_pending_upgrades(self, db_version, current_version) - from haiku.rag.store.upgrades import upgrades as _steps - - highest_step = max((_v(u.version) for u in _steps), default=None) - effective_version = ( - str(max(_v(current_version), highest_step)) - if highest_step is not None - else current_version - ) - except Exception: - effective_version = current_version - - self.set_haiku_version(effective_version) - except Exception as e: - # Avoid hard failure on initial connection; log and continue so CLI remains usable. - logger.warning( - "Skipping upgrade due to error (db=%s -> pkg=%s): %s", - self.get_haiku_version(), - metadata.version("haiku.rag-slim"), - e, - ) + self.set_haiku_version(current_version) + except Exception as e: + # Avoid hard failure on initial connection; log and continue so CLI remains usable. + logger.warning( + "Skipping upgrade due to error (db=%s -> pkg=%s): %s", + self.get_haiku_version(), + metadata.version("haiku.rag-slim"), + e, + ) def get_haiku_version(self) -> str: """Returns the user version stored in settings.""" diff --git a/tests/conftest.py b/tests/conftest.py index 81c9295e..87bf19b8 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -22,7 +22,10 @@ def qa_corpus() -> Dataset: @pytest.fixture def temp_db_path(): - """Create a temporary database path for testing.""" + """Create a temporary database path for testing. + + Note: Tests that need a database should use HaikuRAG with create=True. + """ with tempfile.TemporaryDirectory() as temp_dir: yield Path(temp_dir) / "test.lancedb" @@ -43,11 +46,13 @@ def temp_yaml_config(tmp_path, monkeypatch): "vacuum_retention_seconds": 60, }, "embeddings": { - "provider": "ollama", - "model": "qwen3-embedding:4b", - "vector_dim": 2560, + "model": { + "provider": "ollama", + "name": "qwen3-embedding:4b", + "vector_dim": 2560, + } }, - "qa": {"provider": "ollama", "model": "gpt-oss"}, + "qa": {"model": {"provider": "ollama", "name": "gpt-oss"}}, } with open(config_file, "w") as f: diff --git a/tests/graph/test_deep_qa.py b/tests/graph/test_deep_qa.py index c96cda33..2bcaa221 100644 --- a/tests/graph/test_deep_qa.py +++ b/tests/graph/test_deep_qa.py @@ -32,7 +32,7 @@ async def test_deep_qa_graph_end_to_end(monkeypatch, temp_db_path): ) # Use real client but with TestModel for LLM calls - client = HaikuRAG(temp_db_path) + client = HaikuRAG(temp_db_path, create=True) deps = DeepQADeps(client=client) result = await graph.run(state=state, deps=deps) @@ -67,7 +67,7 @@ async def test_deep_qa_with_citations(monkeypatch, temp_db_path): ) # Use real client but with TestModel for LLM calls - client = HaikuRAG(temp_db_path) + client = HaikuRAG(temp_db_path, create=True) deps = DeepQADeps(client=client) result = await graph.run(state=state, deps=deps) diff --git a/tests/graph/test_research_graph.py b/tests/graph/test_research_graph.py index 6311ba4e..fcca2967 100644 --- a/tests/graph/test_research_graph.py +++ b/tests/graph/test_research_graph.py @@ -47,7 +47,7 @@ async def test_graph_end_to_end_with_test_model(monkeypatch, temp_db_path): ) # Use real client but with TestModel for LLM calls - client = HaikuRAG(temp_db_path) + client = HaikuRAG(temp_db_path, create=True) deps = ResearchDeps(client=client) events = [] diff --git a/tests/test_chunk.py b/tests/test_chunk.py index 08cf5984..cbb341fa 100644 --- a/tests/test_chunk.py +++ b/tests/test_chunk.py @@ -15,7 +15,7 @@ from haiku.rag.store.repositories.document import DocumentRepository async def test_chunk_repository_operations(qa_corpus: Dataset, temp_db_path): """Test ChunkRepository operations.""" # Create client - client = HaikuRAG(db_path=temp_db_path, config=Config) + client = HaikuRAG(db_path=temp_db_path, config=Config, create=True) # Get the first document from the corpus first_doc = qa_corpus[0] @@ -56,7 +56,7 @@ async def test_chunk_repository_operations(qa_corpus: Dataset, temp_db_path): async def test_create_chunks_for_document(qa_corpus: Dataset, temp_db_path): """Test creating chunks for a document.""" # Create a store and repositories - store = Store(temp_db_path) + store = Store(temp_db_path, create=True) chunk_repo = ChunkRepository(store) doc_repo = DocumentRepository(store) @@ -98,7 +98,7 @@ async def test_create_chunks_for_document(qa_corpus: Dataset, temp_db_path): async def test_chunk_repository_crud(temp_db_path): """Test basic CRUD operations in ChunkRepository.""" # Create a store - store = Store(temp_db_path) + store = Store(temp_db_path, create=True) chunk_repo = ChunkRepository(store) doc_repo = DocumentRepository(store) @@ -151,7 +151,7 @@ async def test_chunk_repository_crud(temp_db_path): @pytest.mark.asyncio async def test_adjacent_chunks(temp_db_path): """Test the get_adjacent_chunks repository method.""" - store = Store(temp_db_path) + store = Store(temp_db_path, create=True) doc_repo = DocumentRepository(store) chunk_repo = ChunkRepository(store) diff --git a/tests/test_cli.py b/tests/test_cli.py index 17b22bb1..4ef0521f 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -339,6 +339,18 @@ def test_ask_with_deep_and_verbose(): ) +def test_init(): + with patch("haiku.rag.cli.HaikuRAGApp") as mock_app: + mock_app_instance = MagicMock() + mock_app_instance.init = AsyncMock() + mock_app.return_value = mock_app_instance + + result = runner.invoke(cli, ["init"]) + + assert result.exit_code == 0 + mock_app_instance.init.assert_called_once() + + def test_info(): with patch("haiku.rag.cli.HaikuRAGApp") as mock_app: mock_app_instance = MagicMock() diff --git a/tests/test_client.py b/tests/test_client.py index 9f704557..8d648f70 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -15,7 +15,7 @@ from haiku.rag.store.models.document import Document @pytest.mark.asyncio async def test_client_document_crud(qa_corpus: Dataset, temp_db_path): """Test HaikuRAG CRUD operations for documents.""" - async with HaikuRAG(temp_db_path) as client: + async with HaikuRAG(temp_db_path, create=True) as client: # Get test data first_doc = qa_corpus[0] document_text = first_doc["document_extracted"] @@ -81,7 +81,7 @@ async def test_client_document_crud(qa_corpus: Dataset, temp_db_path): @pytest.mark.asyncio async def test_client_update_document_fields(qa_corpus: Dataset, temp_db_path): """Test updating document with individual parameters.""" - async with HaikuRAG(temp_db_path) as client: + async with HaikuRAG(temp_db_path, create=True) as client: # Get test data first_doc = qa_corpus[0] document_text = first_doc["document_extracted"] @@ -152,7 +152,7 @@ async def test_client_update_document_fields(qa_corpus: Dataset, temp_db_path): @pytest.mark.asyncio async def test_client_create_document_from_source(temp_db_path): """Test creating a document from a file source.""" - async with HaikuRAG(temp_db_path) as client: + async with HaikuRAG(temp_db_path, create=True) as client: with tempfile.TemporaryDirectory() as temp_dir: test_content = "This is test content from a file." temp_path = Path(temp_dir) / "test.txt" @@ -183,7 +183,7 @@ async def test_client_create_document_from_source(temp_db_path): @pytest.mark.asyncio async def test_client_create_document_from_source_with_title(temp_db_path): """Test creating a document from a file source with a title.""" - async with HaikuRAG(temp_db_path) as client: + async with HaikuRAG(temp_db_path, create=True) as client: with tempfile.TemporaryDirectory() as temp_dir: test_content = "This is test content from a file." temp_path = Path(temp_dir) / "test_title.txt" @@ -200,7 +200,7 @@ async def test_client_create_document_from_source_with_title(temp_db_path): @pytest.mark.asyncio async def test_client_update_title_noop_behavior(temp_db_path): """When content is unchanged, updating title should update document without re-chunking.""" - async with HaikuRAG(temp_db_path) as client: + async with HaikuRAG(temp_db_path, create=True) as client: with tempfile.TemporaryDirectory() as temp_dir: temp_path = Path(temp_dir) / "test_update_title.txt" temp_path.write_text("Original content") @@ -222,7 +222,7 @@ async def test_client_update_title_noop_behavior(temp_db_path): @pytest.mark.asyncio async def test_client_create_document_from_source_unsupported(temp_db_path): """Test creating a document from an unsupported file type.""" - async with HaikuRAG(temp_db_path) as client: + async with HaikuRAG(temp_db_path, create=True) as client: # Create a temporary file with unsupported extension with tempfile.NamedTemporaryFile( mode="w", suffix=".unsupported", delete=False @@ -238,7 +238,7 @@ async def test_client_create_document_from_source_unsupported(temp_db_path): @pytest.mark.asyncio async def test_client_create_document_from_source_nonexistent(temp_db_path): """Test creating a document from a non-existent file.""" - async with HaikuRAG(temp_db_path) as client: + async with HaikuRAG(temp_db_path, create=True) as client: non_existent_path = Path("/non/existent/file.txt") # Should raise ValueError when file doesn't exist @@ -249,7 +249,7 @@ async def test_client_create_document_from_source_nonexistent(temp_db_path): @pytest.mark.asyncio async def test_client_create_document_from_directory(temp_db_path): """Test creating documents from a directory recursively.""" - async with HaikuRAG(temp_db_path) as client: + async with HaikuRAG(temp_db_path, create=True) as client: with tempfile.TemporaryDirectory() as temp_dir: test_dir = Path(temp_dir) / "test_docs" test_dir.mkdir() @@ -294,7 +294,7 @@ async def test_client_create_document_from_directory_with_filters( "haiku.rag.client.Config.monitor.include_patterns", ["**/include/**/*.txt"] ) - async with HaikuRAG(temp_db_path) as client: + async with HaikuRAG(temp_db_path, create=True) as client: with tempfile.TemporaryDirectory() as temp_dir: test_dir = Path(temp_dir) / "test_docs" test_dir.mkdir() @@ -333,7 +333,7 @@ async def test_client_create_document_from_directory_with_filters( @pytest.mark.asyncio async def test_client_create_document_from_url(temp_db_path): """Test creating a document from a URL.""" - async with HaikuRAG(temp_db_path) as client: + async with HaikuRAG(temp_db_path, create=True) as client: # Mock the HTTP response mock_response = AsyncMock() mock_response.content = b"

Test Page

This is test content from a webpage.

" @@ -361,7 +361,7 @@ async def test_client_create_document_from_url_with_different_content_types( temp_db_path, ): """Test creating documents from URLs with different content types.""" - async with HaikuRAG(temp_db_path) as client: + async with HaikuRAG(temp_db_path, create=True) as client: # Test JSON content mock_json_response = AsyncMock() mock_json_response.content = ( @@ -406,7 +406,7 @@ async def test_client_create_document_from_url_with_different_content_types( @pytest.mark.asyncio async def test_client_create_document_from_url_unsupported_content(temp_db_path): """Test creating a document from URL with unsupported content type.""" - async with HaikuRAG(temp_db_path) as client: + async with HaikuRAG(temp_db_path, create=True) as client: # Mock response with unsupported content type mock_response = AsyncMock() mock_response.content = b"binary content" @@ -423,7 +423,7 @@ async def test_client_create_document_from_url_unsupported_content(temp_db_path) @pytest.mark.asyncio async def test_client_create_document_from_url_http_error(temp_db_path): """Test handling HTTP errors when creating document from URL.""" - async with HaikuRAG(temp_db_path) as client: + async with HaikuRAG(temp_db_path, create=True) as client: with patch("httpx.AsyncClient.get") as mock_get: mock_get.side_effect = httpx.HTTPStatusError( "404 Not Found", @@ -440,7 +440,7 @@ async def test_client_create_document_from_url_http_error(temp_db_path): @pytest.mark.asyncio async def test_get_extension_from_content_type_or_url(temp_db_path): """Test the helper method for determining file extensions.""" - async with HaikuRAG(temp_db_path) as client: + async with HaikuRAG(temp_db_path, create=True) as client: # Test content type mappings assert ( client._get_extension_from_content_type_or_url("", "text/html") == ".html" @@ -487,7 +487,7 @@ async def test_client_metadata_content_type_and_md5(temp_db_path): """Test that contentType and md5 metadata are correctly set.""" import hashlib - async with HaikuRAG(temp_db_path) as client: + async with HaikuRAG(temp_db_path, create=True) as client: # Create a temporary file with known content test_content = "Test content for MD5 calculation." expected_md5 = hashlib.md5(test_content.encode()).hexdigest() @@ -520,7 +520,7 @@ async def test_client_metadata_content_type_and_md5(temp_db_path): @pytest.mark.asyncio async def test_client_create_update_no_op_behavior(temp_db_path): """Test create/update/no-op behavior based on MD5 changes.""" - async with HaikuRAG(temp_db_path) as client: + async with HaikuRAG(temp_db_path, create=True) as client: # Create a temporary file test_content = "Original content for testing." with tempfile.TemporaryDirectory() as temp_dir: @@ -559,7 +559,7 @@ async def test_client_create_update_no_op_behavior(temp_db_path): @pytest.mark.asyncio async def test_client_unchanged_file_keeps_timestamp(temp_db_path): """Test that unchanged files don't update the updated_at timestamp.""" - async with HaikuRAG(temp_db_path) as client: + async with HaikuRAG(temp_db_path, create=True) as client: # Create a temporary file test_content = "Test content for timestamp check." with tempfile.TemporaryDirectory() as temp_dir: @@ -581,7 +581,7 @@ async def test_client_unchanged_file_keeps_timestamp(temp_db_path): @pytest.mark.asyncio async def test_client_url_create_update_no_op_behavior(temp_db_path): """Test create/update/no-op behavior for URLs based on MD5 changes.""" - async with HaikuRAG(temp_db_path) as client: + async with HaikuRAG(temp_db_path, create=True) as client: url = "https://example.com/test.txt" original_content = b"Original URL content" updated_content = b"Updated URL content" @@ -620,7 +620,7 @@ async def test_client_url_create_update_no_op_behavior(temp_db_path): @pytest.mark.asyncio async def test_client_search(temp_db_path): """Test HaikuRAG search functionality.""" - async with HaikuRAG(temp_db_path) as client: + async with HaikuRAG(temp_db_path, create=True) as client: # Add multiple documents to search from doc1_text = "Python is a high-level programming language known for its simplicity and readability." doc2_text = "Machine learning algorithms help computers learn patterns from data without explicit programming." @@ -665,7 +665,7 @@ async def test_client_async_context_manager(temp_db_path): """Test HaikuRAG as async context manager.""" # Test that context manager works and auto-closes - async with HaikuRAG(temp_db_path) as client: + async with HaikuRAG(temp_db_path, create=True) as client: # Create a document to ensure the client works doc = await client.create_document( content="Test content for context manager", @@ -688,7 +688,7 @@ async def test_client_async_context_manager(temp_db_path): @pytest.mark.asyncio async def test_client_create_document_with_custom_chunks(temp_db_path): """Test creating a document with pre-created chunks.""" - async with HaikuRAG(temp_db_path) as client: + async with HaikuRAG(temp_db_path, create=True) as client: # Create some custom chunks with and without embeddings chunks = [ Chunk( @@ -741,7 +741,7 @@ async def test_client_ask_without_cite(monkeypatch, temp_db_path): "haiku.rag.utils.get_model", lambda *args, **kwargs: TestModel() ) - async with HaikuRAG(temp_db_path) as client: + async with HaikuRAG(temp_db_path, create=True) as client: # Create a test document for the agent to search await client.create_document( content="Python is a high-level programming language.", uri="test.txt" @@ -765,7 +765,7 @@ async def test_client_ask_with_cite(monkeypatch, temp_db_path): "haiku.rag.utils.get_model", lambda *args, **kwargs: TestModel() ) - async with HaikuRAG(temp_db_path) as client: + async with HaikuRAG(temp_db_path, create=True) as client: # Create a test document await client.create_document( content="Python is a high-level programming language.", uri="test.txt" @@ -784,7 +784,7 @@ async def test_client_expand_context(temp_db_path): """Test expanding search results with adjacent chunks.""" # Mock Config to have CONTEXT_CHUNK_RADIUS = 2 with patch("haiku.rag.client.Config.processing.context_chunk_radius", 2): - async with HaikuRAG(temp_db_path) as client: + async with HaikuRAG(temp_db_path, create=True) as client: # Create chunks manually with precomputed embeddings to avoid network dim = client.chunk_repository.embedder._vector_dim z = [0.0] * dim @@ -836,7 +836,7 @@ async def test_client_expand_context(temp_db_path): @pytest.mark.asyncio async def test_client_expand_context_radius_zero(temp_db_path): """Test expand_context with radius 0 returns original results.""" - async with HaikuRAG(temp_db_path) as client: + async with HaikuRAG(temp_db_path, create=True) as client: # Create a simple document doc = await client.create_document(content="Simple test content") assert doc.id is not None @@ -853,7 +853,7 @@ async def test_client_expand_context_radius_zero(temp_db_path): async def test_client_expand_context_multiple_chunks(temp_db_path): """Test expand_context with multiple search results.""" with patch("haiku.rag.client.Config.processing.context_chunk_radius", 1): - async with HaikuRAG(temp_db_path) as client: + async with HaikuRAG(temp_db_path, create=True) as client: # Create first document with manual chunks doc1_chunks = [ Chunk(content="Doc1 Part A", order=0), @@ -906,7 +906,7 @@ async def test_client_expand_context_multiple_chunks(temp_db_path): @pytest.mark.asyncio async def test_client_expand_context_merges_overlapping_chunks(temp_db_path): """Test that overlapping expanded chunks are merged into one.""" - async with HaikuRAG(temp_db_path) as client: + async with HaikuRAG(temp_db_path, create=True) as client: # Create document with 5 chunks manual_chunks = [ Chunk(content="Chunk 0", order=0), @@ -953,7 +953,7 @@ async def test_client_expand_context_merges_overlapping_chunks(temp_db_path): @pytest.mark.asyncio async def test_client_expand_context_keeps_separate_non_overlapping(temp_db_path): """Test that non-overlapping expanded chunks remain separate.""" - async with HaikuRAG(temp_db_path) as client: + async with HaikuRAG(temp_db_path, create=True) as client: # Create document with chunks far apart manual_chunks = [ Chunk(content="Chunk 0", order=0), diff --git a/tests/test_database_autocreate.py b/tests/test_database_autocreate.py index 77ef7c15..aabb6306 100644 --- a/tests/test_database_autocreate.py +++ b/tests/test_database_autocreate.py @@ -7,93 +7,43 @@ from haiku.rag.client import HaikuRAG from haiku.rag.config import AppConfig -def test_read_operations_do_not_create_database(): - """Test that read operations fail if database doesn't exist.""" +def test_database_not_created_without_create_flag(): + """Test that database is not created without create=True.""" with tempfile.TemporaryDirectory() as tmpdir: db_path = Path(tmpdir) / "test.lancedb" config = AppConfig() - # Read operation with read_only=True should fail - with pytest.raises( - FileNotFoundError, - match="Database does not exist.*Use a write operation", - ): - HaikuRAG(db_path=db_path, config=config, read_only=True) + with pytest.raises(FileNotFoundError, match="Database does not exist"): + HaikuRAG(db_path=db_path, config=config) -def test_write_operations_create_database(): - """Test that write operations create the database.""" +def test_database_created_with_create_flag(): + """Test that database is created with create=True.""" with tempfile.TemporaryDirectory() as tmpdir: db_path = Path(tmpdir) / "test.lancedb" config = AppConfig() - # Write operation with read_only=False (default) should succeed - client = HaikuRAG(db_path=db_path, config=config, read_only=False) + client = HaikuRAG(db_path=db_path, config=config, create=True) assert db_path.exists() client.close() -async def test_add_document_creates_database(): - """Test that add operations create the database.""" +@pytest.mark.asyncio +async def test_operations_work_after_database_created(): + """Test that operations work after DB is created.""" with tempfile.TemporaryDirectory() as tmpdir: db_path = Path(tmpdir) / "test.lancedb" config = AppConfig() - # Create a document (write operation) should work and create DB - async with HaikuRAG(db_path=db_path, config=config, read_only=False) as client: - doc = await client.create_document("Test content") - assert doc.id is not None - assert doc.content == "Test content" - assert db_path.exists() - - -async def test_search_fails_if_database_does_not_exist(): - """Test that search operations fail if DB doesn't exist.""" - with tempfile.TemporaryDirectory() as tmpdir: - db_path = Path(tmpdir) / "test.lancedb" - - config = AppConfig() - - # Read operation (search) should fail if DB doesn't exist - with pytest.raises( - FileNotFoundError, - match="Database does not exist.*Use a write operation", - ): - async with HaikuRAG( - db_path=db_path, config=config, read_only=True - ) as client: - await client.search("test query") - - -async def test_read_operations_work_after_database_created(): - """Test that read operations work after DB is created via write operation.""" - with tempfile.TemporaryDirectory() as tmpdir: - db_path = Path(tmpdir) / "test.lancedb" - - config = AppConfig() - - # First, create DB via write operation - async with HaikuRAG(db_path=db_path, config=config, read_only=False) as client: + # First, create DB with create=True and add document + async with HaikuRAG(db_path=db_path, config=config, create=True) as client: await client.create_document("Test content", uri="test://doc1") - # Now read operations should work since DB exists - async with HaikuRAG(db_path=db_path, config=config, read_only=True) as client: + # Re-open without create flag and verify we can read the document + async with HaikuRAG(db_path=db_path, config=config) as client: docs = await client.list_documents() assert len(docs) == 1 assert docs[0].content == "Test content" - - -def test_default_read_only_is_false(): - """Test that read_only defaults to False for backward compatibility.""" - with tempfile.TemporaryDirectory() as tmpdir: - db_path = Path(tmpdir) / "test.lancedb" - - config = AppConfig() - - # Without specifying read_only, it should default to False (allow creation) - client = HaikuRAG(db_path=db_path, config=config) - assert db_path.exists() - client.close() diff --git a/tests/test_document.py b/tests/test_document.py index 61e3ae84..97855a39 100644 --- a/tests/test_document.py +++ b/tests/test_document.py @@ -12,7 +12,7 @@ from haiku.rag.store.repositories.document import DocumentRepository async def test_create_document_with_chunks(qa_corpus: Dataset, temp_db_path): """Test creating a document with chunks from the qa_corpus using repository.""" # Create client - client = HaikuRAG(db_path=temp_db_path, config=Config) + client = HaikuRAG(db_path=temp_db_path, config=Config, create=True) # Get the first document from the corpus first_doc = qa_corpus[0] @@ -44,7 +44,7 @@ async def test_create_document_with_chunks(qa_corpus: Dataset, temp_db_path): async def test_document_repository_crud(qa_corpus: Dataset, temp_db_path): """Test CRUD operations in DocumentRepository.""" # Create a store and repository - store = Store(temp_db_path) + store = Store(temp_db_path, create=True) doc_repo = DocumentRepository(store) # Get the first document from the corpus @@ -100,7 +100,7 @@ async def test_document_repository_crud(qa_corpus: Dataset, temp_db_path): @pytest.mark.asyncio async def test_document_list_with_filter(qa_corpus: Dataset, temp_db_path): """Test listing documents with filter clause.""" - store = Store(temp_db_path) + store = Store(temp_db_path, create=True) doc_repo = DocumentRepository(store) first_doc = qa_corpus[0] diff --git a/tests/test_filter.py b/tests/test_filter.py index 8e8664a8..cc9c4918 100644 --- a/tests/test_filter.py +++ b/tests/test_filter.py @@ -6,7 +6,7 @@ from haiku.rag.client import HaikuRAG @pytest.mark.asyncio async def test_search_with_uri_filter(temp_db_path): """Test filtering by document URI.""" - async with HaikuRAG(db_path=temp_db_path) as client: + async with HaikuRAG(db_path=temp_db_path, create=True) as client: # Add multiple test documents await client.create_document( content="Python tutorial content", @@ -40,7 +40,7 @@ async def test_search_with_uri_filter(temp_db_path): @pytest.mark.asyncio async def test_search_with_title_filter(temp_db_path): """Test filtering by document title.""" - async with HaikuRAG(db_path=temp_db_path) as client: + async with HaikuRAG(db_path=temp_db_path, create=True) as client: # Add test documents await client.create_document( content="Programming content", @@ -66,7 +66,7 @@ async def test_search_with_title_filter(temp_db_path): @pytest.mark.asyncio async def test_search_with_combined_filters(temp_db_path): """Test filtering with AND/OR conditions.""" - async with HaikuRAG(db_path=temp_db_path) as client: + async with HaikuRAG(db_path=temp_db_path, create=True) as client: # Add test documents await client.create_document( content="Content about AI", @@ -105,7 +105,7 @@ async def test_search_with_combined_filters(temp_db_path): @pytest.mark.asyncio async def test_search_with_no_matching_filter(temp_db_path): """Test that search returns empty results when filter matches no documents.""" - async with HaikuRAG(db_path=temp_db_path) as client: + async with HaikuRAG(db_path=temp_db_path, create=True) as client: # Add a test document await client.create_document( content="Test content", @@ -123,7 +123,7 @@ async def test_search_with_no_matching_filter(temp_db_path): @pytest.mark.asyncio async def test_search_with_invalid_filter(temp_db_path): """Test that invalid filter syntax raises an appropriate error.""" - async with HaikuRAG(db_path=temp_db_path) as client: + async with HaikuRAG(db_path=temp_db_path, create=True) as client: # Add a test document await client.create_document( content="Test content", @@ -139,7 +139,7 @@ async def test_search_with_invalid_filter(temp_db_path): @pytest.mark.asyncio async def test_search_filter_with_all_search_types(temp_db_path): """Test that filtering works with all search types (vector, fts, hybrid).""" - async with HaikuRAG(db_path=temp_db_path) as client: + async with HaikuRAG(db_path=temp_db_path, create=True) as client: await client.create_document( content="Machine learning is a subset of artificial intelligence", uri="https://ai.example.com/ml.html", diff --git a/tests/test_info.py b/tests/test_info.py index 7bd6c458..ba909a96 100644 --- a/tests/test_info.py +++ b/tests/test_info.py @@ -6,7 +6,7 @@ from haiku.rag.app import HaikuRAGApp @pytest.mark.asyncio -async def test_app_info_outputs_and_read_only(temp_db_path, capsys): +async def test_app_info_outputs(temp_db_path, capsys): # Build a minimal LanceDB with settings, documents, and chunks without using Store import lancedb from lancedb.pydantic import LanceModel, Vector @@ -32,7 +32,7 @@ async def test_app_info_outputs_and_read_only(temp_db_path, capsys): docs_tbl = db.create_table("documents", schema=DocumentRecord) chunks_tbl = db.create_table("chunks", schema=ChunkRecord) - # Insert one of each + # Insert one of each - using the new config format settings_tbl.add( [ SettingsRecord( @@ -41,9 +41,11 @@ async def test_app_info_outputs_and_read_only(temp_db_path, capsys): { "version": "1.2.3", "embeddings": { - "provider": "openai", - "model": "text-embedding-3-small", - "vector_dim": 3, + "model": { + "provider": "openai", + "name": "text-embedding-3-small", + "vector_dim": 3, + } }, } ), @@ -55,20 +57,13 @@ async def test_app_info_outputs_and_read_only(temp_db_path, capsys): [ChunkRecord(id="c1", document_id="doc-1", content="c", vector=[0.1, 0.2, 0.3])] ) - # Capture versions before - before_versions = { - "settings": int(settings_tbl.version), - "documents": int(docs_tbl.version), - "chunks": int(chunks_tbl.version), - } - app = HaikuRAGApp(db_path=temp_db_path) await app.info() out = capsys.readouterr().out # Validate expected content substrings assert f"path: \n{temp_db_path}" in out - assert "haiku.rag version (db): 1.2.3" in out + assert "haiku.rag version (db):" in out assert "embeddings: openai/text-embedding-3-small (dim: 3)" in out assert "documents: 1" in out assert "chunks: 1" in out @@ -85,13 +80,6 @@ async def test_app_info_outputs_and_read_only(temp_db_path, capsys): assert "lancedb:" in out assert "haiku.rag:" in out - # Verify no versions changed (read-only) - # Re-open to ensure fresh view - db2 = lancedb.connect(temp_db_path) - assert int(db2.open_table("settings").version) == before_versions["settings"] - assert int(db2.open_table("documents").version) == before_versions["documents"] - assert int(db2.open_table("chunks").version) == before_versions["chunks"] - @pytest.mark.asyncio async def test_app_info_with_vector_index(temp_db_path, capsys): @@ -148,13 +136,6 @@ async def test_app_info_with_vector_index(temp_db_path, capsys): # Create vector index chunks_tbl.create_index(metric="cosine", index_type="IVF_PQ") - # Capture versions before - before_versions = { - "settings": int(settings_tbl.version), - "documents": int(docs_tbl.version), - "chunks": int(chunks_tbl.version), - } - app = HaikuRAGApp(db_path=temp_db_path) await app.info() @@ -168,9 +149,3 @@ async def test_app_info_with_vector_index(temp_db_path, capsys): # Check basic info still present assert "documents: 1" in out assert "chunks: 512" in out - - # Verify no versions changed (read-only) - db2 = lancedb.connect(temp_db_path) - assert int(db2.open_table("settings").version) == before_versions["settings"] - assert int(db2.open_table("documents").version) == before_versions["documents"] - assert int(db2.open_table("chunks").version) == before_versions["chunks"] diff --git a/tests/test_lancedb_connection.py b/tests/test_lancedb_connection.py index d1de0877..e9b21ad5 100644 --- a/tests/test_lancedb_connection.py +++ b/tests/test_lancedb_connection.py @@ -10,7 +10,7 @@ from haiku.rag.store.engine import Store async def test_lancedb_cloud_skips_optimization(temp_db_path): """Test that vacuum is skipped when using LanceDB Cloud (db:// URI).""" # Create a store - store = Store(temp_db_path) + store = Store(temp_db_path, create=True) # Mock all cloud config to simulate LanceDB Cloud usage with ( @@ -33,7 +33,7 @@ async def test_lancedb_cloud_skips_optimization(temp_db_path): async def test_local_storage_calls_optimization(temp_db_path): """Test that vacuum calls optimization for local storage.""" # Create a store - store = Store(temp_db_path) + store = Store(temp_db_path, create=True) # Ensure uri is empty (local storage) with patch.object(Config.lancedb, "uri", ""): diff --git a/tests/test_preprocessor.py b/tests/test_preprocessor.py index 9f7d9b07..34339c50 100644 --- a/tests/test_preprocessor.py +++ b/tests/test_preprocessor.py @@ -45,7 +45,7 @@ def add_marker(text: str) -> str: try: Config.processing.markdown_preprocessor = f"{pre_file}:add_marker" - store = Store(temp_db_path) + store = Store(temp_db_path, create=True) chunk_repo = ChunkRepository(store) doc_repo = DocumentRepository(store) diff --git a/tests/test_qa.py b/tests/test_qa.py index 08f41521..113f5219 100644 --- a/tests/test_qa.py +++ b/tests/test_qa.py @@ -17,7 +17,7 @@ VLLM_QA_AVAILABLE = bool(Config.providers.vllm.qa_base_url) @pytest.mark.asyncio async def test_qa_ollama(qa_corpus: Dataset, temp_db_path): """Test Ollama QA with LLM judge.""" - client = HaikuRAG(temp_db_path) + client = HaikuRAG(temp_db_path, create=True) qa = QuestionAnswerAgent( client, ModelConfig(provider="ollama", name="gpt-oss", enable_thinking=False) ) @@ -43,7 +43,7 @@ async def test_qa_ollama(qa_corpus: Dataset, temp_db_path): @pytest.mark.skipif(not OPENAI_AVAILABLE, reason="OpenAI not available") async def test_qa_openai(qa_corpus: Dataset, temp_db_path): """Test OpenAI QA with LLM judge.""" - client = HaikuRAG(temp_db_path) + client = HaikuRAG(temp_db_path, create=True) qa = QuestionAnswerAgent(client, ModelConfig(provider="openai", name="gpt-4o-mini")) llm_judge = LLMJudge() @@ -67,7 +67,7 @@ async def test_qa_openai(qa_corpus: Dataset, temp_db_path): @pytest.mark.skipif(not ANTHROPIC_AVAILABLE, reason="Anthropic not available") async def test_qa_anthropic(qa_corpus: Dataset, temp_db_path): """Test Anthropic QA with LLM judge.""" - client = HaikuRAG(temp_db_path) + client = HaikuRAG(temp_db_path, create=True) qa = QuestionAnswerAgent( client, ModelConfig(provider="anthropic", name="claude-3-5-haiku-20241022") ) @@ -93,7 +93,7 @@ async def test_qa_anthropic(qa_corpus: Dataset, temp_db_path): @pytest.mark.skipif(not VLLM_QA_AVAILABLE, reason="vLLM QA server not configured") async def test_qa_vllm(qa_corpus: Dataset, temp_db_path): """Test vLLM QA with LLM judge.""" - client = HaikuRAG(temp_db_path) + client = HaikuRAG(temp_db_path, create=True) qa = QuestionAnswerAgent(client, ModelConfig(provider="vllm", name="Qwen/Qwen3-4B")) llm_judge = LLMJudge() diff --git a/tests/test_rebuild.py b/tests/test_rebuild.py index d145fd4a..2f749faf 100644 --- a/tests/test_rebuild.py +++ b/tests/test_rebuild.py @@ -7,7 +7,7 @@ from haiku.rag.client import HaikuRAG, RebuildMode @pytest.mark.asyncio async def test_rebuild_full(qa_corpus: Dataset, temp_db_path): """Test full rebuild: converts, chunks, and embeds all documents.""" - async with HaikuRAG(temp_db_path) as client: + async with HaikuRAG(temp_db_path, create=True) as client: doc = await client.create_document(content=qa_corpus["document_extracted"][0]) assert doc.id is not None @@ -30,7 +30,7 @@ async def test_rebuild_full(qa_corpus: Dataset, temp_db_path): @pytest.mark.asyncio async def test_rebuild_embed_only(qa_corpus: Dataset, temp_db_path): """Test embed-only rebuild: keeps chunks, only regenerates embeddings.""" - async with HaikuRAG(temp_db_path) as client: + async with HaikuRAG(temp_db_path, create=True) as client: doc = await client.create_document(content=qa_corpus["document_extracted"][0]) assert doc.id is not None @@ -60,7 +60,7 @@ async def test_rebuild_embed_only(qa_corpus: Dataset, temp_db_path): @pytest.mark.asyncio async def test_rebuild_embed_only_skips_unchanged(qa_corpus: Dataset, temp_db_path): """Test embed-only rebuild skips chunks with unchanged embeddings.""" - async with HaikuRAG(temp_db_path) as client: + async with HaikuRAG(temp_db_path, create=True) as client: doc = await client.create_document(content=qa_corpus["document_extracted"][0]) assert doc.id is not None @@ -96,7 +96,7 @@ async def test_rebuild_embed_only_skips_unchanged(qa_corpus: Dataset, temp_db_pa @pytest.mark.asyncio async def test_rebuild_rechunk(qa_corpus: Dataset, temp_db_path): """Test rechunk rebuild: re-chunks from content without accessing source files.""" - async with HaikuRAG(temp_db_path) as client: + async with HaikuRAG(temp_db_path, create=True) as client: doc = await client.create_document(content=qa_corpus["document_extracted"][0]) assert doc.id is not None diff --git a/tests/test_search.py b/tests/test_search.py index 1783aa90..6436d88f 100644 --- a/tests/test_search.py +++ b/tests/test_search.py @@ -9,7 +9,7 @@ from haiku.rag.config import Config async def test_search_qa_corpus(qa_corpus: Dataset, temp_db_path): """Test that documents can be found by searching with their associated questions.""" # Create client - client = HaikuRAG(db_path=temp_db_path, config=Config) + client = HaikuRAG(db_path=temp_db_path, config=Config, create=True) # Load unique documents (limited to 10) seen_documents = set() @@ -61,7 +61,7 @@ async def test_search_qa_corpus(qa_corpus: Dataset, temp_db_path): @pytest.mark.asyncio async def test_chunks_include_document_info(temp_db_path): """Test that search results include document URI and metadata.""" - client = HaikuRAG(db_path=temp_db_path, config=Config) + client = HaikuRAG(db_path=temp_db_path, config=Config, create=True) # Create a document with URI and metadata created_document = await client.create_document( @@ -93,7 +93,7 @@ async def test_chunks_include_document_info(temp_db_path): @pytest.mark.asyncio async def test_chunks_include_document_title(temp_db_path): """Test that search results include the parent document title when present.""" - client = HaikuRAG(db_path=temp_db_path, config=Config) + client = HaikuRAG(db_path=temp_db_path, config=Config, create=True) # Create a document with URI and title await client.create_document( @@ -119,7 +119,7 @@ async def test_chunks_include_document_title(temp_db_path): @pytest.mark.asyncio async def test_search_score_types(temp_db_path): """Test that different search types return appropriate score ranges.""" - client = HaikuRAG(db_path=temp_db_path, config=Config) + client = HaikuRAG(db_path=temp_db_path, config=Config, create=True) # Create multiple documents with different content documents_content = [ diff --git a/tests/test_settings.py b/tests/test_settings.py index d6e72d29..8e0f2007 100644 --- a/tests/test_settings.py +++ b/tests/test_settings.py @@ -12,7 +12,7 @@ def test_settings_table_populated_on_store_init(temp_db_path): from haiku.rag.store.engine import Store from haiku.rag.store.repositories.settings import SettingsRepository - store = Store(temp_db_path) + store = Store(temp_db_path, create=True) settings_repo = SettingsRepository(store) db_settings = settings_repo.get_current_settings() @@ -32,7 +32,7 @@ def test_settings_save_and_retrieve(temp_db_path): from haiku.rag.store.engine import Store from haiku.rag.store.repositories.settings import SettingsRepository - store = Store(temp_db_path) + store = Store(temp_db_path, create=True) settings_repo = SettingsRepository(store) original_chunk_size = Config.processing.chunk_size @@ -53,7 +53,7 @@ async def test_config_validation_on_db_load(temp_db_path): from haiku.rag.store.repositories.settings import SettingsRepository # Create store and save settings - store1 = Store(temp_db_path) + store1 = Store(temp_db_path, create=True) store1.close() # Change config @@ -63,18 +63,20 @@ async def test_config_validation_on_db_load(temp_db_path): try: # Loading the database should raise ConfigMismatchError with pytest.raises(ConfigMismatchError) as exc_info: - Store(temp_db_path) + Store(temp_db_path, create=True) assert "chunk_size" in str(exc_info.value) assert "rebuild" in str(exc_info.value).lower() # Rebuild - async with HaikuRAG(db_path=temp_db_path, skip_validation=True) as client: + async with HaikuRAG( + db_path=temp_db_path, skip_validation=True, create=True + ) as client: async for _ in client.rebuild_database(): pass # Process all documents # Verify we can now load the database without exception (settings were updated) - store2 = Store(temp_db_path) + store2 = Store(temp_db_path, create=True) settings_repo2 = SettingsRepository(store2) db_settings = settings_repo2.get_current_settings() assert db_settings["processing"]["chunk_size"] == 999 diff --git a/tests/test_versioning.py b/tests/test_versioning.py index e38986b6..2c8a5789 100644 --- a/tests/test_versioning.py +++ b/tests/test_versioning.py @@ -10,7 +10,7 @@ from haiku.rag.store.repositories.document import DocumentRepository @pytest.mark.asyncio async def test_version_rollback_on_create_failure(temp_db_path): - store = Store(temp_db_path) + store = Store(temp_db_path, create=True) repo = DocumentRepository(store) # Ensure chunk repository is instantiated and stub embeddings to avoid network @@ -51,7 +51,7 @@ async def test_version_rollback_on_create_failure(temp_db_path): @pytest.mark.asyncio async def test_version_rollback_on_update_failure(temp_db_path): - store = Store(temp_db_path) + store = Store(temp_db_path, create=True) repo = DocumentRepository(store) # Stub embeddings to avoid network @@ -106,11 +106,11 @@ def test_new_database_does_not_run_upgrades(monkeypatch, temp_db_path): fail_if_called, ) - Store(temp_db_path) + Store(temp_db_path, create=True) def test_existing_database_runs_upgrades(monkeypatch, temp_db_path): - Store(temp_db_path) + Store(temp_db_path, create=True) called = {"value": False} @@ -122,6 +122,7 @@ def test_existing_database_runs_upgrades(monkeypatch, temp_db_path): mark_called, ) + # Opening an existing database should trigger upgrades Store(temp_db_path) assert called["value"] @@ -129,7 +130,7 @@ def test_existing_database_runs_upgrades(monkeypatch, temp_db_path): @pytest.mark.asyncio async def test_vacuum_with_retention_threshold(temp_db_path): - store = Store(temp_db_path) + store = Store(temp_db_path, create=True) repo = DocumentRepository(store) # Stub embeddings to avoid network @@ -209,14 +210,14 @@ async def test_vacuum_completes_before_context_exit(temp_db_path, monkeypatch): # Set aggressive vacuum retention for this test monkeypatch.setattr(Config.storage, "vacuum_retention_seconds", 0) - async with HaikuRAG(db_path=temp_db_path) as client: + async with HaikuRAG(db_path=temp_db_path, create=True) as client: # Create multiple documents - each creation triggers automatic vacuum with retention=0 # This aggressively cleans up old versions between operations for i in range(3): await client.create_document(content=f"Test document {i}") # After context exit, automatic vacuum should have kept versions minimal - store = Store(temp_db_path) + store = Store(temp_db_path, create=True) final_versions = len(list(store.documents_table.list_versions())) # With retention_seconds=0, vacuum aggressively cleans up between operations