From a59b38158d64be0a4118aa3c2800f797488e7aa5 Mon Sep 17 00:00:00 2001 From: Yiorgis Gozadinos Date: Sat, 12 Jul 2025 20:21:34 +0300 Subject: [PATCH 1/6] Settings table & repo, tests --- pyproject.toml | 2 +- src/haiku/rag/store/engine.py | 17 +++++++++ src/haiku/rag/store/repositories/settings.py | 36 ++++++++++++++++++++ tests/test_settings.py | 33 ++++++++++++++++++ uv.lock | 8 ++--- 5 files changed, 91 insertions(+), 5 deletions(-) create mode 100644 src/haiku/rag/store/repositories/settings.py create mode 100644 tests/test_settings.py diff --git a/pyproject.toml b/pyproject.toml index 49237c9a..30dbb443 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -56,7 +56,7 @@ dev = [ "mkdocs>=1.6.1", "mkdocs-material>=9.6.14", "pre-commit>=4.2.0", - "pyright>=1.1.402", + "pyright>=1.1.403", "pytest>=8.4.0", "pytest-asyncio>=1.0.0", "pytest-cov>=6.2.1", diff --git a/src/haiku/rag/store/engine.py b/src/haiku/rag/store/engine.py index cdf1d2ca..4133b6dc 100644 --- a/src/haiku/rag/store/engine.py +++ b/src/haiku/rag/store/engine.py @@ -60,11 +60,28 @@ class Store: ) """) + # Create settings table for storing current configuration + db.execute(""" + CREATE TABLE IF NOT EXISTS settings ( + id INTEGER PRIMARY KEY DEFAULT 1, + settings TEXT NOT NULL DEFAULT '{}' + ) + """) + # Create indexes for better performance db.execute( "CREATE INDEX IF NOT EXISTS idx_chunks_document_id ON chunks(document_id)" ) + # Save current settings to the new database + from haiku.rag.config import Config + + settings_json = Config.model_dump_json() + db.execute( + "INSERT OR IGNORE INTO settings (id, settings) VALUES (1, ?)", + (settings_json,), + ) + db.commit() return db diff --git a/src/haiku/rag/store/repositories/settings.py b/src/haiku/rag/store/repositories/settings.py new file mode 100644 index 00000000..d51cb0d5 --- /dev/null +++ b/src/haiku/rag/store/repositories/settings.py @@ -0,0 +1,36 @@ +import json +from typing import Any + +from haiku.rag.store.engine import Store + + +class SettingsRepository: + def __init__(self, store: Store): + self.store = store + + def get(self) -> dict[str, Any]: + """Get all settings from the database.""" + if self.store._connection is None: + raise ValueError("Store connection is not available") + + cursor = self.store._connection.execute("SELECT settings FROM settings LIMIT 1") + row = cursor.fetchone() + if row: + return json.loads(row[0]) + return {} + + def save(self) -> None: + """Sync settings from the current AppConfig to database.""" + if self.store._connection is None: + raise ValueError("Store connection is not available") + + from haiku.rag.config import Config + + settings_json = Config.model_dump_json() + + self.store._connection.execute( + "INSERT INTO settings (id, settings) VALUES (1, ?) ON CONFLICT(id) DO UPDATE SET settings = excluded.settings", + (settings_json,), + ) + + self.store._connection.commit() diff --git a/tests/test_settings.py b/tests/test_settings.py new file mode 100644 index 00000000..aaec6ac5 --- /dev/null +++ b/tests/test_settings.py @@ -0,0 +1,33 @@ +from haiku.rag.config import Config +from haiku.rag.store.engine import Store +from haiku.rag.store.repositories.settings import SettingsRepository + + +def test_settings_table_populated_on_store_init(): + """Test that settings table is populated with current config when store is initialized.""" + + store = Store(":memory:") + settings_repo = SettingsRepository(store) + + db_settings = settings_repo.get() + config_dict = Config.model_dump(mode="json") + + assert db_settings == config_dict + + store.close() + + +def test_settings_save_and_retrieve(): + """Test saving and retrieving settings after config change.""" + store = Store(":memory:") + settings_repo = SettingsRepository(store) + + original_chunk_size = Config.CHUNK_SIZE + Config.CHUNK_SIZE = 2 * original_chunk_size + + settings_repo.save() + retrieved_settings = settings_repo.get() + assert retrieved_settings["CHUNK_SIZE"] == 2 * original_chunk_size + + Config.CHUNK_SIZE = original_chunk_size + store.close() diff --git a/uv.lock b/uv.lock index 088ad51f..fd6b3649 100644 --- a/uv.lock +++ b/uv.lock @@ -881,7 +881,7 @@ dev = [ { name = "mkdocs", specifier = ">=1.6.1" }, { name = "mkdocs-material", specifier = ">=9.6.14" }, { name = "pre-commit", specifier = ">=4.2.0" }, - { name = "pyright", specifier = ">=1.1.402" }, + { name = "pyright", specifier = ">=1.1.403" }, { name = "pytest", specifier = ">=8.4.0" }, { name = "pytest-asyncio", specifier = ">=1.0.0" }, { name = "pytest-cov", specifier = ">=6.2.1" }, @@ -2305,15 +2305,15 @@ wheels = [ [[package]] name = "pyright" -version = "1.1.402" +version = "1.1.403" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "nodeenv" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/aa/04/ce0c132d00e20f2d2fb3b3e7c125264ca8b909e693841210534b1ea1752f/pyright-1.1.402.tar.gz", hash = "sha256:85a33c2d40cd4439c66aa946fd4ce71ab2f3f5b8c22ce36a623f59ac22937683", size = 3888207, upload-time = "2025-06-11T08:48:35.759Z" } +sdist = { url = "https://files.pythonhosted.org/packages/fe/f6/35f885264ff08c960b23d1542038d8da86971c5d8c955cfab195a4f672d7/pyright-1.1.403.tar.gz", hash = "sha256:3ab69b9f41c67fb5bbb4d7a36243256f0d549ed3608678d381d5f51863921104", size = 3913526, upload-time = "2025-07-09T07:15:52.882Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/fe/37/1a1c62d955e82adae588be8e374c7f77b165b6cb4203f7d581269959abbc/pyright-1.1.402-py3-none-any.whl", hash = "sha256:2c721f11869baac1884e846232800fe021c33f1b4acb3929cff321f7ea4e2982", size = 5624004, upload-time = "2025-06-11T08:48:33.998Z" }, + { url = "https://files.pythonhosted.org/packages/49/b6/b04e5c2f41a5ccad74a1a4759da41adb20b4bc9d59a5e08d29ba60084d07/pyright-1.1.403-py3-none-any.whl", hash = "sha256:c0eeca5aa76cbef3fcc271259bbd785753c7ad7bcac99a9162b4c4c7daed23b3", size = 5684504, upload-time = "2025-07-09T07:15:50.958Z" }, ] [[package]] From 4444ec50da96e236935c7784d731631aabf113ed Mon Sep 17 00:00:00 2001 From: Yiorgis Gozadinos Date: Sat, 12 Jul 2025 20:56:49 +0300 Subject: [PATCH 2/6] Check if settings are compatible when loading a db --- src/haiku/rag/store/engine.py | 6 +++ src/haiku/rag/store/repositories/settings.py | 42 ++++++++++++++++++++ tests/test_settings.py | 42 +++++++++++++++++++- 3 files changed, 89 insertions(+), 1 deletion(-) diff --git a/src/haiku/rag/store/engine.py b/src/haiku/rag/store/engine.py index 4133b6dc..af7b27c4 100644 --- a/src/haiku/rag/store/engine.py +++ b/src/haiku/rag/store/engine.py @@ -13,6 +13,12 @@ class Store: self.db_path: Path | Literal[":memory:"] = db_path self._connection = self.create_db() + # Validate config compatibility after connection is established + from haiku.rag.store.repositories.settings import SettingsRepository + + settings_repo = SettingsRepository(self) + settings_repo.validate_config_compatibility() + def create_db(self) -> sqlite3.Connection: """Create the database and tables with sqlite-vec support for embeddings.""" db = sqlite3.connect(self.db_path) diff --git a/src/haiku/rag/store/repositories/settings.py b/src/haiku/rag/store/repositories/settings.py index d51cb0d5..add87fa5 100644 --- a/src/haiku/rag/store/repositories/settings.py +++ b/src/haiku/rag/store/repositories/settings.py @@ -4,6 +4,12 @@ from typing import Any from haiku.rag.store.engine import Store +class ConfigMismatchError(Exception): + """Raised when current config doesn't match stored settings.""" + + pass + + class SettingsRepository: def __init__(self, store: Store): self.store = store @@ -34,3 +40,39 @@ class SettingsRepository: ) self.store._connection.commit() + + def validate_config_compatibility(self) -> None: + """Check if current config is compatible with stored settings. + + Raises ConfigMismatchError if there are incompatible differences. + If no settings exist, saves current config. + """ + db_settings = self.get() + if not db_settings: + # No settings in DB, save current config + self.save() + return + + from haiku.rag.config import Config + + current_config = Config.model_dump(mode="json") + + # Critical settings that must match + critical_settings = [ + "EMBEDDINGS_PROVIDER", + "EMBEDDINGS_MODEL", + "EMBEDDINGS_VECTOR_DIM", + "CHUNK_SIZE", + "CHUNK_OVERLAP", + ] + + errors = [] + for setting in critical_settings: + if db_settings.get(setting) != current_config.get(setting): + errors.append( + f"{setting}: current={current_config.get(setting)}, stored={db_settings.get(setting)}" + ) + + if errors: + error_msg = f"Config mismatch detected: {'; '.join(errors)}. Consider rebuilding the database with the current configuration." + raise ConfigMismatchError(error_msg) diff --git a/tests/test_settings.py b/tests/test_settings.py index aaec6ac5..8f6f49d7 100644 --- a/tests/test_settings.py +++ b/tests/test_settings.py @@ -1,6 +1,14 @@ +import tempfile +from pathlib import Path + +import pytest + from haiku.rag.config import Config from haiku.rag.store.engine import Store -from haiku.rag.store.repositories.settings import SettingsRepository +from haiku.rag.store.repositories.settings import ( + ConfigMismatchError, + SettingsRepository, +) def test_settings_table_populated_on_store_init(): @@ -31,3 +39,35 @@ def test_settings_save_and_retrieve(): Config.CHUNK_SIZE = original_chunk_size store.close() + + +def test_config_validation_on_db_load(): + """Test that config validation fails when loading db with mismatched settings.""" + # Create a temporary database file + with tempfile.NamedTemporaryFile(suffix=".sqlite", delete=False) as tmp: + db_path = Path(tmp.name) + + try: + # Create store and save settings + store1 = Store(db_path) + SettingsRepository(store1) + store1.close() + + # Change config + original_chunk_size = Config.CHUNK_SIZE + Config.CHUNK_SIZE = 999 + + # Loading the database should raise ConfigMismatchError + with pytest.raises(ConfigMismatchError) as exc_info: + Store(db_path) + + assert "CHUNK_SIZE" in str(exc_info.value) + assert "Consider rebuilding" in str(exc_info.value) + + # Restore original config + Config.CHUNK_SIZE = original_chunk_size + + finally: + # Cleanup + if db_path.exists(): + db_path.unlink() From 62a519b094cb5425e9aa558a139f461999db24df Mon Sep 17 00:00:00 2001 From: Yiorgis Gozadinos Date: Sat, 12 Jul 2025 21:21:14 +0300 Subject: [PATCH 3/6] Prevent rebuild from throwing and recreate the embeddings table --- src/haiku/rag/app.py | 2 +- src/haiku/rag/client.py | 17 +++++++++++------ src/haiku/rag/store/engine.py | 30 ++++++++++++++++++++++++++---- tests/test_settings.py | 16 ++++++++++++++-- 4 files changed, 52 insertions(+), 13 deletions(-) diff --git a/src/haiku/rag/app.py b/src/haiku/rag/app.py index dedfce4c..0e2d1822 100644 --- a/src/haiku/rag/app.py +++ b/src/haiku/rag/app.py @@ -74,7 +74,7 @@ class HaikuRAGApp: self.console.print(f"[red]Error: {e}[/red]") async def rebuild(self): - async with HaikuRAG(db_path=self.db_path) as client: + async with HaikuRAG(db_path=self.db_path, skip_validation=True) as client: try: documents = await client.list_documents() total_docs = len(documents) diff --git a/src/haiku/rag/client.py b/src/haiku/rag/client.py index 74478654..72e5b0a6 100644 --- a/src/haiku/rag/client.py +++ b/src/haiku/rag/client.py @@ -24,12 +24,13 @@ class HaikuRAG: self, db_path: Path | Literal[":memory:"] = Config.DEFAULT_DATA_DIR / "haiku.rag.sqlite", + skip_validation: bool = False, ): """Initialize the RAG client with a database path.""" if isinstance(db_path, Path): if not db_path.parent.exists(): Path.mkdir(db_path.parent, parents=True) - self.store = Store(db_path) + self.store = Store(db_path, skip_validation=skip_validation) self.document_repository = DocumentRepository(self.store) self.chunk_repository = ChunkRepository(self.store) @@ -277,12 +278,16 @@ class HaikuRAG: Yields: int: The ID of the document currently being processed """ - documents = await self.list_documents() - - if not documents: - return - await self.chunk_repository.delete_all() + self.store.recreate_embeddings_table() + + # Update settings to current config + from haiku.rag.store.repositories.settings import SettingsRepository + + settings_repo = SettingsRepository(self.store) + settings_repo.save() + + documents = await self.list_documents() for doc in documents: if doc.id is not None: diff --git a/src/haiku/rag/store/engine.py b/src/haiku/rag/store/engine.py index af7b27c4..a2701ddf 100644 --- a/src/haiku/rag/store/engine.py +++ b/src/haiku/rag/store/engine.py @@ -9,15 +9,18 @@ from haiku.rag.embeddings import get_embedder class Store: - def __init__(self, db_path: Path | Literal[":memory:"]): + def __init__( + self, db_path: Path | Literal[":memory:"], skip_validation: bool = False + ): self.db_path: Path | Literal[":memory:"] = db_path self._connection = self.create_db() # Validate config compatibility after connection is established - from haiku.rag.store.repositories.settings import SettingsRepository + if not skip_validation: + from haiku.rag.store.repositories.settings import SettingsRepository - settings_repo = SettingsRepository(self) - settings_repo.validate_config_compatibility() + settings_repo = SettingsRepository(self) + settings_repo.validate_config_compatibility() def create_db(self) -> sqlite3.Connection: """Create the database and tables with sqlite-vec support for embeddings.""" @@ -91,6 +94,25 @@ class Store: db.commit() return db + def recreate_embeddings_table(self) -> None: + """Recreate the embeddings table with current vector dimensions.""" + if self._connection is None: + raise ValueError("Store connection is not available") + + # Drop existing embeddings table + self._connection.execute("DROP TABLE IF EXISTS chunk_embeddings") + + # Recreate with current dimensions + embedder = get_embedder() + self._connection.execute(f""" + CREATE VIRTUAL TABLE chunk_embeddings USING vec0( + chunk_id INTEGER PRIMARY KEY, + embedding FLOAT[{embedder._vector_dim}] + ) + """) + + self._connection.commit() + @staticmethod def serialize_embedding(embedding: list[float]) -> bytes: """Serialize a list of floats to bytes for sqlite-vec storage.""" diff --git a/tests/test_settings.py b/tests/test_settings.py index 8f6f49d7..0db16cc8 100644 --- a/tests/test_settings.py +++ b/tests/test_settings.py @@ -3,6 +3,7 @@ from pathlib import Path import pytest +from haiku.rag.client import HaikuRAG from haiku.rag.config import Config from haiku.rag.store.engine import Store from haiku.rag.store.repositories.settings import ( @@ -41,7 +42,7 @@ def test_settings_save_and_retrieve(): store.close() -def test_config_validation_on_db_load(): +async def test_config_validation_on_db_load(): """Test that config validation fails when loading db with mismatched settings.""" # Create a temporary database file with tempfile.NamedTemporaryFile(suffix=".sqlite", delete=False) as tmp: @@ -64,7 +65,18 @@ def test_config_validation_on_db_load(): assert "CHUNK_SIZE" in str(exc_info.value) assert "Consider rebuilding" in str(exc_info.value) - # Restore original config + # Rebuild + async with HaikuRAG(db_path=db_path, skip_validation=True) as client: + async for _ in client.rebuild_database(): + pass # Process all documents + + # Verify we can now load the database without exception (settings were updated) + store2 = Store(db_path) + settings_repo2 = SettingsRepository(store2) + db_settings = settings_repo2.get() + assert db_settings["CHUNK_SIZE"] == 999 + store2.close() + Config.CHUNK_SIZE = original_chunk_size finally: From a2e0f2cefd50fabb9f3beac535913feea927ce3f Mon Sep 17 00:00:00 2001 From: Yiorgis Gozadinos Date: Sat, 12 Jul 2025 23:00:49 +0300 Subject: [PATCH 4/6] Remove Claude nonsense, use context manager --- src/haiku/rag/client.py | 27 +- tests/test_client.py | 638 +++++++++++++++++++--------------------- tests/test_monitor.py | 20 +- tests/test_rebuild.py | 71 +++-- tests/test_settings.py | 42 ++- 5 files changed, 366 insertions(+), 432 deletions(-) diff --git a/src/haiku/rag/client.py b/src/haiku/rag/client.py index 72e5b0a6..ca30098c 100644 --- a/src/haiku/rag/client.py +++ b/src/haiku/rag/client.py @@ -166,29 +166,26 @@ class HaikuRAG: # Create a temporary file with the appropriate extension with tempfile.NamedTemporaryFile( - mode="wb", suffix=file_extension, delete=False + mode="wb", suffix=file_extension ) as temp_file: temp_file.write(response.content) + temp_file.flush() # Ensure content is written to disk temp_path = Path(temp_file.name) - try: # Parse the content using FileReader content = FileReader.parse_file(temp_path) - # Merge metadata with contentType and md5 - metadata.update({"contentType": content_type, "md5": md5_hash}) + # Merge metadata with contentType and md5 + metadata.update({"contentType": content_type, "md5": md5_hash}) - if existing_doc: - existing_doc.content = content - existing_doc.metadata = metadata - return await self.update_document(existing_doc) - else: - return await self.create_document( - content=content, uri=url, metadata=metadata - ) - finally: - # Clean up temporary file - temp_path.unlink(missing_ok=True) + if existing_doc: + existing_doc.content = content + existing_doc.metadata = metadata + return await self.update_document(existing_doc) + else: + return await self.create_document( + content=content, uri=url, metadata=metadata + ) def _get_extension_from_content_type_or_url( self, url: str, content_type: str diff --git a/tests/test_client.py b/tests/test_client.py index 086facc0..c5aaeb1c 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -12,300 +12,270 @@ from haiku.rag.client import HaikuRAG @pytest.mark.asyncio async def test_client_document_crud(qa_corpus: Dataset): """Test HaikuRAG CRUD operations for documents.""" - # Create client with in-memory database - client = HaikuRAG(":memory:") + async with HaikuRAG(":memory:") as client: + # Get test data + first_doc = qa_corpus[0] + document_text = first_doc["document_extracted"] + test_uri = "file:///path/to/test.txt" + test_metadata = {"source": "test", "topic": "testing"} - # Get test data - first_doc = qa_corpus[0] - document_text = first_doc["document_extracted"] - test_uri = "file:///path/to/test.txt" - test_metadata = {"source": "test", "topic": "testing"} + # Test create_document + created_doc = await client.create_document( + content=document_text, uri=test_uri, metadata=test_metadata + ) - # Test create_document - created_doc = await client.create_document( - content=document_text, uri=test_uri, metadata=test_metadata - ) + assert created_doc.id is not None + assert created_doc.content == document_text + assert created_doc.uri == test_uri + assert created_doc.metadata == test_metadata - assert created_doc.id is not None - assert created_doc.content == document_text - assert created_doc.uri == test_uri - assert created_doc.metadata == test_metadata + # Test get_document_by_id + retrieved_doc = await client.get_document_by_id(created_doc.id) + assert retrieved_doc is not None + assert retrieved_doc.id == created_doc.id + assert retrieved_doc.content == document_text + assert retrieved_doc.uri == test_uri - # Test get_document_by_id - retrieved_doc = await client.get_document_by_id(created_doc.id) - assert retrieved_doc is not None - assert retrieved_doc.id == created_doc.id - assert retrieved_doc.content == document_text - assert retrieved_doc.uri == test_uri + # Test get_document_by_uri + retrieved_by_uri = await client.get_document_by_uri(test_uri) + assert retrieved_by_uri is not None + assert retrieved_by_uri.id == created_doc.id + assert retrieved_by_uri.content == document_text - # Test get_document_by_uri - retrieved_by_uri = await client.get_document_by_uri(test_uri) - assert retrieved_by_uri is not None - assert retrieved_by_uri.id == created_doc.id - assert retrieved_by_uri.content == document_text + # Test get_document_by_uri with non-existent URI + non_existent = await client.get_document_by_uri("file:///non/existent.txt") + assert non_existent is None - # Test get_document_by_uri with non-existent URI - non_existent = await client.get_document_by_uri("file:///non/existent.txt") - assert non_existent is None + # Test update_document + retrieved_doc.content = "Updated content" + retrieved_doc.uri = "file:///updated/path.txt" + updated_doc = await client.update_document(retrieved_doc) + assert updated_doc.content == "Updated content" + assert updated_doc.uri == "file:///updated/path.txt" - # Test update_document - retrieved_doc.content = "Updated content" - retrieved_doc.uri = "file:///updated/path.txt" - updated_doc = await client.update_document(retrieved_doc) - assert updated_doc.content == "Updated content" - assert updated_doc.uri == "file:///updated/path.txt" + # Test list_documents + all_docs = await client.list_documents() + assert len(all_docs) == 1 + assert all_docs[0].id == created_doc.id - # Test list_documents - all_docs = await client.list_documents() - assert len(all_docs) == 1 - assert all_docs[0].id == created_doc.id + # Test list_documents with pagination + limited_docs = await client.list_documents(limit=10, offset=0) + assert len(limited_docs) == 1 - # Test list_documents with pagination - limited_docs = await client.list_documents(limit=10, offset=0) - assert len(limited_docs) == 1 + # Test delete_document + deleted = await client.delete_document(created_doc.id) + assert deleted is True - # Test delete_document - deleted = await client.delete_document(created_doc.id) - assert deleted is True + # Verify document is gone + retrieved_doc = await client.get_document_by_id(created_doc.id) + assert retrieved_doc is None - # Verify document is gone - retrieved_doc = await client.get_document_by_id(created_doc.id) - assert retrieved_doc is None - - # Test delete non-existent document - deleted_again = await client.delete_document(created_doc.id) - assert deleted_again is False - - client.close() + # Test delete non-existent document + deleted_again = await client.delete_document(created_doc.id) + assert deleted_again is False @pytest.mark.asyncio async def test_client_create_document_from_source(): """Test creating a document from a file source.""" - client = HaikuRAG(":memory:") + async with HaikuRAG(":memory:") as client: + with tempfile.TemporaryDirectory() as temp_dir: + test_content = "This is test content from a file." + temp_path = Path(temp_dir) / "test.txt" + temp_path.write_text(test_content) - # Create a temporary text file - with tempfile.NamedTemporaryFile(mode="w", suffix=".txt", delete=False) as f: - test_content = "This is test content from a file." - f.write(test_content) - temp_path = Path(f.name) + # Test create_document_from_source with Path + doc = await client.create_document_from_source(source=temp_path) - try: - # Test create_document_from_source with Path - doc = await client.create_document_from_source( - source=temp_path, metadata={"source_type": "file"} - ) + assert doc.id is not None + assert doc.content == test_content + assert doc.uri == temp_path.as_uri() + assert "contentType" in doc.metadata + assert "md5" in doc.metadata + assert doc.metadata["contentType"] == "text/plain" - assert doc.id is not None - assert doc.content == test_content - assert doc.uri == temp_path.as_uri() - assert doc.metadata["source_type"] == "file" - assert "contentType" in doc.metadata - assert "md5" in doc.metadata - assert doc.metadata["contentType"] == "text/plain" + # Test create_document_from_source with string path + doc2 = await client.create_document_from_source(source=str(temp_path)) - # Test create_document_from_source with string path - doc2 = await client.create_document_from_source(source=str(temp_path)) - - assert doc2.id is not None - assert doc2.content == test_content - assert doc2.uri == temp_path.as_uri() - assert "contentType" in doc2.metadata - assert "md5" in doc2.metadata - - finally: - # Clean up - temp_path.unlink() - client.close() + assert doc2.id is not None + assert doc2.content == test_content + assert doc2.uri == temp_path.as_uri() + assert "contentType" in doc2.metadata + assert "md5" in doc2.metadata @pytest.mark.asyncio async def test_client_create_document_from_source_unsupported(): """Test creating a document from an unsupported file type.""" - client = HaikuRAG(":memory:") + async with HaikuRAG(":memory:") as client: + # Create a temporary file with unsupported extension + with tempfile.NamedTemporaryFile( + mode="w", suffix=".unsupported", delete=False + ) as f: + f.write("content") + temp_path = Path(f.name) - # Create a temporary file with unsupported extension - with tempfile.NamedTemporaryFile( - mode="w", suffix=".unsupported", delete=False - ) as f: - f.write("content") - temp_path = Path(f.name) - - try: - # Should raise ValueError for unsupported extension - with pytest.raises(ValueError, match="Unsupported file extension"): - await client.create_document_from_source(temp_path) - - finally: - temp_path.unlink() - client.close() + # Should raise ValueError for unsupported extension + with pytest.raises(ValueError, match="Unsupported file extension"): + await client.create_document_from_source(temp_path) @pytest.mark.asyncio async def test_client_create_document_from_source_nonexistent(): """Test creating a document from a non-existent file.""" - client = HaikuRAG(":memory:") + async with HaikuRAG(":memory:") as client: + non_existent_path = Path("/non/existent/file.txt") - non_existent_path = Path("/non/existent/file.txt") - - # Should raise ValueError when file doesn't exist - with pytest.raises(ValueError, match="File does not exist"): - await client.create_document_from_source(non_existent_path) - - client.close() + # Should raise ValueError when file doesn't exist + with pytest.raises(ValueError, match="File does not exist"): + await client.create_document_from_source(non_existent_path) @pytest.mark.asyncio async def test_client_create_document_from_url(): """Test creating a document from a URL.""" - client = HaikuRAG(":memory:") + async with HaikuRAG(":memory:") as client: + # Mock the HTTP response + mock_response = AsyncMock() + mock_response.content = b"

Test Page

This is test content from a webpage.

" + mock_response.headers = {"content-type": "text/html"} + mock_response.raise_for_status = AsyncMock() - # Mock the HTTP response - mock_response = AsyncMock() - mock_response.content = b"

Test Page

This is test content from a webpage.

" - mock_response.headers = {"content-type": "text/html"} - mock_response.raise_for_status = AsyncMock() + with patch("httpx.AsyncClient.get", return_value=mock_response): + doc = await client.create_document_from_source( + source="https://example.com/test.html", metadata={"source_type": "web"} + ) - with patch("httpx.AsyncClient.get", return_value=mock_response): - doc = await client.create_document_from_source( - source="https://example.com/test.html", metadata={"source_type": "web"} - ) - - assert doc.id is not None - assert "Test Page" in doc.content - assert "test content" in doc.content - assert doc.uri == "https://example.com/test.html" - assert doc.metadata["source_type"] == "web" - assert "contentType" in doc.metadata - assert "md5" in doc.metadata - assert doc.metadata["contentType"] == "text/html" - - client.close() + assert doc.id is not None + assert "Test Page" in doc.content + assert "test content" in doc.content + assert doc.uri == "https://example.com/test.html" + assert doc.metadata["source_type"] == "web" + assert "contentType" in doc.metadata + assert "md5" in doc.metadata + assert doc.metadata["contentType"] == "text/html" @pytest.mark.asyncio async def test_client_create_document_from_url_with_different_content_types(): """Test creating documents from URLs with different content types.""" - client = HaikuRAG(":memory:") - - # Test JSON content - mock_json_response = AsyncMock() - mock_json_response.content = ( - b'{"title": "Test JSON", "content": "This is JSON content"}' - ) - mock_json_response.headers = {"content-type": "application/json"} - mock_json_response.raise_for_status = AsyncMock() - - with patch("httpx.AsyncClient.get", return_value=mock_json_response): - doc = await client.create_document_from_source( - "https://api.example.com/data.json" + async with HaikuRAG(":memory:") as client: + # Test JSON content + mock_json_response = AsyncMock() + mock_json_response.content = ( + b'{"title": "Test JSON", "content": "This is JSON content"}' ) + mock_json_response.headers = {"content-type": "application/json"} + mock_json_response.raise_for_status = AsyncMock() - assert doc.id is not None - assert "Test JSON" in doc.content - assert doc.uri == "https://api.example.com/data.json" - assert "contentType" in doc.metadata - assert "md5" in doc.metadata - assert doc.metadata["contentType"] == "application/json" + with patch("httpx.AsyncClient.get", return_value=mock_json_response): + doc = await client.create_document_from_source( + "https://api.example.com/data.json" + ) - # Test plain text content - mock_text_response = AsyncMock() - mock_text_response.content = b"This is plain text content from a URL." - mock_text_response.headers = {"content-type": "text/plain"} - mock_text_response.raise_for_status = AsyncMock() + assert doc.id is not None + assert "Test JSON" in doc.content + assert doc.uri == "https://api.example.com/data.json" + assert "contentType" in doc.metadata + assert "md5" in doc.metadata + assert doc.metadata["contentType"] == "application/json" - with patch("httpx.AsyncClient.get", return_value=mock_text_response): - doc = await client.create_document_from_source("https://example.com/readme.txt") + # Test plain text content + mock_text_response = AsyncMock() + mock_text_response.content = b"This is plain text content from a URL." + mock_text_response.headers = {"content-type": "text/plain"} + mock_text_response.raise_for_status = AsyncMock() - assert doc.id is not None - assert doc.content == "This is plain text content from a URL." - assert doc.uri == "https://example.com/readme.txt" - assert "contentType" in doc.metadata - assert "md5" in doc.metadata - assert doc.metadata["contentType"] == "text/plain" + with patch("httpx.AsyncClient.get", return_value=mock_text_response): + doc = await client.create_document_from_source( + "https://example.com/readme.txt" + ) - client.close() + assert doc.id is not None + assert doc.content == "This is plain text content from a URL." + assert doc.uri == "https://example.com/readme.txt" + assert "contentType" in doc.metadata + assert "md5" in doc.metadata + assert doc.metadata["contentType"] == "text/plain" @pytest.mark.asyncio async def test_client_create_document_from_url_unsupported_content(): """Test creating a document from URL with unsupported content type.""" - client = HaikuRAG(":memory:") + async with HaikuRAG(":memory:") as client: + # Mock response with unsupported content type + mock_response = AsyncMock() + mock_response.content = b"binary content" + mock_response.headers = {"content-type": "application/octet-stream"} + mock_response.raise_for_status = AsyncMock() - # Mock response with unsupported content type - mock_response = AsyncMock() - mock_response.content = b"binary content" - mock_response.headers = {"content-type": "application/octet-stream"} - mock_response.raise_for_status = AsyncMock() - - with patch("httpx.AsyncClient.get", return_value=mock_response): - with pytest.raises(ValueError, match="Unsupported content type"): - await client.create_document_from_source("https://example.com/binary.bin") - - client.close() + with patch("httpx.AsyncClient.get", return_value=mock_response): + with pytest.raises(ValueError, match="Unsupported content type"): + await client.create_document_from_source( + "https://example.com/binary.bin" + ) @pytest.mark.asyncio async def test_client_create_document_from_url_http_error(): """Test handling HTTP errors when creating document from URL.""" - client = HaikuRAG(":memory:") - - with patch("httpx.AsyncClient.get") as mock_get: - mock_get.side_effect = httpx.HTTPStatusError( - "404 Not Found", - request=httpx.Request("GET", "https://example.com/notfound.html"), - response=httpx.Response(404), - ) - - with pytest.raises(httpx.HTTPStatusError): - await client.create_document_from_source( - "https://example.com/notfound.html" + async with HaikuRAG(":memory:") as client: + with patch("httpx.AsyncClient.get") as mock_get: + mock_get.side_effect = httpx.HTTPStatusError( + "404 Not Found", + request=httpx.Request("GET", "https://example.com/notfound.html"), + response=httpx.Response(404), ) - client.close() + with pytest.raises(httpx.HTTPStatusError): + await client.create_document_from_source( + "https://example.com/notfound.html" + ) @pytest.mark.asyncio async def test_get_extension_from_content_type_or_url(): """Test the helper method for determining file extensions.""" - client = HaikuRAG(":memory:") - - # Test content type mappings - assert client._get_extension_from_content_type_or_url("", "text/html") == ".html" - assert ( - client._get_extension_from_content_type_or_url("", "application/pdf") == ".pdf" - ) - assert client._get_extension_from_content_type_or_url("", "text/plain") == ".txt" - - # Test URL extension detection - assert ( - client._get_extension_from_content_type_or_url( - "https://example.com/doc.pdf", "" + async with HaikuRAG(":memory:") as client: + # Test content type mappings + assert ( + client._get_extension_from_content_type_or_url("", "text/html") == ".html" ) - == ".pdf" - ) - assert ( - client._get_extension_from_content_type_or_url( - "https://example.com/data.json", "" + assert ( + client._get_extension_from_content_type_or_url("", "application/pdf") + == ".pdf" ) - == ".json" - ) - - # Test default fallback - assert ( - client._get_extension_from_content_type_or_url("https://example.com/", "") - == ".html" - ) - - # Test content type priority over URL extension - assert ( - client._get_extension_from_content_type_or_url( - "https://example.com/file.txt", "application/pdf" + assert ( + client._get_extension_from_content_type_or_url("", "text/plain") == ".txt" ) - == ".pdf" - ) - client.close() + # Test URL extension detection + assert ( + client._get_extension_from_content_type_or_url( + "https://example.com/doc.pdf", "" + ) + == ".pdf" + ) + assert ( + client._get_extension_from_content_type_or_url( + "https://example.com/data.json", "" + ) + == ".json" + ) + + # Test default fallback + assert ( + client._get_extension_from_content_type_or_url("https://example.com/", "") + == ".html" + ) + + # Test content type priority over URL extension + assert ( + client._get_extension_from_content_type_or_url( + "https://example.com/file.txt", "application/pdf" + ) + == ".pdf" + ) @pytest.mark.asyncio @@ -313,165 +283,147 @@ async def test_client_metadata_content_type_and_md5(): """Test that contentType and md5 metadata are correctly set.""" import hashlib - client = HaikuRAG(":memory:") + async with HaikuRAG(":memory:") as client: + # Create a temporary file with known content + test_content = "Test content for MD5 calculation." + expected_md5 = hashlib.md5(test_content.encode()).hexdigest() - # Create a temporary file with known content - test_content = "Test content for MD5 calculation." - expected_md5 = hashlib.md5(test_content.encode()).hexdigest() + with tempfile.TemporaryDirectory() as temp_dir: + temp_path = Path(temp_dir) / "test.txt" + temp_path.write_text(test_content) - with tempfile.NamedTemporaryFile(mode="w", suffix=".txt", delete=False) as f: - f.write(test_content) - temp_path = Path(f.name) + doc = await client.create_document_from_source(temp_path) - try: - doc = await client.create_document_from_source(temp_path) + assert doc.metadata["contentType"] == "text/plain" + assert doc.metadata["md5"] == expected_md5 - assert doc.metadata["contentType"] == "text/plain" - assert doc.metadata["md5"] == expected_md5 + mock_response = AsyncMock() + mock_response.content = test_content.encode() + mock_response.headers = {"content-type": "text/plain"} + mock_response.raise_for_status = AsyncMock() - mock_response = AsyncMock() - mock_response.content = test_content.encode() - mock_response.headers = {"content-type": "text/plain"} - mock_response.raise_for_status = AsyncMock() + with patch("httpx.AsyncClient.get", return_value=mock_response): + url_doc = await client.create_document_from_source( + "https://example.com/test.txt" + ) - with patch("httpx.AsyncClient.get", return_value=mock_response): - url_doc = await client.create_document_from_source( - "https://example.com/test.txt" - ) - - assert url_doc.metadata["contentType"] == "text/plain" - assert url_doc.metadata["md5"] == expected_md5 - - finally: - temp_path.unlink() - client.close() + assert url_doc.metadata["contentType"] == "text/plain" + assert url_doc.metadata["md5"] == expected_md5 @pytest.mark.asyncio async def test_client_create_update_no_op_behavior(): """Test create/update/no-op behavior based on MD5 changes.""" - client = HaikuRAG(":memory:") + async with HaikuRAG(":memory:") as client: + # Create a temporary file + test_content = "Original content for testing." + with tempfile.TemporaryDirectory() as temp_dir: + temp_path = Path(temp_dir) / "test.txt" + temp_path.write_text(test_content) - # Create a temporary file - test_content = "Original content for testing." - with tempfile.NamedTemporaryFile(mode="w", suffix=".txt", delete=False) as f: - f.write(test_content) - temp_path = Path(f.name) + # First call - should create new document + doc1 = await client.create_document_from_source(temp_path) + assert doc1.id is not None + assert doc1.content == test_content + original_id = doc1.id - try: - # First call - should create new document - doc1 = await client.create_document_from_source(temp_path) - assert doc1.id is not None - assert doc1.content == test_content - original_id = doc1.id + # Second call with same content - should return existing document (no-op) + doc2 = await client.create_document_from_source(temp_path) + assert doc2.id == original_id # Same document + assert doc2.content == test_content - # Second call with same content - should return existing document (no-op) - doc2 = await client.create_document_from_source(temp_path) - assert doc2.id == original_id # Same document - assert doc2.content == test_content + # Modify file content + updated_content = "Updated content for testing." + temp_path.write_text(updated_content) - # Modify file content - updated_content = "Updated content for testing." - temp_path.write_text(updated_content) + # Third call with changed content - should update existing document + doc3 = await client.create_document_from_source(temp_path) + assert doc3.id == original_id # Same document ID + assert doc3.content == updated_content # Updated content - # Third call with changed content - should update existing document - doc3 = await client.create_document_from_source(temp_path) - assert doc3.id == original_id # Same document ID - assert doc3.content == updated_content # Updated content - - # Verify the document was actually updated in database - retrieved_doc = await client.get_document_by_id(original_id) - assert retrieved_doc is not None - assert retrieved_doc.content == updated_content - - finally: - temp_path.unlink() - client.close() + # Verify the document was actually updated in database + retrieved_doc = await client.get_document_by_id(original_id) + assert retrieved_doc is not None + assert retrieved_doc.content == updated_content @pytest.mark.asyncio async def test_client_url_create_update_no_op_behavior(): """Test create/update/no-op behavior for URLs based on MD5 changes.""" - client = HaikuRAG(":memory:") + async with HaikuRAG(":memory:") as client: + url = "https://example.com/test.txt" + original_content = b"Original URL content" + updated_content = b"Updated URL content" - url = "https://example.com/test.txt" - original_content = b"Original URL content" - updated_content = b"Updated URL content" + # Mock first response + mock_response1 = AsyncMock() + mock_response1.content = original_content + mock_response1.headers = {"content-type": "text/plain"} + mock_response1.raise_for_status = AsyncMock() - # Mock first response - mock_response1 = AsyncMock() - mock_response1.content = original_content - mock_response1.headers = {"content-type": "text/plain"} - mock_response1.raise_for_status = AsyncMock() + with patch("httpx.AsyncClient.get", return_value=mock_response1): + # First call - should create new document + doc1 = await client.create_document_from_source(url) + assert doc1.id is not None + original_id = doc1.id - with patch("httpx.AsyncClient.get", return_value=mock_response1): - # First call - should create new document - doc1 = await client.create_document_from_source(url) - assert doc1.id is not None - original_id = doc1.id + # Second call with same content - should return existing document (no-op) + doc2 = await client.create_document_from_source(url) + assert doc2.id == original_id # Same document - # Second call with same content - should return existing document (no-op) - doc2 = await client.create_document_from_source(url) - assert doc2.id == original_id # Same document + mock_response2 = AsyncMock() + mock_response2.content = updated_content + mock_response2.headers = {"content-type": "text/plain"} + mock_response2.raise_for_status = AsyncMock() - mock_response2 = AsyncMock() - mock_response2.content = updated_content - mock_response2.headers = {"content-type": "text/plain"} - mock_response2.raise_for_status = AsyncMock() - - with patch("httpx.AsyncClient.get", return_value=mock_response2): - # Third call with changed content - should update existing document - doc3 = await client.create_document_from_source(url) - assert doc3.id == original_id # Same document ID - assert doc3.content == updated_content.decode() # Updated content - - client.close() + with patch("httpx.AsyncClient.get", return_value=mock_response2): + # Third call with changed content - should update existing document + doc3 = await client.create_document_from_source(url) + assert doc3.id == original_id # Same document ID + assert doc3.content == updated_content.decode() # Updated content @pytest.mark.asyncio async def test_client_search(): """Test HaikuRAG search functionality.""" - client = HaikuRAG(":memory:") + async with HaikuRAG(":memory:") as client: + # Add multiple documents to search from + doc1_text = "Python is a high-level programming language known for its simplicity and readability." + doc2_text = "Machine learning algorithms help computers learn patterns from data without explicit programming." + doc3_text = "Data science combines statistics, programming, and domain expertise to extract insights." - # Add multiple documents to search from - doc1_text = "Python is a high-level programming language known for its simplicity and readability." - doc2_text = "Machine learning algorithms help computers learn patterns from data without explicit programming." - doc3_text = "Data science combines statistics, programming, and domain expertise to extract insights." + # Create documents + doc1 = await client.create_document( + content=doc1_text, uri="doc1.txt", metadata={"topic": "python"} + ) + doc2 = await client.create_document( + content=doc2_text, uri="doc2.txt", metadata={"topic": "ml"} + ) + await client.create_document( + content=doc3_text, uri="doc3.txt", metadata={"topic": "data_science"} + ) - # Create documents - doc1 = await client.create_document( - content=doc1_text, uri="doc1.txt", metadata={"topic": "python"} - ) - doc2 = await client.create_document( - content=doc2_text, uri="doc2.txt", metadata={"topic": "ml"} - ) - await client.create_document( - content=doc3_text, uri="doc3.txt", metadata={"topic": "data_science"} - ) + # Test search with keyword that should match doc1 + results = await client.search("Python programming", limit=3) - # Test search with keyword that should match doc1 - results = await client.search("Python programming", limit=3) + assert len(results) > 0 + assert all(len(result) == 2 for result in results) - assert len(results) > 0 - assert all(len(result) == 2 for result in results) + # Verify first result is from the Python document (doc1) + first_chunk, _ = results[0] + assert first_chunk.document_id == doc1.id - # Verify first result is from the Python document (doc1) - first_chunk, _ = results[0] - assert first_chunk.document_id == doc1.id + # Test search with different query + ml_results = await client.search("machine learning data", limit=2) + assert len(ml_results) > 0 - # Test search with different query - ml_results = await client.search("machine learning data", limit=2) - assert len(ml_results) > 0 + # Verify first result is from the machine learning document (doc2) + first_ml_chunk, _ = ml_results[0] + assert first_ml_chunk.document_id == doc2.id - # Verify first result is from the machine learning document (doc2) - first_ml_chunk, _ = ml_results[0] - assert first_ml_chunk.document_id == doc2.id - - # Test search with limit parameter - limited_results = await client.search("programming", limit=1) - assert len(limited_results) <= 1 - - client.close() + # Test search with limit parameter + limited_results = await client.search("programming", limit=1) + assert len(limited_results) <= 1 @pytest.mark.asyncio diff --git a/tests/test_monitor.py b/tests/test_monitor.py index ac909631..73992416 100644 --- a/tests/test_monitor.py +++ b/tests/test_monitor.py @@ -13,11 +13,10 @@ from haiku.rag.store.models.document import Document async def test_file_watcher_upsert_document(): """Test FileWatcher._upsert_document method.""" - with tempfile.NamedTemporaryFile(mode="w", suffix=".txt", delete=False) as f: - f.write("Test content for file watcher") - temp_path = Path(f.name) + with tempfile.TemporaryDirectory() as temp_dir: + temp_path = Path(temp_dir) / "test.txt" + temp_path.write_text("Test content for file watcher") - try: mock_client = AsyncMock(spec=HaikuRAG) mock_doc = Document(id=1, content="Test content", uri=temp_path.as_uri()) mock_client.create_document_from_source.return_value = mock_doc @@ -32,19 +31,15 @@ async def test_file_watcher_upsert_document(): mock_client.get_document_by_uri.assert_called_once_with(temp_path.as_uri()) mock_client.create_document_from_source.assert_called_once_with(str(temp_path)) - finally: - temp_path.unlink(missing_ok=True) - @pytest.mark.asyncio async def test_file_watcher_upsert_existing_document(): """Test FileWatcher._upsert_document with existing document.""" - with tempfile.NamedTemporaryFile(mode="w", suffix=".txt", delete=False) as f: - f.write("Test content for file watcher") - temp_path = Path(f.name) + with tempfile.TemporaryDirectory() as temp_dir: + temp_path = Path(temp_dir) / "test.txt" + temp_path.write_text("Test content for file watcher") - try: mock_client = AsyncMock(spec=HaikuRAG) existing_doc = Document(id=1, content="Old content", uri=temp_path.as_uri()) updated_doc = Document(id=1, content="Updated content", uri=temp_path.as_uri()) @@ -61,9 +56,6 @@ async def test_file_watcher_upsert_existing_document(): mock_client.get_document_by_uri.assert_called_once_with(temp_path.as_uri()) mock_client.create_document_from_source.assert_called_once_with(str(temp_path)) - finally: - temp_path.unlink(missing_ok=True) - @pytest.mark.asyncio async def test_file_watcher_delete_document(): diff --git a/tests/test_rebuild.py b/tests/test_rebuild.py index 3254ce1d..04fd8533 100644 --- a/tests/test_rebuild.py +++ b/tests/test_rebuild.py @@ -8,45 +8,42 @@ from haiku.rag.store.models.document import Document @pytest.mark.asyncio async def test_rebuild_database(qa_corpus: Dataset): """Test rebuild functionality with existing documents.""" - client = HaikuRAG(":memory:") + async with HaikuRAG(":memory:") as client: + created_docs: list[Document] = [] + for content in qa_corpus["document_extracted"][:3]: + doc = await client.create_document( + content=content, + ) + created_docs.append(doc) - created_docs: list[Document] = [] - for content in qa_corpus["document_extracted"][:3]: - doc = await client.create_document( - content=content, - ) - created_docs.append(doc) + documents_before = await client.list_documents() + assert len(documents_before) == 3 - documents_before = await client.list_documents() - assert len(documents_before) == 3 - - chunks_before = [] - for doc in created_docs: - assert doc.id is not None - doc_chunks = await client.chunk_repository.get_by_document_id(doc.id) - chunks_before.extend(doc_chunks) - - assert len(chunks_before) > 0 - - # Perform rebuild - processed_doc_ids = [] - async for doc_id in client.rebuild_database(): - processed_doc_ids.append(doc_id) - - # Verify all documents were processed - expected_doc_ids = [doc.id for doc in created_docs] - assert set(processed_doc_ids) == set(expected_doc_ids) - - documents_after = await client.list_documents() - assert len(documents_after) == 3 - - # Verify chunks were recreated - chunks_after = [] - for doc in documents_after: - if doc.id is not None: + chunks_before = [] + for doc in created_docs: + assert doc.id is not None doc_chunks = await client.chunk_repository.get_by_document_id(doc.id) - chunks_after.extend(doc_chunks) + chunks_before.extend(doc_chunks) - assert len(chunks_after) > 0 + assert len(chunks_before) > 0 - client.close() + # Perform rebuild + processed_doc_ids = [] + async for doc_id in client.rebuild_database(): + processed_doc_ids.append(doc_id) + + # Verify all documents were processed + expected_doc_ids = [doc.id for doc in created_docs] + assert set(processed_doc_ids) == set(expected_doc_ids) + + documents_after = await client.list_documents() + assert len(documents_after) == 3 + + # Verify chunks were recreated + chunks_after = [] + for doc in documents_after: + if doc.id is not None: + doc_chunks = await client.chunk_repository.get_by_document_id(doc.id) + chunks_after.extend(doc_chunks) + + assert len(chunks_after) > 0 diff --git a/tests/test_settings.py b/tests/test_settings.py index 0db16cc8..6a531685 100644 --- a/tests/test_settings.py +++ b/tests/test_settings.py @@ -45,10 +45,9 @@ def test_settings_save_and_retrieve(): async def test_config_validation_on_db_load(): """Test that config validation fails when loading db with mismatched settings.""" # Create a temporary database file - with tempfile.NamedTemporaryFile(suffix=".sqlite", delete=False) as tmp: + with tempfile.NamedTemporaryFile(suffix=".sqlite") as tmp: db_path = Path(tmp.name) - try: # Create store and save settings store1 = Store(db_path) SettingsRepository(store1) @@ -58,28 +57,25 @@ async def test_config_validation_on_db_load(): original_chunk_size = Config.CHUNK_SIZE Config.CHUNK_SIZE = 999 - # Loading the database should raise ConfigMismatchError - with pytest.raises(ConfigMismatchError) as exc_info: - Store(db_path) + try: + # Loading the database should raise ConfigMismatchError + with pytest.raises(ConfigMismatchError) as exc_info: + Store(db_path) - assert "CHUNK_SIZE" in str(exc_info.value) - assert "Consider rebuilding" in str(exc_info.value) + assert "CHUNK_SIZE" in str(exc_info.value) + assert "Consider rebuilding" in str(exc_info.value) - # Rebuild - async with HaikuRAG(db_path=db_path, skip_validation=True) as client: - async for _ in client.rebuild_database(): - pass # Process all documents + # Rebuild + async with HaikuRAG(db_path=db_path, skip_validation=True) as client: + async for _ in client.rebuild_database(): + pass # Process all documents - # Verify we can now load the database without exception (settings were updated) - store2 = Store(db_path) - settings_repo2 = SettingsRepository(store2) - db_settings = settings_repo2.get() - assert db_settings["CHUNK_SIZE"] == 999 - store2.close() + # Verify we can now load the database without exception (settings were updated) + store2 = Store(db_path) + settings_repo2 = SettingsRepository(store2) + db_settings = settings_repo2.get() + assert db_settings["CHUNK_SIZE"] == 999 + store2.close() - Config.CHUNK_SIZE = original_chunk_size - - finally: - # Cleanup - if db_path.exists(): - db_path.unlink() + finally: + Config.CHUNK_SIZE = original_chunk_size From a2d7e527d193ea4677d2fe813d87ed7fda8307a8 Mon Sep 17 00:00:00 2001 From: Yiorgis Gozadinos Date: Mon, 14 Jul 2025 12:00:03 +0300 Subject: [PATCH 5/6] Update docs on configuration changes --- docs/configuration.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/configuration.md b/docs/configuration.md index a9846508..5dbba71b 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -2,6 +2,9 @@ Configuration is done through the use of environment variables. +!!! note + If you create a db with certain settings and later change them, `haiku.rag` will detect incompatibilities (for example, if you change embedding provider) and will exit. You can **rebuild** the database to apply the new settings, see [Rebuild Database](./cli.md#rebuild-database). + ## File Monitoring Set directories to monitor for automatic indexing: From f4aef4eb9c55d4ad6fb7e9c7732f201a64c4077e Mon Sep 17 00:00:00 2001 From: Yiorgis Gozadinos Date: Mon, 14 Jul 2025 12:13:01 +0300 Subject: [PATCH 6/6] Check pypi version on startup --- src/haiku/rag/cli.py | 19 ++++++++++++++++++- src/haiku/rag/utils.py | 24 ++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/src/haiku/rag/cli.py b/src/haiku/rag/cli.py index 426af784..80ee047e 100644 --- a/src/haiku/rag/cli.py +++ b/src/haiku/rag/cli.py @@ -5,7 +5,7 @@ import typer from rich.console import Console from haiku.rag.app import HaikuRAGApp -from haiku.rag.utils import get_default_data_dir +from haiku.rag.utils import get_default_data_dir, is_up_to_date cli = typer.Typer( context_settings={"help_option_names": ["-h", "--help"]}, no_args_is_help=True @@ -15,6 +15,23 @@ console = Console() event_loop = asyncio.get_event_loop() +async def check_version(): + """Check if haiku.rag is up to date and show warning if not.""" + up_to_date, current_version, latest_version = await is_up_to_date() + if not up_to_date: + console.print( + f"[yellow]Warning: haiku.rag is outdated. Current: {current_version}, Latest: {latest_version}[/yellow]" + ) + console.print("[yellow]Please update.[/yellow]") + + +@cli.callback() +def main(): + """haiku.rag CLI - SQLite-based RAG system""" + # Run version check before any command + event_loop.run_until_complete(check_version()) + + @cli.command("list", help="List all stored documents") def list_documents( db: Path = typer.Option( diff --git a/src/haiku/rag/utils.py b/src/haiku/rag/utils.py index 03c160bf..8b78f008 100644 --- a/src/haiku/rag/utils.py +++ b/src/haiku/rag/utils.py @@ -1,6 +1,10 @@ import sys +from importlib import metadata from pathlib import Path +import httpx +from packaging.version import Version, parse + def get_default_data_dir() -> Path: """ @@ -23,3 +27,23 @@ def get_default_data_dir() -> Path: data_path = system_paths[sys.platform] return data_path + + +async def is_up_to_date() -> tuple[bool, Version, Version]: + """ + Checks whether haiku.rag is current. + + :return: A tuple containing a boolean indicating whether haiku.rag is current, the running version and the latest version + :rtype: tuple[bool, Version, Version] + """ + + async with httpx.AsyncClient() as client: + running_version = parse(metadata.version("haiku.rag")) + try: + response = await client.get("https://pypi.org/pypi/haiku.rag/json") + data = response.json() + pypi_version = parse(data["info"]["version"]) + except Exception: + # If no network connection, do not raise alarms. + pypi_version = running_version + return running_version >= pypi_version, running_version, pypi_version