diff --git a/docs/benchmarks.md b/docs/benchmarks.md index 86103edd..4096931c 100644 --- a/docs/benchmarks.md +++ b/docs/benchmarks.md @@ -7,7 +7,7 @@ You can perform your own evaluations using as example the script found at ## Recall -In order to calculate recall, we load the `News Stories` from `repliqa_3` which is 1035 documents and index them in a sqlite db. Subsequently, we run a search over the `question` field for each row of the dataset and check whether we match the document that answers the question. +In order to calculate recall, we load the `News Stories` from `repliqa_3` which is 1035 documents and index them. Subsequently, we run a search over the `question` field for each row of the dataset and check whether we match the document that answers the question. The recall obtained is ~0.73 for matching in the top result, raising to ~0.75 for the top 3 results. diff --git a/src/haiku/rag/store/engine.py b/src/haiku/rag/store/engine.py index cf0ed233..4ed8023b 100644 --- a/src/haiku/rag/store/engine.py +++ b/src/haiku/rag/store/engine.py @@ -60,9 +60,6 @@ class Store: settings_repo = SettingsRepository(self) settings_repo.validate_config_compatibility() - current_version = metadata.version("haiku.rag") - self.set_user_version(current_version) - def create_or_update_db(self): """Create the database tables.""" @@ -98,6 +95,10 @@ class Store: [SettingsRecord(id="settings", settings=json.dumps(settings_data))] ) + # Set current version in settings + current_version = metadata.version("haiku.rag") + self.set_haiku_version(current_version) + # Check if we need to perform upgrades try: existing_settings = list( @@ -105,7 +106,7 @@ class Store: ) if existing_settings: console = Console() - db_version = self.get_user_version() + db_version = self.get_haiku_version() # Future: Add upgrade logic here similar to SQLite version console.print( f"[green]LanceDB store initialized (version: {db_version})[/green]" @@ -113,7 +114,7 @@ class Store: except Exception: pass - def get_user_version(self) -> str: + def get_haiku_version(self) -> str: """Returns the user version stored in settings.""" try: settings_records = list( @@ -130,7 +131,7 @@ class Store: pass return "0.0.0" - def set_user_version(self, version: str) -> None: + def set_haiku_version(self, version: str) -> None: """Updates the user version in settings.""" try: settings_records = list( @@ -145,7 +146,7 @@ class Store: settings["version"] = version # Update the record self.settings_table.update( - where="id = 1", values={"settings": json.dumps(settings)} + where="id = 'settings'", values={"settings": json.dumps(settings)} ) else: # Create new settings record diff --git a/src/haiku/rag/store/repositories/chunk.py b/src/haiku/rag/store/repositories/chunk.py index 23bee6fd..6c8bfaca 100644 --- a/src/haiku/rag/store/repositories/chunk.py +++ b/src/haiku/rag/store/repositories/chunk.py @@ -140,29 +140,23 @@ class ChunkRepository: return created_chunks - async def delete_all(self) -> bool: + async def delete_all(self) -> None: """Delete all chunks from the database.""" - try: - count = len( - list( - self.store.chunks_table.search() - .limit(1) - .to_pydantic(self.store.ChunkRecord) - ) + count = len( + list( + self.store.chunks_table.search() + .limit(1) + .to_pydantic(self.store.ChunkRecord) ) - if count > 0: - # Drop and recreate table to clear all data - self.store.db.drop_table("chunks") - self.store.chunks_table = self.store.db.create_table( - "chunks", schema=self.store.ChunkRecord - ) - # Create FTS index on the new table - self.store.chunks_table.create_fts_index("content", replace=True) - - return True - return False - except Exception: - return False + ) + if count > 0: + # Drop and recreate table to clear all data + self.store.db.drop_table("chunks") + self.store.chunks_table = self.store.db.create_table( + "chunks", schema=self.store.ChunkRecord + ) + # Create FTS index on the new table + self.store.chunks_table.create_fts_index("content", replace=True) async def delete_by_document_id(self, document_id: str) -> bool: """Delete all chunks for a document.""" diff --git a/src/haiku/rag/store/repositories/document.py b/src/haiku/rag/store/repositories/document.py index 93d25e4d..d9c20490 100644 --- a/src/haiku/rag/store/repositories/document.py +++ b/src/haiku/rag/store/repositories/document.py @@ -15,13 +15,12 @@ if TYPE_CHECKING: class DocumentRepository: """Repository for Document operations.""" - def __init__(self, store: Store, chunk_repository=None) -> None: + def __init__(self, store: Store) -> None: self.store = store - # Avoid circular import by using late import if not provided - if chunk_repository is None: - from haiku.rag.store.repositories.chunk import ChunkRepository - chunk_repository = ChunkRepository(store) + from haiku.rag.store.repositories.chunk import ChunkRepository + + chunk_repository = ChunkRepository(store) self.chunk_repository = chunk_repository async def create(self, entity: Document) -> Document: @@ -169,33 +168,26 @@ class DocumentRepository: else datetime.now(), ) - async def delete_all(self) -> bool: + async def delete_all(self) -> None: """Delete all documents from the database.""" - try: - # Delete all chunks first - from haiku.rag.store.repositories.chunk import ChunkRepository + # Delete all chunks first + from haiku.rag.store.repositories.chunk import ChunkRepository - chunk_repo = ChunkRepository(self.store) - await chunk_repo.delete_all() + chunk_repo = ChunkRepository(self.store) + await chunk_repo.delete_all() - # Get count before deletion - count = len( - list( - self.store.documents_table.search() - .limit(1) - .to_pydantic(DocumentRecord) - ) + # Get count before deletion + count = len( + list( + self.store.documents_table.search().limit(1).to_pydantic(DocumentRecord) + ) + ) + if count > 0: + # Drop and recreate table to clear all data + self.store.db.drop_table("documents") + self.store.documents_table = self.store.db.create_table( + "documents", schema=DocumentRecord ) - if count > 0: - # Drop and recreate table to clear all data - self.store.db.drop_table("documents") - self.store.documents_table = self.store.db.create_table( - "documents", schema=DocumentRecord - ) - return True - return False - except Exception: - return False async def _create_with_docling( self, diff --git a/src/haiku/rag/utils.py b/src/haiku/rag/utils.py index 3cde88b3..edf381a5 100644 --- a/src/haiku/rag/utils.py +++ b/src/haiku/rag/utils.py @@ -32,37 +32,6 @@ def get_default_data_dir() -> Path: return data_path -def semantic_version_to_int(version: str) -> int: - """Convert a semantic version string to an integer. - - Args: - version: Semantic version string. - - Returns: - Integer representation of semantic version. - """ - major, minor, patch = version.split(".") - major = int(major) << 16 - minor = int(minor) << 8 - patch = int(patch) - return major + minor + patch - - -def int_to_semantic_version(version: int) -> str: - """Convert an integer to a semantic version string. - - Args: - version: Integer representation of semantic version. - - Returns: - Semantic version string. - """ - major = version >> 16 - minor = (version >> 8) & 255 - patch = version & 255 - return f"{major}.{minor}.{patch}" - - async def is_up_to_date() -> tuple[bool, Version, Version]: """Check whether haiku.rag is current. diff --git a/tests/generate_benchmark_db.py b/tests/generate_benchmark_db.py index 5404f6b9..28dfe271 100644 --- a/tests/generate_benchmark_db.py +++ b/tests/generate_benchmark_db.py @@ -144,7 +144,7 @@ async def main(): await populate_db() console.print("Running retrieval benchmarks...", style="bold blue") - await run_match_benchmark() + # await run_match_benchmark() console.print("\nRunning QA benchmarks...", style="bold yellow") await run_qa_benchmark() diff --git a/tests/test_utils.py b/tests/test_utils.py index 51201b89..3bfced8e 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,22 +1,4 @@ -from haiku.rag.utils import ( - int_to_semantic_version, - semantic_version_to_int, - text_to_docling_document, -) - - -def test_sqlite_user_version(): - version = "0.1.5" - assert semantic_version_to_int(version) == 261 - assert int_to_semantic_version(261) == version - - version = "0.0.0" - assert semantic_version_to_int(version) == 0 - assert int_to_semantic_version(0) == version - - version = "255.255.255" - assert semantic_version_to_int(version) == 16777215 - assert int_to_semantic_version(16777215) == version +from haiku.rag.utils import text_to_docling_document def test_text_to_docling_document():