danvics/haiku.rag
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Set/get haiku version to db

Browse source
This commit is contained in:
Yiorgis Gozadinos 2025-09-01 17:41:41 +03:00
parent 58a04db556
commit b05e5f5408
No known key found for this signature in database
7 changed files with 46 additions and 108 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
docs/benchmarks.md
View file

@ -7,7 +7,7 @@ You can perform your own evaluations using as example the script found at
## Recall
In order to calculate recall, we load the `News Stories` from `repliqa_3` which is 1035 documents and index them in a sqlite db. Subsequently, we run a search over the `question` field for each row of the dataset and check whether we match the document that answers the question.
In order to calculate recall, we load the `News Stories` from `repliqa_3` which is 1035 documents and index them. Subsequently, we run a search over the `question` field for each row of the dataset and check whether we match the document that answers the question.
The recall obtained is ~0.73 for matching in the top result, raising to ~0.75 for the top 3 results.

15
src/haiku/rag/store/engine.py
View file

@ -60,9 +60,6 @@ class Store:
settings_repo = SettingsRepository(self)
settings_repo.validate_config_compatibility()
current_version = metadata.version("haiku.rag")
self.set_user_version(current_version)
def create_or_update_db(self):
"""Create the database tables."""
@ -98,6 +95,10 @@ class Store:
[SettingsRecord(id="settings", settings=json.dumps(settings_data))]
)
# Set current version in settings
current_version = metadata.version("haiku.rag")
self.set_haiku_version(current_version)
# Check if we need to perform upgrades
try:
existing_settings = list(
@ -105,7 +106,7 @@ class Store:
)
if existing_settings:
console = Console()
db_version = self.get_user_version()
db_version = self.get_haiku_version()
# Future: Add upgrade logic here similar to SQLite version
console.print(
f"[green]LanceDB store initialized (version: {db_version})[/green]"
@ -113,7 +114,7 @@ class Store:
except Exception:
pass
def get_user_version(self) -> str:
def get_haiku_version(self) -> str:
"""Returns the user version stored in settings."""
try:
settings_records = list(
@ -130,7 +131,7 @@ class Store:
pass
return "0.0.0"
def set_user_version(self, version: str) -> None:
def set_haiku_version(self, version: str) -> None:
"""Updates the user version in settings."""
try:
settings_records = list(
@ -145,7 +146,7 @@ class Store:
settings["version"] = version
# Update the record
self.settings_table.update(
where="id = 1", values={"settings": json.dumps(settings)}
where="id = 'settings'", values={"settings": json.dumps(settings)}
)
else:
# Create new settings record

36
src/haiku/rag/store/repositories/chunk.py
View file

@ -140,29 +140,23 @@ class ChunkRepository:
return created_chunks
async def delete_all(self) -> bool:
async def delete_all(self) -> None:
"""Delete all chunks from the database."""
try:
count = len(
list(
self.store.chunks_table.search()
.limit(1)
.to_pydantic(self.store.ChunkRecord)
)
count = len(
list(
self.store.chunks_table.search()
.limit(1)
.to_pydantic(self.store.ChunkRecord)
)
if count > 0:
# Drop and recreate table to clear all data
self.store.db.drop_table("chunks")
self.store.chunks_table = self.store.db.create_table(
"chunks", schema=self.store.ChunkRecord
)
# Create FTS index on the new table
self.store.chunks_table.create_fts_index("content", replace=True)
return True
return False
except Exception:
return False
)
if count > 0:
# Drop and recreate table to clear all data
self.store.db.drop_table("chunks")
self.store.chunks_table = self.store.db.create_table(
"chunks", schema=self.store.ChunkRecord
)
# Create FTS index on the new table
self.store.chunks_table.create_fts_index("content", replace=True)
async def delete_by_document_id(self, document_id: str) -> bool:
"""Delete all chunks for a document."""

48
src/haiku/rag/store/repositories/document.py
View file

@ -15,13 +15,12 @@ if TYPE_CHECKING:
class DocumentRepository:
"""Repository for Document operations."""
def __init__(self, store: Store, chunk_repository=None) -> None:
def __init__(self, store: Store) -> None:
self.store = store
# Avoid circular import by using late import if not provided
if chunk_repository is None:
from haiku.rag.store.repositories.chunk import ChunkRepository
chunk_repository = ChunkRepository(store)
from haiku.rag.store.repositories.chunk import ChunkRepository
chunk_repository = ChunkRepository(store)
self.chunk_repository = chunk_repository
async def create(self, entity: Document) -> Document:
@ -169,33 +168,26 @@ class DocumentRepository:
else datetime.now(),
)
async def delete_all(self) -> bool:
async def delete_all(self) -> None:
"""Delete all documents from the database."""
try:
# Delete all chunks first
from haiku.rag.store.repositories.chunk import ChunkRepository
# Delete all chunks first
from haiku.rag.store.repositories.chunk import ChunkRepository
chunk_repo = ChunkRepository(self.store)
await chunk_repo.delete_all()
chunk_repo = ChunkRepository(self.store)
await chunk_repo.delete_all()
# Get count before deletion
count = len(
list(
self.store.documents_table.search()
.limit(1)
.to_pydantic(DocumentRecord)
)
# Get count before deletion
count = len(
list(
self.store.documents_table.search().limit(1).to_pydantic(DocumentRecord)
)
)
if count > 0:
# Drop and recreate table to clear all data
self.store.db.drop_table("documents")
self.store.documents_table = self.store.db.create_table(
"documents", schema=DocumentRecord
)
if count > 0:
# Drop and recreate table to clear all data
self.store.db.drop_table("documents")
self.store.documents_table = self.store.db.create_table(
"documents", schema=DocumentRecord
)
return True
return False
except Exception:
return False
async def _create_with_docling(
self,

31
src/haiku/rag/utils.py
View file

@ -32,37 +32,6 @@ def get_default_data_dir() -> Path:
return data_path
def semantic_version_to_int(version: str) -> int:
"""Convert a semantic version string to an integer.
Args:
version: Semantic version string.
Returns:
Integer representation of semantic version.
"""
major, minor, patch = version.split(".")
major = int(major) << 16
minor = int(minor) << 8
patch = int(patch)
return major + minor + patch
def int_to_semantic_version(version: int) -> str:
"""Convert an integer to a semantic version string.
Args:
version: Integer representation of semantic version.
Returns:
Semantic version string.
"""
major = version >> 16
minor = (version >> 8) & 255
patch = version & 255
return f"{major}.{minor}.{patch}"
async def is_up_to_date() -> tuple[bool, Version, Version]:
"""Check whether haiku.rag is current.

2
tests/generate_benchmark_db.py
View file

@ -144,7 +144,7 @@ async def main():
await populate_db()
console.print("Running retrieval benchmarks...", style="bold blue")
await run_match_benchmark()
# await run_match_benchmark()
console.print("\nRunning QA benchmarks...", style="bold yellow")
await run_qa_benchmark()

20
tests/test_utils.py
View file

@ -1,22 +1,4 @@
from haiku.rag.utils import (
int_to_semantic_version,
semantic_version_to_int,
text_to_docling_document,
)
def test_sqlite_user_version():
version = "0.1.5"
assert semantic_version_to_int(version) == 261
assert int_to_semantic_version(261) == version
version = "0.0.0"
assert semantic_version_to_int(version) == 0
assert int_to_semantic_version(0) == version
version = "255.255.255"
assert semantic_version_to_int(version) == 16777215
assert int_to_semantic_version(16777215) == version
from haiku.rag.utils import text_to_docling_document
def test_text_to_docling_document():