Read a created database's embedder with its settings
Creating re-read the settings blob and left `stored_embedding` at None, so a client that created a database compared as though it recorded no embedder. `_remember_settings` takes both, and the comment no longer says one follows the other.
This commit is contained in:
parent
18c22f1ddb
commit
5ecffdedf2
2 changed files with 32 additions and 7 deletions
|
|
@ -214,11 +214,19 @@ class Store:
|
||||||
|
|
||||||
# Create embedder (sync — no LanceDB needed)
|
# Create embedder (sync — no LanceDB needed)
|
||||||
self.embedder = get_embedder(config=self._config)
|
self.embedder = get_embedder(config=self._config)
|
||||||
self.stored_embedding: tuple[str | None, str | None, int | None] | None = None
|
# The settings blob as of open, and the embedder it records, so
|
||||||
# The settings blob as of open, so reporting on a database costs no
|
# reporting on a database and comparing it against another cost no
|
||||||
# second read of it. Not refreshed by a later write, the same as
|
# second read. Neither follows a later write.
|
||||||
# `stored_embedding`.
|
|
||||||
self.stored_settings: dict = {}
|
self.stored_settings: dict = {}
|
||||||
|
self.stored_embedding: tuple[str | None, str | None, int | None] | None = None
|
||||||
|
|
||||||
|
def _remember_settings(self, settings: dict) -> None:
|
||||||
|
"""Hold the settings blob and the embedder it records.
|
||||||
|
|
||||||
|
Together, so nothing reports on one reading while comparing the other.
|
||||||
|
"""
|
||||||
|
self.stored_settings = settings
|
||||||
|
self.stored_embedding = _stored_embedding(settings)
|
||||||
|
|
||||||
async def _initialize(self):
|
async def _initialize(self):
|
||||||
"""Perform async initialization: connect to LanceDB, init tables, validate."""
|
"""Perform async initialization: connect to LanceDB, init tables, validate."""
|
||||||
|
|
@ -234,12 +242,11 @@ class Store:
|
||||||
|
|
||||||
if not is_new_db and "settings" in existing_tables:
|
if not is_new_db and "settings" in existing_tables:
|
||||||
self.settings_table = await self.db.open_table("settings")
|
self.settings_table = await self.db.open_table("settings")
|
||||||
self.stored_settings = await self._read_stored_settings()
|
self._remember_settings(await self._read_stored_settings())
|
||||||
|
|
||||||
# An existing database's chunks can only be read with the dimension they
|
# An existing database's chunks can only be read with the dimension they
|
||||||
# were written at.
|
# were written at.
|
||||||
stored_vector_dim = _stored_vector_dim(self.stored_settings)
|
stored_vector_dim = _stored_vector_dim(self.stored_settings)
|
||||||
self.stored_embedding = _stored_embedding(self.stored_settings)
|
|
||||||
chunk_vector_dim = stored_vector_dim or self.embedder._vector_dim
|
chunk_vector_dim = stored_vector_dim or self.embedder._vector_dim
|
||||||
self.ChunkRecord: type[ChunkRecordBase] = create_chunk_model(chunk_vector_dim)
|
self.ChunkRecord: type[ChunkRecordBase] = create_chunk_model(chunk_vector_dim)
|
||||||
|
|
||||||
|
|
@ -251,7 +258,9 @@ class Store:
|
||||||
# Set version for new databases.
|
# Set version for new databases.
|
||||||
if is_new_db and not self._read_only:
|
if is_new_db and not self._read_only:
|
||||||
await self._set_initial_version()
|
await self._set_initial_version()
|
||||||
self.stored_settings = await self._read_stored_settings()
|
# Creating wrote the settings this database will be read with, so
|
||||||
|
# both readings of them are taken again together.
|
||||||
|
self._remember_settings(await self._read_stored_settings())
|
||||||
|
|
||||||
# Validate config compatibility after connection is established
|
# Validate config compatibility after connection is established
|
||||||
if not self._skip_validation:
|
if not self._skip_validation:
|
||||||
|
|
|
||||||
|
|
@ -482,6 +482,22 @@ class TestFusingWhatARerankerReturns:
|
||||||
await _fuse(rag, clients, "cats", per_source, 5)
|
await _fuse(rag, clients, "cats", per_source, 5)
|
||||||
|
|
||||||
|
|
||||||
|
class TestRememberingTheStoredEmbedder:
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_creating_a_database_records_the_embedder_it_wrote(self, tmp_path):
|
||||||
|
"""Creating writes the settings this database will be read with, so a
|
||||||
|
client that created one can be compared against a client that opened
|
||||||
|
one."""
|
||||||
|
config = _config(tmp_path, ["alpha", "beta"])
|
||||||
|
|
||||||
|
async with HaikuRAG(config=config, create=True, sources=["alpha"]) as created:
|
||||||
|
assert created.store.stored_embedding is not None
|
||||||
|
written = created.store.stored_embedding
|
||||||
|
|
||||||
|
async with HaikuRAG(config=config, sources=["alpha"]) as reopened:
|
||||||
|
assert reopened.store.stored_embedding == written
|
||||||
|
|
||||||
|
|
||||||
class TestComparingEmbedders:
|
class TestComparingEmbedders:
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_a_database_recording_no_embedder_is_not_compared(self, tmp_path):
|
async def test_a_database_recording_no_embedder_is_not_compared(self, tmp_path):
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue