Infer vector dimension
This commit is contained in:
parent
d1e2470d94
commit
cf8b87246b
1 changed files with 17 additions and 1 deletions
|
|
@ -12,9 +12,25 @@ if TYPE_CHECKING: # pragma: no cover - for type hints only
|
||||||
from haiku.rag.store.engine import Store
|
from haiku.rag.store.engine import Store
|
||||||
|
|
||||||
|
|
||||||
|
def _infer_vector_dim(store: Store) -> int:
|
||||||
|
"""Infer vector dimension from existing data; fallback to embedder config."""
|
||||||
|
try:
|
||||||
|
arrow = store.chunks_table.search().limit(1).to_arrow()
|
||||||
|
rows = arrow.to_pylist()
|
||||||
|
if rows:
|
||||||
|
vec = rows[0].get("vector")
|
||||||
|
if isinstance(vec, list) and vec:
|
||||||
|
return len(vec)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
# Fallback to configured embedder vector dim
|
||||||
|
return getattr(store.embedder, "_vector_dim", 1024)
|
||||||
|
|
||||||
|
|
||||||
def _apply_chunk_order(store: Store) -> None:
|
def _apply_chunk_order(store: Store) -> None:
|
||||||
"""Add integer 'order' column to chunks and backfill from metadata."""
|
"""Add integer 'order' column to chunks and backfill from metadata."""
|
||||||
vector_dim = store.embedder._vector_dim
|
|
||||||
|
vector_dim = _infer_vector_dim(store)
|
||||||
|
|
||||||
# ============== Chunks: add 'order' column and backfill ==============
|
# ============== Chunks: add 'order' column and backfill ==============
|
||||||
class ChunkRecordV2(LanceModel):
|
class ChunkRecordV2(LanceModel):
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue