Cover sandbox VFS reads, binary part dedup and picture spans
This commit is contained in:
parent
2afe1bd28c
commit
7c120587f0
4 changed files with 119 additions and 2 deletions
|
|
@ -356,7 +356,8 @@ class Sandbox:
|
||||||
return read_toc
|
return read_toc
|
||||||
|
|
||||||
for doc in docs:
|
for doc in docs:
|
||||||
if not doc.id:
|
if not doc.id: # pragma: no cover - rows read from LanceDB always
|
||||||
|
# carry an id; only a hand-built Document could be id-less.
|
||||||
continue
|
continue
|
||||||
doc_id: str = doc.id
|
doc_id: str = doc.id
|
||||||
doc_dir = f"/documents/{doc_id}"
|
doc_dir = f"/documents/{doc_id}"
|
||||||
|
|
@ -438,7 +439,10 @@ class Sandbox:
|
||||||
stdout_lines: list[str] = []
|
stdout_lines: list[str] = []
|
||||||
|
|
||||||
def print_callback(_stream: Literal["stdout"], text: str) -> None:
|
def print_callback(_stream: Literal["stdout"], text: str) -> None:
|
||||||
stdout_lines.append(text)
|
# pragma: no cover - Monty invokes this from its own Rust-owned
|
||||||
|
# thread, which coverage cannot trace. The captured stdout is
|
||||||
|
# asserted by test_execute_simple_code.
|
||||||
|
stdout_lines.append(text) # pragma: no cover
|
||||||
|
|
||||||
max_chars = self._config.analysis.max_output_chars
|
max_chars = self._config.analysis.max_output_chars
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -364,3 +364,57 @@ class TestItemsJsonlSurfacesNewFields:
|
||||||
assert expected <= set(r)
|
assert expected <= set(r)
|
||||||
assert "position" not in r
|
assert "position" not in r
|
||||||
assert "tree_depth" not in r
|
assert "tree_depth" not in r
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
class TestVfsReadPaths:
|
||||||
|
"""The synchronous VFS readers bridge back to the event loop; drive them
|
||||||
|
through a worker thread the way execute() does."""
|
||||||
|
|
||||||
|
async def test_content_txt_is_read_lazily_per_document(self, temp_db_path):
|
||||||
|
async with HaikuRAG(temp_db_path, create=True) as client:
|
||||||
|
doc = await client.document_repository.create(
|
||||||
|
Document(content="the stored body", uri="test://body", title="Body")
|
||||||
|
)
|
||||||
|
doc_id = doc.id
|
||||||
|
|
||||||
|
sandbox = Sandbox(temp_db_path, AppConfig(), AnalysisContext())
|
||||||
|
|
||||||
|
content = await _read_vfs_text(sandbox, f"/documents/{doc_id}/content.txt")
|
||||||
|
|
||||||
|
assert content == "the stored body"
|
||||||
|
|
||||||
|
async def test_document_files_are_read_only(self, temp_db_path):
|
||||||
|
async with HaikuRAG(temp_db_path, create=True) as client:
|
||||||
|
doc = await client.document_repository.create(
|
||||||
|
Document(content="x", uri="test://ro", title="RO")
|
||||||
|
)
|
||||||
|
doc_id = doc.id
|
||||||
|
|
||||||
|
sandbox = Sandbox(temp_db_path, AppConfig(), AnalysisContext())
|
||||||
|
vfs = await sandbox._build_vfs()
|
||||||
|
sandbox._loop = asyncio.get_running_loop()
|
||||||
|
|
||||||
|
with pytest.raises(PermissionError, match="read-only"):
|
||||||
|
await asyncio.to_thread(
|
||||||
|
vfs.path_write_text,
|
||||||
|
PurePosixPath(f"/documents/{doc_id}/content.txt"),
|
||||||
|
"nope",
|
||||||
|
)
|
||||||
|
|
||||||
|
async def test_toc_skips_gaps_in_item_positions(self, temp_db_path):
|
||||||
|
"""Positions need not be contiguous — a heading's span may cover
|
||||||
|
positions that carry no item."""
|
||||||
|
async with HaikuRAG(temp_db_path, create=True) as client:
|
||||||
|
doc_id = await _empty_doc(client, uri="test://gaps", title="Gaps")
|
||||||
|
# Positions 1 and 2 are absent between the header and the paragraph.
|
||||||
|
items = [
|
||||||
|
_header(doc_id, 0, 1, "Intro"),
|
||||||
|
_para(doc_id, 3),
|
||||||
|
]
|
||||||
|
await client.document_item_repository.create_items(doc_id, items)
|
||||||
|
|
||||||
|
sandbox = Sandbox(temp_db_path, AppConfig(), AnalysisContext())
|
||||||
|
toc = await _read_toc(sandbox, doc_id)
|
||||||
|
|
||||||
|
assert [n["title"] for n in toc["tree"]] == ["Intro"]
|
||||||
|
|
|
||||||
|
|
@ -1360,3 +1360,16 @@ class TestExpandWithItemsPictureBytes:
|
||||||
e_low = by_chunk["c-low"]
|
e_low = by_chunk["c-low"]
|
||||||
assert e_low.image_data == {"#/pictures/0": "LOWBYTES"}
|
assert e_low.image_data == {"#/pictures/0": "LOWBYTES"}
|
||||||
assert "HIGHBYTES" not in (e_low.image_data or {}).values()
|
assert "HIGHBYTES" not in (e_low.image_data or {}).values()
|
||||||
|
|
||||||
|
|
||||||
|
class TestSpanInWindow:
|
||||||
|
def test_zero_width_span_is_inside_when_position_is_in_window(self):
|
||||||
|
from haiku.rag.context import _span_in_window
|
||||||
|
from haiku.rag.store.models.document_item import DocumentItem
|
||||||
|
|
||||||
|
item = DocumentItem(
|
||||||
|
document_id="d1", position=0, self_ref="#/pictures/0", label="picture"
|
||||||
|
)
|
||||||
|
# A picture occupies no characters, so containment is by position.
|
||||||
|
assert _span_in_window((10, 10, item), 0, 20) is True
|
||||||
|
assert _span_in_window((30, 30, item), 0, 20) is False
|
||||||
|
|
|
||||||
|
|
@ -214,3 +214,49 @@ def search_config():
|
||||||
from haiku.rag.config import Config
|
from haiku.rag.config import Config
|
||||||
|
|
||||||
return Config
|
return Config
|
||||||
|
|
||||||
|
|
||||||
|
class TestBuildBinaryPartsFromResults:
|
||||||
|
"""Picture bytes are attached once per (document, self_ref) pair."""
|
||||||
|
|
||||||
|
def test_results_without_image_data_contribute_nothing(self):
|
||||||
|
from haiku.rag.tools.search import build_binary_parts_from_results
|
||||||
|
|
||||||
|
results = [
|
||||||
|
SearchResult(content="text only", score=0.5, chunk_id="c1", image_data=None)
|
||||||
|
]
|
||||||
|
|
||||||
|
assert build_binary_parts_from_results(results) == []
|
||||||
|
|
||||||
|
def test_duplicate_document_and_ref_is_attached_once(self):
|
||||||
|
import base64
|
||||||
|
from io import BytesIO
|
||||||
|
|
||||||
|
from PIL import Image as PILImage
|
||||||
|
|
||||||
|
from haiku.rag.tools.search import build_binary_parts_from_results
|
||||||
|
|
||||||
|
buf = BytesIO()
|
||||||
|
PILImage.new("RGB", (4, 4), "red").save(buf, format="PNG")
|
||||||
|
png = base64.b64encode(buf.getvalue()).decode()
|
||||||
|
shared = {"#/pictures/0": png}
|
||||||
|
results = [
|
||||||
|
SearchResult(
|
||||||
|
content="a",
|
||||||
|
score=0.9,
|
||||||
|
chunk_id="c1",
|
||||||
|
document_id="doc-1",
|
||||||
|
image_data=shared,
|
||||||
|
),
|
||||||
|
SearchResult(
|
||||||
|
content="b",
|
||||||
|
score=0.8,
|
||||||
|
chunk_id="c2",
|
||||||
|
document_id="doc-1",
|
||||||
|
image_data=shared,
|
||||||
|
),
|
||||||
|
]
|
||||||
|
|
||||||
|
parts = build_binary_parts_from_results(results)
|
||||||
|
|
||||||
|
assert len(parts) == 1
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue