remove defensive code
This commit is contained in:
parent
e8d89aa035
commit
ee41bc676f
4 changed files with 30 additions and 29 deletions
|
|
@ -182,11 +182,9 @@ def build_picture_chunks(
|
|||
text = extract_item_text(picture, docling_document) or ""
|
||||
|
||||
page_numbers: list[int] = []
|
||||
if prov := getattr(picture, "prov", None):
|
||||
for p in prov:
|
||||
page_no = getattr(p, "page_no", None)
|
||||
if page_no is not None and page_no not in page_numbers:
|
||||
page_numbers.append(page_no)
|
||||
for p in picture.prov:
|
||||
if p.page_no not in page_numbers:
|
||||
page_numbers.append(p.page_no)
|
||||
|
||||
metadata = {
|
||||
"doc_item_refs": [picture.self_ref],
|
||||
|
|
|
|||
|
|
@ -1,5 +1,4 @@
|
|||
import base64
|
||||
import binascii
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
|
@ -44,27 +43,16 @@ def _decode_picture_bytes(item: "PictureItem") -> bytes | None:
|
|||
"""Decode a PictureItem's embedded image into raw bytes.
|
||||
|
||||
Reads ``item.image.uri`` and base64-decodes it when it is a ``data:`` URI.
|
||||
Returns None for items whose image is absent, stripped, or not a data URI
|
||||
(e.g. file references). Tolerant of malformed data — returns None on any
|
||||
decode failure rather than raising.
|
||||
Returns None for items whose image is absent or stripped, or whose URI is
|
||||
a file reference rather than inline data.
|
||||
"""
|
||||
image = getattr(item, "image", None)
|
||||
if image is None:
|
||||
if item.image is None:
|
||||
return None
|
||||
uri = getattr(image, "uri", None)
|
||||
if uri is None:
|
||||
return None
|
||||
uri_str = str(uri)
|
||||
if not uri_str.startswith("data:"):
|
||||
return None
|
||||
try:
|
||||
_, encoded = uri_str.split(",", 1)
|
||||
except ValueError:
|
||||
return None
|
||||
try:
|
||||
return base64.b64decode(encoded, validate=False)
|
||||
except (ValueError, binascii.Error):
|
||||
uri = str(item.image.uri)
|
||||
if not uri.startswith("data:"):
|
||||
return None
|
||||
_, encoded = uri.split(",", 1)
|
||||
return base64.b64decode(encoded, validate=False)
|
||||
|
||||
|
||||
def extract_item_text(item: "NodeItem", docling_doc: "DoclingDocument") -> str | None:
|
||||
|
|
|
|||
|
|
@ -101,13 +101,9 @@ def create_search_toolset(
|
|||
for self_ref, b64 in result.image_data.items():
|
||||
if self_ref in seen:
|
||||
continue
|
||||
try:
|
||||
raw = base64.b64decode(b64)
|
||||
except (ValueError, TypeError):
|
||||
continue
|
||||
binary_parts.append(
|
||||
BinaryContent(
|
||||
data=raw,
|
||||
data=base64.b64decode(b64),
|
||||
media_type="image/png",
|
||||
identifier=self_ref,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -160,6 +160,25 @@ async def test_embed_chunks_empty_list():
|
|||
assert result == []
|
||||
|
||||
|
||||
async def test_embed_chunks_picture_with_text_only_embedder_raises():
|
||||
"""A picture chunk fed through a text-only embedder must surface a
|
||||
clear error, not silently drop the chunk or call ``embed_image_query``
|
||||
on something that doesn't support it."""
|
||||
chunk = Chunk(id="pic", content="x")
|
||||
chunk._picture_data = b"\x89PNG\r\n\x1a\nfake"
|
||||
|
||||
config = AppConfig(
|
||||
embeddings=EmbeddingsConfig(
|
||||
model=EmbeddingModelConfig(
|
||||
provider="ollama", name="qwen3-embedding:4b", vector_dim=2560
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="multimodal embedder"):
|
||||
await embed_chunks([chunk], config)
|
||||
|
||||
|
||||
async def test_embed_chunks_batches_large_inputs(monkeypatch):
|
||||
"""Test that embed_chunks batches calls when chunk count exceeds batch size."""
|
||||
from haiku.rag.embeddings import EMBEDDING_BATCH_SIZE, EmbedderWrapper
|
||||
|
|
|
|||
Loading…
Reference in a new issue