remove defensive code
This commit is contained in:
parent
e8d89aa035
commit
ee41bc676f
4 changed files with 30 additions and 29 deletions
|
|
@ -182,11 +182,9 @@ def build_picture_chunks(
|
||||||
text = extract_item_text(picture, docling_document) or ""
|
text = extract_item_text(picture, docling_document) or ""
|
||||||
|
|
||||||
page_numbers: list[int] = []
|
page_numbers: list[int] = []
|
||||||
if prov := getattr(picture, "prov", None):
|
for p in picture.prov:
|
||||||
for p in prov:
|
if p.page_no not in page_numbers:
|
||||||
page_no = getattr(p, "page_no", None)
|
page_numbers.append(p.page_no)
|
||||||
if page_no is not None and page_no not in page_numbers:
|
|
||||||
page_numbers.append(page_no)
|
|
||||||
|
|
||||||
metadata = {
|
metadata = {
|
||||||
"doc_item_refs": [picture.self_ref],
|
"doc_item_refs": [picture.self_ref],
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,4 @@
|
||||||
import base64
|
import base64
|
||||||
import binascii
|
|
||||||
from typing import TYPE_CHECKING
|
from typing import TYPE_CHECKING
|
||||||
|
|
||||||
from pydantic import BaseModel
|
from pydantic import BaseModel
|
||||||
|
|
@ -44,27 +43,16 @@ def _decode_picture_bytes(item: "PictureItem") -> bytes | None:
|
||||||
"""Decode a PictureItem's embedded image into raw bytes.
|
"""Decode a PictureItem's embedded image into raw bytes.
|
||||||
|
|
||||||
Reads ``item.image.uri`` and base64-decodes it when it is a ``data:`` URI.
|
Reads ``item.image.uri`` and base64-decodes it when it is a ``data:`` URI.
|
||||||
Returns None for items whose image is absent, stripped, or not a data URI
|
Returns None for items whose image is absent or stripped, or whose URI is
|
||||||
(e.g. file references). Tolerant of malformed data — returns None on any
|
a file reference rather than inline data.
|
||||||
decode failure rather than raising.
|
|
||||||
"""
|
"""
|
||||||
image = getattr(item, "image", None)
|
if item.image is None:
|
||||||
if image is None:
|
|
||||||
return None
|
return None
|
||||||
uri = getattr(image, "uri", None)
|
uri = str(item.image.uri)
|
||||||
if uri is None:
|
if not uri.startswith("data:"):
|
||||||
return None
|
|
||||||
uri_str = str(uri)
|
|
||||||
if not uri_str.startswith("data:"):
|
|
||||||
return None
|
|
||||||
try:
|
|
||||||
_, encoded = uri_str.split(",", 1)
|
|
||||||
except ValueError:
|
|
||||||
return None
|
|
||||||
try:
|
|
||||||
return base64.b64decode(encoded, validate=False)
|
|
||||||
except (ValueError, binascii.Error):
|
|
||||||
return None
|
return None
|
||||||
|
_, encoded = uri.split(",", 1)
|
||||||
|
return base64.b64decode(encoded, validate=False)
|
||||||
|
|
||||||
|
|
||||||
def extract_item_text(item: "NodeItem", docling_doc: "DoclingDocument") -> str | None:
|
def extract_item_text(item: "NodeItem", docling_doc: "DoclingDocument") -> str | None:
|
||||||
|
|
|
||||||
|
|
@ -101,13 +101,9 @@ def create_search_toolset(
|
||||||
for self_ref, b64 in result.image_data.items():
|
for self_ref, b64 in result.image_data.items():
|
||||||
if self_ref in seen:
|
if self_ref in seen:
|
||||||
continue
|
continue
|
||||||
try:
|
|
||||||
raw = base64.b64decode(b64)
|
|
||||||
except (ValueError, TypeError):
|
|
||||||
continue
|
|
||||||
binary_parts.append(
|
binary_parts.append(
|
||||||
BinaryContent(
|
BinaryContent(
|
||||||
data=raw,
|
data=base64.b64decode(b64),
|
||||||
media_type="image/png",
|
media_type="image/png",
|
||||||
identifier=self_ref,
|
identifier=self_ref,
|
||||||
)
|
)
|
||||||
|
|
|
||||||
|
|
@ -160,6 +160,25 @@ async def test_embed_chunks_empty_list():
|
||||||
assert result == []
|
assert result == []
|
||||||
|
|
||||||
|
|
||||||
|
async def test_embed_chunks_picture_with_text_only_embedder_raises():
|
||||||
|
"""A picture chunk fed through a text-only embedder must surface a
|
||||||
|
clear error, not silently drop the chunk or call ``embed_image_query``
|
||||||
|
on something that doesn't support it."""
|
||||||
|
chunk = Chunk(id="pic", content="x")
|
||||||
|
chunk._picture_data = b"\x89PNG\r\n\x1a\nfake"
|
||||||
|
|
||||||
|
config = AppConfig(
|
||||||
|
embeddings=EmbeddingsConfig(
|
||||||
|
model=EmbeddingModelConfig(
|
||||||
|
provider="ollama", name="qwen3-embedding:4b", vector_dim=2560
|
||||||
|
)
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
with pytest.raises(ValueError, match="multimodal embedder"):
|
||||||
|
await embed_chunks([chunk], config)
|
||||||
|
|
||||||
|
|
||||||
async def test_embed_chunks_batches_large_inputs(monkeypatch):
|
async def test_embed_chunks_batches_large_inputs(monkeypatch):
|
||||||
"""Test that embed_chunks batches calls when chunk count exceeds batch size."""
|
"""Test that embed_chunks batches calls when chunk count exceeds batch size."""
|
||||||
from haiku.rag.embeddings import EMBEDDING_BATCH_SIZE, EmbedderWrapper
|
from haiku.rag.embeddings import EMBEDDING_BATCH_SIZE, EmbedderWrapper
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue