always store picture bytes; collapse converter to a single zip path
This commit is contained in:
parent
17e36fc069
commit
e88321767d
14 changed files with 144945 additions and 131207 deletions
|
|
@ -80,10 +80,10 @@ async def _update_document_with_chunks(
|
|||
assert document.id is not None, "Document ID is required for update"
|
||||
|
||||
# Snapshot existing picture bytes before deleting items so the post-delete
|
||||
# extract_items can merge them back. Skip under pictures="none" so updates
|
||||
# reclaim storage.
|
||||
# extract_items can merge them back when the live docling has had its
|
||||
# picture URIs stripped (rebuild / re-extract via the stored blob).
|
||||
existing_picture_data: dict[str, bytes] | None = None
|
||||
if docling_document is not None and client._config.processing.pictures != "none":
|
||||
if docling_document is not None:
|
||||
existing_picture_data = (
|
||||
await client.document_item_repository.get_all_picture_data(document.id)
|
||||
)
|
||||
|
|
|
|||
|
|
@ -197,19 +197,14 @@ async def _flush_rebuild_batch(
|
|||
|
||||
# Repopulate document items from stored docling data. The stored docling
|
||||
# blob has had its picture URIs stripped (compress_docling_split), so
|
||||
# re-extracting from it would lose picture_data; under modes that retain
|
||||
# bytes (`description`/`image`) we snapshot the existing bytes per
|
||||
# document and merge them back. Under `none`, we deliberately skip the
|
||||
# snapshot so the rebuild reclaims storage.
|
||||
keep_picture_data = client._config.processing.pictures != "none"
|
||||
# re-extracting from it would lose picture_data — snapshot the existing
|
||||
# bytes per document and merge them back.
|
||||
for doc in documents:
|
||||
assert doc.id is not None
|
||||
docling_doc = doc.get_docling_document()
|
||||
if docling_doc is not None:
|
||||
existing_picture_data = (
|
||||
await client.document_item_repository.get_all_picture_data(doc.id)
|
||||
if keep_picture_data
|
||||
else None
|
||||
)
|
||||
await client.document_item_repository.delete_by_document_id(doc.id)
|
||||
items = extract_items(
|
||||
|
|
|
|||
|
|
@ -123,16 +123,14 @@ class DoclingLocalConverter(DocumentConverter):
|
|||
|
||||
opts = self.config.processing.conversion_options
|
||||
pic_desc = opts.picture_description
|
||||
pictures_mode = self.config.processing.pictures
|
||||
wants_picture_images = pictures_mode != "none"
|
||||
runs_vlm = pictures_mode == "description"
|
||||
runs_vlm = self.config.processing.pictures == "description"
|
||||
|
||||
pipeline_options = PdfPipelineOptions(
|
||||
do_ocr=opts.do_ocr,
|
||||
do_table_structure=opts.do_table_structure,
|
||||
images_scale=opts.images_scale,
|
||||
generate_page_images=opts.generate_page_images,
|
||||
generate_picture_images=wants_picture_images,
|
||||
generate_picture_images=True,
|
||||
table_structure_options=TableStructureOptions(
|
||||
do_cell_matching=opts.table_cell_matching,
|
||||
mode=(
|
||||
|
|
|
|||
|
|
@ -84,35 +84,22 @@ class DoclingServeConverter(DocumentConverter):
|
|||
|
||||
raise ValueError(f"Unsupported VLM provider: {model.provider}")
|
||||
|
||||
def _picture_images_enabled(self) -> bool:
|
||||
"""Whether the conversion should produce embedded picture images."""
|
||||
return self.config.processing.pictures != "none"
|
||||
|
||||
def _build_conversion_data(self) -> dict[str, str | list[str]]:
|
||||
"""Build form data for conversion request.
|
||||
|
||||
When picture images are requested, switch to
|
||||
``image_export_mode="referenced"`` + ``target_type="zip"``. Per
|
||||
docling-jobkit, ``generate_picture_images=True`` is only set when
|
||||
``image_export_mode == "referenced"``; with ``"embedded"`` the server
|
||||
emits page images but leaves ``PictureItem.image=None`` (upstream
|
||||
issue docling-project/docling-serve#576). The zip target then ships
|
||||
the picture (and page) bytes as files under ``artifacts/`` which the
|
||||
caller rehydrates back into ``data:`` URIs for parity with the local
|
||||
Picture bytes are always retrieved via ``image_export_mode="referenced"``
|
||||
+ ``target_type="zip"``. Per docling-jobkit, ``generate_picture_images=True``
|
||||
is only honored when ``image_export_mode == "referenced"``; with
|
||||
``"embedded"`` the server emits page images but leaves
|
||||
``PictureItem.image=None`` (upstream issue
|
||||
docling-project/docling-serve#576). The zip target ships the picture
|
||||
(and page) bytes as files under ``artifacts/`` which the caller
|
||||
rehydrates back into ``data:`` URIs for parity with the local
|
||||
converter.
|
||||
"""
|
||||
opts = self.config.processing.conversion_options
|
||||
pic_desc = opts.picture_description
|
||||
pictures_mode = self.config.processing.pictures
|
||||
picture_images_enabled = pictures_mode != "none"
|
||||
runs_vlm = pictures_mode == "description"
|
||||
|
||||
if picture_images_enabled:
|
||||
image_export_mode = "referenced"
|
||||
else:
|
||||
image_export_mode = (
|
||||
"embedded" if opts.generate_page_images else "placeholder"
|
||||
)
|
||||
runs_vlm = self.config.processing.pictures == "description"
|
||||
|
||||
data: dict[str, str | list[str]] = {
|
||||
"to_formats": "json",
|
||||
|
|
@ -123,14 +110,12 @@ class DoclingServeConverter(DocumentConverter):
|
|||
"table_mode": opts.table_mode,
|
||||
"table_cell_matching": str(opts.table_cell_matching).lower(),
|
||||
"images_scale": str(opts.images_scale),
|
||||
"image_export_mode": image_export_mode,
|
||||
"include_images": str(picture_images_enabled).lower(),
|
||||
"image_export_mode": "referenced",
|
||||
"include_images": "true",
|
||||
"do_picture_description": str(runs_vlm).lower(),
|
||||
"target_type": "zip",
|
||||
}
|
||||
|
||||
if picture_images_enabled:
|
||||
data["target_type"] = "zip"
|
||||
|
||||
if opts.ocr_lang:
|
||||
data["ocr_lang"] = opts.ocr_lang
|
||||
|
||||
|
|
@ -200,9 +185,21 @@ class DoclingServeConverter(DocumentConverter):
|
|||
|
||||
for picture in doc_json.get("pictures") or []:
|
||||
_inline(picture.get("image"))
|
||||
|
||||
# Docling-serve always bundles page rasters when
|
||||
# image_export_mode="referenced". Honor the local
|
||||
# ``generate_page_images`` flag by stripping them when the user
|
||||
# didn't ask for whole-page images.
|
||||
keep_page_images = (
|
||||
self.config.processing.conversion_options.generate_page_images
|
||||
)
|
||||
for page in (doc_json.get("pages") or {}).values():
|
||||
if isinstance(page, dict):
|
||||
if not isinstance(page, dict):
|
||||
continue
|
||||
if keep_page_images:
|
||||
_inline(page.get("image"))
|
||||
else:
|
||||
page["image"] = None
|
||||
|
||||
return DoclingDocument.model_validate(doc_json)
|
||||
|
||||
|
|
@ -219,39 +216,15 @@ class DoclingServeConverter(DocumentConverter):
|
|||
Raises:
|
||||
ValueError: If conversion fails or service is unavailable
|
||||
"""
|
||||
from docling_core.types.doc.document import DoclingDocument
|
||||
|
||||
data = self._build_conversion_data()
|
||||
|
||||
if self._picture_images_enabled():
|
||||
zip_bytes = await self.client.submit_and_poll_zip(
|
||||
endpoint="/v1/convert/file/async",
|
||||
files=files,
|
||||
data=data,
|
||||
name=name,
|
||||
)
|
||||
return self._parse_zip_to_docling(zip_bytes, name)
|
||||
|
||||
result = await self.client.submit_and_poll(
|
||||
zip_bytes = await self.client.submit_and_poll_zip(
|
||||
endpoint="/v1/convert/file/async",
|
||||
files=files,
|
||||
data=data,
|
||||
name=name,
|
||||
)
|
||||
|
||||
if result.get("status") not in ("success", "partial_success", None):
|
||||
errors = result.get("errors", [])
|
||||
raise ValueError(f"Conversion failed: {errors}")
|
||||
|
||||
json_content = result.get("document", {}).get("json_content")
|
||||
|
||||
if json_content is None:
|
||||
raise ValueError(
|
||||
f"docling-serve did not return JSON content for {name}. "
|
||||
"This may indicate an unsupported file format."
|
||||
)
|
||||
|
||||
return DoclingDocument.model_validate(json_content)
|
||||
return self._parse_zip_to_docling(zip_bytes, name)
|
||||
|
||||
async def convert_file(self, path: Path) -> "DoclingDocument":
|
||||
"""Convert a file to DoclingDocument using docling-serve.
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
|
|
@ -1,15 +1,15 @@
|
|||
interactions:
|
||||
- request:
|
||||
body: "--b484380e1e52463dfd1441de1832f57a\r\nContent-Disposition: form-data; name=\"to_formats\"\r\n\r\njson\r\n--b484380e1e52463dfd1441de1832f57a\r\nContent-Disposition:
|
||||
form-data; name=\"do_ocr\"\r\n\r\ntrue\r\n--b484380e1e52463dfd1441de1832f57a\r\nContent-Disposition: form-data; name=\"force_ocr\"\r\n\r\nfalse\r\n--b484380e1e52463dfd1441de1832f57a\r\nContent-Disposition:
|
||||
form-data; name=\"ocr_engine\"\r\n\r\nauto\r\n--b484380e1e52463dfd1441de1832f57a\r\nContent-Disposition: form-data;
|
||||
name=\"do_table_structure\"\r\n\r\ntrue\r\n--b484380e1e52463dfd1441de1832f57a\r\nContent-Disposition: form-data; name=\"table_mode\"\r\n\r\naccurate\r\n--b484380e1e52463dfd1441de1832f57a\r\nContent-Disposition:
|
||||
form-data; name=\"table_cell_matching\"\r\n\r\ntrue\r\n--b484380e1e52463dfd1441de1832f57a\r\nContent-Disposition: form-data;
|
||||
name=\"images_scale\"\r\n\r\n2.0\r\n--b484380e1e52463dfd1441de1832f57a\r\nContent-Disposition: form-data; name=\"image_export_mode\"\r\n\r\nembedded\r\n--b484380e1e52463dfd1441de1832f57a\r\nContent-Disposition:
|
||||
form-data; name=\"include_images\"\r\n\r\nfalse\r\n--b484380e1e52463dfd1441de1832f57a\r\nContent-Disposition: form-data;
|
||||
name=\"do_picture_description\"\r\n\r\nfalse\r\n--b484380e1e52463dfd1441de1832f57a\r\nContent-Disposition: form-data;
|
||||
name=\"files\"; filename=\"tmpx3posbtx.md\"\r\nContent-Type: text/markdown\r\n\r\n```python\ndef test():\n return
|
||||
42\n```\r\n--b484380e1e52463dfd1441de1832f57a--\r\n"
|
||||
body: "--531488c6ec8fd3f3dc8d82965be34756\r\nContent-Disposition: form-data; name=\"to_formats\"\r\n\r\njson\r\n--531488c6ec8fd3f3dc8d82965be34756\r\nContent-Disposition:
|
||||
form-data; name=\"do_ocr\"\r\n\r\ntrue\r\n--531488c6ec8fd3f3dc8d82965be34756\r\nContent-Disposition: form-data; name=\"force_ocr\"\r\n\r\nfalse\r\n--531488c6ec8fd3f3dc8d82965be34756\r\nContent-Disposition:
|
||||
form-data; name=\"ocr_engine\"\r\n\r\nauto\r\n--531488c6ec8fd3f3dc8d82965be34756\r\nContent-Disposition: form-data;
|
||||
name=\"do_table_structure\"\r\n\r\ntrue\r\n--531488c6ec8fd3f3dc8d82965be34756\r\nContent-Disposition: form-data; name=\"table_mode\"\r\n\r\naccurate\r\n--531488c6ec8fd3f3dc8d82965be34756\r\nContent-Disposition:
|
||||
form-data; name=\"table_cell_matching\"\r\n\r\ntrue\r\n--531488c6ec8fd3f3dc8d82965be34756\r\nContent-Disposition: form-data;
|
||||
name=\"images_scale\"\r\n\r\n2.0\r\n--531488c6ec8fd3f3dc8d82965be34756\r\nContent-Disposition: form-data; name=\"image_export_mode\"\r\n\r\nreferenced\r\n--531488c6ec8fd3f3dc8d82965be34756\r\nContent-Disposition:
|
||||
form-data; name=\"include_images\"\r\n\r\ntrue\r\n--531488c6ec8fd3f3dc8d82965be34756\r\nContent-Disposition: form-data;
|
||||
name=\"do_picture_description\"\r\n\r\nfalse\r\n--531488c6ec8fd3f3dc8d82965be34756\r\nContent-Disposition: form-data;
|
||||
name=\"target_type\"\r\n\r\nzip\r\n--531488c6ec8fd3f3dc8d82965be34756\r\nContent-Disposition: form-data; name=\"files\";
|
||||
filename=\"tmpjqdhf3ne.md\"\r\nContent-Type: text/markdown\r\n\r\n```python\ndef test():\n return 42\n```\r\n--531488c6ec8fd3f3dc8d82965be34756--\r\n"
|
||||
headers:
|
||||
accept:
|
||||
- '*/*'
|
||||
|
|
@ -18,9 +18,9 @@ interactions:
|
|||
connection:
|
||||
- keep-alive
|
||||
content-length:
|
||||
- '1311'
|
||||
- '1407'
|
||||
content-type:
|
||||
- multipart/form-data; boundary=b484380e1e52463dfd1441de1832f57a
|
||||
- multipart/form-data; boundary=531488c6ec8fd3f3dc8d82965be34756
|
||||
host:
|
||||
- localhost:5001
|
||||
method: POST
|
||||
|
|
@ -33,7 +33,7 @@ interactions:
|
|||
- application/json
|
||||
parsed_body:
|
||||
error_message: null
|
||||
task_id: 5e82f2a0-0775-4d1b-b204-7de29b2b4a1a
|
||||
task_id: 5a0f33d5-4b34-48c2-ab7c-19fda353f926
|
||||
task_meta: null
|
||||
task_position: 1
|
||||
task_status: pending
|
||||
|
|
@ -53,19 +53,19 @@ interactions:
|
|||
host:
|
||||
- localhost:5001
|
||||
method: GET
|
||||
uri: http://localhost:5001/v1/status/poll/5e82f2a0-0775-4d1b-b204-7de29b2b4a1a
|
||||
uri: http://localhost:5001/v1/status/poll/5a0f33d5-4b34-48c2-ab7c-19fda353f926
|
||||
response:
|
||||
headers:
|
||||
content-length:
|
||||
- '155'
|
||||
- '152'
|
||||
content-type:
|
||||
- application/json
|
||||
parsed_body:
|
||||
error_message: null
|
||||
task_id: 5e82f2a0-0775-4d1b-b204-7de29b2b4a1a
|
||||
task_id: 5a0f33d5-4b34-48c2-ab7c-19fda353f926
|
||||
task_meta: null
|
||||
task_position: null
|
||||
task_status: started
|
||||
task_position: 1
|
||||
task_status: pending
|
||||
task_type: convert
|
||||
status:
|
||||
code: 200
|
||||
|
|
@ -82,7 +82,36 @@ interactions:
|
|||
host:
|
||||
- localhost:5001
|
||||
method: GET
|
||||
uri: http://localhost:5001/v1/status/poll/5e82f2a0-0775-4d1b-b204-7de29b2b4a1a
|
||||
uri: http://localhost:5001/v1/status/poll/5a0f33d5-4b34-48c2-ab7c-19fda353f926
|
||||
response:
|
||||
headers:
|
||||
content-length:
|
||||
- '152'
|
||||
content-type:
|
||||
- application/json
|
||||
parsed_body:
|
||||
error_message: null
|
||||
task_id: 5a0f33d5-4b34-48c2-ab7c-19fda353f926
|
||||
task_meta: null
|
||||
task_position: 1
|
||||
task_status: pending
|
||||
task_type: convert
|
||||
status:
|
||||
code: 200
|
||||
message: OK
|
||||
- request:
|
||||
body: ''
|
||||
headers:
|
||||
accept:
|
||||
- '*/*'
|
||||
accept-encoding:
|
||||
- gzip, deflate, zstd
|
||||
connection:
|
||||
- keep-alive
|
||||
host:
|
||||
- localhost:5001
|
||||
method: GET
|
||||
uri: http://localhost:5001/v1/status/poll/5a0f33d5-4b34-48c2-ab7c-19fda353f926
|
||||
response:
|
||||
headers:
|
||||
content-length:
|
||||
|
|
@ -91,7 +120,7 @@ interactions:
|
|||
- application/json
|
||||
parsed_body:
|
||||
error_message: null
|
||||
task_id: 5e82f2a0-0775-4d1b-b204-7de29b2b4a1a
|
||||
task_id: 5a0f33d5-4b34-48c2-ab7c-19fda353f926
|
||||
task_meta: null
|
||||
task_position: null
|
||||
task_status: success
|
||||
|
|
@ -111,78 +140,29 @@ interactions:
|
|||
host:
|
||||
- localhost:5001
|
||||
method: GET
|
||||
uri: http://localhost:5001/v1/result/5e82f2a0-0775-4d1b-b204-7de29b2b4a1a
|
||||
uri: http://localhost:5001/v1/result/5a0f33d5-4b34-48c2-ab7c-19fda353f926
|
||||
response:
|
||||
body:
|
||||
string: !!binary |
|
||||
UEsDBBQAAAAAAIU7pVwAAAAAAAAAAAAAAAAKAAAAYXJ0aWZhY3RzL1BLAwQUAAAACACFO6VcRNYR
|
||||
GJUBAAAoBAAAEAAAAHRtcGpxZGhmM25lLmpzb26dU11PwjAUfedXLNMHTQgMERGe/RdqlrLdbpV+
|
||||
2XboQvjv9m4UxpzRmCxpeu85vef0dPtRFMU2K0GQVBIB8TqKn1TGmSz8UgmQLh4jZgfGMiWxP5vM
|
||||
kknSlgPHCf32npd0LqFtKMMKhvC93/m9YAJcrVswfLqpIGabqw/ZwD1gwyQxdVoSW3rMKkkeV8vF
|
||||
cj5PHhb+W86SI44yDgNTJyKPPeDQDKeVkcxVBs7zLXCaGqBIu5qeAcdTs5Lx3AAKfn4NNSWd959y
|
||||
UoNB3jdW0JEapVwaqpxsgGO5klZDxiiDjraNyuufZDW9AUVNJTpymu71iYOXaadJfOwdmvVHC90J
|
||||
/1FfGFVpe7qlJkp7khgE9mwFiePQ1sTgw1oPO2o0BjsnzkBCvxjsmslUDp35Ru0uz8HnirgcaOTA
|
||||
upvb9YvEjgEfuIzu785stPNXbEa087+NvZzmrYK3kkGvTn0QUrl+GcV7h7KoSAFtNFuJv84oJN6G
|
||||
oVmGr7MbD9nw7n4LdbojvIKUORCdBlVG9GvaT8Pt/jA6fAFQSwECFAMUAAAAAACFO6VcAAAAAAAA
|
||||
AAAAAAAACgAAAAAAAAAAABAA7UEAAAAAYXJ0aWZhY3RzL1BLAQIUAxQAAAAIAIU7pVxE1hEYlQEA
|
||||
ACgEAAAQAAAAAAAAAAAAAACkgSgAAAB0bXBqcWRoZjNuZS5qc29uUEsFBgAAAAACAAIAdgAAAOsB
|
||||
AAAAAA==
|
||||
headers:
|
||||
content-disposition:
|
||||
- attachment; filename="converted_docs.zip"
|
||||
content-length:
|
||||
- '1114'
|
||||
- '631'
|
||||
content-type:
|
||||
- application/json
|
||||
parsed_body:
|
||||
document:
|
||||
doctags_content: null
|
||||
filename: tmpx3posbtx.md
|
||||
html_content: null
|
||||
json_content:
|
||||
body:
|
||||
children:
|
||||
- $ref: '#/texts/0'
|
||||
content_layer: body
|
||||
label: unspecified
|
||||
meta: null
|
||||
name: _root_
|
||||
parent: null
|
||||
self_ref: '#/body'
|
||||
form_items: []
|
||||
furniture:
|
||||
children: []
|
||||
content_layer: furniture
|
||||
label: unspecified
|
||||
meta: null
|
||||
name: _root_
|
||||
parent: null
|
||||
self_ref: '#/furniture'
|
||||
groups: []
|
||||
key_value_items: []
|
||||
name: tmpx3posbtx
|
||||
origin:
|
||||
binary_hash: 9008975733065065710
|
||||
filename: tmpx3posbtx.md
|
||||
mimetype: text/markdown
|
||||
uri: null
|
||||
pages: {}
|
||||
pictures: []
|
||||
schema_name: DoclingDocument
|
||||
tables: []
|
||||
texts:
|
||||
- captions: []
|
||||
children: []
|
||||
code_language: unknown
|
||||
content_layer: body
|
||||
footnotes: []
|
||||
formatting: null
|
||||
hyperlink: null
|
||||
image: null
|
||||
label: code
|
||||
meta: null
|
||||
orig: |-
|
||||
def test():
|
||||
return 42
|
||||
parent:
|
||||
$ref: '#/body'
|
||||
prov: []
|
||||
references: []
|
||||
self_ref: '#/texts/0'
|
||||
text: |-
|
||||
def test():
|
||||
return 42
|
||||
version: 1.10.0
|
||||
md_content: null
|
||||
text_content: null
|
||||
errors: []
|
||||
processing_time: 0.001853875000961125
|
||||
status: success
|
||||
timings: {}
|
||||
- application/zip
|
||||
status:
|
||||
code: 200
|
||||
message: OK
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because it is too large
Load diff
File diff suppressed because one or more lines are too long
|
|
@ -1,14 +1,15 @@
|
|||
interactions:
|
||||
- request:
|
||||
body: "--b76818aa08785867d7feea5e663b3613\r\nContent-Disposition: form-data; name=\"to_formats\"\r\n\r\njson\r\n--b76818aa08785867d7feea5e663b3613\r\nContent-Disposition:
|
||||
form-data; name=\"do_ocr\"\r\n\r\ntrue\r\n--b76818aa08785867d7feea5e663b3613\r\nContent-Disposition: form-data; name=\"force_ocr\"\r\n\r\nfalse\r\n--b76818aa08785867d7feea5e663b3613\r\nContent-Disposition:
|
||||
form-data; name=\"ocr_engine\"\r\n\r\nauto\r\n--b76818aa08785867d7feea5e663b3613\r\nContent-Disposition: form-data;
|
||||
name=\"do_table_structure\"\r\n\r\ntrue\r\n--b76818aa08785867d7feea5e663b3613\r\nContent-Disposition: form-data; name=\"table_mode\"\r\n\r\naccurate\r\n--b76818aa08785867d7feea5e663b3613\r\nContent-Disposition:
|
||||
form-data; name=\"table_cell_matching\"\r\n\r\ntrue\r\n--b76818aa08785867d7feea5e663b3613\r\nContent-Disposition: form-data;
|
||||
name=\"images_scale\"\r\n\r\n2.0\r\n--b76818aa08785867d7feea5e663b3613\r\nContent-Disposition: form-data; name=\"image_export_mode\"\r\n\r\nembedded\r\n--b76818aa08785867d7feea5e663b3613\r\nContent-Disposition:
|
||||
form-data; name=\"include_images\"\r\n\r\nfalse\r\n--b76818aa08785867d7feea5e663b3613\r\nContent-Disposition: form-data;
|
||||
name=\"do_picture_description\"\r\n\r\nfalse\r\n--b76818aa08785867d7feea5e663b3613\r\nContent-Disposition: form-data;
|
||||
name=\"files\"; filename=\"content.md\"\r\nContent-Type: text/markdown\r\n\r\n# Test Document\n\nThis is a test.\r\n--b76818aa08785867d7feea5e663b3613--\r\n"
|
||||
body: "--e8cb45d53bf32f9efdc69acc0aab5ba7\r\nContent-Disposition: form-data; name=\"to_formats\"\r\n\r\njson\r\n--e8cb45d53bf32f9efdc69acc0aab5ba7\r\nContent-Disposition:
|
||||
form-data; name=\"do_ocr\"\r\n\r\ntrue\r\n--e8cb45d53bf32f9efdc69acc0aab5ba7\r\nContent-Disposition: form-data; name=\"force_ocr\"\r\n\r\nfalse\r\n--e8cb45d53bf32f9efdc69acc0aab5ba7\r\nContent-Disposition:
|
||||
form-data; name=\"ocr_engine\"\r\n\r\nauto\r\n--e8cb45d53bf32f9efdc69acc0aab5ba7\r\nContent-Disposition: form-data;
|
||||
name=\"do_table_structure\"\r\n\r\ntrue\r\n--e8cb45d53bf32f9efdc69acc0aab5ba7\r\nContent-Disposition: form-data; name=\"table_mode\"\r\n\r\naccurate\r\n--e8cb45d53bf32f9efdc69acc0aab5ba7\r\nContent-Disposition:
|
||||
form-data; name=\"table_cell_matching\"\r\n\r\ntrue\r\n--e8cb45d53bf32f9efdc69acc0aab5ba7\r\nContent-Disposition: form-data;
|
||||
name=\"images_scale\"\r\n\r\n2.0\r\n--e8cb45d53bf32f9efdc69acc0aab5ba7\r\nContent-Disposition: form-data; name=\"image_export_mode\"\r\n\r\nreferenced\r\n--e8cb45d53bf32f9efdc69acc0aab5ba7\r\nContent-Disposition:
|
||||
form-data; name=\"include_images\"\r\n\r\ntrue\r\n--e8cb45d53bf32f9efdc69acc0aab5ba7\r\nContent-Disposition: form-data;
|
||||
name=\"do_picture_description\"\r\n\r\nfalse\r\n--e8cb45d53bf32f9efdc69acc0aab5ba7\r\nContent-Disposition: form-data;
|
||||
name=\"target_type\"\r\n\r\nzip\r\n--e8cb45d53bf32f9efdc69acc0aab5ba7\r\nContent-Disposition: form-data; name=\"files\";
|
||||
filename=\"content.md\"\r\nContent-Type: text/markdown\r\n\r\n# Test Document\n\nThis is a test.\r\n--e8cb45d53bf32f9efdc69acc0aab5ba7--\r\n"
|
||||
headers:
|
||||
accept:
|
||||
- '*/*'
|
||||
|
|
@ -17,9 +18,9 @@ interactions:
|
|||
connection:
|
||||
- keep-alive
|
||||
content-length:
|
||||
- '1300'
|
||||
- '1396'
|
||||
content-type:
|
||||
- multipart/form-data; boundary=b76818aa08785867d7feea5e663b3613
|
||||
- multipart/form-data; boundary=e8cb45d53bf32f9efdc69acc0aab5ba7
|
||||
host:
|
||||
- localhost:5001
|
||||
method: POST
|
||||
|
|
@ -32,65 +33,7 @@ interactions:
|
|||
- application/json
|
||||
parsed_body:
|
||||
error_message: null
|
||||
task_id: 1ad62ce1-5260-4fdb-aab7-e17f7adc8ee5
|
||||
task_meta: null
|
||||
task_position: 2
|
||||
task_status: pending
|
||||
task_type: convert
|
||||
status:
|
||||
code: 200
|
||||
message: OK
|
||||
- request:
|
||||
body: ''
|
||||
headers:
|
||||
accept:
|
||||
- '*/*'
|
||||
accept-encoding:
|
||||
- gzip, deflate, zstd
|
||||
connection:
|
||||
- keep-alive
|
||||
host:
|
||||
- localhost:5001
|
||||
method: GET
|
||||
uri: http://localhost:5001/v1/status/poll/1ad62ce1-5260-4fdb-aab7-e17f7adc8ee5
|
||||
response:
|
||||
headers:
|
||||
content-length:
|
||||
- '152'
|
||||
content-type:
|
||||
- application/json
|
||||
parsed_body:
|
||||
error_message: null
|
||||
task_id: 1ad62ce1-5260-4fdb-aab7-e17f7adc8ee5
|
||||
task_meta: null
|
||||
task_position: 2
|
||||
task_status: pending
|
||||
task_type: convert
|
||||
status:
|
||||
code: 200
|
||||
message: OK
|
||||
- request:
|
||||
body: ''
|
||||
headers:
|
||||
accept:
|
||||
- '*/*'
|
||||
accept-encoding:
|
||||
- gzip, deflate, zstd
|
||||
connection:
|
||||
- keep-alive
|
||||
host:
|
||||
- localhost:5001
|
||||
method: GET
|
||||
uri: http://localhost:5001/v1/status/poll/1ad62ce1-5260-4fdb-aab7-e17f7adc8ee5
|
||||
response:
|
||||
headers:
|
||||
content-length:
|
||||
- '152'
|
||||
content-type:
|
||||
- application/json
|
||||
parsed_body:
|
||||
error_message: null
|
||||
task_id: 1ad62ce1-5260-4fdb-aab7-e17f7adc8ee5
|
||||
task_id: 4f56c428-78b8-4a62-815c-9da38f85e1e7
|
||||
task_meta: null
|
||||
task_position: 1
|
||||
task_status: pending
|
||||
|
|
@ -110,297 +53,7 @@ interactions:
|
|||
host:
|
||||
- localhost:5001
|
||||
method: GET
|
||||
uri: http://localhost:5001/v1/status/poll/1ad62ce1-5260-4fdb-aab7-e17f7adc8ee5
|
||||
response:
|
||||
headers:
|
||||
content-length:
|
||||
- '152'
|
||||
content-type:
|
||||
- application/json
|
||||
parsed_body:
|
||||
error_message: null
|
||||
task_id: 1ad62ce1-5260-4fdb-aab7-e17f7adc8ee5
|
||||
task_meta: null
|
||||
task_position: 1
|
||||
task_status: pending
|
||||
task_type: convert
|
||||
status:
|
||||
code: 200
|
||||
message: OK
|
||||
- request:
|
||||
body: ''
|
||||
headers:
|
||||
accept:
|
||||
- '*/*'
|
||||
accept-encoding:
|
||||
- gzip, deflate, zstd
|
||||
connection:
|
||||
- keep-alive
|
||||
host:
|
||||
- localhost:5001
|
||||
method: GET
|
||||
uri: http://localhost:5001/v1/status/poll/1ad62ce1-5260-4fdb-aab7-e17f7adc8ee5
|
||||
response:
|
||||
headers:
|
||||
content-length:
|
||||
- '152'
|
||||
content-type:
|
||||
- application/json
|
||||
parsed_body:
|
||||
error_message: null
|
||||
task_id: 1ad62ce1-5260-4fdb-aab7-e17f7adc8ee5
|
||||
task_meta: null
|
||||
task_position: 1
|
||||
task_status: pending
|
||||
task_type: convert
|
||||
status:
|
||||
code: 200
|
||||
message: OK
|
||||
- request:
|
||||
body: ''
|
||||
headers:
|
||||
accept:
|
||||
- '*/*'
|
||||
accept-encoding:
|
||||
- gzip, deflate, zstd
|
||||
connection:
|
||||
- keep-alive
|
||||
host:
|
||||
- localhost:5001
|
||||
method: GET
|
||||
uri: http://localhost:5001/v1/status/poll/1ad62ce1-5260-4fdb-aab7-e17f7adc8ee5
|
||||
response:
|
||||
headers:
|
||||
content-length:
|
||||
- '152'
|
||||
content-type:
|
||||
- application/json
|
||||
parsed_body:
|
||||
error_message: null
|
||||
task_id: 1ad62ce1-5260-4fdb-aab7-e17f7adc8ee5
|
||||
task_meta: null
|
||||
task_position: 1
|
||||
task_status: pending
|
||||
task_type: convert
|
||||
status:
|
||||
code: 200
|
||||
message: OK
|
||||
- request:
|
||||
body: ''
|
||||
headers:
|
||||
accept:
|
||||
- '*/*'
|
||||
accept-encoding:
|
||||
- gzip, deflate, zstd
|
||||
connection:
|
||||
- keep-alive
|
||||
host:
|
||||
- localhost:5001
|
||||
method: GET
|
||||
uri: http://localhost:5001/v1/status/poll/1ad62ce1-5260-4fdb-aab7-e17f7adc8ee5
|
||||
response:
|
||||
headers:
|
||||
content-length:
|
||||
- '152'
|
||||
content-type:
|
||||
- application/json
|
||||
parsed_body:
|
||||
error_message: null
|
||||
task_id: 1ad62ce1-5260-4fdb-aab7-e17f7adc8ee5
|
||||
task_meta: null
|
||||
task_position: 1
|
||||
task_status: pending
|
||||
task_type: convert
|
||||
status:
|
||||
code: 200
|
||||
message: OK
|
||||
- request:
|
||||
body: ''
|
||||
headers:
|
||||
accept:
|
||||
- '*/*'
|
||||
accept-encoding:
|
||||
- gzip, deflate, zstd
|
||||
connection:
|
||||
- keep-alive
|
||||
host:
|
||||
- localhost:5001
|
||||
method: GET
|
||||
uri: http://localhost:5001/v1/status/poll/1ad62ce1-5260-4fdb-aab7-e17f7adc8ee5
|
||||
response:
|
||||
headers:
|
||||
content-length:
|
||||
- '152'
|
||||
content-type:
|
||||
- application/json
|
||||
parsed_body:
|
||||
error_message: null
|
||||
task_id: 1ad62ce1-5260-4fdb-aab7-e17f7adc8ee5
|
||||
task_meta: null
|
||||
task_position: 1
|
||||
task_status: pending
|
||||
task_type: convert
|
||||
status:
|
||||
code: 200
|
||||
message: OK
|
||||
- request:
|
||||
body: ''
|
||||
headers:
|
||||
accept:
|
||||
- '*/*'
|
||||
accept-encoding:
|
||||
- gzip, deflate, zstd
|
||||
connection:
|
||||
- keep-alive
|
||||
host:
|
||||
- localhost:5001
|
||||
method: GET
|
||||
uri: http://localhost:5001/v1/status/poll/1ad62ce1-5260-4fdb-aab7-e17f7adc8ee5
|
||||
response:
|
||||
headers:
|
||||
content-length:
|
||||
- '152'
|
||||
content-type:
|
||||
- application/json
|
||||
parsed_body:
|
||||
error_message: null
|
||||
task_id: 1ad62ce1-5260-4fdb-aab7-e17f7adc8ee5
|
||||
task_meta: null
|
||||
task_position: 1
|
||||
task_status: pending
|
||||
task_type: convert
|
||||
status:
|
||||
code: 200
|
||||
message: OK
|
||||
- request:
|
||||
body: ''
|
||||
headers:
|
||||
accept:
|
||||
- '*/*'
|
||||
accept-encoding:
|
||||
- gzip, deflate, zstd
|
||||
connection:
|
||||
- keep-alive
|
||||
host:
|
||||
- localhost:5001
|
||||
method: GET
|
||||
uri: http://localhost:5001/v1/status/poll/1ad62ce1-5260-4fdb-aab7-e17f7adc8ee5
|
||||
response:
|
||||
headers:
|
||||
content-length:
|
||||
- '152'
|
||||
content-type:
|
||||
- application/json
|
||||
parsed_body:
|
||||
error_message: null
|
||||
task_id: 1ad62ce1-5260-4fdb-aab7-e17f7adc8ee5
|
||||
task_meta: null
|
||||
task_position: 1
|
||||
task_status: pending
|
||||
task_type: convert
|
||||
status:
|
||||
code: 200
|
||||
message: OK
|
||||
- request:
|
||||
body: ''
|
||||
headers:
|
||||
accept:
|
||||
- '*/*'
|
||||
accept-encoding:
|
||||
- gzip, deflate, zstd
|
||||
connection:
|
||||
- keep-alive
|
||||
host:
|
||||
- localhost:5001
|
||||
method: GET
|
||||
uri: http://localhost:5001/v1/status/poll/1ad62ce1-5260-4fdb-aab7-e17f7adc8ee5
|
||||
response:
|
||||
headers:
|
||||
content-length:
|
||||
- '152'
|
||||
content-type:
|
||||
- application/json
|
||||
parsed_body:
|
||||
error_message: null
|
||||
task_id: 1ad62ce1-5260-4fdb-aab7-e17f7adc8ee5
|
||||
task_meta: null
|
||||
task_position: 1
|
||||
task_status: pending
|
||||
task_type: convert
|
||||
status:
|
||||
code: 200
|
||||
message: OK
|
||||
- request:
|
||||
body: ''
|
||||
headers:
|
||||
accept:
|
||||
- '*/*'
|
||||
accept-encoding:
|
||||
- gzip, deflate, zstd
|
||||
connection:
|
||||
- keep-alive
|
||||
host:
|
||||
- localhost:5001
|
||||
method: GET
|
||||
uri: http://localhost:5001/v1/status/poll/1ad62ce1-5260-4fdb-aab7-e17f7adc8ee5
|
||||
response:
|
||||
headers:
|
||||
content-length:
|
||||
- '152'
|
||||
content-type:
|
||||
- application/json
|
||||
parsed_body:
|
||||
error_message: null
|
||||
task_id: 1ad62ce1-5260-4fdb-aab7-e17f7adc8ee5
|
||||
task_meta: null
|
||||
task_position: 1
|
||||
task_status: pending
|
||||
task_type: convert
|
||||
status:
|
||||
code: 200
|
||||
message: OK
|
||||
- request:
|
||||
body: ''
|
||||
headers:
|
||||
accept:
|
||||
- '*/*'
|
||||
accept-encoding:
|
||||
- gzip, deflate, zstd
|
||||
connection:
|
||||
- keep-alive
|
||||
host:
|
||||
- localhost:5001
|
||||
method: GET
|
||||
uri: http://localhost:5001/v1/status/poll/1ad62ce1-5260-4fdb-aab7-e17f7adc8ee5
|
||||
response:
|
||||
headers:
|
||||
content-length:
|
||||
- '152'
|
||||
content-type:
|
||||
- application/json
|
||||
parsed_body:
|
||||
error_message: null
|
||||
task_id: 1ad62ce1-5260-4fdb-aab7-e17f7adc8ee5
|
||||
task_meta: null
|
||||
task_position: 1
|
||||
task_status: pending
|
||||
task_type: convert
|
||||
status:
|
||||
code: 200
|
||||
message: OK
|
||||
- request:
|
||||
body: ''
|
||||
headers:
|
||||
accept:
|
||||
- '*/*'
|
||||
accept-encoding:
|
||||
- gzip, deflate, zstd
|
||||
connection:
|
||||
- keep-alive
|
||||
host:
|
||||
- localhost:5001
|
||||
method: GET
|
||||
uri: http://localhost:5001/v1/status/poll/1ad62ce1-5260-4fdb-aab7-e17f7adc8ee5
|
||||
uri: http://localhost:5001/v1/status/poll/4f56c428-78b8-4a62-815c-9da38f85e1e7
|
||||
response:
|
||||
headers:
|
||||
content-length:
|
||||
|
|
@ -409,7 +62,65 @@ interactions:
|
|||
- application/json
|
||||
parsed_body:
|
||||
error_message: null
|
||||
task_id: 1ad62ce1-5260-4fdb-aab7-e17f7adc8ee5
|
||||
task_id: 4f56c428-78b8-4a62-815c-9da38f85e1e7
|
||||
task_meta: null
|
||||
task_position: null
|
||||
task_status: started
|
||||
task_type: convert
|
||||
status:
|
||||
code: 200
|
||||
message: OK
|
||||
- request:
|
||||
body: ''
|
||||
headers:
|
||||
accept:
|
||||
- '*/*'
|
||||
accept-encoding:
|
||||
- gzip, deflate, zstd
|
||||
connection:
|
||||
- keep-alive
|
||||
host:
|
||||
- localhost:5001
|
||||
method: GET
|
||||
uri: http://localhost:5001/v1/status/poll/4f56c428-78b8-4a62-815c-9da38f85e1e7
|
||||
response:
|
||||
headers:
|
||||
content-length:
|
||||
- '155'
|
||||
content-type:
|
||||
- application/json
|
||||
parsed_body:
|
||||
error_message: null
|
||||
task_id: 4f56c428-78b8-4a62-815c-9da38f85e1e7
|
||||
task_meta: null
|
||||
task_position: null
|
||||
task_status: started
|
||||
task_type: convert
|
||||
status:
|
||||
code: 200
|
||||
message: OK
|
||||
- request:
|
||||
body: ''
|
||||
headers:
|
||||
accept:
|
||||
- '*/*'
|
||||
accept-encoding:
|
||||
- gzip, deflate, zstd
|
||||
connection:
|
||||
- keep-alive
|
||||
host:
|
||||
- localhost:5001
|
||||
method: GET
|
||||
uri: http://localhost:5001/v1/status/poll/4f56c428-78b8-4a62-815c-9da38f85e1e7
|
||||
response:
|
||||
headers:
|
||||
content-length:
|
||||
- '155'
|
||||
content-type:
|
||||
- application/json
|
||||
parsed_body:
|
||||
error_message: null
|
||||
task_id: 4f56c428-78b8-4a62-815c-9da38f85e1e7
|
||||
task_meta: null
|
||||
task_position: null
|
||||
task_status: success
|
||||
|
|
@ -429,82 +140,28 @@ interactions:
|
|||
host:
|
||||
- localhost:5001
|
||||
method: GET
|
||||
uri: http://localhost:5001/v1/result/1ad62ce1-5260-4fdb-aab7-e17f7adc8ee5
|
||||
uri: http://localhost:5001/v1/result/4f56c428-78b8-4a62-815c-9da38f85e1e7
|
||||
response:
|
||||
body:
|
||||
string: !!binary |
|
||||
UEsDBBQAAAAAAIU7pVwAAAAAAAAAAAAAAAAKAAAAYXJ0aWZhY3RzL1BLAwQUAAAACACFO6VcZbiO
|
||||
d4ABAADNBAAADAAAAGNvbnRlbnQuanNvbsWT306DMBTG73kKgl4ujKqZuGsfwTtjSIEDNOsf0pYp
|
||||
WXh32w4YY8wZb0xISM/3nfb7lcPB8/1AZRUwnHDMINj6wavIKOGleTUMuA5W1rMHqYjgVkchisLo
|
||||
WB56MsH16BWSlMRaD2Zl1oww0G3tjBq+9JphucvFJ3d2Y0gJx7JNKqwq40Exenp5ROhhs4kRip6f
|
||||
URz3xoJQmB0ZsjwwYudOLhrJiW4knA5XQItEQmFb7tYnQ79jVhGaS7Bp3z+G2nHnhOIWpO276Boy
|
||||
JFIInQxVilOgttxwVUNGCgKTbKnI22uxnLaQyFX8vsep92OPvUm1joJe61a3zWg0u/dV3mmcv6CW
|
||||
UjS1Gq/UfXQ18gwBZ3cw8AwcQY2lHantMpHLOGdf+pw3AKcwmmgKkwBS7M83spNtjW+gtH/2gzjZ
|
||||
IlzK3iTij+zoX9lt9t+gV0T55sG+NpThAvzM4A0Dd5yFmmT2T5pOB07pdL2DNtlj2kBCNLCJUAjJ
|
||||
5rUal6730HndN1BLAQIUAxQAAAAAAIU7pVwAAAAAAAAAAAAAAAAKAAAAAAAAAAAAEADtQQAAAABh
|
||||
cnRpZmFjdHMvUEsBAhQDFAAAAAgAhTulXGW4jneAAQAAzQQAAAwAAAAAAAAAAAAAAKSBKAAAAGNv
|
||||
bnRlbnQuanNvblBLBQYAAAAAAgACAHIAAADSAQAAAAA=
|
||||
headers:
|
||||
content-disposition:
|
||||
- attachment; filename="converted_docs.zip"
|
||||
content-length:
|
||||
- '1226'
|
||||
- '602'
|
||||
content-type:
|
||||
- application/json
|
||||
parsed_body:
|
||||
document:
|
||||
doctags_content: null
|
||||
filename: content.md
|
||||
html_content: null
|
||||
json_content:
|
||||
body:
|
||||
children:
|
||||
- $ref: '#/texts/0'
|
||||
- $ref: '#/texts/1'
|
||||
content_layer: body
|
||||
label: unspecified
|
||||
meta: null
|
||||
name: _root_
|
||||
parent: null
|
||||
self_ref: '#/body'
|
||||
form_items: []
|
||||
furniture:
|
||||
children: []
|
||||
content_layer: furniture
|
||||
label: unspecified
|
||||
meta: null
|
||||
name: _root_
|
||||
parent: null
|
||||
self_ref: '#/furniture'
|
||||
groups: []
|
||||
key_value_items: []
|
||||
name: content
|
||||
origin:
|
||||
binary_hash: 18149311266811077188
|
||||
filename: content.md
|
||||
mimetype: text/markdown
|
||||
uri: null
|
||||
pages: {}
|
||||
pictures: []
|
||||
schema_name: DoclingDocument
|
||||
tables: []
|
||||
texts:
|
||||
- children: []
|
||||
content_layer: body
|
||||
formatting: null
|
||||
hyperlink: null
|
||||
label: title
|
||||
meta: null
|
||||
orig: Test Document
|
||||
parent:
|
||||
$ref: '#/body'
|
||||
prov: []
|
||||
self_ref: '#/texts/0'
|
||||
text: Test Document
|
||||
- children: []
|
||||
content_layer: body
|
||||
formatting: null
|
||||
hyperlink: null
|
||||
label: text
|
||||
meta: null
|
||||
orig: This is a test.
|
||||
parent:
|
||||
$ref: '#/body'
|
||||
prov: []
|
||||
self_ref: '#/texts/1'
|
||||
text: This is a test.
|
||||
version: 1.10.0
|
||||
md_content: null
|
||||
text_content: null
|
||||
errors: []
|
||||
processing_time: 0.003092042010393925
|
||||
status: success
|
||||
timings: {}
|
||||
- application/zip
|
||||
status:
|
||||
code: 200
|
||||
message: OK
|
||||
|
|
|
|||
|
|
@ -54,31 +54,6 @@ def create_mock_docling_document(name: str = "test") -> dict:
|
|||
}
|
||||
|
||||
|
||||
def create_async_workflow_mocks(
|
||||
doc_json: dict, task_id: str = "test-task-123"
|
||||
) -> tuple[Mock, Mock, Mock]:
|
||||
"""Create mock responses for docling-serve async workflow.
|
||||
|
||||
Returns tuple of (submit_response, poll_response, result_response).
|
||||
"""
|
||||
submit_response = Mock()
|
||||
submit_response.status_code = 200
|
||||
submit_response.json.return_value = {"task_id": task_id, "task_status": "pending"}
|
||||
submit_response.raise_for_status = Mock()
|
||||
|
||||
poll_response = Mock()
|
||||
poll_response.status_code = 200
|
||||
poll_response.json.return_value = {"task_id": task_id, "task_status": "success"}
|
||||
poll_response.raise_for_status = Mock()
|
||||
|
||||
result_response = Mock()
|
||||
result_response.status_code = 200
|
||||
result_response.json.return_value = {"document": {"json_content": doc_json}}
|
||||
result_response.raise_for_status = Mock()
|
||||
|
||||
return submit_response, poll_response, result_response
|
||||
|
||||
|
||||
def create_async_workflow_zip_mocks(
|
||||
doc_json: dict,
|
||||
artifacts: dict[str, bytes] | None = None,
|
||||
|
|
@ -374,37 +349,20 @@ class TestDoclingLocalConverter:
|
|||
converter.config.processing.conversion_options.generate_page_images is False
|
||||
)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_convert_pdf_without_picture_images(self, config):
|
||||
"""Test PDF conversion excludes embedded images by default."""
|
||||
pdf_path = Path("tests/data/doclaynet.pdf")
|
||||
config.processing.pictures = "none"
|
||||
converter = DoclingLocalConverter(config)
|
||||
|
||||
doc = await converter.convert_file(pdf_path)
|
||||
assert isinstance(doc, DoclingDocument)
|
||||
|
||||
# Check that pictures don't have image data
|
||||
for picture in doc.pictures:
|
||||
assert picture.image is None, (
|
||||
'Pictures should not have image data when pictures="none"'
|
||||
)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_convert_pdf_with_picture_images(self, config):
|
||||
"""Test PDF conversion includes embedded images when enabled."""
|
||||
"""Picture bytes are produced by the local converter for PDFs that
|
||||
contain figures."""
|
||||
pdf_path = Path("tests/data/doclaynet.pdf")
|
||||
config.processing.pictures = "image"
|
||||
converter = DoclingLocalConverter(config)
|
||||
|
||||
doc = await converter.convert_file(pdf_path)
|
||||
assert isinstance(doc, DoclingDocument)
|
||||
|
||||
# Check that at least some pictures have image data
|
||||
pictures_with_images = [p for p in doc.pictures if p.image is not None]
|
||||
if doc.pictures:
|
||||
assert len(pictures_with_images) > 0, (
|
||||
'Pictures should have image data when pictures="image"'
|
||||
"Pictures should carry image data after conversion"
|
||||
)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -652,7 +610,7 @@ class TestDoclingServeConverter:
|
|||
async def test_convert_text_success(self, converter):
|
||||
"""Test successful text conversion via docling-serve async workflow."""
|
||||
doc_json = create_mock_docling_document("test")
|
||||
submit_resp, poll_resp, result_resp = create_async_workflow_mocks(doc_json)
|
||||
submit_resp, poll_resp, result_resp = create_async_workflow_zip_mocks(doc_json)
|
||||
|
||||
with patch("httpx.AsyncClient") as mock_client_class:
|
||||
mock_client = AsyncMock()
|
||||
|
|
@ -674,7 +632,7 @@ class TestDoclingServeConverter:
|
|||
converter = DoclingServeConverter(config)
|
||||
|
||||
doc_json = create_mock_docling_document("test")
|
||||
submit_resp, poll_resp, result_resp = create_async_workflow_mocks(doc_json)
|
||||
submit_resp, poll_resp, result_resp = create_async_workflow_zip_mocks(doc_json)
|
||||
|
||||
with patch("httpx.AsyncClient") as mock_client_class:
|
||||
mock_client = AsyncMock()
|
||||
|
|
@ -700,11 +658,10 @@ class TestDoclingServeConverter:
|
|||
config.processing.conversion_options.table_cell_matching = False
|
||||
config.processing.conversion_options.do_table_structure = False
|
||||
config.processing.conversion_options.images_scale = 3.0
|
||||
config.processing.pictures = "none"
|
||||
converter = DoclingServeConverter(config)
|
||||
|
||||
doc_json = create_mock_docling_document("test")
|
||||
submit_resp, poll_resp, result_resp = create_async_workflow_mocks(doc_json)
|
||||
submit_resp, poll_resp, result_resp = create_async_workflow_zip_mocks(doc_json)
|
||||
|
||||
with patch("httpx.AsyncClient") as mock_client_class:
|
||||
mock_client = AsyncMock()
|
||||
|
|
@ -726,8 +683,9 @@ class TestDoclingServeConverter:
|
|||
assert data["table_cell_matching"] == "false"
|
||||
assert data["do_table_structure"] == "false"
|
||||
assert data["images_scale"] == "3.0"
|
||||
assert data["include_images"] == "false"
|
||||
assert data["image_export_mode"] == "embedded"
|
||||
assert data["include_images"] == "true"
|
||||
assert data["image_export_mode"] == "referenced"
|
||||
assert data["target_type"] == "zip"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ocr_engine_passed_to_api(self, config):
|
||||
|
|
@ -735,32 +693,6 @@ class TestDoclingServeConverter:
|
|||
config.processing.conversion_options.ocr_engine = "rapidocr"
|
||||
converter = DoclingServeConverter(config)
|
||||
|
||||
doc_json = create_mock_docling_document("test")
|
||||
submit_resp, poll_resp, result_resp = create_async_workflow_mocks(doc_json)
|
||||
|
||||
with patch("httpx.AsyncClient") as mock_client_class:
|
||||
mock_client = AsyncMock()
|
||||
mock_client.post = AsyncMock(return_value=submit_resp)
|
||||
mock_client.get = AsyncMock(side_effect=[poll_resp, result_resp])
|
||||
mock_client.__aenter__ = AsyncMock(return_value=mock_client)
|
||||
mock_client.__aexit__ = AsyncMock(return_value=None)
|
||||
mock_client_class.return_value = mock_client
|
||||
|
||||
await converter.convert_text("# Test")
|
||||
|
||||
call_kwargs = mock_client.post.call_args.kwargs
|
||||
data = call_kwargs["data"]
|
||||
assert data["ocr_engine"] == "rapidocr"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_picture_images_request_uses_referenced_zip(self, config):
|
||||
"""When pictures="image" the request flips to
|
||||
image_export_mode=referenced + target_type=zip and consumes a zip
|
||||
response — mirrors the upstream docling-serve#576 workaround.
|
||||
"""
|
||||
config.processing.pictures = "image"
|
||||
converter = DoclingServeConverter(config)
|
||||
|
||||
doc_json = create_mock_docling_document("test")
|
||||
submit_resp, poll_resp, result_resp = create_async_workflow_zip_mocks(doc_json)
|
||||
|
||||
|
|
@ -774,10 +706,9 @@ class TestDoclingServeConverter:
|
|||
|
||||
await converter.convert_text("# Test")
|
||||
|
||||
data = mock_client.post.call_args.kwargs["data"]
|
||||
assert data["image_export_mode"] == "referenced"
|
||||
assert data["target_type"] == "zip"
|
||||
assert data["include_images"] == "true"
|
||||
call_kwargs = mock_client.post.call_args.kwargs
|
||||
data = call_kwargs["data"]
|
||||
assert data["ocr_engine"] == "rapidocr"
|
||||
|
||||
def test_parse_zip_rehydrates_picture_uri(self, config):
|
||||
"""Zip path inlines artifact bytes as data: URIs on PictureItem.image.
|
||||
|
|
@ -916,31 +847,11 @@ class TestDoclingServeConverter:
|
|||
with pytest.raises(ValueError, match="Authentication failed"):
|
||||
await converter.convert_text("# Test")
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_convert_text_no_json_content(self, converter):
|
||||
"""Test handling when docling-serve returns no JSON content."""
|
||||
submit_resp, poll_resp, _ = create_async_workflow_mocks({})
|
||||
result_resp = Mock()
|
||||
result_resp.status_code = 200
|
||||
result_resp.json.return_value = {"document": {"json_content": None}}
|
||||
result_resp.raise_for_status = Mock()
|
||||
|
||||
with patch("httpx.AsyncClient") as mock_client_class:
|
||||
mock_client = AsyncMock()
|
||||
mock_client.post = AsyncMock(return_value=submit_resp)
|
||||
mock_client.get = AsyncMock(side_effect=[poll_resp, result_resp])
|
||||
mock_client.__aenter__ = AsyncMock(return_value=mock_client)
|
||||
mock_client.__aexit__ = AsyncMock(return_value=None)
|
||||
mock_client_class.return_value = mock_client
|
||||
|
||||
with pytest.raises(ValueError, match="did not return JSON content"):
|
||||
await converter.convert_text("# Test")
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_convert_file_pdf(self, converter):
|
||||
"""Test converting PDF file via docling-serve async workflow."""
|
||||
doc_json = create_mock_docling_document("test")
|
||||
submit_resp, poll_resp, result_resp = create_async_workflow_mocks(doc_json)
|
||||
submit_resp, poll_resp, result_resp = create_async_workflow_zip_mocks(doc_json)
|
||||
|
||||
with patch("httpx.AsyncClient") as mock_client_class:
|
||||
mock_client = AsyncMock()
|
||||
|
|
@ -963,7 +874,7 @@ class TestDoclingServeConverter:
|
|||
async def test_convert_file_text(self, converter):
|
||||
"""Test converting text file (reads locally, sends to docling-serve)."""
|
||||
doc_json = create_mock_docling_document("test")
|
||||
submit_resp, poll_resp, result_resp = create_async_workflow_mocks(doc_json)
|
||||
submit_resp, poll_resp, result_resp = create_async_workflow_zip_mocks(doc_json)
|
||||
|
||||
with patch("httpx.AsyncClient") as mock_client_class:
|
||||
mock_client = AsyncMock()
|
||||
|
|
@ -1092,7 +1003,7 @@ class TestDoclingServeConverterPictureDescription:
|
|||
converter = DoclingServeConverter(config)
|
||||
|
||||
doc_json = create_mock_docling_document("test")
|
||||
submit_resp, poll_resp, result_resp = create_async_workflow_mocks(doc_json)
|
||||
submit_resp, poll_resp, result_resp = create_async_workflow_zip_mocks(doc_json)
|
||||
|
||||
with patch("httpx.AsyncClient") as mock_client_class:
|
||||
mock_client = AsyncMock()
|
||||
|
|
@ -1223,23 +1134,6 @@ class TestDoclingServeConverterIntegration:
|
|||
"Pages should have image data when generate_page_images=True"
|
||||
)
|
||||
|
||||
@pytest.mark.vcr()
|
||||
@pytest.mark.asyncio
|
||||
async def test_convert_pdf_without_picture_images(self, config):
|
||||
"""Test PDF conversion excludes picture images when disabled."""
|
||||
pdf_path = Path("tests/data/doclaynet.pdf")
|
||||
config.processing.pictures = "none"
|
||||
converter = DoclingServeConverter(config)
|
||||
|
||||
doc = await converter.convert_file(pdf_path)
|
||||
assert isinstance(doc, DoclingDocument)
|
||||
|
||||
# Check that pictures don't have image data
|
||||
for picture in doc.pictures:
|
||||
assert picture.image is None, (
|
||||
'Pictures should not have image data when pictures="none"'
|
||||
)
|
||||
|
||||
@pytest.mark.vcr()
|
||||
@pytest.mark.asyncio
|
||||
async def test_convert_pdf_with_ocr_engine(self, config):
|
||||
|
|
@ -1256,17 +1150,16 @@ class TestDoclingServeConverterIntegration:
|
|||
@pytest.mark.vcr()
|
||||
@pytest.mark.asyncio
|
||||
async def test_convert_pdf_with_picture_images(self, config):
|
||||
"""Test PDF conversion includes picture images when enabled.
|
||||
"""Picture bytes are produced for PDFs that contain figures.
|
||||
|
||||
docling-serve only emits picture image bytes when
|
||||
``image_export_mode="referenced"`` (upstream issue
|
||||
docling-project/docling-serve#576). The converter switches to
|
||||
``referenced`` + ``target_type="zip"`` when picture images are
|
||||
wanted, then rehydrates the bundled artifact files into ``data:``
|
||||
URIs so the result is shape-equivalent to the local converter.
|
||||
docling-serve only emits picture image bytes via the
|
||||
``image_export_mode="referenced"`` + ``target_type="zip"`` path
|
||||
(upstream issue docling-project/docling-serve#576). The converter
|
||||
always uses that path and rehydrates the bundled artifact files
|
||||
into ``data:`` URIs so the result is shape-equivalent to the local
|
||||
converter.
|
||||
"""
|
||||
pdf_path = Path("tests/data/doclaynet.pdf")
|
||||
config.processing.pictures = "image"
|
||||
converter = DoclingServeConverter(config)
|
||||
|
||||
doc = await converter.convert_file(pdf_path)
|
||||
|
|
@ -1275,7 +1168,7 @@ class TestDoclingServeConverterIntegration:
|
|||
pictures_with_images = [p for p in doc.pictures if p.image is not None]
|
||||
assert doc.pictures, "doclaynet.pdf is expected to contain at least one picture"
|
||||
assert len(pictures_with_images) > 0, (
|
||||
'Pictures should have image data when pictures="image"'
|
||||
"Pictures should carry image data after conversion"
|
||||
)
|
||||
sample = pictures_with_images[0]
|
||||
assert sample.image is not None
|
||||
|
|
|
|||
|
|
@ -196,45 +196,6 @@ async def test_rechunk_preserves_picture_data(temp_db_path):
|
|||
assert after.get("#/pictures/0") == before.get("#/pictures/0")
|
||||
|
||||
|
||||
@pytest.mark.vcr()
|
||||
@pytest.mark.asyncio
|
||||
async def test_update_clears_picture_data_when_mode_none(temp_db_path):
|
||||
"""Switching to ``pictures="none"`` and re-running update_document
|
||||
drops picture_data — the snapshot/merge gate is what gives users a
|
||||
"rebuild reclaims storage" path when they downgrade modes."""
|
||||
from haiku.rag.client.documents import (
|
||||
_store_document_with_chunks,
|
||||
_update_document_with_chunks,
|
||||
)
|
||||
from haiku.rag.store.models.document import Document
|
||||
from tests.store.test_document_items import _docling_doc_with_picture
|
||||
|
||||
docling_doc = _docling_doc_with_picture()
|
||||
|
||||
config = AppConfig()
|
||||
config.processing.pictures = "image"
|
||||
|
||||
async with HaikuRAG(temp_db_path, config=config, create=True) as rag:
|
||||
# Ingest under "image" so picture bytes land in document_items.
|
||||
document = Document(content="x", uri="test://doc")
|
||||
document.set_docling(docling_doc)
|
||||
created = await _store_document_with_chunks(rag, document, [], docling_doc)
|
||||
assert created.id is not None
|
||||
before = await rag.document_item_repository.get_all_picture_data(created.id)
|
||||
assert before.get("#/pictures/0") is not None
|
||||
|
||||
# Downgrade to "none" and re-run update with the (already stripped)
|
||||
# docling pulled from storage. The snapshot/merge must be skipped so
|
||||
# picture_data is cleared on the new items rows.
|
||||
rag._config.processing.pictures = "none"
|
||||
from_blob = created.get_docling_document()
|
||||
assert from_blob is not None
|
||||
await _update_document_with_chunks(rag, created, [], from_blob)
|
||||
|
||||
after = await rag.document_item_repository.get_all_picture_data(created.id)
|
||||
assert after.get("#/pictures/0") is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_expand_context_repopulates_image_data(temp_db_path):
|
||||
"""expand_context rebuilds SearchResult objects via expand_with_items, so
|
||||
|
|
|
|||
Loading…
Reference in a new issue