Do not generate base64 images by default
This commit is contained in:
parent
2e06cf5277
commit
68457f251e
5 changed files with 46 additions and 1 deletions
|
|
@ -35,6 +35,7 @@ processing:
|
||||||
|
|
||||||
# Image settings
|
# Image settings
|
||||||
images_scale: 2.0 # Image scale factor
|
images_scale: 2.0 # Image scale factor
|
||||||
|
generate_picture_images: false # Include embedded images in output
|
||||||
```
|
```
|
||||||
|
|
||||||
### Conversion Options
|
### Conversion Options
|
||||||
|
|
@ -73,10 +74,12 @@ conversion_options:
|
||||||
|
|
||||||
```yaml
|
```yaml
|
||||||
conversion_options:
|
conversion_options:
|
||||||
images_scale: 2.0 # Image resolution scale factor
|
images_scale: 2.0 # Image resolution scale factor
|
||||||
|
generate_picture_images: false # Include embedded images in output
|
||||||
```
|
```
|
||||||
|
|
||||||
- **images_scale**: Scale factor for extracted images. Higher values = better quality but larger size. Typical range: 1.0-3.0.
|
- **images_scale**: Scale factor for extracted images. Higher values = better quality but larger size. Typical range: 1.0-3.0.
|
||||||
|
- **generate_picture_images**: When `true`, embedded images (figures, diagrams) are included as base64-encoded data in the document. When `false` (default), images are excluded to reduce chunk size and avoid context bloat.
|
||||||
|
|
||||||
### Local vs Remote Processing
|
### Local vs Remote Processing
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -106,6 +106,7 @@ class ConversionOptions(BaseModel):
|
||||||
|
|
||||||
# Image options
|
# Image options
|
||||||
images_scale: float = 2.0
|
images_scale: float = 2.0
|
||||||
|
generate_picture_images: bool = False
|
||||||
|
|
||||||
|
|
||||||
class ProcessingConfig(BaseModel):
|
class ProcessingConfig(BaseModel):
|
||||||
|
|
|
||||||
|
|
@ -79,6 +79,7 @@ class DoclingLocalConverter(DocumentConverter):
|
||||||
do_table_structure=opts.do_table_structure,
|
do_table_structure=opts.do_table_structure,
|
||||||
images_scale=opts.images_scale,
|
images_scale=opts.images_scale,
|
||||||
generate_page_images=True,
|
generate_page_images=True,
|
||||||
|
generate_picture_images=opts.generate_picture_images,
|
||||||
table_structure_options=TableStructureOptions(
|
table_structure_options=TableStructureOptions(
|
||||||
do_cell_matching=opts.table_cell_matching,
|
do_cell_matching=opts.table_cell_matching,
|
||||||
mode=(
|
mode=(
|
||||||
|
|
|
||||||
|
|
@ -88,6 +88,7 @@ class DoclingServeConverter(DocumentConverter):
|
||||||
"table_mode": opts.table_mode,
|
"table_mode": opts.table_mode,
|
||||||
"table_cell_matching": str(opts.table_cell_matching).lower(),
|
"table_cell_matching": str(opts.table_cell_matching).lower(),
|
||||||
"images_scale": str(opts.images_scale),
|
"images_scale": str(opts.images_scale),
|
||||||
|
"generate_picture_images": str(opts.generate_picture_images).lower(),
|
||||||
}
|
}
|
||||||
|
|
||||||
if opts.ocr_lang:
|
if opts.ocr_lang:
|
||||||
|
|
|
||||||
|
|
@ -237,6 +237,45 @@ class TestDoclingLocalConverter:
|
||||||
assert converter.config.processing.conversion_options.table_mode == "fast"
|
assert converter.config.processing.conversion_options.table_mode == "fast"
|
||||||
assert converter.config.processing.conversion_options.images_scale == 3.0
|
assert converter.config.processing.conversion_options.images_scale == 3.0
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_convert_pdf_without_picture_images(self, config):
|
||||||
|
"""Test PDF conversion excludes embedded images by default."""
|
||||||
|
pdf_path = Path("tests/data/doclaynet.pdf")
|
||||||
|
if not pdf_path.exists():
|
||||||
|
pytest.skip("doclaynet.pdf not found")
|
||||||
|
|
||||||
|
config.processing.conversion_options.generate_picture_images = False
|
||||||
|
converter = DoclingLocalConverter(config)
|
||||||
|
|
||||||
|
doc = await converter.convert_file(pdf_path)
|
||||||
|
assert isinstance(doc, DoclingDocument)
|
||||||
|
|
||||||
|
# Check that pictures don't have image data
|
||||||
|
for picture in doc.pictures:
|
||||||
|
assert picture.image is None, (
|
||||||
|
"Pictures should not have image data when generate_picture_images=False"
|
||||||
|
)
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_convert_pdf_with_picture_images(self, config):
|
||||||
|
"""Test PDF conversion includes embedded images when enabled."""
|
||||||
|
pdf_path = Path("tests/data/doclaynet.pdf")
|
||||||
|
if not pdf_path.exists():
|
||||||
|
pytest.skip("doclaynet.pdf not found")
|
||||||
|
|
||||||
|
config.processing.conversion_options.generate_picture_images = True
|
||||||
|
converter = DoclingLocalConverter(config)
|
||||||
|
|
||||||
|
doc = await converter.convert_file(pdf_path)
|
||||||
|
assert isinstance(doc, DoclingDocument)
|
||||||
|
|
||||||
|
# Check that at least some pictures have image data
|
||||||
|
pictures_with_images = [p for p in doc.pictures if p.image is not None]
|
||||||
|
if doc.pictures:
|
||||||
|
assert len(pictures_with_images) > 0, (
|
||||||
|
"Pictures should have image data when generate_picture_images=True"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class TestDoclingServeConverter:
|
class TestDoclingServeConverter:
|
||||||
"""Tests for DoclingServeConverter (mocked)."""
|
"""Tests for DoclingServeConverter (mocked)."""
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue