fdbab2f49e
- Add e2e/ui/ as peer project to e2e/api/ (own pom.xml, runner, config) - 5 critical UI journeys: upload, delete, analysis, batch-progress, rechunk - 4 local-only tests: sidebar, i18n, error-states, pipeline-options - 1 full happy path workflow covering all modes - Add data-e2e attributes on all tested Vue components (decoupled from CSS) - Add CONVENTIONS.md with 7 golden rules for writing Karate UI tests - Update CI with dedicated e2e-ui job (Chrome headless, --no-sandbox) - Update docs/contributing.md with UI test instructions Closes #124
80 lines
2.9 KiB
Python
80 lines
2.9 KiB
Python
"""Generate deterministic test PDFs for E2E tests.
|
|
|
|
Uses fpdf2 to create valid PDFs with real text content so Docling
|
|
can extract and chunk them. No binary files committed to the repo.
|
|
|
|
Usage:
|
|
python e2e/generate-test-data.py
|
|
|
|
Dependencies:
|
|
pip install fpdf2
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
|
|
from fpdf import FPDF
|
|
|
|
OUTPUT_DIRS = [
|
|
os.path.join(os.path.dirname(__file__), "api", "src", "test", "resources", "common", "data", "generated"),
|
|
os.path.join(os.path.dirname(__file__), "ui", "src", "test", "resources", "common", "data", "generated"),
|
|
]
|
|
|
|
_PARAGRAPHS = [
|
|
"Document processing is a critical step in building retrieval-augmented generation systems.",
|
|
"Docling Studio provides tools for analyzing PDF documents and extracting structured content.",
|
|
"The conversion pipeline supports OCR, table detection, and formula enrichment features.",
|
|
"Chunking splits document content into semantically meaningful segments for vector indexing.",
|
|
"Each chunk preserves metadata such as page number, bounding boxes, and heading hierarchy.",
|
|
"The hybrid chunker combines hierarchical document structure with token-based splitting.",
|
|
"Vector stores like OpenSearch enable fast similarity search over embedded chunk vectors.",
|
|
"Quality control requires visual inspection of chunk boundaries and extracted text accuracy.",
|
|
"Batch processing of large documents uses page ranges to prevent memory exhaustion.",
|
|
"Progress reporting allows users to monitor long-running document conversion tasks.",
|
|
]
|
|
|
|
|
|
def _make_pdf(page_count: int, path: str) -> None:
|
|
"""Create a valid PDF with N pages containing text paragraphs."""
|
|
pdf = FPDF()
|
|
pdf.set_auto_page_break(auto=True, margin=25)
|
|
|
|
for page_num in range(page_count):
|
|
pdf.add_page()
|
|
pdf.set_font("Helvetica", "B", 16)
|
|
pdf.cell(0, 10, f"Page {page_num + 1} of {page_count}", new_x="LMARGIN", new_y="NEXT")
|
|
pdf.ln(5)
|
|
pdf.set_font("Helvetica", "", 11)
|
|
for i in range(5):
|
|
para = _PARAGRAPHS[(page_num * 5 + i) % len(_PARAGRAPHS)]
|
|
pdf.multi_cell(0, 6, para)
|
|
pdf.ln(3)
|
|
|
|
pdf.output(path)
|
|
size_kb = os.path.getsize(path) / 1024
|
|
print(f" {os.path.basename(path)}: {page_count} pages, {size_kb:.1f} KB")
|
|
|
|
|
|
def _make_non_pdf(path: str) -> None:
|
|
"""Create a non-PDF file for negative testing."""
|
|
with open(path, "wb") as f:
|
|
f.write(b"This is not a PDF file.\n")
|
|
print(f" {os.path.basename(path)}: not-a-pdf")
|
|
|
|
|
|
def main() -> None:
|
|
for output_dir in OUTPUT_DIRS:
|
|
os.makedirs(output_dir, exist_ok=True)
|
|
print(f"Generating test data in {output_dir}")
|
|
|
|
_make_pdf(1, os.path.join(output_dir, "small.pdf"))
|
|
_make_pdf(5, os.path.join(output_dir, "medium.pdf"))
|
|
_make_pdf(25, os.path.join(output_dir, "large.pdf"))
|
|
_make_non_pdf(os.path.join(output_dir, "not-a-pdf.txt"))
|
|
|
|
print("Done.")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|