danvics/docling-studio
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

Fix batch processing

Browse source
This commit is contained in:
pjmalandrino 2025-05-30 15:27:04 +02:00
parent 40a4c6e5cc
commit 46b22770dc
3 changed files with 70 additions and 23 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

13
backend/app.py
View file

@ -491,6 +491,19 @@ if batch_processing_available:
if not pdf_file or not os.path.exists(pdf_file):
return jsonify({'success': False, 'error': 'Invalid PDF file'}), 400
# Check if there's already an active batch for this PDF
from backend.batch_treatment.batch_processor import batch_processors, batch_lock
with batch_lock:
for batch_id, processor in batch_processors.items():
if (processor.pdf_file == pdf_file and
not processor.state['completed'] and
not processor.state['cancelled']):
return jsonify({
'success': False,
'error': 'A batch process is already running for this PDF',
'existing_batch_id': batch_id
}), 409
options = {
'adjust': request.form.get('adjust') == 'true',
'parallel': request.form.get('parallel') == 'true',

33
backend/batch_treatment/batch_processor.py
View file

@ -163,6 +163,7 @@ class BatchProcessor:
def run_analyzer(self, page_num):
"""Run the analyzer for a specific page"""
try:
# Use page-specific output to avoid conflicts
command = (f"python backend/page_treatment/analyzer.py "
f"--image {self.pdf_file} --page {page_num} "
f"--start-page {page_num} --end-page {page_num}")
@ -174,16 +175,34 @@ class BatchProcessor:
if not success:
raise Exception(f"Analyzer failed: {stderr}")
# Copy doctags to batch directory
doctags_src = ensure_results_folder() / "output.doctags.txt"
doctags_dst = self.results_dir / f"page_{page_num}.doctags.txt"
# The analyzer will create either output.doctags.txt or output_pageN.doctags.txt
# Check both locations
results_dir = ensure_results_folder()
possible_paths = [
results_dir / "output.doctags.txt",
results_dir / f"output_page{page_num}.doctags.txt"
]
if doctags_src.exists():
shutil.copy2(doctags_src, doctags_dst)
self.log_message(f"DocTags saved for page {page_num}")
else:
doctags_src = None
for path in possible_paths:
if path.exists():
doctags_src = path
break
if not doctags_src:
raise Exception("DocTags file not generated")
# Copy to batch directory with page-specific name
doctags_dst = self.results_dir / f"page_{page_num}.doctags.txt"
shutil.copy2(doctags_src, doctags_dst)
# Verify the file has content
with open(doctags_dst, 'r') as f:
content = f.read().strip()
if not content or '<doctag>' not in content:
raise Exception("DocTags file is empty or invalid")
self.log_message(f"DocTags saved for page {page_num}")
return True
except Exception as e:

47
backend/page_treatment/visualizer.py
View file

@ -25,8 +25,8 @@ def parse_arguments():
results_dir = ensure_results_folder()
parser = argparse.ArgumentParser(description='Visualize zones identified in DocTags format')
parser.add_argument('--doctags', '-d', type=str, required=True,
help='Path to DocTags file')
parser.add_argument('--doctags', '-d', type=str, required=False,
help='Path to DocTags file (optional, will auto-detect if not provided)')
parser.add_argument('--pdf', '-p', type=str, required=True,
help='Path to original PDF file')
parser.add_argument('--page', type=int, default=1,
@ -47,6 +47,10 @@ def parse_doctags(doctags_path):
with open(doctags_path, 'r', encoding='utf-8') as f:
doctags_content = f.read()
# Check if file is empty or invalid
if not doctags_content.strip():
raise ValueError("DocTags file is empty")
# Extract content between <doctag> tags
doctag_match = re.search(r'<doctag>(.*?)</doctag>', doctags_content, re.DOTALL)
if not doctag_match:
@ -88,6 +92,10 @@ def parse_doctags(doctags_path):
'content': text_content
})
# If no zones found, it might be a page with no detectable content
if not zones:
print(f"Warning: No zones with location data found in {doctags_path}")
return zones
def create_visualization(image, zones, page_num, output_path):
@ -153,23 +161,30 @@ def process_page(pdf_path, page_num, doctags_path, output_path, dpi, adjust):
image = load_pdf_page(pdf_path, page_num, dpi)
print(f"Page {page_num} loaded: {image.size}")
# Parse DocTags
zones = parse_doctags(doctags_path)
print(f"Found {len(zones)} zones in DocTags")
try:
# Parse DocTags
zones = parse_doctags(doctags_path)
print(f"Found {len(zones)} zones in DocTags")
if zones:
# Check if we need to adjust coordinates
max_x = max([zone['x2'] for zone in zones])
max_y = max([zone['y2'] for zone in zones])
if zones:
# Check if we need to adjust coordinates
max_x = max([zone['x2'] for zone in zones])
max_y = max([zone['y2'] for zone in zones])
# Auto-adjust if needed
if max_x <= DEFAULT_GRID_SIZE and max_y <= DEFAULT_GRID_SIZE:
print(f"Detected normalized coordinates (0-{DEFAULT_GRID_SIZE} grid)")
zones = normalize_coordinates(zones, image.width, image.height)
elif adjust:
zones = auto_adjust_coordinates(zones, image.width, image.height)
# Auto-adjust if needed
if max_x <= DEFAULT_GRID_SIZE and max_y <= DEFAULT_GRID_SIZE:
print(f"Detected normalized coordinates (0-{DEFAULT_GRID_SIZE} grid)")
zones = normalize_coordinates(zones, image.width, image.height)
elif adjust:
zones = auto_adjust_coordinates(zones, image.width, image.height)
else:
print(f"Warning: No zones found for page {page_num}, creating blank visualization")
# Create visualization
except ValueError as e:
print(f"Warning: {e} for page {page_num}, creating blank visualization")
zones = []
# Create visualization (even if no zones)
create_visualization(image, zones, page_num, output_path)
return True