From 40a4c6e5cc24d802e7b96138c1e081616c06103d Mon Sep 17 00:00:00 2001 From: pjmalandrino Date: Fri, 30 May 2025 15:16:42 +0200 Subject: [PATCH] Fix single page bugs --- backend/app.py | 206 ++++++++++++++++++++++++----- backend/page_treatment/analyzer.py | 17 ++- backend/utils.py | 1 + frontend/static/app.js | 56 ++++---- 4 files changed, 216 insertions(+), 64 deletions(-) diff --git a/backend/app.py b/backend/app.py index 4b2d5c1..26e6080 100644 --- a/backend/app.py +++ b/backend/app.py @@ -4,6 +4,7 @@ Flask application for DocTags web interface """ from flask import Flask, request, send_file, jsonify +import subprocess import os import sys import time @@ -37,7 +38,6 @@ app.config['MAX_CONTENT_LENGTH'] = MAX_CONTENT_LENGTH # Task results storage task_results = {} -task_lock = threading.Lock() # Import batch processor if available try: @@ -134,49 +134,178 @@ def pdf_preview(pdf_file, page_num): logger.error(f"Error generating PDF preview: {e}") return jsonify({'error': str(e)}), 500 -def run_background_task(task_id, command): - """Run a command in background and store results""" - logger.info(f"Running task {task_id}: {command}") - +def run_command(task_id, command): + """Run a command in a background thread and store result""" + logger.info(f"Running command: {command}") try: - success, stdout, stderr = run_command_with_timeout(command, PROCESSING_TIMEOUT) + # Run the command with pipe input to automatically answer "n" to prompts + process = subprocess.Popen( + command, + shell=True, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + universal_newlines=True + ) - with task_lock: + # Send "n" to the process to bypass prompts + stdout, stderr = process.communicate(input="n\n") + + # Log output for debugging + logger.info(f"Command stdout: {stdout[:500]}...") + if stderr: + logger.error(f"Command stderr: {stderr}") + + # Update task result + if process.returncode == 0: task_results[task_id] = { - 'success': success, + 'success': True, 'output': stdout, - 'error': stderr if not success else None, 'done': True } - - logger.info(f"Task {task_id} completed: {'success' if success else 'failed'}") - - except Exception as e: - logger.error(f"Task {task_id} error: {e}") - with task_lock: + logger.info(f"Command completed successfully: {task_id}") + else: + error_message = stderr or f"Command failed with return code {process.returncode}" task_results[task_id] = { 'success': False, - 'error': str(e), + 'error': error_message, 'done': True } + logger.error(f"Command failed: {task_id} - {error_message}") + + except Exception as e: + import traceback + logger.error(f"Unexpected error: {task_id} - {str(e)}") + logger.error(traceback.format_exc()) + task_results[task_id] = { + 'success': False, + 'error': f"Exception: {str(e)}", + 'done': True + } @app.route('/run-analyzer', methods=['POST']) def run_analyzer(): - return run_processing_task('analyzer', request.form) + try: + pdf_file = request.form.get('pdf_file') + page_num = request.form.get('page_num', 1) + + if not pdf_file: + return jsonify({'success': False, 'error': 'PDF file not specified'}), 400 + + if not os.path.exists(pdf_file): + return jsonify({'success': False, 'error': f'PDF file not found: {pdf_file}'}), 404 + + command = f"python backend/page_treatment/analyzer.py --image {pdf_file} --page {page_num} --start-page {page_num} --end-page {page_num}" + + # Generate task ID + task_id = f"analyzer_{int(time.time())}" + + # Initialize task result + task_results[task_id] = { + 'success': None, + 'output': "Running analyzer...", + 'done': False + } + + # Start background thread + thread = threading.Thread(target=run_command, args=(task_id, command)) + thread.daemon = True + thread.start() + + return jsonify({ + 'success': True, + 'task_id': task_id, + 'message': f"Analyzer started. Processing {pdf_file} page {page_num}..." + }) + + except Exception as e: + logger.error(f"Error starting analyzer: {e}") + return jsonify({'success': False, 'error': str(e)}), 500 @app.route('/run-visualizer', methods=['POST']) def run_visualizer(): - return run_processing_task('visualizer', request.form) + try: + pdf_file = request.form.get('pdf_file') + page_num = request.form.get('page_num', 1) + adjust = request.form.get('adjust') == 'true' + + if not pdf_file: + return jsonify({'success': False, 'error': 'PDF file not specified'}), 400 + + command = f"python backend/page_treatment/visualizer.py --doctags results/output.doctags.txt --pdf {pdf_file} --page {page_num}" + if adjust: + command += " --adjust" + + # Generate task ID with page number + task_id = f"visualizer_{int(time.time())}_{page_num}" + + # Initialize task result + task_results[task_id] = { + 'success': None, + 'output': "Running visualizer...", + 'done': False + } + + # Start background thread + thread = threading.Thread(target=run_command, args=(task_id, command)) + thread.daemon = True + thread.start() + + return jsonify({ + 'success': True, + 'task_id': task_id, + 'message': f"Visualizer started. Processing {pdf_file} page {page_num}..." + }) + + except Exception as e: + logger.error(f"Error starting visualizer: {e}") + return jsonify({'success': False, 'error': str(e)}), 500 @app.route('/run-extractor', methods=['POST']) def run_extractor(): - return run_processing_task('extractor', request.form) + try: + pdf_file = request.form.get('pdf_file') + page_num = request.form.get('page_num', 1) + adjust = request.form.get('adjust') == 'true' + + if not pdf_file: + return jsonify({'success': False, 'error': 'PDF file not specified'}), 400 + + command = f"python backend/page_treatment/picture_extractor.py --doctags results/output.doctags.txt --pdf {pdf_file} --page {page_num}" + if adjust: + command += " --adjust" + + # Generate task ID with page number + task_id = f"extractor_{int(time.time())}_{page_num}" + + # Initialize task result + task_results[task_id] = { + 'success': None, + 'output': "Running picture extractor...", + 'done': False + } + + # Start background thread + thread = threading.Thread(target=run_command, args=(task_id, command)) + thread.daemon = True + thread.start() + + return jsonify({ + 'success': True, + 'task_id': task_id, + 'message': f"Picture extractor started. Processing {pdf_file} page {page_num}..." + }) + + except Exception as e: + logger.error(f"Error starting extractor: {e}") + return jsonify({'success': False, 'error': str(e)}), 500 def run_processing_task(task_type, form_data): """Generic function to run processing tasks""" try: pdf_file = form_data.get('pdf_file') - page_num = form_data.get('page_num', 1) + page_num = form_data.get('page_num', '1') if not pdf_file: return jsonify({'success': False, 'error': 'PDF file not specified'}), 400 @@ -187,7 +316,7 @@ def run_processing_task(task_type, form_data): # Build command based on task type command = build_command(task_type, pdf_file, page_num, form_data) - # Generate task ID + # Generate task ID with page number task_id = f"{task_type}_{int(time.time() * 1000)}_{page_num}" # Initialize task result @@ -198,6 +327,8 @@ def run_processing_task(task_type, form_data): 'done': False } + logger.info(f"Created task {task_id} for {task_type} on page {page_num}") + # Start background thread thread = threading.Thread(target=run_background_task, args=(task_id, command)) thread.daemon = True @@ -211,6 +342,8 @@ def run_processing_task(task_type, form_data): except Exception as e: logger.error(f"Error starting {task_type}: {e}") + import traceback + traceback.print_exc() return jsonify({'success': False, 'error': str(e)}), 500 def build_command(task_type, pdf_file, page_num, form_data): @@ -240,18 +373,31 @@ def build_command(task_type, pdf_file, page_num, form_data): @app.route('/task-status/') def task_status(task_id): - """Get status of a background task""" - with task_lock: - if task_id not in task_results: - return jsonify({'success': False, 'error': 'Task not found'}), 404 + if task_id not in task_results: + logger.warning(f"Task {task_id} not found in task_results") + return jsonify({'success': False, 'error': 'Task not found'}), 404 - result = task_results[task_id].copy() + result = task_results[task_id].copy() - # Add file paths for completed tasks + # Log the complete result for debugging + logger.info(f"Task {task_id} result: {result}") + + # If task is completed successfully, add file paths if result.get('done') and result.get('success'): - if 'visualizer' in task_id: - page_num = task_id.split('_')[-1] - result['image_file'] = f"results/visualization_page_{page_num}.png" + if task_id.startswith('analyzer_'): + doctags_path = Path("results") / "output.doctags.txt" + if doctags_path.exists(): + result['doctags_file'] = "results/output.doctags.txt" + logger.info(f"Added doctags_file to result") + else: + logger.warning(f"DocTags file not found: {doctags_path}") + + elif task_id.startswith('visualizer_'): + # Extract page number from the end of task_id if it exists + page_num = task_id.split('_')[-1] if len(task_id.split('_')) > 2 else "1" + viz_filename = f"visualization_page_{page_num}.png" + result['image_file'] = f"results/{viz_filename}" + logger.info(f"Found visualization at: {result['image_file']}") return jsonify(result) diff --git a/backend/page_treatment/analyzer.py b/backend/page_treatment/analyzer.py index 5f34a49..d6bdb4b 100644 --- a/backend/page_treatment/analyzer.py +++ b/backend/page_treatment/analyzer.py @@ -81,15 +81,19 @@ def process_page(model, processor, config, args, pil_image, page_num=1): from mlx_vlm.utils import stream_generate results_dir = ensure_results_folder() - output_base = Path(args.output) - # Handle multi-page output naming - if Path(args.image).suffix.lower() == '.pdf' and page_num > 1: - output_path = results_dir / f"{output_base.stem}_page{page_num}{output_base.suffix}" + # For web interface, always use output.doctags.txt + # For command line with specific pages, use page-specific names + if args.start_page == args.end_page and args.start_page == page_num: + # Single page processing + output_path = results_dir / "output.html" + doctags_path = results_dir / "output.doctags.txt" else: - output_path = output_base + # Multi-page processing + output_path = results_dir / f"output_page{page_num}.html" + doctags_path = results_dir / f"output_page{page_num}.doctags.txt" - print(f"Processing page {page_num}, output will be saved to {output_path}") + print(f"Processing page {page_num}") # Save image temporarily with tempfile.NamedTemporaryFile(suffix='.png', delete=False) as temp_img_file: @@ -117,7 +121,6 @@ def process_page(model, processor, config, args, pil_image, page_num=1): os.unlink(temp_img_path) # Save DocTags output - doctags_path = results_dir / f"{output_path.stem}.doctags.txt" with open(doctags_path, 'w', encoding='utf-8') as f: f.write(output) print(f"Raw DocTags saved to: {doctags_path}") diff --git a/backend/utils.py b/backend/utils.py index 557a0a1..f5f9c82 100644 --- a/backend/utils.py +++ b/backend/utils.py @@ -169,6 +169,7 @@ def calculate_scale_factor(max_coord: float, image_size: float) -> float: else: return max(image_size / max_coord, 0.5) +# In backend/utils.py, make sure this function exists: def run_command_with_timeout(command: str, timeout: int = 300, input_text: str = "n\n") -> Tuple[bool, str, str]: """ Run a command with timeout and return success, stdout, stderr. diff --git a/frontend/static/app.js b/frontend/static/app.js index 717d05a..c51a81c 100644 --- a/frontend/static/app.js +++ b/frontend/static/app.js @@ -8,21 +8,6 @@ const appState = { } }; -// API Client -const api = { - async get(url) { - const response = await fetch(url); - if (!response.ok) throw new Error(`HTTP error! status: ${response.status}`); - return response.json(); - }, - - async post(url, formData) { - const response = await fetch(url, { method: 'POST', body: formData }); - if (!response.ok) throw new Error(`HTTP error! status: ${response.status}`); - return response.json(); - } -}; - // UI Helper Functions const ui = { show(elementId) { @@ -158,7 +143,12 @@ function stopPolling() { async function pollTasks() { for (const taskId in appState.activeTasks) { try { - const data = await api.get(`/task-status/${taskId}`); + const response = await fetch(`/task-status/${taskId}`); + const data = await response.json(); + + // Debug log + console.log(`Task ${taskId} status:`, data); + updateTaskStatus(taskId, data); } catch (error) { console.error('Error polling task:', error); @@ -179,14 +169,14 @@ function updateTaskStatus(taskId, data) { delete appState.activeTasks[taskId]; stopPolling(); } else { - ui.setHtml(statusElement, '
Running...'); - ui.show(statusElement.id); + ui.setHtml(`${taskInfo.type}-status`, '
Running...'); + ui.show(`${taskInfo.type}-status`); } } function handleTaskSuccess(taskId, taskInfo, data, statusElement) { - ui.setHtml(statusElement, '✓ Completed successfully!'); - ui.show(statusElement.id); + ui.setHtml(`${taskInfo.type}-status`, '✓ Completed successfully!'); + ui.show(`${taskInfo.type}-status`); ui.enable(`${taskInfo.type}-btn`); // Display output @@ -215,9 +205,10 @@ function handleTaskSuccess(taskId, taskInfo, data, statusElement) { } function handleTaskFailure(taskId, data, statusElement) { - ui.setHtml(statusElement, '✗ Failed: ' + (data.error || 'Unknown error') + ''); - ui.show(statusElement.id); - ui.enable(statusElement.id.replace('-status', '-btn')); + const taskInfo = appState.activeTasks[taskId]; + ui.setHtml(`${taskInfo.type}-status`, '✗ Failed: ' + (data.error || 'Unknown error') + ''); + ui.show(`${taskInfo.type}-status`); + ui.enable(`${taskInfo.type}-btn`); ui.setText('output', 'Error: ' + (data.error || 'Unknown error')); ui.show('output'); @@ -257,7 +248,11 @@ async function runScript(script) { formData.append('adjust', adjust); try { - const data = await api.post(`/run-${script}`, formData); + const response = await fetch(`/run-${script}`, { + method: 'POST', + body: formData + }); + const data = await response.json(); if (data.success && data.task_id) { appState.activeTasks[data.task_id] = { @@ -375,7 +370,8 @@ async function checkEnvironment() { ui.setHtml('env-details', '
Checking environment...'); try { - const data = await api.get('/check-environment'); + const response = await fetch('/check-environment'); + const data = await response.json(); let html = '