diff --git a/.gitignore b/.gitignore index f9f4e88..0b1edcf 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,7 @@ document_output results handbook.pdf +ariston.pdf +.DS_Store +__pycache__/ +perplexity.pdf \ No newline at end of file diff --git a/app.py b/app.py index 246479d..b809d7a 100644 --- a/app.py +++ b/app.py @@ -5,6 +5,9 @@ import time import threading import logging from pathlib import Path +import sys +import uuid +import pdf2image # Configure logging logging.basicConfig(level=logging.INFO, @@ -19,6 +22,14 @@ app = Flask(__name__, # Dictionary to store background task results task_results = {} +# Import batch processor if available +try: + from batch_processor import start_batch_processing, get_batch_processor, cleanup_old_batches + batch_processing_available = True +except ImportError: + logger.warning("batch_processor.py not found. Batch processing features will be disabled.") + batch_processing_available = False + # Ensure results folder exists def ensure_results_folder(): results_dir = Path("results") @@ -44,6 +55,11 @@ def ensure_frontend_folders(): def index(): return send_file('frontend/index.html') +@app.route('/batch') +def batch_interface(): + """Serve the batch processing interface""" + return send_file('frontend/batch.html') + @app.route('/static/') def serve_static(filename): return send_file(os.path.join('frontend/static', filename)) @@ -59,6 +75,46 @@ def pdf_files(): logger.error(f"Error listing PDF files: {str(e)}") return jsonify({"error": str(e)}), 500 +@app.route('/pdf-info/') +def pdf_info(pdf_file): + """Get information about a PDF file (page count, etc.)""" + try: + if not os.path.exists(pdf_file): + return jsonify({'error': f'PDF file not found: {pdf_file}'}), 404 + + # Get page count using pdf2image + try: + from pdf2image.pdf2image import pdfinfo_from_path + info = pdfinfo_from_path(pdf_file) + page_count = info["Pages"] + except Exception as e: + # Fallback method + logger.warning(f"pdfinfo failed, using fallback: {e}") + images = pdf2image.convert_from_path(pdf_file, dpi=72, first_page=1, last_page=1) + page_count = 1 # At least one page if we got here + + # Try to get actual count by checking last pages + for i in range(2, 1000): # Reasonable upper limit + try: + images = pdf2image.convert_from_path(pdf_file, dpi=72, first_page=i, last_page=i) + if not images: + page_count = i - 1 + break + page_count = i + except: + page_count = i - 1 + break + + return jsonify({ + 'pageCount': page_count, + 'filename': os.path.basename(pdf_file), + 'size': os.path.getsize(pdf_file) + }) + + except Exception as e: + logger.error(f"Error getting PDF info: {str(e)}") + return jsonify({'error': str(e)}), 500 + def run_command(task_id, command): """Run a command in a background thread and store result""" logger.info(f"Running command: {command}") @@ -383,7 +439,8 @@ def check_environment(): 'pdf_files': pdf_files, 'results_dir_exists': results_dir_exists, 'results_dir_writable': results_writable, - 'python_version': python_version + 'python_version': python_version, + 'batch_processing_available': batch_processing_available }) except Exception as e: import traceback @@ -440,8 +497,303 @@ def run_manual_command(): logger.error(f"Error running manual command: {str(e)}") return jsonify({'success': False, 'error': str(e)}), 500 +# Batch processing endpoints +@app.route('/run-batch-processor', methods=['POST']) +def run_batch_processor(): + """Start a new batch processing job""" + if not batch_processing_available: + return jsonify({'success': False, 'error': 'Batch processing not available'}), 503 + + try: + pdf_file = request.form.get('pdf_file') + start_page = int(request.form.get('start_page', 1)) + end_page = int(request.form.get('end_page', 1)) + + if not pdf_file: + return jsonify({'success': False, 'error': 'PDF file not specified'}), 400 + + if not os.path.exists(pdf_file): + return jsonify({'success': False, 'error': f'PDF file not found: {pdf_file}'}), 404 + + # Processing options + options = { + 'adjust': request.form.get('adjust') == 'true', + 'parallel': request.form.get('parallel') == 'true', + 'generate_report': request.form.get('generate_report') == 'true' + } + + # Generate batch ID + batch_id = str(uuid.uuid4())[:8] + + # Start batch processing + if start_batch_processing(batch_id, pdf_file, start_page, end_page, options): + logger.info(f"Started batch processing with ID: {batch_id}") + return jsonify({ + 'success': True, + 'batch_id': batch_id, + 'message': f'Batch processing started for {end_page - start_page + 1} pages' + }) + else: + return jsonify({'success': False, 'error': 'Failed to start batch processing'}), 500 + + except Exception as e: + logger.error(f"Error starting batch processor: {str(e)}") + return jsonify({'success': False, 'error': str(e)}), 500 + +@app.route('/batch-status/') +def batch_status(batch_id): + """Get the status of a batch processing job""" + if not batch_processing_available: + return jsonify({'error': 'Batch processing not available'}), 503 + + try: + processor = get_batch_processor(batch_id) + + if not processor: + return jsonify({'error': 'Batch not found'}), 404 + + state = processor.get_state() + + # Only return recent logs (last 20) + state['logs'] = state['logs'][-20:] + + return jsonify(state) + + except Exception as e: + logger.error(f"Error getting batch status: {str(e)}") + return jsonify({'error': str(e)}), 500 + +@app.route('/pause-batch/', methods=['POST']) +def pause_batch(batch_id): + """Pause a batch processing job""" + if not batch_processing_available: + return jsonify({'error': 'Batch processing not available'}), 503 + + try: + processor = get_batch_processor(batch_id) + + if not processor: + return jsonify({'error': 'Batch not found'}), 404 + + processor.pause() + return jsonify({'success': True}) + + except Exception as e: + logger.error(f"Error pausing batch: {str(e)}") + return jsonify({'error': str(e)}), 500 + +@app.route('/resume-batch/', methods=['POST']) +def resume_batch(batch_id): + """Resume a paused batch processing job""" + if not batch_processing_available: + return jsonify({'error': 'Batch processing not available'}), 503 + + try: + processor = get_batch_processor(batch_id) + + if not processor: + return jsonify({'error': 'Batch not found'}), 404 + + processor.resume() + return jsonify({'success': True}) + + except Exception as e: + logger.error(f"Error resuming batch: {str(e)}") + return jsonify({'error': str(e)}), 500 + +@app.route('/cancel-batch/', methods=['POST']) +def cancel_batch(batch_id): + """Cancel a batch processing job""" + if not batch_processing_available: + return jsonify({'error': 'Batch processing not available'}), 503 + + try: + processor = get_batch_processor(batch_id) + + if not processor: + return jsonify({'error': 'Batch not found'}), 404 + + processor.cancel() + return jsonify({'success': True}) + + except Exception as e: + logger.error(f"Error cancelling batch: {str(e)}") + return jsonify({'error': str(e)}), 500 + +@app.route('/retry-page', methods=['POST']) +def retry_page(): + """Retry processing a single failed page""" + if not batch_processing_available: + return jsonify({'success': False, 'error': 'Batch processing not available'}), 503 + + try: + pdf_file = request.form.get('pdf_file') + page_num = int(request.form.get('page_num')) + adjust = request.form.get('adjust') == 'true' + + if not pdf_file or not page_num: + return jsonify({'success': False, 'error': 'Missing parameters'}), 400 + + # Create a single-page batch for retry + batch_id = f"retry_{uuid.uuid4().hex[:8]}" + options = {'adjust': adjust, 'parallel': False, 'generate_report': False} + + if start_batch_processing(batch_id, pdf_file, page_num, page_num, options): + # Wait for completion (since it's just one page) + processor = get_batch_processor(batch_id) + timeout = 60 # 60 seconds timeout + start_time = time.time() + + while not processor.state['completed'] and (time.time() - start_time) < timeout: + time.sleep(0.5) + + if processor.state['results']['successful'] > 0: + return jsonify({'success': True}) + else: + return jsonify({'success': False, 'error': 'Page processing failed'}) + else: + return jsonify({'success': False, 'error': 'Failed to start retry'}), 500 + + except Exception as e: + logger.error(f"Error retrying page: {str(e)}") + return jsonify({'success': False, 'error': str(e)}), 500 + +@app.route('/download-batch-results/') +def download_batch_results(batch_id): + """Download all batch results as a ZIP file""" + if not batch_processing_available: + return jsonify({'error': 'Batch processing not available'}), 503 + + try: + processor = get_batch_processor(batch_id) + + if not processor: + return jsonify({'error': 'Batch not found'}), 404 + + # Create ZIP archive + zip_path = processor.create_zip_archive() + + if zip_path and os.path.exists(zip_path): + return send_file( + zip_path, + mimetype='application/zip', + as_attachment=True, + download_name=f'batch_results_{batch_id}.zip' + ) + else: + return jsonify({'error': 'Failed to create archive'}), 500 + + except Exception as e: + logger.error(f"Error downloading batch results: {str(e)}") + return jsonify({'error': str(e)}), 500 + +@app.route('/batch-report/') +def batch_report(batch_id): + """View the batch processing report""" + if not batch_processing_available: + return jsonify({'error': 'Batch processing not available'}), 503 + + try: + processor = get_batch_processor(batch_id) + + if not processor: + return jsonify({'error': 'Batch not found'}), 404 + + report_path = processor.results_dir / "report.html" + + if report_path.exists(): + # Read and modify the HTML to fix image paths + with open(report_path, 'r') as f: + html_content = f.read() + + # Replace relative image paths with absolute Flask routes + html_content = html_content.replace( + 'src="visualization_page_', + f'src="/batch-report-image/{batch_id}/visualization_page_' + ) + html_content = html_content.replace( + 'href="visualization_page_', + f'href="/batch-report-image/{batch_id}/visualization_page_' + ) + + return html_content + else: + return jsonify({'error': 'Report not found'}), 404 + + except Exception as e: + logger.error(f"Error viewing batch report: {str(e)}") + return jsonify({'error': str(e)}), 500 + +@app.route('/batch-report-image//') +def batch_report_image(batch_id, filename): + """Serve images from batch report directory""" + if not batch_processing_available: + return jsonify({'error': 'Batch processing not available'}), 503 + + try: + processor = get_batch_processor(batch_id) + + if not processor: + return jsonify({'error': 'Batch not found'}), 404 + + image_path = processor.results_dir / filename + + if image_path.exists() and image_path.is_file(): + return send_file(image_path) + else: + return jsonify({'error': f'Image not found: {filename}'}), 404 + + except Exception as e: + logger.error(f"Error serving batch report image: {str(e)}") + return jsonify({'error': str(e)}), 500 + +@app.route('/open-results-folder', methods=['POST']) +def open_results_folder(): + """Open the results folder in the system file explorer""" + try: + results_path = os.path.abspath("results") + + if sys.platform == 'darwin': # macOS + subprocess.run(['open', results_path]) + elif sys.platform == 'win32': # Windows + os.startfile(results_path) + else: # Linux + subprocess.run(['xdg-open', results_path]) + + return jsonify({'success': True}) + + except Exception as e: + logger.error(f"Error opening results folder: {str(e)}") + return jsonify({'success': False, 'error': str(e)}), 500 + +# Cleanup task for batch processors +def cleanup_task(): + """Periodic cleanup of old batch processors""" + if not batch_processing_available: + return + + while True: + time.sleep(3600) # Run every hour + try: + cleanup_old_batches(24) # Clean up batches older than 24 hours + except Exception as e: + logger.error(f"Error in cleanup task: {str(e)}") + +# Start cleanup thread when app starts (only if batch processing is available) +if batch_processing_available: + cleanup_thread = threading.Thread(target=cleanup_task) + cleanup_thread.daemon = True + cleanup_thread.start() + if __name__ == '__main__': # Ensure folders exist ensure_results_folder() ensure_frontend_folders() + + # Check environment + if not batch_processing_available: + logger.warning("batch_processor.py not found. Batch processing features will be disabled.") + else: + logger.info("Batch processing features enabled.") + app.run(debug=True, host='127.0.0.1', port=5000) \ No newline at end of file diff --git a/batch_processor.py b/batch_processor.py index c482836..032e84e 100644 --- a/batch_processor.py +++ b/batch_processor.py @@ -1,471 +1,645 @@ #!/usr/bin/env python3 """ -DocTags Batch Processor - Process multiple pages from a PDF in sequence. - -Usage: - python batch_processor.py --pdf document.pdf --start 1 --end 10 --adjust +Batch Processor for DocTags +Handles batch processing of PDF documents with parallel processing support """ -import argparse -import subprocess import os -import sys -import time import json +import time +import threading +import queue +import logging from pathlib import Path from datetime import datetime +from concurrent.futures import ThreadPoolExecutor, as_completed +import subprocess +import shutil +import zipfile -def ensure_results_folder(): - """Create the results folder if it doesn't exist.""" - results_dir = Path("results") - if not results_dir.exists(): - results_dir.mkdir() - print(f"Created results directory: {results_dir}") - return results_dir +# Configure logging +logging.basicConfig( + level=logging.INFO, + format='%(asctime)s - %(name)s - %(levelname)s - %(message)s' +) +logger = logging.getLogger(__name__) -def parse_arguments(): - """Parse command line arguments.""" - parser = argparse.ArgumentParser(description='Batch process multiple PDF pages') - parser.add_argument('--pdf', '-p', type=str, required=True, - help='Path to PDF file') - parser.add_argument('--start', type=int, default=1, - help='Starting page number (default: 1)') - parser.add_argument('--end', type=int, default=None, - help='Ending page number (default: last page)') - parser.add_argument('--pages', type=str, default=None, - help='Comma-separated list of specific pages (e.g., "1,3,5-7,10")') - parser.add_argument('--adjust', action='store_true', - help='Auto-adjust coordinates for visualizer and extractor') - parser.add_argument('--skip-analyzer', action='store_true', - help='Skip analyzer step (use existing doctags)') - parser.add_argument('--skip-visualizer', action='store_true', - help='Skip visualizer step') - parser.add_argument('--skip-extractor', action='store_true', - help='Skip extractor step') - parser.add_argument('--output-dir', type=str, default='results', - help='Output directory for all results') - parser.add_argument('--dpi', type=int, default=200, - help='DPI for PDF rendering') - parser.add_argument('--max-pages', type=int, default=None, - help='Maximum number of pages to process') - parser.add_argument('--verbose', '-v', action='store_true', - help='Verbose output') - return parser.parse_args() -def get_pdf_page_count(pdf_path): - """Get the number of pages in a PDF file.""" - try: - from pdf2image.pdf2image import pdfinfo_from_path - info = pdfinfo_from_path(pdf_path) - return info["Pages"] - except Exception as e: - print(f"Warning: Could not determine page count: {e}") - return None +class BatchProcessor: + def __init__(self, batch_id, pdf_file, start_page, end_page, options): + self.batch_id = batch_id + self.pdf_file = pdf_file + self.start_page = start_page + self.end_page = end_page + self.total_pages = end_page - start_page + 1 + self.options = options -def parse_page_list(pages_str): - """Parse a page list string like "1,3,5-7,10" into a list of page numbers.""" - pages = [] - parts = pages_str.split(',') - - for part in parts: - part = part.strip() - if '-' in part: - # Range like "5-7" - try: - start, end = map(int, part.split('-')) - pages.extend(range(start, end + 1)) - except ValueError: - print(f"Warning: Invalid page range '{part}'") - else: - # Single page - try: - pages.append(int(part)) - except ValueError: - print(f"Warning: Invalid page number '{part}'") - - # Remove duplicates and sort - return sorted(set(pages)) - -def run_command(command, verbose=False): - """Run a command and return success status and output.""" - if verbose: - print(f"Running: {command}") - - try: - process = subprocess.Popen( - command, - shell=True, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - text=True, - universal_newlines=True - ) - - # Send "n" to bypass any prompts - stdout, stderr = process.communicate(input="n\n", timeout=300) # 5 minute timeout - - if verbose and stdout: - print("Output:", stdout[:500], "..." if len(stdout) > 500 else "") - - if process.returncode != 0: - print(f"Error: {stderr}") - return False, stdout, stderr - - return True, stdout, stderr - - except subprocess.TimeoutExpired: - print("Error: Command timed out") - return False, "", "Command timed out" - except Exception as e: - print(f"Error running command: {e}") - return False, "", str(e) - -def process_page(pdf_path, page_num, args, results_dir): - """Process a single page through all three steps.""" - print(f"\n{'='*60}") - print(f"Processing page {page_num}") - print(f"{'='*60}") - - page_results = { - 'page': page_num, - 'analyzer': {'success': False}, - 'visualizer': {'success': False}, - 'extractor': {'success': False}, - 'start_time': datetime.now().isoformat() - } - - # Step 1: Analyzer - if not args.skip_analyzer: - print(f"\n[1/3] Running analyzer for page {page_num}...") - - # Create page-specific output name - output_name = f"page_{page_num}" - analyzer_cmd = f"python analyzer.py --image {pdf_path} --page {page_num} --output {results_dir}/{output_name}.html --dpi {args.dpi}" - - success, stdout, stderr = run_command(analyzer_cmd, args.verbose) - - page_results['analyzer'] = { - 'success': success, - 'doctags_file': f"{results_dir}/{output_name}.doctags.txt" if success else None, - 'error': stderr if not success else None + # State management + self.state = { + 'status': 'initializing', + 'processed': 0, + 'total': self.total_pages, + 'start_time': time.time(), + 'completed': False, + 'paused': False, + 'cancelled': False, + 'page_statuses': {}, + 'stages': { + 'analysis': {'completed': 0, 'total': self.total_pages}, + 'visualization': {'completed': 0, 'total': self.total_pages}, + 'extraction': {'completed': 0, 'total': self.total_pages} + }, + 'results': { + 'successful': 0, + 'failed': 0, + 'totalImages': 0, + 'failedPages': [] + }, + 'logs': [] } - if success: - print(f"✓ Analyzer completed for page {page_num}") - else: - print(f"✗ Analyzer failed for page {page_num}") - if not args.skip_visualizer or not args.skip_extractor: - print(f"Skipping remaining steps for page {page_num} due to analyzer failure") - return page_results - else: - # Check if doctags file exists - doctags_file = f"{results_dir}/page_{page_num}.doctags.txt" - if os.path.exists(doctags_file): - page_results['analyzer'] = { - 'success': True, - 'doctags_file': doctags_file, - 'skipped': True - } - else: - print(f"✗ No existing doctags file found for page {page_num}") - return page_results + # Initialize page statuses + for page in range(start_page, end_page + 1): + self.state['page_statuses'][str(page)] = 'pending' - # Step 2: Visualizer - if not args.skip_visualizer: - print(f"\n[2/3] Running visualizer for page {page_num}...") + # Threading + self.lock = threading.Lock() + self.pause_event = threading.Event() + self.pause_event.set() # Start unpaused - doctags_file = page_results['analyzer']['doctags_file'] - visualizer_cmd = f"python visualizer.py --doctags {doctags_file} --pdf {pdf_path} --page {page_num}" + # Create batch results directory + self.results_dir = Path("results") / f"batch_{batch_id}" + self.results_dir.mkdir(parents=True, exist_ok=True) - if args.adjust: - visualizer_cmd += " --adjust" + # Log file + self.log_file = self.results_dir / "batch_processing.log" - success, stdout, stderr = run_command(visualizer_cmd, args.verbose) - - page_results['visualizer'] = { - 'success': success, - 'image_file': f"{results_dir}/visualization_page_{page_num}.png" if success else None, - 'error': stderr if not success else None + def log_message(self, message, level='info'): + """Add a log message to the state""" + timestamp = datetime.now().isoformat() + log_entry = { + 'timestamp': timestamp, + 'level': level, + 'message': message } - if success: - print(f"✓ Visualizer completed for page {page_num}") - else: - print(f"✗ Visualizer failed for page {page_num}") + with self.lock: + self.state['logs'].append(log_entry) + # Keep only last 100 log entries in memory + if len(self.state['logs']) > 100: + self.state['logs'] = self.state['logs'][-100:] - # Step 3: Picture Extractor - if not args.skip_extractor: - print(f"\n[3/3] Running picture extractor for page {page_num}...") + # Also write to log file + with open(self.log_file, 'a') as f: + f.write(f"[{timestamp}] [{level.upper()}] {message}\n") - doctags_file = page_results['analyzer']['doctags_file'] - pictures_dir = f"{results_dir}/pictures_page_{page_num}" - extractor_cmd = f"python picture_extractor.py --doctags {doctags_file} --pdf {pdf_path} --page {page_num} --output {pictures_dir}" + logger.info(f"[{level}] {message}") - if args.adjust: - extractor_cmd += " --adjust" + def update_page_status(self, page_num, status): + """Update the status of a specific page""" + with self.lock: + self.state['page_statuses'][str(page_num)] = status - success, stdout, stderr = run_command(extractor_cmd, args.verbose) + def update_stage_progress(self, stage, increment=1): + """Update progress for a specific stage""" + with self.lock: + self.state['stages'][stage]['completed'] += increment - page_results['extractor'] = { - 'success': success, - 'output_dir': pictures_dir if success else None, - 'error': stderr if not success else None - } + def process_page(self, page_num): + """Process a single page through all stages""" + try: + # Check if paused or cancelled + self.pause_event.wait() + if self.state['cancelled']: + return False - if success: - print(f"✓ Picture extractor completed for page {page_num}") - else: - print(f"✗ Picture extractor failed for page {page_num}") + self.log_message(f"Starting processing for page {page_num}") + self.update_page_status(page_num, 'processing') - page_results['end_time'] = datetime.now().isoformat() - return page_results + # Stage 1: Analysis + if not self.run_analyzer(page_num): + raise Exception("Analyzer failed") + self.update_stage_progress('analysis') -def create_batch_report(results, pdf_path, output_dir): - """Create an HTML report summarizing the batch processing results.""" - report_path = Path(output_dir) / "batch_report.html" + # Check if paused or cancelled + self.pause_event.wait() + if self.state['cancelled']: + return False - total_pages = len(results) - successful_pages = sum(1 for r in results if all( - r.get(step, {}).get('success', False) or r.get(step, {}).get('skipped', False) - for step in ['analyzer', 'visualizer', 'extractor'] - )) + # Stage 2: Visualization + if not self.run_visualizer(page_num): + raise Exception("Visualizer failed") + self.update_stage_progress('visualization') - html = f""" + # Check if paused or cancelled + self.pause_event.wait() + if self.state['cancelled']: + return False + + # Stage 3: Extraction + image_count = self.run_extractor(page_num) + self.update_stage_progress('extraction') + + # Update results + with self.lock: + self.state['results']['successful'] += 1 + self.state['results']['totalImages'] += image_count + self.state['processed'] += 1 + + self.update_page_status(page_num, 'completed') + self.log_message(f"Successfully processed page {page_num}", 'success') + + return True + + except Exception as e: + # Handle failure + with self.lock: + self.state['results']['failed'] += 1 + self.state['results']['failedPages'].append({ + 'pageNum': page_num, + 'reason': str(e) + }) + self.state['processed'] += 1 + + self.update_page_status(page_num, 'failed') + self.log_message(f"Failed to process page {page_num}: {str(e)}", 'error') + + return False + + def run_analyzer(self, page_num): + """Run the analyzer for a specific page""" + try: + # Use standard results directory for analyzer output + output_base = Path("results") / f"output" + + command = [ + "python", "analyzer.py", + "--image", self.pdf_file, + "--page", str(page_num), + "--output", str(output_base), + "--start-page", str(page_num), + "--end-page", str(page_num) + ] + + self.log_message(f"Running analyzer for page {page_num}") + + # Run the command + process = subprocess.Popen( + command, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True + ) + + # Send "n" to bypass prompts + stdout, stderr = process.communicate(input="n\n", timeout=300) # 5 minute timeout + + if process.returncode != 0: + raise Exception(f"Analyzer failed: {stderr}") + + # Copy doctags to batch directory for archiving + doctags_src = Path("results") / "output.doctags.txt" + doctags_dst = self.results_dir / f"page_{page_num}.doctags.txt" + if doctags_src.exists(): + shutil.copy2(doctags_src, doctags_dst) + self.log_message(f"DocTags saved for page {page_num}") + else: + raise Exception("DocTags file not generated") + + return True + + except Exception as e: + self.log_message(f"Analyzer error for page {page_num}: {str(e)}", 'error') + return False + + def run_visualizer(self, page_num): + """Run the visualizer for a specific page""" + try: + # The visualizer expects doctags in the standard location + doctags_path = Path("results") / "output.doctags.txt" + + # First, ensure we have the right doctags file for this page + page_doctags = self.results_dir / f"page_{page_num}.doctags.txt" + if page_doctags.exists(): + shutil.copy2(page_doctags, doctags_path) + + command = [ + "python", "visualizer.py", + "--doctags", str(doctags_path), + "--pdf", self.pdf_file, + "--page", str(page_num) + ] + + if self.options.get('adjust', True): + command.append("--adjust") + + self.log_message(f"Running visualizer for page {page_num}") + + # Run the command + process = subprocess.run( + command, + capture_output=True, + text=True, + timeout=300 + ) + + if process.returncode != 0: + raise Exception(f"Visualizer failed: {process.stderr}") + + # The visualizer should have created the file in results/ + viz_src = Path("results") / f"visualization_page_{page_num}.png" + viz_dst = self.results_dir / f"visualization_page_{page_num}.png" + + if viz_src.exists(): + shutil.copy2(viz_src, viz_dst) + self.log_message(f"Visualization saved for page {page_num}") + else: + self.log_message(f"Warning: Visualization file not found for page {page_num}", 'warning') + + return True + + except Exception as e: + self.log_message(f"Visualizer error for page {page_num}: {str(e)}", 'error') + return False + + def run_extractor(self, page_num): + """Run the picture extractor for a specific page""" + try: + # Ensure we have the right doctags file for this page + doctags_path = Path("results") / "output.doctags.txt" + page_doctags = self.results_dir / f"page_{page_num}.doctags.txt" + if page_doctags.exists(): + shutil.copy2(page_doctags, doctags_path) + + # Let extractor create files in its default location first + command = [ + "python", "picture_extractor.py", + "--doctags", str(doctags_path), + "--pdf", self.pdf_file, + "--page", str(page_num) + ] + + if self.options.get('adjust', True): + command.append("--adjust") + + self.log_message(f"Running extractor for page {page_num}") + + # Run the command + process = subprocess.run( + command, + capture_output=True, + text=True, + timeout=300 + ) + + if process.returncode != 0: + self.log_message(f"Extractor warning for page {page_num}: {process.stderr}", 'warning') + + # Count and copy extracted images + image_count = 0 + pics_src = Path("results") / "pictures" + pics_dst = self.results_dir / f"pictures_page_{page_num}" + + if pics_src.exists(): + # Copy to batch directory + if pics_dst.exists(): + shutil.rmtree(pics_dst) + shutil.copytree(pics_src, pics_dst) + + # Also copy to page-specific location for web interface + pics_web = Path("results") / f"pictures_page_{page_num}" + if pics_web.exists(): + shutil.rmtree(pics_web) + shutil.copytree(pics_src, pics_web) + + # Count PNG files + image_count = len(list(pics_dst.glob("*.png"))) + self.log_message(f"Extracted {image_count} images from page {page_num}") + else: + self.log_message(f"No images extracted from page {page_num}", 'info') + + return image_count + + except Exception as e: + self.log_message(f"Extractor error for page {page_num}: {str(e)}", 'error') + return 0 + + def run(self): + """Main batch processing loop""" + try: + self.state['status'] = 'processing' + self.log_message(f"Starting batch processing for {self.pdf_file} (pages {self.start_page}-{self.end_page})") + + # Determine number of workers + max_workers = 4 if self.options.get('parallel', True) else 1 + + # Create page list + pages = list(range(self.start_page, self.end_page + 1)) + + # Process pages + if max_workers > 1: + # Parallel processing + with ThreadPoolExecutor(max_workers=max_workers) as executor: + futures = {executor.submit(self.process_page, page): page for page in pages} + + for future in as_completed(futures): + if self.state['cancelled']: + executor.shutdown(wait=False) + break + + page = futures[future] + try: + future.result() + except Exception as e: + self.log_message(f"Unexpected error processing page {page}: {str(e)}", 'error') + else: + # Sequential processing + for page in pages: + if self.state['cancelled']: + break + self.process_page(page) + + # Generate report if requested + if self.options.get('generate_report', True) and not self.state['cancelled']: + self.generate_report() + + # Update final state + with self.lock: + self.state['completed'] = True + self.state['status'] = 'cancelled' if self.state['cancelled'] else 'completed' + + duration = time.time() - self.state['start_time'] + self.log_message(f"Batch processing completed in {self.format_duration(duration)}", 'success') + + except Exception as e: + self.log_message(f"Critical error in batch processing: {str(e)}", 'error') + with self.lock: + self.state['completed'] = True + self.state['status'] = 'error' + + def generate_report(self): + """Generate a comprehensive HTML report""" + try: + self.log_message("Generating batch processing report") + + report_path = self.results_dir / "report.html" + + # Calculate statistics + duration = time.time() - self.state['start_time'] + success_rate = (self.state['results']['successful'] / self.total_pages * 100) if self.total_pages > 0 else 0 + + html = f""" - Batch Processing Report - {os.path.basename(pdf_path)} + Batch Processing Report - {self.pdf_file}

Batch Processing Report

-

Generated on {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}

+

Generated on {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}

-
-

Summary

-
- {total_pages}
- Total Pages +
+
+
{self.total_pages}
+
Total Pages
-
- {successful_pages}
- Successful +
+
{self.state['results']['successful']}
+
Successful
-
- {total_pages - successful_pages}
- Failed +
+
{self.state['results']['failed']}
+
Failed
-
- {os.path.basename(pdf_path)}
- PDF File +
+
{self.state['results']['totalImages']}
+
Images Extracted
+
+
+
{self.format_duration(duration)}
+
Processing Time
+
+
+
{success_rate:.1f}%
+
Success Rate
-

Processing Results

+

Processing Details

- - - - - + + + + + + + + + + + + + + + + + + + + + + +
PageAnalyzerVisualizerExtractorActionsParameterValue
PDF File{self.pdf_file}
Page Range{self.start_page} - {self.end_page}
Batch ID{self.batch_id}
Parallel Processing{'Enabled' if self.options.get('parallel', True) else 'Disabled'}
Auto-adjust Coordinates{'Enabled' if self.options.get('adjust', True) else 'Disabled'}
""" - for result in results: - page_num = result['page'] - - # Determine status for each step - analyzer_status = "✓ Success" if result['analyzer']['success'] else "✗ Failed" - if result['analyzer'].get('skipped'): - analyzer_status = "— Skipped" - - visualizer_status = "✓ Success" if result.get('visualizer', {}).get('success') else "✗ Failed" - if 'visualizer' not in result: - visualizer_status = "— Skipped" - - extractor_status = "✓ Success" if result.get('extractor', {}).get('success') else "✗ Failed" - if 'extractor' not in result: - extractor_status = "— Skipped" - - # Build action links - actions = [] - if result['analyzer']['success']: - doctags_file = result['analyzer'].get('doctags_file') - if doctags_file: - actions.append(f'DocTags') - - if result.get('visualizer', {}).get('success'): - viz_file = result['visualizer'].get('image_file') - if viz_file: - actions.append(f'Visualization') - - if result.get('extractor', {}).get('success'): - extract_dir = result['extractor'].get('output_dir') - if extract_dir: - index_file = f"{os.path.basename(extract_dir)}/index.html" - actions.append(f'Extracted Images') - - # Determine row styling - analyzer_class = "success" if result['analyzer']['success'] else "failed" - if result['analyzer'].get('skipped'): - analyzer_class = "skipped" - - visualizer_class = "success" if result.get('visualizer', {}).get('success') else "failed" - if 'visualizer' not in result: - visualizer_class = "skipped" - - extractor_class = "success" if result.get('extractor', {}).get('success') else "failed" - if 'extractor' not in result: - extractor_class = "skipped" - - html += f""" - Page {page_num} - {analyzer_status} - {visualizer_status} - {extractor_status} - {' | '.join(actions) if actions else 'No outputs'} - + # Add failed pages section if any + if self.state['results']['failedPages']: + html += """ +
+

Failed Pages

+ + + + + +""" + for failed in self.state['results']['failedPages']: + html += f""" + + + + +""" + html += """ +
Page NumberReason
{failed['pageNum']}{failed['reason']}
+
""" - html += """ + # Add successful pages preview + html += """ +

Processed Pages

+
+""" + + for page in range(self.start_page, self.end_page + 1): + if self.state['page_statuses'].get(str(page)) == 'completed': + viz_path = f"visualization_page_{page}.png" + html += f""" +
+ + Page {page} + +

Page {page}

+
+""" + + html += """ +
""" - with open(report_path, 'w', encoding='utf-8') as f: - f.write(html) + with open(report_path, 'w') as f: + f.write(html) - return report_path + self.log_message("Report generated successfully") -def main(): - # Parse arguments - args = parse_arguments() + except Exception as e: + self.log_message(f"Error generating report: {str(e)}", 'error') - # Check if PDF exists - if not os.path.exists(args.pdf): - print(f"Error: PDF file not found: {args.pdf}") - sys.exit(1) + def pause(self): + """Pause the batch processing""" + self.pause_event.clear() + self.state['paused'] = True + self.log_message("Batch processing paused") - # Create results directory - results_dir = ensure_results_folder() - if args.output_dir != 'results': - results_dir = Path(args.output_dir) - if not results_dir.exists(): - results_dir.mkdir(parents=True) - print(f"Created output directory: {results_dir}") + def resume(self): + """Resume the batch processing""" + self.pause_event.set() + self.state['paused'] = False + self.log_message("Batch processing resumed") - # Determine which pages to process - pages_to_process = [] + def cancel(self): + """Cancel the batch processing""" + self.state['cancelled'] = True + self.pause_event.set() # Ensure not stuck on pause + self.log_message("Batch processing cancelled") - if args.pages: - # Specific pages provided - pages_to_process = parse_page_list(args.pages) - print(f"Processing specific pages: {pages_to_process}") - else: - # Range of pages - pdf_page_count = get_pdf_page_count(args.pdf) + def get_state(self): + """Get the current state with calculated fields""" + with self.lock: + state = self.state.copy() - if pdf_page_count: - print(f"PDF has {pdf_page_count} pages") + # Calculate ETA + if state['processed'] > 0 and not state['completed']: + elapsed = time.time() - state['start_time'] + rate = state['processed'] / elapsed + remaining = state['total'] - state['processed'] + eta = remaining / rate if rate > 0 else 0 + state['eta'] = eta * 1000 # Convert to milliseconds + else: + state['eta'] = 0 - start_page = args.start - end_page = args.end or pdf_page_count or 1 + return state - if args.max_pages and (end_page - start_page + 1) > args.max_pages: - end_page = start_page + args.max_pages - 1 - print(f"Limiting to {args.max_pages} pages") + def format_duration(self, seconds): + """Format duration in seconds to human readable format""" + hours = int(seconds // 3600) + minutes = int((seconds % 3600) // 60) + secs = int(seconds % 60) - pages_to_process = list(range(start_page, end_page + 1)) - print(f"Processing pages {start_page} to {end_page}") + if hours > 0: + return f"{hours}:{minutes:02d}:{secs:02d}" + else: + return f"{minutes}:{secs:02d}" - # Start batch processing - print(f"\nStarting batch processing of {len(pages_to_process)} pages") - print(f"PDF: {args.pdf}") - print(f"Output directory: {results_dir}") + def create_zip_archive(self): + """Create a ZIP archive of all results""" + try: + zip_path = self.results_dir / f"batch_results_{self.batch_id}.zip" - if args.skip_analyzer: - print("Note: Skipping analyzer step") - if args.skip_visualizer: - print("Note: Skipping visualizer step") - if args.skip_extractor: - print("Note: Skipping extractor step") + with zipfile.ZipFile(zip_path, 'w', zipfile.ZIP_DEFLATED) as zipf: + # Add all files in the results directory + for file_path in self.results_dir.rglob('*'): + if file_path.is_file() and file_path != zip_path: + arcname = file_path.relative_to(self.results_dir) + zipf.write(file_path, arcname) - # Process each page - all_results = [] - start_time = time.time() + self.log_message(f"Created ZIP archive: {zip_path}") + return zip_path - for i, page_num in enumerate(pages_to_process, 1): - print(f"\n[{i}/{len(pages_to_process)}] Processing page {page_num}") + except Exception as e: + self.log_message(f"Error creating ZIP archive: {str(e)}", 'error') + return None - page_result = process_page(args.pdf, page_num, args, results_dir) - all_results.append(page_result) - # Save intermediate results - with open(results_dir / "batch_results.json", 'w') as f: - json.dump(all_results, f, indent=2) +# Global batch processors storage +batch_processors = {} +batch_lock = threading.Lock() - # Calculate processing time - end_time = time.time() - processing_time = end_time - start_time - # Create summary report - print(f"\n{'='*60}") - print("BATCH PROCESSING COMPLETE") - print(f"{'='*60}") - print(f"Total pages processed: {len(pages_to_process)}") - print(f"Total time: {processing_time:.1f} seconds") - print(f"Average time per page: {processing_time/len(pages_to_process):.1f} seconds") +def start_batch_processing(batch_id, pdf_file, start_page, end_page, options): + """Start a new batch processing job""" + try: + processor = BatchProcessor(batch_id, pdf_file, start_page, end_page, options) - # Create HTML report - report_path = create_batch_report(all_results, args.pdf, results_dir) - print(f"\nBatch report created: {report_path}") + with batch_lock: + batch_processors[batch_id] = processor - # Summary statistics - successful_count = sum(1 for r in all_results if all( - r.get(step, {}).get('success', False) or r.get(step, {}).get('skipped', False) - for step in ['analyzer', 'visualizer', 'extractor'] if step in r - )) + # Start processing in a separate thread + thread = threading.Thread(target=processor.run) + thread.daemon = True + thread.start() - print(f"\nSuccess rate: {successful_count}/{len(pages_to_process)} pages ({successful_count/len(pages_to_process)*100:.1f}%)") + return True - # Print any errors - errors = [] - for result in all_results: - page_num = result['page'] - for step in ['analyzer', 'visualizer', 'extractor']: - if step in result and not result[step]['success'] and not result[step].get('skipped'): - error = result[step].get('error', 'Unknown error') - errors.append(f"Page {page_num} - {step}: {error}") + except Exception as e: + logger.error(f"Error starting batch processing: {str(e)}") + return False - if errors: - print("\nErrors encountered:") - for error in errors[:5]: # Show first 5 errors - print(f" - {error}") - if len(errors) > 5: - print(f" ... and {len(errors) - 5} more errors") - print(f"\nAll results saved to: {results_dir}") +def get_batch_processor(batch_id): + """Get a batch processor by ID""" + with batch_lock: + return batch_processors.get(batch_id) -if __name__ == "__main__": - main() \ No newline at end of file + +def cleanup_old_batches(max_age_hours=24): + """Clean up old batch processors""" + current_time = time.time() + max_age_seconds = max_age_hours * 3600 + + with batch_lock: + to_remove = [] + for batch_id, processor in batch_processors.items(): + if processor.state['completed']: + age = current_time - processor.state['start_time'] + if age > max_age_seconds: + to_remove.append(batch_id) + + for batch_id in to_remove: + del batch_processors[batch_id] \ No newline at end of file diff --git a/frontend/batch.html b/frontend/batch.html new file mode 100644 index 0000000..7732b6d --- /dev/null +++ b/frontend/batch.html @@ -0,0 +1,251 @@ + + + + + + DocTags Batch Processor + + + + +
+

DocTags Batch Processor

+ + +
+ ← Back to Single Page Processor +
+ + +
+

Batch Processing Configuration

+ + +
+
+ + + +
+ + +
+ +
+ + +
+ +
+ + +
+ +
+ + + +
+
+
+
+ + + + + +
+ + + + +
+ + + + + + + + + + + + +
+ + +
+
+ × +
+ + +
+
+ Page Visualization +
+
+ +
+
+

+
+
+
+ + + + \ No newline at end of file diff --git a/frontend/static/batch-processor.js b/frontend/static/batch-processor.js new file mode 100644 index 0000000..f996e3c --- /dev/null +++ b/frontend/static/batch-processor.js @@ -0,0 +1,718 @@ +// Batch Processor State Management +const batchState = { + isProcessing: false, + isPaused: false, + currentBatchId: null, + totalPages: 0, + processedPages: 0, + startTime: null, + pageStatuses: {}, + results: { + successful: 0, + failed: 0, + totalImages: 0, + failedPages: [] + }, + autoScroll: true +}; + +// Timer for elapsed time +let elapsedTimer = null; + +// Initialize on page load +window.addEventListener('DOMContentLoaded', function() { + loadPDFFiles(); + setupEventListeners(); + checkBatchEnvironment(); +}); + +// Load available PDF files +function loadPDFFiles() { + fetch('/pdf-files') + .then(response => response.json()) + .then(data => { + const select = document.getElementById('batch_pdf_file'); + data.forEach(file => { + const option = document.createElement('option'); + option.value = file; + option.textContent = file; + select.appendChild(option); + }); + }) + .catch(error => { + console.error('Error loading PDFs:', error); + addConsoleMessage('Error loading PDF files: ' + error.message, 'error'); + }); +} + +// Setup event listeners +function setupEventListeners() { + // PDF selection change + document.getElementById('batch_pdf_file').addEventListener('change', function() { + if (this.value) { + fetchPDFInfo(this.value); + } else { + document.getElementById('batch-pdf-info').classList.add('hidden'); + } + }); + + // Page range radio buttons + document.querySelectorAll('input[name="page_range"]').forEach(radio => { + radio.addEventListener('change', function() { + const customRange = document.getElementById('custom-range'); + if (this.value === 'range') { + customRange.classList.remove('hidden'); + } else { + customRange.classList.add('hidden'); + } + }); + }); +} + +// Fetch PDF information (page count) +function fetchPDFInfo(pdfFile) { + fetch(`/pdf-info/${encodeURIComponent(pdfFile)}`) + .then(response => response.json()) + .then(data => { + document.getElementById('total-pages').textContent = data.pageCount; + document.getElementById('batch-pdf-info').classList.remove('hidden'); + + // Update max values for custom range + document.getElementById('start_page').max = data.pageCount; + document.getElementById('end_page').max = data.pageCount; + document.getElementById('end_page').value = data.pageCount; + }) + .catch(error => { + console.error('Error fetching PDF info:', error); + addConsoleMessage('Error fetching PDF info: ' + error.message, 'error'); + }); +} + +// Start batch processing +function startBatchProcessing() { + const pdfFile = document.getElementById('batch_pdf_file').value; + if (!pdfFile) { + alert('Please select a PDF file'); + return; + } + + // Get page range + const pageRangeType = document.querySelector('input[name="page_range"]:checked').value; + let startPage = 1; + let endPage = parseInt(document.getElementById('total-pages').textContent); + + if (pageRangeType === 'range') { + startPage = parseInt(document.getElementById('start_page').value) || 1; + endPage = parseInt(document.getElementById('end_page').value) || endPage; + + if (startPage > endPage) { + alert('Invalid page range: start page must be less than or equal to end page'); + return; + } + } + + // Reset state + resetBatchState(); + batchState.isProcessing = true; + batchState.totalPages = endPage - startPage + 1; + batchState.startTime = Date.now(); + + // Update UI + updateUIForProcessing(true); + initializePageGrid(startPage, endPage); + + // Start elapsed timer + startElapsedTimer(); + + // Show console + document.getElementById('console-output').classList.remove('hidden'); + addConsoleMessage(`Starting batch processing for ${pdfFile} (pages ${startPage}-${endPage})`, 'info'); + + // Prepare request data + const formData = new FormData(); + formData.append('pdf_file', pdfFile); + formData.append('start_page', startPage); + formData.append('end_page', endPage); + formData.append('adjust', document.getElementById('batch_adjust').checked); + formData.append('parallel', document.getElementById('parallel_processing').checked); + formData.append('generate_report', document.getElementById('generate_report').checked); + + // Start batch processing + fetch('/run-batch-processor', { + method: 'POST', + body: formData + }) + .then(response => response.json()) + .then(data => { + if (data.success) { + batchState.currentBatchId = data.batch_id; + addConsoleMessage(`Batch processing started with ID: ${data.batch_id}`, 'success'); + + // Start polling for updates + pollBatchStatus(); + } else { + throw new Error(data.error || 'Failed to start batch processing'); + } + }) + .catch(error => { + console.error('Error starting batch processing:', error); + addConsoleMessage('Error: ' + error.message, 'error'); + finishBatchProcessing(false); + }); +} + +// Poll for batch status updates +function pollBatchStatus() { + if (!batchState.isProcessing || !batchState.currentBatchId) { + return; + } + + fetch(`/batch-status/${batchState.currentBatchId}`) + .then(response => response.json()) + .then(data => { + updateBatchProgress(data); + + if (data.completed) { + finishBatchProcessing(true); + } else if (!batchState.isPaused) { + // Continue polling + setTimeout(pollBatchStatus, 1000); + } + }) + .catch(error => { + console.error('Error polling batch status:', error); + if (batchState.isProcessing) { + setTimeout(pollBatchStatus, 2000); // Retry with longer delay + } + }); +} + +// Update batch progress +function updateBatchProgress(data) { + // Update overall progress + const percentage = Math.round((data.processed / data.total) * 100); + document.getElementById('overall-percentage').textContent = percentage + '%'; + document.getElementById('overall-progress-fill').style.width = percentage + '%'; + document.getElementById('pages-completed').textContent = data.processed; + document.getElementById('pages-total').textContent = data.total; + + // Update stage progress + if (data.stages) { + updateStageProgress('analysis', data.stages.analysis); + updateStageProgress('visualization', data.stages.visualization); + updateStageProgress('extraction', data.stages.extraction); + } + + // Update page statuses + if (data.page_statuses) { + for (const [pageNum, status] of Object.entries(data.page_statuses)) { + updatePageStatus(pageNum, status); + } + } + + // Update results + if (data.results) { + batchState.results = data.results; + updateResultsSummary(); + } + + // Add log messages + if (data.logs && data.logs.length > 0) { + data.logs.forEach(log => { + addConsoleMessage(log.message, log.level); + }); + } + + // Update ETA + if (data.eta) { + document.getElementById('eta-time').textContent = formatTime(data.eta); + } +} + +// Update stage progress +function updateStageProgress(stage, data) { + if (!data) return; + + const percentage = Math.round((data.completed / data.total) * 100); + document.getElementById(`${stage}-percentage`).textContent = percentage + '%'; + document.getElementById(`${stage}-progress`).style.width = percentage + '%'; +} + +// Initialize page grid +function initializePageGrid(startPage, endPage) { + const grid = document.getElementById('page-grid'); + grid.innerHTML = ''; + + document.getElementById('batch-progress-panel').classList.remove('hidden'); + document.getElementById('page-status-grid').classList.remove('hidden'); + + for (let i = startPage; i <= endPage; i++) { + const pageItem = createPageStatusItem(i); + grid.appendChild(pageItem); + batchState.pageStatuses[i] = 'pending'; + } +} + +// Create page status item +function createPageStatusItem(pageNum) { + const item = document.createElement('div'); + item.className = 'page-status-item pending'; + item.id = `page-status-${pageNum}`; + item.innerHTML = ` +
Page ${pageNum}
+
⏳
+ + `; + item.onclick = () => viewPageResults(pageNum); + return item; +} + +// Update page status +function updatePageStatus(pageNum, status) { + const item = document.getElementById(`page-status-${pageNum}`); + if (!item) return; + + // Remove all status classes + item.classList.remove('pending', 'processing', 'completed', 'failed'); + + // Add new status class + item.classList.add(status.toLowerCase()); + + // Update icon + const icons = { + 'pending': '⏳', + 'processing': '🔄', + 'completed': '✅', + 'failed': '❌' + }; + + const iconElement = item.querySelector('.page-status-icon'); + iconElement.textContent = icons[status.toLowerCase()] || '❓'; + + // Show actions for completed pages + if (status.toLowerCase() === 'completed') { + item.querySelector('.page-actions').classList.remove('hidden'); + } + + batchState.pageStatuses[pageNum] = status.toLowerCase(); +} + +// View page results +function viewPageResults(pageNum) { + // Check if we have a current batch ID + if (!batchState.currentBatchId) { + addConsoleMessage('No batch ID available', 'error'); + return; + } + + // Open modal with page visualization and extracted images + const modal = document.getElementById('batch-image-modal'); + modal.classList.add('active'); + + // Load visualization from the batch report directory + const vizImage = document.getElementById('modal-visualization-image'); + vizImage.src = `/batch-report-image/${batchState.currentBatchId}/visualization_page_${pageNum}.png?t=${Date.now()}`; + + // Handle loading errors + vizImage.onerror = function() { + // Try the regular results directory as fallback + this.src = `/results/visualization_page_${pageNum}.png?t=${Date.now()}`; + + this.onerror = function() { + this.alt = 'Visualization not found'; + console.error('Failed to load visualization'); + }; + }; + + // Load extracted images + loadExtractedImagesForPage(pageNum); + + // Update page info + document.getElementById('modal-page-info').textContent = `Page ${pageNum} Results`; +} + +// Load extracted images for a specific page +function loadExtractedImagesForPage(pageNum) { + const gallery = document.getElementById('modal-extracted-gallery'); + gallery.innerHTML = '
Loading extracted images...'; + + // Fetch extracted images for this page + fetch(`/results/pictures_page_${pageNum}/index.html`) + .then(response => { + if (!response.ok) throw new Error('No extracted images'); + return response.text(); + }) + .then(html => { + // Parse and display images + const parser = new DOMParser(); + const doc = parser.parseFromString(html, 'text/html'); + const images = doc.querySelectorAll('.picture-card img'); + + if (images.length === 0) { + gallery.innerHTML = '

No images extracted from this page

'; + return; + } + + let galleryHTML = ''; + images.forEach(img => { + const src = img.getAttribute('src'); + galleryHTML += ` +
+ Extracted image +
+ `; + }); + + gallery.innerHTML = galleryHTML; + }) + .catch(error => { + gallery.innerHTML = '

No extracted images available

'; + }); +} + +// Switch modal tabs +function switchModalTab(tabName) { + // Update tab buttons + document.querySelectorAll('.modal-tab').forEach(tab => { + tab.classList.remove('active'); + }); + event.target.classList.add('active'); + + // Update content + document.querySelectorAll('.modal-tab-content').forEach(content => { + content.classList.remove('active'); + }); + document.getElementById(`modal-${tabName}-content`).classList.add('active'); +} + +// Close modal +function closeBatchImageModal() { + document.getElementById('batch-image-modal').classList.remove('active'); +} + +// Pause batch processing +function pauseBatchProcessing() { + if (!batchState.isProcessing || batchState.isPaused) return; + + batchState.isPaused = true; + + fetch(`/pause-batch/${batchState.currentBatchId}`, { method: 'POST' }) + .then(response => response.json()) + .then(data => { + if (data.success) { + addConsoleMessage('Batch processing paused', 'info'); + document.getElementById('pause-batch-btn').classList.add('hidden'); + document.getElementById('resume-batch-btn').classList.remove('hidden'); + } + }) + .catch(error => { + console.error('Error pausing batch:', error); + addConsoleMessage('Error pausing batch: ' + error.message, 'error'); + }); +} + +// Resume batch processing +function resumeBatchProcessing() { + if (!batchState.isProcessing || !batchState.isPaused) return; + + batchState.isPaused = false; + + fetch(`/resume-batch/${batchState.currentBatchId}`, { method: 'POST' }) + .then(response => response.json()) + .then(data => { + if (data.success) { + addConsoleMessage('Batch processing resumed', 'info'); + document.getElementById('resume-batch-btn').classList.add('hidden'); + document.getElementById('pause-batch-btn').classList.remove('hidden'); + + // Resume polling + pollBatchStatus(); + } + }) + .catch(error => { + console.error('Error resuming batch:', error); + addConsoleMessage('Error resuming batch: ' + error.message, 'error'); + }); +} + +// Cancel batch processing +function cancelBatchProcessing() { + if (!batchState.isProcessing) return; + + if (!confirm('Are you sure you want to cancel the batch processing?')) { + return; + } + + fetch(`/cancel-batch/${batchState.currentBatchId}`, { method: 'POST' }) + .then(response => response.json()) + .then(data => { + if (data.success) { + addConsoleMessage('Batch processing cancelled', 'warning'); + finishBatchProcessing(false); + } + }) + .catch(error => { + console.error('Error cancelling batch:', error); + addConsoleMessage('Error cancelling batch: ' + error.message, 'error'); + }); +} + +// Finish batch processing +function finishBatchProcessing(success) { + batchState.isProcessing = false; + stopElapsedTimer(); + + // Update UI + updateUIForProcessing(false); + + // Show results + document.getElementById('batch-results').classList.remove('hidden'); + updateResultsSummary(); + + if (success) { + addConsoleMessage('Batch processing completed successfully!', 'success'); + } else { + addConsoleMessage('Batch processing stopped', 'warning'); + } + + // Update failed pages list if any + if (batchState.results.failedPages.length > 0) { + showFailedPages(); + } +} + +// Update results summary +function updateResultsSummary() { + document.getElementById('stat-success').textContent = batchState.results.successful; + document.getElementById('stat-failed').textContent = batchState.results.failed; + document.getElementById('stat-duration').textContent = formatTime(Date.now() - batchState.startTime); + document.getElementById('stat-images').textContent = batchState.results.totalImages; +} + +// Show failed pages +function showFailedPages() { + const failedSection = document.getElementById('failed-pages'); + const failedList = document.getElementById('failed-list'); + + failedSection.classList.remove('hidden'); + failedList.innerHTML = ''; + + batchState.results.failedPages.forEach(page => { + const item = document.createElement('div'); + item.className = 'failed-item'; + item.innerHTML = ` + Page ${page.pageNum} + ${page.reason} + + `; + failedList.appendChild(item); + }); +} + +// Retry failed page +function retryPage(pageNum) { + addConsoleMessage(`Retrying page ${pageNum}...`, 'info'); + + const formData = new FormData(); + formData.append('pdf_file', document.getElementById('batch_pdf_file').value); + formData.append('page_num', pageNum); + formData.append('adjust', document.getElementById('batch_adjust').checked); + + // Update page status + updatePageStatus(pageNum, 'processing'); + + fetch('/retry-page', { + method: 'POST', + body: formData + }) + .then(response => response.json()) + .then(data => { + if (data.success) { + updatePageStatus(pageNum, 'completed'); + addConsoleMessage(`Page ${pageNum} processed successfully`, 'success'); + + // Update results + batchState.results.successful++; + batchState.results.failed--; + batchState.results.failedPages = batchState.results.failedPages.filter(p => p.pageNum !== pageNum); + updateResultsSummary(); + + if (batchState.results.failedPages.length === 0) { + document.getElementById('failed-pages').classList.add('hidden'); + } + } else { + updatePageStatus(pageNum, 'failed'); + addConsoleMessage(`Failed to process page ${pageNum}: ${data.error}`, 'error'); + } + }) + .catch(error => { + updatePageStatus(pageNum, 'failed'); + addConsoleMessage(`Error retrying page ${pageNum}: ${error.message}`, 'error'); + }); +} + +// Download all results +function downloadResults() { + addConsoleMessage('Preparing results for download...', 'info'); + + fetch(`/download-batch-results/${batchState.currentBatchId}`) + .then(response => { + if (!response.ok) throw new Error('Failed to prepare download'); + return response.blob(); + }) + .then(blob => { + const url = window.URL.createObjectURL(blob); + const a = document.createElement('a'); + a.href = url; + a.download = `doctags_batch_results_${new Date().toISOString().split('T')[0]}.zip`; + document.body.appendChild(a); + a.click(); + window.URL.revokeObjectURL(url); + document.body.removeChild(a); + + addConsoleMessage('Download started', 'success'); + }) + .catch(error => { + addConsoleMessage('Error downloading results: ' + error.message, 'error'); + }); +} + +// View detailed report +function viewReport() { + window.open(`/batch-report/${batchState.currentBatchId}`, '_blank'); +} + +// Open results folder +function openResultsFolder() { + addConsoleMessage('Opening results folder...', 'info'); + + fetch('/open-results-folder', { method: 'POST' }) + .then(response => response.json()) + .then(data => { + if (data.success) { + addConsoleMessage('Results folder opened', 'success'); + } else { + addConsoleMessage('Could not open results folder: ' + data.error, 'error'); + } + }) + .catch(error => { + addConsoleMessage('Error: ' + error.message, 'error'); + }); +} + +// Console functions +function addConsoleMessage(message, level = 'info') { + const console = document.getElementById('console-content'); + const timestamp = new Date().toLocaleTimeString(); + + const messageDiv = document.createElement('div'); + messageDiv.className = `console-message ${level}`; + messageDiv.innerHTML = `[${timestamp}] ${message}`; + + console.appendChild(messageDiv); + + if (batchState.autoScroll) { + console.scrollTop = console.scrollHeight; + } +} + +function clearConsole() { + document.getElementById('console-content').innerHTML = ''; +} + +function toggleAutoScroll() { + batchState.autoScroll = !batchState.autoScroll; + document.getElementById('autoscroll-status').textContent = + `Auto-scroll: ${batchState.autoScroll ? 'ON' : 'OFF'}`; +} + +// Timer functions +function startElapsedTimer() { + elapsedTimer = setInterval(() => { + const elapsed = Date.now() - batchState.startTime; + document.getElementById('elapsed-time').textContent = formatTime(elapsed); + }, 1000); +} + +function stopElapsedTimer() { + if (elapsedTimer) { + clearInterval(elapsedTimer); + elapsedTimer = null; + } +} + +// Helper functions +function formatTime(milliseconds) { + const seconds = Math.floor(milliseconds / 1000); + const minutes = Math.floor(seconds / 60); + const hours = Math.floor(minutes / 60); + + if (hours > 0) { + return `${hours}:${String(minutes % 60).padStart(2, '0')}:${String(seconds % 60).padStart(2, '0')}`; + } else { + return `${minutes}:${String(seconds % 60).padStart(2, '0')}`; + } +} + +function resetBatchState() { + batchState.isProcessing = false; + batchState.isPaused = false; + batchState.currentBatchId = null; + batchState.totalPages = 0; + batchState.processedPages = 0; + batchState.startTime = null; + batchState.pageStatuses = {}; + batchState.results = { + successful: 0, + failed: 0, + totalImages: 0, + failedPages: [] + }; +} + +function updateUIForProcessing(isProcessing) { + // Toggle button visibility + document.getElementById('start-batch-btn').classList.toggle('hidden', isProcessing); + document.getElementById('pause-batch-btn').classList.toggle('hidden', !isProcessing); + document.getElementById('cancel-batch-btn').classList.toggle('hidden', !isProcessing); + + // Disable form inputs during processing + const inputs = document.querySelectorAll('.settings-panel input, .settings-panel select'); + inputs.forEach(input => { + input.disabled = isProcessing; + }); +} + +// Check batch environment +function checkBatchEnvironment() { + fetch('/check-environment') + .then(response => response.json()) + .then(data => { + const envDiv = document.getElementById('batch-environment-check'); + const envDetails = document.getElementById('batch-env-details'); + + if (data.missing_scripts.length > 0 || data.pdf_files.length === 0) { + envDiv.classList.remove('hidden'); + + let html = '
    '; + if (data.missing_scripts.length > 0) { + html += '
  • Missing scripts: ' + data.missing_scripts.join(', ') + '
    • '; + } + if (data.pdf_files.length === 0) { + html += '
  • No PDF files found in working directory
    • '; + } + html += '
'; + + envDetails.innerHTML = html; + } + }) + .catch(error => { + console.error('Error checking environment:', error); + }); +} \ No newline at end of file diff --git a/frontend/static/batch-styles.css b/frontend/static/batch-styles.css new file mode 100644 index 0000000..b397723 --- /dev/null +++ b/frontend/static/batch-styles.css @@ -0,0 +1,616 @@ +/* Batch Processor Specific Styles */ + +/* Navigation link */ +.nav-link { + text-align: center; + margin-bottom: var(--spacing-lg); +} + +.nav-link a { + color: var(--primary-medium); + text-decoration: none; + font-weight: 500; + transition: color 0.2s ease; +} + +.nav-link a:hover { + color: var(--primary-dark); +} + +/* Batch Settings Grid */ +.batch-settings-grid { + display: grid; + grid-template-columns: 1fr 1fr 1fr; + gap: var(--spacing-xl); + margin-top: var(--spacing-lg); +} + +.pdf-info { + margin-top: var(--spacing-sm); + padding: var(--spacing-sm); + background: var(--surface-light); + border-radius: var(--radius-sm); + font-size: 0.9rem; +} + +.info-label { + color: var(--text-medium); + font-weight: 500; +} + +/* Page Range Controls */ +.page-range-controls { + display: flex; + flex-direction: column; + gap: var(--spacing-sm); +} + +.radio-label { + display: flex; + align-items: center; + cursor: pointer; + font-weight: 400; + color: var(--text-medium); +} + +.radio-label input[type="radio"] { + margin-right: var(--spacing-xs); +} + +.custom-range { + display: flex; + align-items: center; + gap: var(--spacing-sm); + margin-top: var(--spacing-sm); +} + +.custom-range input { + width: 80px; +} + +/* Processing Options */ +.processing-options { + display: flex; + flex-direction: column; + gap: var(--spacing-sm); +} + +/* Batch Progress Panel */ +.batch-progress-panel { + background: var(--surface-white); + padding: var(--spacing-xl); + border-radius: var(--radius-lg); + margin: var(--spacing-xl) 0; + box-shadow: var(--shadow-md); + border: 1px solid var(--border-light); +} + +/* Overall Progress */ +.overall-progress { + margin-bottom: var(--spacing-xl); +} + +.progress-header { + display: flex; + justify-content: space-between; + align-items: center; + margin-bottom: var(--spacing-sm); +} + +.progress-header span:first-child { + font-weight: 600; + color: var(--primary-dark); +} + +#overall-percentage { + font-weight: 600; + color: var(--primary-medium); + font-size: 1.125rem; +} + +.progress-bar { + height: 24px; + background: var(--surface-light); + border-radius: 12px; + overflow: hidden; + position: relative; + box-shadow: inset 0 2px 4px rgba(0, 0, 0, 0.1); +} + +.progress-bar.small { + height: 16px; + border-radius: 8px; +} + +.progress-fill { + height: 100%; + background: linear-gradient(90deg, var(--primary-medium), var(--primary-dark)); + width: 0%; + transition: width 0.3s ease; + position: relative; + overflow: hidden; +} + +.progress-fill::after { + content: ''; + position: absolute; + top: 0; + left: 0; + bottom: 0; + right: 0; + background: linear-gradient( + 45deg, + rgba(255, 255, 255, 0.2) 25%, + transparent 25%, + transparent 50%, + rgba(255, 255, 255, 0.2) 50%, + rgba(255, 255, 255, 0.2) 75%, + transparent 75%, + transparent + ); + background-size: 30px 30px; + animation: progress-stripes 1s linear infinite; +} + +@keyframes progress-stripes { + 0% { background-position: 0 0; } + 100% { background-position: 30px 0; } +} + +.progress-stats { + margin-top: var(--spacing-sm); + font-size: 0.875rem; + color: var(--text-medium); + display: flex; + align-items: center; + gap: var(--spacing-sm); +} + +.separator { + color: var(--border-light); +} + +/* Stage Progress */ +.stage-progress { + display: grid; + grid-template-columns: repeat(3, 1fr); + gap: var(--spacing-lg); +} + +.stage-item { + padding: var(--spacing-md); + background: var(--surface-light); + border-radius: var(--radius-md); +} + +.stage-header { + display: flex; + align-items: center; + gap: var(--spacing-sm); + margin-bottom: var(--spacing-sm); +} + +.stage-icon { + font-size: 1.25rem; +} + +.stage-name { + flex: 1; + font-weight: 500; + color: var(--text-dark); +} + +.stage-percentage { + font-weight: 600; + color: var(--text-medium); + font-size: 0.875rem; +} + +/* Stage-specific progress colors */ +.progress-fill.analysis { + background: linear-gradient(90deg, #3498db, #2980b9); +} + +.progress-fill.visualization { + background: linear-gradient(90deg, #e74c3c, #c0392b); +} + +.progress-fill.extraction { + background: linear-gradient(90deg, #2ecc71, #27ae60); +} + +/* Batch Controls */ +.batch-controls { + display: flex; + justify-content: center; + gap: var(--spacing-md); + margin: var(--spacing-xl) 0; +} + +.primary-btn { + background: var(--primary-medium); + color: white; + padding: var(--spacing-md) var(--spacing-xl); + font-weight: 600; +} + +.primary-btn:hover { + background: var(--primary-dark); +} + +.secondary-btn { + background: var(--accent-amber); + color: var(--primary-dark); +} + +.secondary-btn:hover { + background: #f0c674; +} + +.danger-btn { + background: var(--error); + color: white; +} + +.danger-btn:hover { + background: #c0392b; +} + +/* Page Status Grid */ +.page-status-grid { + background: var(--surface-white); + padding: var(--spacing-xl); + border-radius: var(--radius-lg); + margin: var(--spacing-xl) 0; + box-shadow: var(--shadow-md); + border: 1px solid var(--border-light); +} + +.page-grid { + display: grid; + grid-template-columns: repeat(auto-fill, minmax(80px, 1fr)); + gap: var(--spacing-sm); + margin-top: var(--spacing-lg); +} + +.page-status-item { + aspect-ratio: 1; + display: flex; + flex-direction: column; + align-items: center; + justify-content: center; + background: var(--surface-light); + border-radius: var(--radius-md); + border: 2px solid transparent; + cursor: pointer; + transition: all 0.2s ease; + position: relative; +} + +.page-status-item:hover { + transform: translateY(-2px); + box-shadow: var(--shadow-md); +} + +.page-status-item.pending { + border-color: var(--border-light); +} + +.page-status-item.processing { + border-color: var(--accent-amber); + background: linear-gradient(135deg, var(--accent-amber) 0%, transparent 100%); + animation: pulse 2s infinite; +} + +@keyframes pulse { + 0%, 100% { opacity: 1; } + 50% { opacity: 0.7; } +} + +.page-status-item.completed { + border-color: var(--success); + background: var(--accent-mint); +} + +.page-status-item.failed { + border-color: var(--error); + background: var(--accent-rose); +} + +.page-number { + font-size: 0.75rem; + font-weight: 600; + color: var(--text-dark); +} + +.page-status-icon { + font-size: 1.5rem; + margin: var(--spacing-xs) 0; +} + +.page-actions { + position: absolute; + bottom: 4px; + right: 4px; +} + +.mini-btn { + padding: 2px 4px; + font-size: 0.75rem; + background: var(--surface-white); + border: 1px solid var(--border-light); + border-radius: 4px; + cursor: pointer; +} + +/* Batch Results */ +.batch-results { + background: var(--surface-white); + padding: var(--spacing-xl); + border-radius: var(--radius-lg); + margin: var(--spacing-xl) 0; + box-shadow: var(--shadow-md); + border: 1px solid var(--border-light); +} + +.results-summary { + display: grid; + grid-template-columns: repeat(4, 1fr); + gap: var(--spacing-lg); + margin: var(--spacing-lg) 0; +} + +.stat-card { + text-align: center; + padding: var(--spacing-lg); + background: var(--surface-light); + border-radius: var(--radius-md); +} + +.stat-value { + font-size: 2rem; + font-weight: 700; + color: var(--primary-dark); + margin-bottom: var(--spacing-xs); +} + +.stat-label { + font-size: 0.875rem; + color: var(--text-medium); + text-transform: uppercase; + letter-spacing: 0.05em; +} + +.results-actions { + display: flex; + justify-content: center; + gap: var(--spacing-md); + margin: var(--spacing-xl) 0; +} + +.action-btn { + display: flex; + align-items: center; + gap: var(--spacing-sm); + padding: var(--spacing-md) var(--spacing-lg); + background: var(--primary-medium); + color: white; + border: none; + border-radius: var(--radius-sm); + cursor: pointer; + font-weight: 500; + transition: all 0.2s ease; +} + +.action-btn:hover { + background: var(--primary-dark); + transform: translateY(-1px); +} + +/* Failed Pages */ +.failed-pages { + margin-top: var(--spacing-xl); + padding: var(--spacing-lg); + background: var(--accent-rose); + border-radius: var(--radius-md); + border: 1px solid var(--error); +} + +.failed-pages h4 { + color: var(--error); + margin-top: 0; +} + +.failed-list { + display: flex; + flex-direction: column; + gap: var(--spacing-sm); +} + +.failed-item { + display: flex; + align-items: center; + gap: var(--spacing-md); + padding: var(--spacing-sm); + background: var(--surface-white); + border-radius: var(--radius-sm); +} + +.failed-reason { + flex: 1; + font-size: 0.875rem; + color: var(--text-medium); +} + +.retry-btn { + padding: var(--spacing-xs) var(--spacing-md); + background: var(--warning); + color: white; + font-size: 0.875rem; +} + +/* Console Output */ +.console-output { + background: var(--text-dark); + border-radius: var(--radius-lg); + margin: var(--spacing-xl) 0; + box-shadow: var(--shadow-md); + overflow: hidden; +} + +.console-header { + display: flex; + align-items: center; + gap: var(--spacing-md); + padding: var(--spacing-md) var(--spacing-lg); + background: rgba(0, 0, 0, 0.3); + border-bottom: 1px solid rgba(255, 255, 255, 0.1); +} + +.console-header h4 { + flex: 1; + margin: 0; + color: white; + font-size: 1rem; +} + +.console-btn { + padding: var(--spacing-xs) var(--spacing-md); + background: rgba(255, 255, 255, 0.1); + color: white; + border: 1px solid rgba(255, 255, 255, 0.2); + font-size: 0.8rem; +} + +.console-btn:hover { + background: rgba(255, 255, 255, 0.2); +} + +.console-content { + height: 300px; + overflow-y: auto; + padding: var(--spacing-md); + font-family: 'Monaco', 'Menlo', 'Ubuntu Mono', monospace; + font-size: 0.8rem; +} + +.console-message { + margin-bottom: var(--spacing-xs); + line-height: 1.4; +} + +.console-message.info { + color: #64b5f6; +} + +.console-message.success { + color: #81c784; +} + +.console-message.warning { + color: #ffb74d; +} + +.console-message.error { + color: #e57373; +} + +.timestamp { + color: #90a4ae; + margin-right: var(--spacing-sm); +} + +/* Modal Enhancements */ +.modal-tabs { + display: flex; + border-bottom: 1px solid var(--border-light); + background: var(--surface-light); +} + +.modal-tab { + flex: 1; + padding: var(--spacing-md); + background: transparent; + border: none; + cursor: pointer; + font-weight: 500; + color: var(--text-medium); + transition: all 0.2s ease; +} + +.modal-tab.active { + color: var(--primary-dark); + border-bottom: 2px solid var(--primary-medium); +} + +.modal-tab-content { + display: none; + padding: var(--spacing-lg); +} + +.modal-tab-content.active { + display: block; +} + +.modal-gallery { + display: grid; + grid-template-columns: repeat(auto-fill, minmax(150px, 1fr)); + gap: var(--spacing-md); + max-height: 400px; + overflow-y: auto; +} + +.modal-gallery-item { + cursor: pointer; + transition: transform 0.2s ease; +} + +.modal-gallery-item:hover { + transform: scale(1.05); +} + +.modal-gallery-item img { + width: 100%; + height: 150px; + object-fit: cover; + border-radius: var(--radius-sm); +} + +/* Responsive Design */ +@media (max-width: 1024px) { + .batch-settings-grid { + grid-template-columns: 1fr; + } + + .stage-progress { + grid-template-columns: 1fr; + gap: var(--spacing-md); + } + + .results-summary { + grid-template-columns: repeat(2, 1fr); + } +} + +@media (max-width: 768px) { + .page-grid { + grid-template-columns: repeat(auto-fill, minmax(60px, 1fr)); + } + + .results-actions { + flex-direction: column; + } + + .action-btn { + width: 100%; + justify-content: center; + } + + .batch-controls { + flex-wrap: wrap; + } + + .console-content { + height: 200px; + } +} \ No newline at end of file