diff --git a/backend/app.py b/backend/app.py index d00b054..9ce19da 100644 --- a/backend/app.py +++ b/backend/app.py @@ -8,6 +8,7 @@ import logging from pathlib import Path import uuid import pdf2image +from multipart_handler import MultipartHandler, default_handler # Add the parent directory to Python path to allow imports sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) @@ -24,6 +25,8 @@ app = Flask(__name__, static_folder=os.path.join(frontend_path, 'static'), static_url_path='/static') +app.config['MAX_CONTENT_LENGTH'] = 100 * 1024 * 1024 # 100MB + # Dictionary to store background task results task_results = {} @@ -876,6 +879,508 @@ def debug_results(): return jsonify({'error': str(e)}), 500 # Cleanup task for batch processors + + +@app.route('/api/upload/analyze', methods=['POST']) +def api_upload_analyze(): + """ + API endpoint for uploading and analyzing a PDF file. + Expects multipart/form-data with: + - file: PDF file + - page_num: (optional) Page number to analyze (default: 1) + - adjust: (optional) Auto-adjust coordinates (default: true) + """ + try: + # Check if file is in request + if 'file' not in request.files: + return jsonify(default_handler.create_multipart_response( + False, {'error': 'No file part in request'} + )), 400 + + file = request.files['file'] + + # Save uploaded file + success, result = default_handler.save_uploaded_file(file, permanent=True) + if not success: + return jsonify(default_handler.create_multipart_response(False, result)), 400 + + # Get processing parameters + page_num = request.form.get('page_num', '1') + adjust = request.form.get('adjust', 'true').lower() == 'true' + + # Run analyzer + filepath = result['filepath'] + command = f"python backend/page_treatment/analyzer.py --image {filepath} --page {page_num} --start-page {page_num} --end-page {page_num}" + + # Generate task ID + task_id = f"api_analyzer_{int(time.time() * 1000)}" + + # Initialize task result + task_results[task_id] = { + 'success': None, + 'output': "Processing uploaded file...", + 'done': False, + 'file_info': result + } + + # Start background processing + thread = threading.Thread(target=run_command, args=(task_id, command)) + thread.daemon = True + thread.start() + + # Return response + return jsonify(default_handler.create_multipart_response(True, { + 'task_id': task_id, + 'file_info': result, + 'message': f"Processing started for {result['filename']}, page {page_num}" + })) + + except Exception as e: + logger.error(f"Error in api_upload_analyze: {str(e)}") + return jsonify(default_handler.create_multipart_response( + False, {'error': str(e)} + )), 500 + + +@app.route('/api/upload/process', methods=['POST']) +def api_upload_process(): + """ + API endpoint for uploading and processing a PDF through all stages. + Expects multipart/form-data with: + - file: PDF file + - page_num: (optional) Page number (default: 1) + - adjust: (optional) Auto-adjust coordinates (default: true) + - stages: (optional) Comma-separated list of stages to run (default: analyzer,visualizer,extractor) + """ + try: + # Check if file is in request + if 'file' not in request.files: + return jsonify(default_handler.create_multipart_response( + False, {'error': 'No file part in request'} + )), 400 + + file = request.files['file'] + + # Save uploaded file + success, result = default_handler.save_uploaded_file(file, permanent=True) + if not success: + return jsonify(default_handler.create_multipart_response(False, result)), 400 + + # Get processing parameters + page_num = request.form.get('page_num', '1') + adjust = request.form.get('adjust', 'true').lower() == 'true' + stages = request.form.get('stages', 'analyzer,visualizer,extractor').split(',') + + # Create a combined task ID + task_id = f"api_process_{int(time.time() * 1000)}" + + # Process through all stages + filepath = result['filepath'] + processing_info = { + 'task_id': task_id, + 'file_info': result, + 'page_num': page_num, + 'adjust': adjust, + 'stages': stages, + 'current_stage': None, + 'completed_stages': [], + 'results': {} + } + + # Start processing in background + def process_all_stages(): + try: + # Run analyzer if requested + if 'analyzer' in stages: + processing_info['current_stage'] = 'analyzer' + command = f"python backend/page_treatment/analyzer.py --image {filepath} --page {page_num} --start-page {page_num} --end-page {page_num}" + + process = subprocess.run( + command, + shell=True, + capture_output=True, + text=True, + input="n\n" + ) + + if process.returncode == 0: + processing_info['completed_stages'].append('analyzer') + processing_info['results']['analyzer'] = { + 'success': True, + 'doctags_file': 'results/output.doctags.txt' + } + else: + processing_info['results']['analyzer'] = { + 'success': False, + 'error': process.stderr + } + return + + # Run visualizer if requested + if 'visualizer' in stages: + processing_info['current_stage'] = 'visualizer' + command = f"python backend/page_treatment/visualizer.py --doctags results/output.doctags.txt --pdf {filepath} --page {page_num}" + if adjust: + command += " --adjust" + + process = subprocess.run( + command, + shell=True, + capture_output=True, + text=True + ) + + if process.returncode == 0: + processing_info['completed_stages'].append('visualizer') + processing_info['results']['visualizer'] = { + 'success': True, + 'visualization_file': f'results/visualization_page_{page_num}.png' + } + else: + processing_info['results']['visualizer'] = { + 'success': False, + 'error': process.stderr + } + return + + # Run extractor if requested + if 'extractor' in stages: + processing_info['current_stage'] = 'extractor' + command = f"python backend/page_treatment/picture_extractor.py --doctags results/output.doctags.txt --pdf {filepath} --page {page_num}" + if adjust: + command += " --adjust" + + process = subprocess.run( + command, + shell=True, + capture_output=True, + text=True + ) + + if process.returncode == 0: + processing_info['completed_stages'].append('extractor') + + # Count extracted images + pics_dir = Path("results") / "pictures" + image_count = len(list(pics_dir.glob("*.png"))) if pics_dir.exists() else 0 + + processing_info['results']['extractor'] = { + 'success': True, + 'images_extracted': image_count, + 'pictures_folder': 'results/pictures' + } + else: + processing_info['results']['extractor'] = { + 'success': False, + 'error': process.stderr + } + + processing_info['current_stage'] = 'completed' + + except Exception as e: + processing_info['current_stage'] = 'error' + processing_info['error'] = str(e) + + # Store processing info + task_results[task_id] = processing_info + + # Start background thread + thread = threading.Thread(target=process_all_stages) + thread.daemon = True + thread.start() + + # Return response + return jsonify(default_handler.create_multipart_response(True, { + 'task_id': task_id, + 'file_info': result, + 'stages': stages, + 'message': f"Processing started for {result['filename']}, page {page_num}" + })) + + except Exception as e: + logger.error(f"Error in api_upload_process: {str(e)}") + return jsonify(default_handler.create_multipart_response( + False, {'error': str(e)} + )), 500 + + +@app.route('/api/upload/batch', methods=['POST']) +def api_upload_batch(): + """ + API endpoint for uploading and batch processing a PDF. + Expects multipart/form-data with: + - file: PDF file + - start_page: (optional) Start page (default: 1) + - end_page: (optional) End page (default: all) + - parallel: (optional) Enable parallel processing (default: true) + - adjust: (optional) Auto-adjust coordinates (default: true) + """ + if not batch_processing_available: + return jsonify(default_handler.create_multipart_response( + False, {'error': 'Batch processing not available'} + )), 503 + + try: + # Check if file is in request + if 'file' not in request.files: + return jsonify(default_handler.create_multipart_response( + False, {'error': 'No file part in request'} + )), 400 + + file = request.files['file'] + + # Save uploaded file + success, result = default_handler.save_uploaded_file(file, permanent=True) + if not success: + return jsonify(default_handler.create_multipart_response(False, result)), 400 + + # Get PDF info + filepath = result['filepath'] + try: + from pdf2image.pdf2image import pdfinfo_from_path + info = pdfinfo_from_path(filepath) + total_pages = info["Pages"] + except Exception as e: + return jsonify(default_handler.create_multipart_response( + False, {'error': f'Failed to read PDF info: {str(e)}'} + )), 400 + + # Get processing parameters + start_page = int(request.form.get('start_page', '1')) + end_page = int(request.form.get('end_page', str(total_pages))) + parallel = request.form.get('parallel', 'true').lower() == 'true' + adjust = request.form.get('adjust', 'true').lower() == 'true' + + # Validate page range + start_page = max(1, min(start_page, total_pages)) + end_page = max(start_page, min(end_page, total_pages)) + + # Start batch processing + batch_id = f"api_{uuid.uuid4().hex[:8]}" + options = { + 'adjust': adjust, + 'parallel': parallel, + 'generate_report': True + } + + if start_batch_processing(batch_id, filepath, start_page, end_page, options): + return jsonify(default_handler.create_multipart_response(True, { + 'batch_id': batch_id, + 'file_info': result, + 'total_pages': total_pages, + 'processing_pages': f"{start_page}-{end_page}", + 'page_count': end_page - start_page + 1, + 'message': f'Batch processing started for {result["filename"]}' + })) + else: + return jsonify(default_handler.create_multipart_response( + False, {'error': 'Failed to start batch processing'} + )), 500 + + except Exception as e: + logger.error(f"Error in api_upload_batch: {str(e)}") + return jsonify(default_handler.create_multipart_response( + False, {'error': str(e)} + )), 500 + + +@app.route('/api/task/', methods=['GET']) +def api_task_status(task_id): + """Get status of a processing task""" + if task_id not in task_results: + return jsonify({ + 'success': False, + 'error': 'Task not found' + }), 404 + + return jsonify({ + 'success': True, + 'task': task_results[task_id] + }) + + +@app.route('/api/batch/', methods=['GET']) +def api_batch_status(batch_id): + """Get status of a batch processing job""" + if not batch_processing_available: + return jsonify({ + 'success': False, + 'error': 'Batch processing not available' + }), 503 + + processor = get_batch_processor(batch_id) + if not processor: + return jsonify({ + 'success': False, + 'error': 'Batch not found' + }), 404 + + state = processor.get_state() + return jsonify({ + 'success': True, + 'batch': state + }) + + +@app.route('/api/cleanup', methods=['POST']) +def api_cleanup(): + """Cleanup old uploaded files""" + try: + max_age_hours = int(request.json.get('max_age_hours', 24)) + folder = request.json.get('folder', 'both') + + removed_count = default_handler.cleanup_old_files(max_age_hours, folder) + + return jsonify({ + 'success': True, + 'removed_files': removed_count, + 'message': f'Removed {removed_count} old files' + }) + except Exception as e: + return jsonify({ + 'success': False, + 'error': str(e) + }), 500 + +# Add this streamlined endpoint to your app.py + +@app.route('/api/upload/doctags', methods=['POST']) +def api_upload_doctags(): + """ + Simple API endpoint for uploading a PDF and getting DocTags output. + Automatically cleans up the uploaded file after processing. + + Expects multipart/form-data with: + - file: PDF file + - page_num: (optional) Page number to analyze (default: 1) + + Returns JSON with: + - success: boolean + - filename: original filename + - page: page number processed + - doctags: the DocTags content + """ + uploaded_file_path = None + + try: + # Check if file is in request + if 'file' not in request.files: + return jsonify({'success': False, 'error': 'No file part in request'}), 400 + + file = request.files['file'] + + # Save uploaded file temporarily + success, result = default_handler.save_uploaded_file(file, permanent=True) + if not success: + return jsonify({'success': False, 'error': result.get('error', 'Failed to save file')}), 400 + + # Store the uploaded file path for cleanup + uploaded_file_path = result['filepath'] + + # Get page number + page_num = request.form.get('page_num', '1') + + # Run analyzer directly (synchronously) + command = f"python backend/page_treatment/analyzer.py --image {uploaded_file_path} --page {page_num} --start-page {page_num} --end-page {page_num}" + + try: + # Run the command with timeout + process = subprocess.run( + command, + shell=True, + capture_output=True, + text=True, + input="n\n", + timeout=60 # 60 second timeout + ) + + if process.returncode != 0: + error_msg = process.stderr or 'Analyzer failed' + logger.error(f"Analyzer error: {error_msg}") + return jsonify({ + 'success': False, + 'error': 'Analyzer failed', + 'details': error_msg + }), 500 + + # Check if doctags file was created + doctags_path = Path("results") / "output.doctags.txt" + if not doctags_path.exists(): + return jsonify({ + 'success': False, + 'error': 'DocTags file not generated' + }), 500 + + # Read the doctags content + with open(doctags_path, 'r', encoding='utf-8') as f: + doctags_content = f.read() + + # Prepare successful response + response = { + 'success': True, + 'filename': result['filename'], + 'page': int(page_num), + 'doctags': doctags_content + } + + # Clean up the uploaded file + try: + if uploaded_file_path and os.path.exists(uploaded_file_path): + os.remove(uploaded_file_path) + logger.info(f"Cleaned up uploaded file: {uploaded_file_path}") + except Exception as cleanup_error: + logger.error(f"Error cleaning up file: {cleanup_error}") + # Don't fail the request due to cleanup error + + return jsonify(response) + + except subprocess.TimeoutExpired: + return jsonify({ + 'success': False, + 'error': 'Processing timeout - page took too long to analyze' + }), 500 + + except Exception as e: + logger.error(f"Processing error: {str(e)}") + return jsonify({ + 'success': False, + 'error': f'Processing error: {str(e)}' + }), 500 + + except Exception as e: + logger.error(f"Error in api_upload_doctags: {str(e)}") + return jsonify({ + 'success': False, + 'error': str(e) + }), 500 + + finally: + # Ensure cleanup happens even if there's an error + try: + if uploaded_file_path and os.path.exists(uploaded_file_path): + os.remove(uploaded_file_path) + logger.info(f"Cleaned up uploaded file in finally block: {uploaded_file_path}") + except Exception as cleanup_error: + logger.error(f"Error in finally cleanup: {cleanup_error}") + +# Add periodic cleanup task +def periodic_cleanup(): + """Run cleanup every hour""" + while True: + time.sleep(3600) # Wait 1 hour + try: + removed = default_handler.cleanup_old_files(24, 'both') + if removed > 0: + logger.info(f"Periodic cleanup removed {removed} old files") + except Exception as e: + logger.error(f"Error in periodic cleanup: {str(e)}") + +# Start cleanup thread +cleanup_thread = threading.Thread(target=periodic_cleanup) +cleanup_thread.daemon = True +cleanup_thread.start() + + def cleanup_task(): """Periodic cleanup of old batch processors""" if not batch_processing_available: @@ -905,4 +1410,4 @@ if __name__ == '__main__': else: logger.info("Batch processing features enabled.") - app.run(debug=True, host='127.0.0.1', port=5000) \ No newline at end of file + app.run(debug=True, host='127.0.0.1', port=5000) diff --git a/backend/multipart_handler.py b/backend/multipart_handler.py new file mode 100644 index 0000000..a115342 --- /dev/null +++ b/backend/multipart_handler.py @@ -0,0 +1,217 @@ +#!/usr/bin/env python3 +""" +Multipart File Upload Handler for DocTags +Handles file upload processing and validation +""" + +import os +import time +import logging +from pathlib import Path +from werkzeug.utils import secure_filename +from werkzeug.datastructures import FileStorage +import tempfile +import shutil +from typing import Optional, Dict, Tuple, List + +logger = logging.getLogger(__name__) + +class MultipartHandler: + """Handle multipart file uploads with validation and storage""" + + ALLOWED_EXTENSIONS = {'pdf'} + MAX_FILE_SIZE = 100 * 1024 * 1024 # 100MB + UPLOAD_FOLDER = 'uploads' + TEMP_FOLDER = 'temp_uploads' + + def __init__(self, upload_folder: str = None, temp_folder: str = None): + self.upload_folder = Path(upload_folder or self.UPLOAD_FOLDER) + self.temp_folder = Path(temp_folder or self.TEMP_FOLDER) + self._ensure_folders() + + def _ensure_folders(self): + """Ensure upload and temp folders exist""" + for folder in [self.upload_folder, self.temp_folder]: + if not folder.exists(): + folder.mkdir(parents=True) + logger.info(f"Created folder: {folder}") + + def allowed_file(self, filename: str) -> bool: + """Check if file extension is allowed""" + return '.' in filename and \ + filename.rsplit('.', 1)[1].lower() in self.ALLOWED_EXTENSIONS + + def validate_file(self, file: FileStorage) -> Tuple[bool, Optional[str]]: + """ + Validate uploaded file + Returns: (is_valid, error_message) + """ + if not file: + return False, "No file provided" + + if file.filename == '': + return False, "No file selected" + + if not self.allowed_file(file.filename): + return False, f"Invalid file type. Allowed types: {', '.join(self.ALLOWED_EXTENSIONS)}" + + # Check file size (if possible) + file.seek(0, os.SEEK_END) + file_size = file.tell() + file.seek(0) # Reset file pointer + + if file_size > self.MAX_FILE_SIZE: + return False, f"File too large. Maximum size: {self.MAX_FILE_SIZE / (1024*1024):.1f}MB" + + return True, None + + def save_uploaded_file(self, file: FileStorage, permanent: bool = True) -> Tuple[bool, Dict]: + """ + Save uploaded file + + Args: + file: The uploaded file + permanent: If True, save to uploads folder; if False, save to temp folder + + Returns: + (success, result_dict) where result_dict contains: + - filepath: Path to saved file + - filename: Original filename + - unique_filename: Saved filename + - size: File size in bytes + - error: Error message if failed + """ + # Validate file + is_valid, error_msg = self.validate_file(file) + if not is_valid: + return False, {'error': error_msg} + + try: + # Secure the filename + original_filename = secure_filename(file.filename) + timestamp = int(time.time() * 1000) # Millisecond timestamp + + # Create unique filename + name_parts = original_filename.rsplit('.', 1) + if len(name_parts) == 2: + unique_filename = f"{name_parts[0]}_{timestamp}.{name_parts[1]}" + else: + unique_filename = f"{original_filename}_{timestamp}" + + # Determine save location + save_folder = self.upload_folder if permanent else self.temp_folder + filepath = save_folder / unique_filename + + # Save file + file.save(str(filepath)) + + # Get file size + file_size = os.path.getsize(filepath) + + logger.info(f"Saved file: {filepath} (size: {file_size} bytes)") + + return True, { + 'filepath': str(filepath), + 'filename': original_filename, + 'unique_filename': unique_filename, + 'size': file_size, + 'permanent': permanent + } + + except Exception as e: + logger.error(f"Error saving file: {str(e)}") + return False, {'error': f"Failed to save file: {str(e)}"} + + def save_to_temp(self, file: FileStorage) -> Tuple[bool, Dict]: + """Save file to temporary folder""" + return self.save_uploaded_file(file, permanent=False) + + def move_to_permanent(self, temp_filepath: str) -> Tuple[bool, Dict]: + """Move file from temp to permanent storage""" + try: + temp_path = Path(temp_filepath) + if not temp_path.exists(): + return False, {'error': 'Temporary file not found'} + + permanent_path = self.upload_folder / temp_path.name + shutil.move(str(temp_path), str(permanent_path)) + + return True, { + 'filepath': str(permanent_path), + 'moved_from': str(temp_path) + } + except Exception as e: + logger.error(f"Error moving file: {str(e)}") + return False, {'error': f"Failed to move file: {str(e)}"} + + def cleanup_old_files(self, max_age_hours: int = 24, folder: str = 'both'): + """ + Remove old files from upload/temp folders + + Args: + max_age_hours: Maximum age of files in hours + folder: 'uploads', 'temp', or 'both' + """ + folders_to_clean = [] + if folder in ['uploads', 'both']: + folders_to_clean.append(self.upload_folder) + if folder in ['temp', 'both']: + folders_to_clean.append(self.temp_folder) + + current_time = time.time() + max_age_seconds = max_age_hours * 3600 + removed_count = 0 + + for folder_path in folders_to_clean: + if not folder_path.exists(): + continue + + for file_path in folder_path.glob('*.pdf'): + try: + file_age = current_time - file_path.stat().st_mtime + if file_age > max_age_seconds: + file_path.unlink() + removed_count += 1 + logger.info(f"Deleted old file: {file_path}") + except Exception as e: + logger.error(f"Error deleting file {file_path}: {str(e)}") + + return removed_count + + def get_file_info(self, filepath: str) -> Optional[Dict]: + """Get information about an uploaded file""" + try: + path = Path(filepath) + if not path.exists(): + return None + + stats = path.stat() + return { + 'filepath': str(path), + 'filename': path.name, + 'size': stats.st_size, + 'created': stats.st_ctime, + 'modified': stats.st_mtime, + 'exists': True + } + except Exception as e: + logger.error(f"Error getting file info: {str(e)}") + return None + + def create_multipart_response(self, success: bool, data: Dict) -> Dict: + """Create standardized response for multipart operations""" + response = { + 'success': success, + 'timestamp': time.time() + } + + if success: + response['data'] = data + else: + response['error'] = data.get('error', 'Unknown error') + + return response + + +# Create a default instance +default_handler = MultipartHandler() \ No newline at end of file