-
-
\n"
-
- html += """
-
-
-
-"""
- return html
-
- def pause(self):
- """Pause the batch processing"""
- self.pause_event.clear()
- self.state['paused'] = True
- self.log_message("Batch processing paused")
-
- def resume(self):
- """Resume the batch processing"""
- self.pause_event.set()
- self.state['paused'] = False
- self.log_message("Batch processing resumed")
-
- def cancel(self):
- """Cancel the batch processing"""
- self.state['cancelled'] = True
- self.pause_event.set()
- self.log_message("Batch processing cancelled")
-
- def get_state(self):
- """Get current state with calculated fields"""
- with self.lock:
- state = self.state.copy()
-
- # Calculate ETA
- if state['processed'] > 0 and not state['completed']:
- elapsed = time.time() - state['start_time']
- rate = state['processed'] / elapsed
- remaining = state['total'] - state['processed']
- eta = remaining / rate if rate > 0 else 0
- state['eta'] = eta * 1000 # Convert to milliseconds
- else:
- state['eta'] = 0
-
- return state
-
- def create_zip_archive(self):
- """Create ZIP archive of all results"""
- try:
- zip_path = self.results_dir / f"batch_results_{self.batch_id}.zip"
-
- with zipfile.ZipFile(zip_path, 'w', zipfile.ZIP_DEFLATED) as zipf:
- for file_path in self.results_dir.rglob('*'):
- if file_path.is_file() and file_path != zip_path:
- arcname = file_path.relative_to(self.results_dir)
- zipf.write(file_path, arcname)
-
- self.log_message(f"Created ZIP archive: {zip_path}")
- return zip_path
-
- except Exception as e:
- self.log_message(f"Error creating ZIP archive: {str(e)}", 'error')
- return None
-
-
-# Global batch processors storage
-batch_processors = {}
-batch_lock = threading.Lock()
-
-
-def start_batch_processing(batch_id, pdf_file, start_page, end_page, options):
- """Start a new batch processing job"""
- try:
- processor = BatchProcessor(batch_id, pdf_file, start_page, end_page, options)
-
- with batch_lock:
- batch_processors[batch_id] = processor
-
- # Start processing in a separate thread
- thread = threading.Thread(target=processor.run)
- thread.daemon = True
- thread.start()
-
- return True
-
- except Exception as e:
- logger.error(f"Error starting batch processing: {str(e)}")
- return False
-
-
-def get_batch_processor(batch_id):
- """Get a batch processor by ID"""
- with batch_lock:
- return batch_processors.get(batch_id)
-
-
-def cleanup_old_batches(max_age_hours=24):
- """Clean up old batch processors"""
- current_time = time.time()
- max_age_seconds = max_age_hours * 3600
-
- with batch_lock:
- to_remove = []
- for batch_id, processor in batch_processors.items():
- if processor.state['completed']:
- age = current_time - processor.state['start_time']
- if age > max_age_seconds:
- to_remove.append(batch_id)
-
- for batch_id in to_remove:
- del batch_processors[batch_id]
- logger.info(f"Cleaned up old batch processor: {batch_id}")
\ No newline at end of file
diff --git a/backend/config.py b/backend/config.py
deleted file mode 100644
index 8f35d0e..0000000
--- a/backend/config.py
+++ /dev/null
@@ -1,72 +0,0 @@
-#!/usr/bin/env python3
-"""
-Configuration settings for DocTags
-"""
-
-import os
-from pathlib import Path
-
-# Application settings
-APP_NAME = "DocTags Intelligence Suite"
-APP_VERSION = "1.0.0"
-DEBUG = os.environ.get('DEBUG', 'False').lower() == 'true'
-
-# Server settings
-HOST = '0.0.0.0'
-PORT = 5000
-MAX_CONTENT_LENGTH = 100 * 1024 * 1024 # 100MB
-
-# Processing settings
-DEFAULT_DPI = 200
-PREVIEW_DPI = 150
-DEFAULT_GRID_SIZE = 500
-MAX_IMAGE_WIDTH = 1200
-DEFAULT_PAGE = 1
-PROCESSING_TIMEOUT = 300 # 5 minutes
-BATCH_WORKERS = 4
-
-# File settings
-ALLOWED_EXTENSIONS = {'pdf'}
-RESULTS_DIR = 'results'
-UPLOAD_DIR = 'uploads'
-TEMP_DIR = 'temp_uploads'
-
-# Cleanup settings
-CLEANUP_AGE_HOURS = 24
-CLEANUP_INTERVAL = 3600 # 1 hour
-
-# Model settings
-MODEL_PATH = "ds4sd/SmolDocling-256M-preview"
-MAX_TOKENS = 4096
-
-# Zone colors for visualization
-ZONE_COLORS = {
- 'section_header_level_1': (255, 87, 34), # Orange
- 'text': (33, 150, 243), # Blue
- 'picture': (76, 175, 80), # Green
- 'table': (156, 39, 176), # Purple
- 'page_header': (255, 193, 7), # Amber
- 'page_footer': (121, 85, 72), # Brown
- 'default': (96, 125, 139) # Blue Grey
-}
-
-# Logging configuration
-LOGGING_CONFIG = {
- 'version': 1,
- 'disable_existing_loggers': False,
- 'formatters': {
- 'default': {
- 'format': '%(asctime)s - %(name)s - %(levelname)s - %(message)s',
- },
- },
- 'handlers': {
- 'console': {
- 'class': 'logging.StreamHandler',
- 'formatter': 'default',
- },
- },
- 'root': {
- 'level': 'INFO',
- 'handlers': ['console'],
- },
-}
\ No newline at end of file
diff --git a/backend/multipart_handler.py b/backend/multipart_handler.py
deleted file mode 100644
index a115342..0000000
--- a/backend/multipart_handler.py
+++ /dev/null
@@ -1,217 +0,0 @@
-#!/usr/bin/env python3
-"""
-Multipart File Upload Handler for DocTags
-Handles file upload processing and validation
-"""
-
-import os
-import time
-import logging
-from pathlib import Path
-from werkzeug.utils import secure_filename
-from werkzeug.datastructures import FileStorage
-import tempfile
-import shutil
-from typing import Optional, Dict, Tuple, List
-
-logger = logging.getLogger(__name__)
-
-class MultipartHandler:
- """Handle multipart file uploads with validation and storage"""
-
- ALLOWED_EXTENSIONS = {'pdf'}
- MAX_FILE_SIZE = 100 * 1024 * 1024 # 100MB
- UPLOAD_FOLDER = 'uploads'
- TEMP_FOLDER = 'temp_uploads'
-
- def __init__(self, upload_folder: str = None, temp_folder: str = None):
- self.upload_folder = Path(upload_folder or self.UPLOAD_FOLDER)
- self.temp_folder = Path(temp_folder or self.TEMP_FOLDER)
- self._ensure_folders()
-
- def _ensure_folders(self):
- """Ensure upload and temp folders exist"""
- for folder in [self.upload_folder, self.temp_folder]:
- if not folder.exists():
- folder.mkdir(parents=True)
- logger.info(f"Created folder: {folder}")
-
- def allowed_file(self, filename: str) -> bool:
- """Check if file extension is allowed"""
- return '.' in filename and \
- filename.rsplit('.', 1)[1].lower() in self.ALLOWED_EXTENSIONS
-
- def validate_file(self, file: FileStorage) -> Tuple[bool, Optional[str]]:
- """
- Validate uploaded file
- Returns: (is_valid, error_message)
- """
- if not file:
- return False, "No file provided"
-
- if file.filename == '':
- return False, "No file selected"
-
- if not self.allowed_file(file.filename):
- return False, f"Invalid file type. Allowed types: {', '.join(self.ALLOWED_EXTENSIONS)}"
-
- # Check file size (if possible)
- file.seek(0, os.SEEK_END)
- file_size = file.tell()
- file.seek(0) # Reset file pointer
-
- if file_size > self.MAX_FILE_SIZE:
- return False, f"File too large. Maximum size: {self.MAX_FILE_SIZE / (1024*1024):.1f}MB"
-
- return True, None
-
- def save_uploaded_file(self, file: FileStorage, permanent: bool = True) -> Tuple[bool, Dict]:
- """
- Save uploaded file
-
- Args:
- file: The uploaded file
- permanent: If True, save to uploads folder; if False, save to temp folder
-
- Returns:
- (success, result_dict) where result_dict contains:
- - filepath: Path to saved file
- - filename: Original filename
- - unique_filename: Saved filename
- - size: File size in bytes
- - error: Error message if failed
- """
- # Validate file
- is_valid, error_msg = self.validate_file(file)
- if not is_valid:
- return False, {'error': error_msg}
-
- try:
- # Secure the filename
- original_filename = secure_filename(file.filename)
- timestamp = int(time.time() * 1000) # Millisecond timestamp
-
- # Create unique filename
- name_parts = original_filename.rsplit('.', 1)
- if len(name_parts) == 2:
- unique_filename = f"{name_parts[0]}_{timestamp}.{name_parts[1]}"
- else:
- unique_filename = f"{original_filename}_{timestamp}"
-
- # Determine save location
- save_folder = self.upload_folder if permanent else self.temp_folder
- filepath = save_folder / unique_filename
-
- # Save file
- file.save(str(filepath))
-
- # Get file size
- file_size = os.path.getsize(filepath)
-
- logger.info(f"Saved file: {filepath} (size: {file_size} bytes)")
-
- return True, {
- 'filepath': str(filepath),
- 'filename': original_filename,
- 'unique_filename': unique_filename,
- 'size': file_size,
- 'permanent': permanent
- }
-
- except Exception as e:
- logger.error(f"Error saving file: {str(e)}")
- return False, {'error': f"Failed to save file: {str(e)}"}
-
- def save_to_temp(self, file: FileStorage) -> Tuple[bool, Dict]:
- """Save file to temporary folder"""
- return self.save_uploaded_file(file, permanent=False)
-
- def move_to_permanent(self, temp_filepath: str) -> Tuple[bool, Dict]:
- """Move file from temp to permanent storage"""
- try:
- temp_path = Path(temp_filepath)
- if not temp_path.exists():
- return False, {'error': 'Temporary file not found'}
-
- permanent_path = self.upload_folder / temp_path.name
- shutil.move(str(temp_path), str(permanent_path))
-
- return True, {
- 'filepath': str(permanent_path),
- 'moved_from': str(temp_path)
- }
- except Exception as e:
- logger.error(f"Error moving file: {str(e)}")
- return False, {'error': f"Failed to move file: {str(e)}"}
-
- def cleanup_old_files(self, max_age_hours: int = 24, folder: str = 'both'):
- """
- Remove old files from upload/temp folders
-
- Args:
- max_age_hours: Maximum age of files in hours
- folder: 'uploads', 'temp', or 'both'
- """
- folders_to_clean = []
- if folder in ['uploads', 'both']:
- folders_to_clean.append(self.upload_folder)
- if folder in ['temp', 'both']:
- folders_to_clean.append(self.temp_folder)
-
- current_time = time.time()
- max_age_seconds = max_age_hours * 3600
- removed_count = 0
-
- for folder_path in folders_to_clean:
- if not folder_path.exists():
- continue
-
- for file_path in folder_path.glob('*.pdf'):
- try:
- file_age = current_time - file_path.stat().st_mtime
- if file_age > max_age_seconds:
- file_path.unlink()
- removed_count += 1
- logger.info(f"Deleted old file: {file_path}")
- except Exception as e:
- logger.error(f"Error deleting file {file_path}: {str(e)}")
-
- return removed_count
-
- def get_file_info(self, filepath: str) -> Optional[Dict]:
- """Get information about an uploaded file"""
- try:
- path = Path(filepath)
- if not path.exists():
- return None
-
- stats = path.stat()
- return {
- 'filepath': str(path),
- 'filename': path.name,
- 'size': stats.st_size,
- 'created': stats.st_ctime,
- 'modified': stats.st_mtime,
- 'exists': True
- }
- except Exception as e:
- logger.error(f"Error getting file info: {str(e)}")
- return None
-
- def create_multipart_response(self, success: bool, data: Dict) -> Dict:
- """Create standardized response for multipart operations"""
- response = {
- 'success': success,
- 'timestamp': time.time()
- }
-
- if success:
- response['data'] = data
- else:
- response['error'] = data.get('error', 'Unknown error')
-
- return response
-
-
-# Create a default instance
-default_handler = MultipartHandler()
\ No newline at end of file
diff --git a/backend/page_treatment/__init__.py b/backend/page_treatment/__init__.py
deleted file mode 100644
index e69de29..0000000
diff --git a/backend/page_treatment/analyzer.py b/backend/page_treatment/analyzer.py
deleted file mode 100644
index 36762d3..0000000
--- a/backend/page_treatment/analyzer.py
+++ /dev/null
@@ -1,146 +0,0 @@
-#!/usr/bin/env python3
-# /// script
-# requires-python = ">=3.12"
-# dependencies = [
-# "transformers>=4.50",
-# "torch",
-# "pillow",
-# "requests",
-# "argparse",
-# "pdf2image",
-# "docling_core",
-# ]
-# ///
-
-import argparse
-import os
-import tempfile
-from pathlib import Path
-from urllib.parse import urlparse
-import requests
-from PIL import Image
-from pdf2image import convert_from_bytes
-import torch
-from transformers import AutoProcessor, AutoModelForVision2Seq
-from docling_core.types.doc import DoclingDocument
-from docling_core.types.doc.document import DocTagsDocument
-
-# Add parent directory to path for imports
-import sys
-sys.path.append(str(Path(__file__).parent.parent.parent))
-
-from backend.utils import ensure_results_folder, load_pdf_page
-from backend.config import MODEL_PATH, MAX_TOKENS, DEFAULT_DPI
-
-def parse_arguments():
- results_dir = ensure_results_folder()
-
- parser = argparse.ArgumentParser(description='Convert an image or PDF to docling format')
- parser.add_argument('--image', '-i', type=str, required=True,
- help='Path to local image file, PDF file, or URL')
- parser.add_argument('--prompt', '-p', type=str, default="Convert this page to docling.",
- help='Prompt for the model')
- parser.add_argument('--output', '-o', type=str, default=str(results_dir / "output.html"),
- help='Output file path')
- parser.add_argument('--page', type=int, default=1,
- help='Page number to process for PDF files (starts at 1)')
- parser.add_argument('--dpi', type=int, default=DEFAULT_DPI,
- help='DPI for PDF rendering')
- parser.add_argument('--start-page', type=int, default=1,
- help='Start processing PDF from this page number')
- parser.add_argument('--end-page', type=int, default=None,
- help='Stop processing PDF at this page number')
- return parser.parse_args()
-
-def load_image(image_path, page_num=1, dpi=DEFAULT_DPI):
- if urlparse(image_path).scheme in ['http', 'https']:
- response = requests.get(image_path, stream=True, timeout=10)
- response.raise_for_status()
-
- if image_path.lower().endswith('.pdf') or response.headers.get('Content-Type') == 'application/pdf':
- print(f"Converting PDF from URL (page {page_num})...")
- pdf_images = convert_from_bytes(response.content, dpi=dpi, first_page=page_num, last_page=page_num)
- if not pdf_images:
- raise Exception(f"Could not extract page {page_num} from PDF")
- return pdf_images[0].convert("RGB")
- else:
- return Image.open(response.raw).convert("RGB")
- else:
- image_path = Path(image_path)
- if not image_path.exists():
- raise FileNotFoundError(f"File not found: {image_path}")
-
- if image_path.suffix.lower() == '.pdf':
- return load_pdf_page(str(image_path), page_num, dpi).convert("RGB")
- else:
- return Image.open(image_path).convert("RGB")
-
-def process_page(model, processor, args, pil_image, page_num=1):
- results_dir = ensure_results_folder()
-
- if args.start_page == args.end_page and args.start_page == page_num:
- doctags_path = results_dir / "output.doctags.txt"
- output_path = results_dir / "output.html"
- else:
- doctags_path = results_dir / f"output_page{page_num}.doctags.txt"
- output_path = results_dir / f"output_page{page_num}.html"
-
- print(f"Processing page {page_num}")
-
- # Préparer les messages
- messages = [
- {
- "role": "user",
- "content": [
- {"type": "image"},
- {"type": "text", "text": args.prompt}
- ]
- }
- ]
-
- prompt = processor.apply_chat_template(messages, add_generation_prompt=True)
-
- device = next(model.parameters()).device
- inputs = processor(text=prompt, images=[pil_image], return_tensors="pt").to(device)
-
- # Génération
- generated_ids = model.generate(**inputs, max_new_tokens=MAX_TOKENS)
- prompt_length = inputs.input_ids.shape[1]
- trimmed_generated_ids = generated_ids[:, prompt_length:]
-
- doctags = processor.batch_decode(trimmed_generated_ids, skip_special_tokens=False)[0].lstrip()
- with open(doctags_path, "w", encoding="utf-8") as f:
- f.write(doctags)
- print(f"DocTags saved to {doctags_path}")
-
- doctags_doc = DocTagsDocument.from_doctags_and_image_pairs([doctags], [pil_image])
- doc = DoclingDocument.load_from_doctags(doctags_doc, document_name=f"Page {page_num}")
- html = doc.export_to_html()
-
- with open(output_path, "w", encoding="utf-8") as f:
- f.write(html)
- print(f"HTML exported to {output_path}")
-
- return output_path
-
-def main():
- args = parse_arguments()
- print("Loading model and processor...")
-
- device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
- model = AutoModelForVision2Seq.from_pretrained(
- MODEL_PATH,
- torch_dtype=torch.bfloat16,
- _attn_implementation="flash_attention_2" if device.type == "cuda" else "eager"
- ).to(device)
- processor = AutoProcessor.from_pretrained(MODEL_PATH)
-
- start_page = args.start_page
- end_page = args.end_page or args.page
-
- for page_num in range(start_page, end_page + 1):
- pil_image = load_image(args.image, page_num=page_num, dpi=args.dpi)
- process_page(model, processor, args, pil_image, page_num)
-
-if __name__ == "__main__":
- main()
diff --git a/backend/page_treatment/picture_extractor.py b/backend/page_treatment/picture_extractor.py
deleted file mode 100644
index a624f0d..0000000
--- a/backend/page_treatment/picture_extractor.py
+++ /dev/null
@@ -1,246 +0,0 @@
-#!/usr/bin/env python3
-"""
-DocTags Picture Extractor - Extract Batch Processing Report
-Generated on {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}
- -
-
-"""
-
- # Add failed pages if any
- if self.state['results']['failedPages']:
- html += """
-
-
- {self.total_pages}
- Total Pages
-
-
- {self.state['results']['successful']}
- Successful
-
-
- {self.state['results']['failed']}
- Failed
-
-
- {self.state['results']['totalImages']}
- Images Extracted
-
-
- {format_duration(duration)}
- Processing Time
-
-
- {success_rate:.1f}%
- Success Rate
- Failed Pages
-| Page Number | Reason |
|---|---|
| {failed['pageNum']} | {failed['reason']} |
Extracted Pictures from {pdf_name} - Page {page_num}
-Total pictures found: {len(pictures)}
-""" - - if pictures: - html += '\n'
-
- for picture, file_path in zip(pictures, saved_files):
- rel_path = file_path.name
- html += f"""
\n'
- else:
- html += '
- ![Picture {picture['id']}]({rel_path})
-
-"""
-
- html += '
-
- Picture {picture['id']}
- {f'{picture["caption"]}
' if picture['caption'] else ''}
- Coordinates: ({picture['x1']},{picture['y1']})-({picture['x2']},{picture['y2']})
- \n
\n'
-
- html += '\n\n'
-
- with open(index_file, 'w', encoding='utf-8') as f:
- f.write(html)
-
- print(f"Created index file: {index_file}")
- return index_file
-
-def main():
- args = parse_arguments()
- output_dir = ensure_results_folder(args.output)
-
- try:
- # Extract pictures from DocTags
- print(f"Extracting pictures from {args.doctags}...")
- pictures = extract_pictures_from_doctags(args.doctags)
-
- if not pictures:
- print("No picture elements found in the DocTags file.")
- return
-
- print(f"Found {len(pictures)} picture elements.")
-
- # Load the image from PDF
- page_image = load_pdf_page(args.pdf, args.page, args.dpi)
- print(f"Loaded page {args.page} image: {page_image.size[0]}x{page_image.size[1]}")
-
- # Adjust coordinates if needed
- if args.adjust:
- # Check if coordinates need normalization
- max_x = max([p['x2'] for p in pictures])
- max_y = max([p['y2'] for p in pictures])
-
- if max_x <= DEFAULT_GRID_SIZE and max_y <= DEFAULT_GRID_SIZE:
- print(f"Detected normalized coordinates (0-{DEFAULT_GRID_SIZE} grid)")
- pictures = normalize_coordinates(pictures, page_image.width, page_image.height)
- else:
- pictures = auto_adjust_coordinates(pictures, page_image.width, page_image.height)
-
- # Extract and save pictures
- saved_files = extract_and_save_pictures(
- page_image, pictures, output_dir,
- args.max_width, args.margin
- )
-
- # Create HTML index
- pdf_name = Path(args.pdf).stem
- create_html_index(pictures, saved_files, pdf_name, args.page, output_dir)
-
- except Exception as e:
- print(f"Error: {e}")
- import traceback
- traceback.print_exc()
-
-if __name__ == "__main__":
- main()
\ No newline at end of file
diff --git a/backend/page_treatment/visualizer.py b/backend/page_treatment/visualizer.py
deleted file mode 100644
index e10635d..0000000
--- a/backend/page_treatment/visualizer.py
+++ /dev/null
@@ -1,362 +0,0 @@
-#!/usr/bin/env python3
-"""
-DocTags Zone Visualizer - Visualize zones identified in DocTags format.
-"""
-
-import argparse
-import os
-import re
-from pathlib import Path
-from PIL import Image, ImageDraw, ImageFont
-
-# Add parent directory to path for imports
-import sys
-sys.path.append(str(Path(__file__).parent.parent.parent))
-
-from backend.utils import (ensure_results_folder, load_pdf_page, count_pdf_pages,
- normalize_coordinates, auto_adjust_coordinates)
-from backend.config import ZONE_COLORS, DEFAULT_DPI, DEFAULT_GRID_SIZE
-
-# Regular expression to extract location data
-LOC_PATTERN = r'No pictures found on this page
\n
-
-
-
-DocTags Batch Processor
- - - - - -
-
-
-
-
-
-
- Batch Processing Configuration
- - -
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-System Environment
- -
-
-
-
-
-
\ No newline at end of file
diff --git a/frontend/index.html b/frontend/index.html
index 72dda17..b5dfc1a 100644
--- a/frontend/index.html
+++ b/frontend/index.html
@@ -1,176 +1,13 @@
-
-
-
- ×
-
-
-
-
-
-
- ![Page Visualization]()
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
-
\ No newline at end of file
+
-
-
-
-
-
-
- - DocTags Intelligence Suite - AI powered -
-Enterprise-grade document processing and analysis
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- 1
-
- 2
-
- 3
-
-
-
-
-
-
-
-
-
-
-
-
- Document Analysis
-Extract comprehensive document structure from your PDF using advanced AI-powered analysis. This process identifies headers, paragraphs, images, tables, and other document elements with precise coordinate information.
- - - - - - -
- ![PDF Preview]()
-
-
- PDF Preview - Page 1
-
-
-
-
-
-
-
-
-
-
- Analysis Process:
-
- -
-
- Converts your PDF page into a high-resolution image -
- Applies machine learning to identify document structure -
- Maps elements with precise pixel coordinates -
- Generates structured DocTags format output -
- Saves results to the
results/directory
-
-
-
- Structure Visualization
-Generate a visual representation of the analyzed document structure with color-coded bounding boxes around each detected element type.
- - - - - -
- ![Document Structure Visualization]()
-
-
- Visualization Result
-
- Visualization Features:
-
- -
-
- Color-coded element boundaries (headers, text, images, tables) -
- Element type labels for easy identification -
- Accurate coordinate mapping and scaling -
- High-quality PNG output for documentation -
- Interactive overlay on original document -
-
- Image Extraction
-Automatically extract and save individual images from your document based on the detected picture regions, complete with metadata and captions.
- - - - - - -
-
-
- Extracted Images
-
-
-
-
- Extraction Capabilities:
-
- -
-
- Precise image boundary detection and cropping -
- Caption and metadata preservation -
- Organized file naming and directory structure -
- HTML index page for easy browsing -
- Automatic quality optimization and resizing -