Fix single page bugs
This commit is contained in:
pjmalandrino
parent
ffa2f168e1
commit
40a4c6e5cc
4 changed files with 216 additions and 64 deletions
206
backend/app.py
206
backend/app.py
|
|
@ -4,6 +4,7 @@ Flask application for DocTags web interface
|
|||
"""
|
||||
|
||||
from flask import Flask, request, send_file, jsonify
|
||||
import subprocess
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
|
|
@ -37,7 +38,6 @@ app.config['MAX_CONTENT_LENGTH'] = MAX_CONTENT_LENGTH
|
|||
|
||||
# Task results storage
|
||||
task_results = {}
|
||||
task_lock = threading.Lock()
|
||||
|
||||
# Import batch processor if available
|
||||
try:
|
||||
|
|
@ -134,49 +134,178 @@ def pdf_preview(pdf_file, page_num):
|
|||
logger.error(f"Error generating PDF preview: {e}")
|
||||
return jsonify({'error': str(e)}), 500
|
||||
|
||||
def run_background_task(task_id, command):
|
||||
"""Run a command in background and store results"""
|
||||
logger.info(f"Running task {task_id}: {command}")
|
||||
|
||||
def run_command(task_id, command):
|
||||
"""Run a command in a background thread and store result"""
|
||||
logger.info(f"Running command: {command}")
|
||||
try:
|
||||
success, stdout, stderr = run_command_with_timeout(command, PROCESSING_TIMEOUT)
|
||||
# Run the command with pipe input to automatically answer "n" to prompts
|
||||
process = subprocess.Popen(
|
||||
command,
|
||||
shell=True,
|
||||
stdin=subprocess.PIPE,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True,
|
||||
universal_newlines=True
|
||||
)
|
||||
|
||||
with task_lock:
|
||||
# Send "n" to the process to bypass prompts
|
||||
stdout, stderr = process.communicate(input="n\n")
|
||||
|
||||
# Log output for debugging
|
||||
logger.info(f"Command stdout: {stdout[:500]}...")
|
||||
if stderr:
|
||||
logger.error(f"Command stderr: {stderr}")
|
||||
|
||||
# Update task result
|
||||
if process.returncode == 0:
|
||||
task_results[task_id] = {
|
||||
'success': success,
|
||||
'success': True,
|
||||
'output': stdout,
|
||||
'error': stderr if not success else None,
|
||||
'done': True
|
||||
}
|
||||
|
||||
logger.info(f"Task {task_id} completed: {'success' if success else 'failed'}")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Task {task_id} error: {e}")
|
||||
with task_lock:
|
||||
logger.info(f"Command completed successfully: {task_id}")
|
||||
else:
|
||||
error_message = stderr or f"Command failed with return code {process.returncode}"
|
||||
task_results[task_id] = {
|
||||
'success': False,
|
||||
'error': str(e),
|
||||
'error': error_message,
|
||||
'done': True
|
||||
}
|
||||
logger.error(f"Command failed: {task_id} - {error_message}")
|
||||
|
||||
except Exception as e:
|
||||
import traceback
|
||||
logger.error(f"Unexpected error: {task_id} - {str(e)}")
|
||||
logger.error(traceback.format_exc())
|
||||
task_results[task_id] = {
|
||||
'success': False,
|
||||
'error': f"Exception: {str(e)}",
|
||||
'done': True
|
||||
}
|
||||
|
||||
@app.route('/run-analyzer', methods=['POST'])
|
||||
def run_analyzer():
|
||||
return run_processing_task('analyzer', request.form)
|
||||
try:
|
||||
pdf_file = request.form.get('pdf_file')
|
||||
page_num = request.form.get('page_num', 1)
|
||||
|
||||
if not pdf_file:
|
||||
return jsonify({'success': False, 'error': 'PDF file not specified'}), 400
|
||||
|
||||
if not os.path.exists(pdf_file):
|
||||
return jsonify({'success': False, 'error': f'PDF file not found: {pdf_file}'}), 404
|
||||
|
||||
command = f"python backend/page_treatment/analyzer.py --image {pdf_file} --page {page_num} --start-page {page_num} --end-page {page_num}"
|
||||
|
||||
# Generate task ID
|
||||
task_id = f"analyzer_{int(time.time())}"
|
||||
|
||||
# Initialize task result
|
||||
task_results[task_id] = {
|
||||
'success': None,
|
||||
'output': "Running analyzer...",
|
||||
'done': False
|
||||
}
|
||||
|
||||
# Start background thread
|
||||
thread = threading.Thread(target=run_command, args=(task_id, command))
|
||||
thread.daemon = True
|
||||
thread.start()
|
||||
|
||||
return jsonify({
|
||||
'success': True,
|
||||
'task_id': task_id,
|
||||
'message': f"Analyzer started. Processing {pdf_file} page {page_num}..."
|
||||
})
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error starting analyzer: {e}")
|
||||
return jsonify({'success': False, 'error': str(e)}), 500
|
||||
|
||||
@app.route('/run-visualizer', methods=['POST'])
|
||||
def run_visualizer():
|
||||
return run_processing_task('visualizer', request.form)
|
||||
try:
|
||||
pdf_file = request.form.get('pdf_file')
|
||||
page_num = request.form.get('page_num', 1)
|
||||
adjust = request.form.get('adjust') == 'true'
|
||||
|
||||
if not pdf_file:
|
||||
return jsonify({'success': False, 'error': 'PDF file not specified'}), 400
|
||||
|
||||
command = f"python backend/page_treatment/visualizer.py --doctags results/output.doctags.txt --pdf {pdf_file} --page {page_num}"
|
||||
if adjust:
|
||||
command += " --adjust"
|
||||
|
||||
# Generate task ID with page number
|
||||
task_id = f"visualizer_{int(time.time())}_{page_num}"
|
||||
|
||||
# Initialize task result
|
||||
task_results[task_id] = {
|
||||
'success': None,
|
||||
'output': "Running visualizer...",
|
||||
'done': False
|
||||
}
|
||||
|
||||
# Start background thread
|
||||
thread = threading.Thread(target=run_command, args=(task_id, command))
|
||||
thread.daemon = True
|
||||
thread.start()
|
||||
|
||||
return jsonify({
|
||||
'success': True,
|
||||
'task_id': task_id,
|
||||
'message': f"Visualizer started. Processing {pdf_file} page {page_num}..."
|
||||
})
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error starting visualizer: {e}")
|
||||
return jsonify({'success': False, 'error': str(e)}), 500
|
||||
|
||||
@app.route('/run-extractor', methods=['POST'])
|
||||
def run_extractor():
|
||||
return run_processing_task('extractor', request.form)
|
||||
try:
|
||||
pdf_file = request.form.get('pdf_file')
|
||||
page_num = request.form.get('page_num', 1)
|
||||
adjust = request.form.get('adjust') == 'true'
|
||||
|
||||
if not pdf_file:
|
||||
return jsonify({'success': False, 'error': 'PDF file not specified'}), 400
|
||||
|
||||
command = f"python backend/page_treatment/picture_extractor.py --doctags results/output.doctags.txt --pdf {pdf_file} --page {page_num}"
|
||||
if adjust:
|
||||
command += " --adjust"
|
||||
|
||||
# Generate task ID with page number
|
||||
task_id = f"extractor_{int(time.time())}_{page_num}"
|
||||
|
||||
# Initialize task result
|
||||
task_results[task_id] = {
|
||||
'success': None,
|
||||
'output': "Running picture extractor...",
|
||||
'done': False
|
||||
}
|
||||
|
||||
# Start background thread
|
||||
thread = threading.Thread(target=run_command, args=(task_id, command))
|
||||
thread.daemon = True
|
||||
thread.start()
|
||||
|
||||
return jsonify({
|
||||
'success': True,
|
||||
'task_id': task_id,
|
||||
'message': f"Picture extractor started. Processing {pdf_file} page {page_num}..."
|
||||
})
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error starting extractor: {e}")
|
||||
return jsonify({'success': False, 'error': str(e)}), 500
|
||||
|
||||
def run_processing_task(task_type, form_data):
|
||||
"""Generic function to run processing tasks"""
|
||||
try:
|
||||
pdf_file = form_data.get('pdf_file')
|
||||
page_num = form_data.get('page_num', 1)
|
||||
page_num = form_data.get('page_num', '1')
|
||||
|
||||
if not pdf_file:
|
||||
return jsonify({'success': False, 'error': 'PDF file not specified'}), 400
|
||||
|
|
@ -187,7 +316,7 @@ def run_processing_task(task_type, form_data):
|
|||
# Build command based on task type
|
||||
command = build_command(task_type, pdf_file, page_num, form_data)
|
||||
|
||||
# Generate task ID
|
||||
# Generate task ID with page number
|
||||
task_id = f"{task_type}_{int(time.time() * 1000)}_{page_num}"
|
||||
|
||||
# Initialize task result
|
||||
|
|
@ -198,6 +327,8 @@ def run_processing_task(task_type, form_data):
|
|||
'done': False
|
||||
}
|
||||
|
||||
logger.info(f"Created task {task_id} for {task_type} on page {page_num}")
|
||||
|
||||
# Start background thread
|
||||
thread = threading.Thread(target=run_background_task, args=(task_id, command))
|
||||
thread.daemon = True
|
||||
|
|
@ -211,6 +342,8 @@ def run_processing_task(task_type, form_data):
|
|||
|
||||
except Exception as e:
|
||||
logger.error(f"Error starting {task_type}: {e}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
return jsonify({'success': False, 'error': str(e)}), 500
|
||||
|
||||
def build_command(task_type, pdf_file, page_num, form_data):
|
||||
|
|
@ -240,18 +373,31 @@ def build_command(task_type, pdf_file, page_num, form_data):
|
|||
|
||||
@app.route('/task-status/<task_id>')
|
||||
def task_status(task_id):
|
||||
"""Get status of a background task"""
|
||||
with task_lock:
|
||||
if task_id not in task_results:
|
||||
return jsonify({'success': False, 'error': 'Task not found'}), 404
|
||||
if task_id not in task_results:
|
||||
logger.warning(f"Task {task_id} not found in task_results")
|
||||
return jsonify({'success': False, 'error': 'Task not found'}), 404
|
||||
|
||||
result = task_results[task_id].copy()
|
||||
result = task_results[task_id].copy()
|
||||
|
||||
# Add file paths for completed tasks
|
||||
# Log the complete result for debugging
|
||||
logger.info(f"Task {task_id} result: {result}")
|
||||
|
||||
# If task is completed successfully, add file paths
|
||||
if result.get('done') and result.get('success'):
|
||||
if 'visualizer' in task_id:
|
||||
page_num = task_id.split('_')[-1]
|
||||
result['image_file'] = f"results/visualization_page_{page_num}.png"
|
||||
if task_id.startswith('analyzer_'):
|
||||
doctags_path = Path("results") / "output.doctags.txt"
|
||||
if doctags_path.exists():
|
||||
result['doctags_file'] = "results/output.doctags.txt"
|
||||
logger.info(f"Added doctags_file to result")
|
||||
else:
|
||||
logger.warning(f"DocTags file not found: {doctags_path}")
|
||||
|
||||
elif task_id.startswith('visualizer_'):
|
||||
# Extract page number from the end of task_id if it exists
|
||||
page_num = task_id.split('_')[-1] if len(task_id.split('_')) > 2 else "1"
|
||||
viz_filename = f"visualization_page_{page_num}.png"
|
||||
result['image_file'] = f"results/{viz_filename}"
|
||||
logger.info(f"Found visualization at: {result['image_file']}")
|
||||
|
||||
return jsonify(result)
|
||||
|
||||
|
|
|
|||
|
|
@ -81,15 +81,19 @@ def process_page(model, processor, config, args, pil_image, page_num=1):
|
|||
from mlx_vlm.utils import stream_generate
|
||||
|
||||
results_dir = ensure_results_folder()
|
||||
output_base = Path(args.output)
|
||||
|
||||
# Handle multi-page output naming
|
||||
if Path(args.image).suffix.lower() == '.pdf' and page_num > 1:
|
||||
output_path = results_dir / f"{output_base.stem}_page{page_num}{output_base.suffix}"
|
||||
# For web interface, always use output.doctags.txt
|
||||
# For command line with specific pages, use page-specific names
|
||||
if args.start_page == args.end_page and args.start_page == page_num:
|
||||
# Single page processing
|
||||
output_path = results_dir / "output.html"
|
||||
doctags_path = results_dir / "output.doctags.txt"
|
||||
else:
|
||||
output_path = output_base
|
||||
# Multi-page processing
|
||||
output_path = results_dir / f"output_page{page_num}.html"
|
||||
doctags_path = results_dir / f"output_page{page_num}.doctags.txt"
|
||||
|
||||
print(f"Processing page {page_num}, output will be saved to {output_path}")
|
||||
print(f"Processing page {page_num}")
|
||||
|
||||
# Save image temporarily
|
||||
with tempfile.NamedTemporaryFile(suffix='.png', delete=False) as temp_img_file:
|
||||
|
|
@ -117,7 +121,6 @@ def process_page(model, processor, config, args, pil_image, page_num=1):
|
|||
os.unlink(temp_img_path)
|
||||
|
||||
# Save DocTags output
|
||||
doctags_path = results_dir / f"{output_path.stem}.doctags.txt"
|
||||
with open(doctags_path, 'w', encoding='utf-8') as f:
|
||||
f.write(output)
|
||||
print(f"Raw DocTags saved to: {doctags_path}")
|
||||
|
|
|
|||
|
|
@ -169,6 +169,7 @@ def calculate_scale_factor(max_coord: float, image_size: float) -> float:
|
|||
else:
|
||||
return max(image_size / max_coord, 0.5)
|
||||
|
||||
# In backend/utils.py, make sure this function exists:
|
||||
def run_command_with_timeout(command: str, timeout: int = 300, input_text: str = "n\n") -> Tuple[bool, str, str]:
|
||||
"""
|
||||
Run a command with timeout and return success, stdout, stderr.
|
||||
|
|
|
|||
|
|
@ -8,21 +8,6 @@ const appState = {
|
|||
}
|
||||
};
|
||||
|
||||
// API Client
|
||||
const api = {
|
||||
async get(url) {
|
||||
const response = await fetch(url);
|
||||
if (!response.ok) throw new Error(`HTTP error! status: ${response.status}`);
|
||||
return response.json();
|
||||
},
|
||||
|
||||
async post(url, formData) {
|
||||
const response = await fetch(url, { method: 'POST', body: formData });
|
||||
if (!response.ok) throw new Error(`HTTP error! status: ${response.status}`);
|
||||
return response.json();
|
||||
}
|
||||
};
|
||||
|
||||
// UI Helper Functions
|
||||
const ui = {
|
||||
show(elementId) {
|
||||
|
|
@ -158,7 +143,12 @@ function stopPolling() {
|
|||
async function pollTasks() {
|
||||
for (const taskId in appState.activeTasks) {
|
||||
try {
|
||||
const data = await api.get(`/task-status/${taskId}`);
|
||||
const response = await fetch(`/task-status/${taskId}`);
|
||||
const data = await response.json();
|
||||
|
||||
// Debug log
|
||||
console.log(`Task ${taskId} status:`, data);
|
||||
|
||||
updateTaskStatus(taskId, data);
|
||||
} catch (error) {
|
||||
console.error('Error polling task:', error);
|
||||
|
|
@ -179,14 +169,14 @@ function updateTaskStatus(taskId, data) {
|
|||
delete appState.activeTasks[taskId];
|
||||
stopPolling();
|
||||
} else {
|
||||
ui.setHtml(statusElement, '<div class="loader"></div><span class="working">Running...</span>');
|
||||
ui.show(statusElement.id);
|
||||
ui.setHtml(`${taskInfo.type}-status`, '<div class="loader"></div><span class="working">Running...</span>');
|
||||
ui.show(`${taskInfo.type}-status`);
|
||||
}
|
||||
}
|
||||
|
||||
function handleTaskSuccess(taskId, taskInfo, data, statusElement) {
|
||||
ui.setHtml(statusElement, '<span class="success">✓ Completed successfully!</span>');
|
||||
ui.show(statusElement.id);
|
||||
ui.setHtml(`${taskInfo.type}-status`, '<span class="success">✓ Completed successfully!</span>');
|
||||
ui.show(`${taskInfo.type}-status`);
|
||||
ui.enable(`${taskInfo.type}-btn`);
|
||||
|
||||
// Display output
|
||||
|
|
@ -215,9 +205,10 @@ function handleTaskSuccess(taskId, taskInfo, data, statusElement) {
|
|||
}
|
||||
|
||||
function handleTaskFailure(taskId, data, statusElement) {
|
||||
ui.setHtml(statusElement, '<span class="error">✗ Failed: ' + (data.error || 'Unknown error') + '</span>');
|
||||
ui.show(statusElement.id);
|
||||
ui.enable(statusElement.id.replace('-status', '-btn'));
|
||||
const taskInfo = appState.activeTasks[taskId];
|
||||
ui.setHtml(`${taskInfo.type}-status`, '<span class="error">✗ Failed: ' + (data.error || 'Unknown error') + '</span>');
|
||||
ui.show(`${taskInfo.type}-status`);
|
||||
ui.enable(`${taskInfo.type}-btn`);
|
||||
|
||||
ui.setText('output', 'Error: ' + (data.error || 'Unknown error'));
|
||||
ui.show('output');
|
||||
|
|
@ -257,7 +248,11 @@ async function runScript(script) {
|
|||
formData.append('adjust', adjust);
|
||||
|
||||
try {
|
||||
const data = await api.post(`/run-${script}`, formData);
|
||||
const response = await fetch(`/run-${script}`, {
|
||||
method: 'POST',
|
||||
body: formData
|
||||
});
|
||||
const data = await response.json();
|
||||
|
||||
if (data.success && data.task_id) {
|
||||
appState.activeTasks[data.task_id] = {
|
||||
|
|
@ -375,7 +370,8 @@ async function checkEnvironment() {
|
|||
ui.setHtml('env-details', '<div class="loader"></div> Checking environment...');
|
||||
|
||||
try {
|
||||
const data = await api.get('/check-environment');
|
||||
const response = await fetch('/check-environment');
|
||||
const data = await response.json();
|
||||
|
||||
let html = '<ul>';
|
||||
html += `<li>Working directory: <code>${data.cwd}</code></li>`;
|
||||
|
|
@ -413,7 +409,12 @@ async function manuallyRunScript() {
|
|||
formData.append('command', command);
|
||||
|
||||
try {
|
||||
const data = await api.post('/run-manual-command', formData);
|
||||
const response = await fetch('/run-manual-command', {
|
||||
method: 'POST',
|
||||
body: formData
|
||||
});
|
||||
const data = await response.json();
|
||||
|
||||
ui.setText('output',
|
||||
'Command: ' + command + '\n\n' +
|
||||
(data.success ? 'Success!\n\n' : 'Failed!\n\n') +
|
||||
|
|
@ -432,7 +433,8 @@ window.addEventListener('DOMContentLoaded', async function() {
|
|||
ui.setHtml('pdf-load-status', '<div class="loader"></div> Loading PDF files...');
|
||||
|
||||
try {
|
||||
const data = await api.get('/pdf-files');
|
||||
const response = await fetch('/pdf-files');
|
||||
const data = await response.json();
|
||||
const select = document.getElementById('pdf_file');
|
||||
|
||||
if (data.length === 0) {
|
||||
|
|
|
|||
Loading…
Reference in a new issue