Update project structure to isolate backend files in order to preprare containerisation properly
This commit is contained in:
parent
4d998251a9
commit
9945a29a8e
12 changed files with 360 additions and 47 deletions
0
backend/__init__.py
Normal file
0
backend/__init__.py
Normal file
|
|
@ -1,22 +1,27 @@
|
||||||
from flask import Flask, request, send_file, jsonify, Response
|
from flask import Flask, request, send_file, jsonify, Response
|
||||||
import subprocess
|
import subprocess
|
||||||
import os
|
import os
|
||||||
|
import sys
|
||||||
import time
|
import time
|
||||||
import threading
|
import threading
|
||||||
import logging
|
import logging
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
import sys
|
|
||||||
import uuid
|
import uuid
|
||||||
import pdf2image
|
import pdf2image
|
||||||
|
|
||||||
|
# Add the parent directory to Python path to allow imports
|
||||||
|
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||||
|
|
||||||
# Configure logging
|
# Configure logging
|
||||||
logging.basicConfig(level=logging.INFO,
|
logging.basicConfig(level=logging.INFO,
|
||||||
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
|
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
# Initialize Flask app with frontend folder structure
|
# Initialize Flask app with frontend folder structure
|
||||||
|
# Frontend is in the parent directory
|
||||||
|
frontend_path = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), 'frontend')
|
||||||
app = Flask(__name__,
|
app = Flask(__name__,
|
||||||
static_folder='frontend/static',
|
static_folder=os.path.join(frontend_path, 'static'),
|
||||||
static_url_path='/static')
|
static_url_path='/static')
|
||||||
|
|
||||||
# Dictionary to store background task results
|
# Dictionary to store background task results
|
||||||
|
|
@ -24,7 +29,7 @@ task_results = {}
|
||||||
|
|
||||||
# Import batch processor if available
|
# Import batch processor if available
|
||||||
try:
|
try:
|
||||||
from batch_processor import start_batch_processing, get_batch_processor, cleanup_old_batches
|
from backend.batch_treatment.batch_processor import start_batch_processing, get_batch_processor, cleanup_old_batches
|
||||||
batch_processing_available = True
|
batch_processing_available = True
|
||||||
except ImportError:
|
except ImportError:
|
||||||
logger.warning("batch_processor.py not found. Batch processing features will be disabled.")
|
logger.warning("batch_processor.py not found. Batch processing features will be disabled.")
|
||||||
|
|
@ -32,14 +37,18 @@ except ImportError:
|
||||||
|
|
||||||
# Ensure results folder exists
|
# Ensure results folder exists
|
||||||
def ensure_results_folder():
|
def ensure_results_folder():
|
||||||
results_dir = Path("results")
|
# Always create results folder relative to where app.py is run from
|
||||||
|
results_dir = Path.cwd() / "results"
|
||||||
if not results_dir.exists():
|
if not results_dir.exists():
|
||||||
results_dir.mkdir()
|
results_dir.mkdir()
|
||||||
|
logger.info(f"Created results directory at: {results_dir.absolute()}")
|
||||||
|
else:
|
||||||
|
logger.info(f"Results directory exists at: {results_dir.absolute()}")
|
||||||
return results_dir
|
return results_dir
|
||||||
|
|
||||||
# Ensure frontend folders exist
|
# Ensure frontend folders exist
|
||||||
def ensure_frontend_folders():
|
def ensure_frontend_folders():
|
||||||
frontend_dir = Path("frontend")
|
frontend_dir = Path(frontend_path)
|
||||||
if not frontend_dir.exists():
|
if not frontend_dir.exists():
|
||||||
frontend_dir.mkdir()
|
frontend_dir.mkdir()
|
||||||
logger.info(f"Created frontend directory: {frontend_dir}")
|
logger.info(f"Created frontend directory: {frontend_dir}")
|
||||||
|
|
@ -53,16 +62,16 @@ def ensure_frontend_folders():
|
||||||
|
|
||||||
@app.route('/')
|
@app.route('/')
|
||||||
def index():
|
def index():
|
||||||
return send_file('frontend/index.html')
|
return send_file(os.path.join(frontend_path, 'index.html'))
|
||||||
|
|
||||||
@app.route('/batch')
|
@app.route('/batch')
|
||||||
def batch_interface():
|
def batch_interface():
|
||||||
"""Serve the batch processing interface"""
|
"""Serve the batch processing interface"""
|
||||||
return send_file('frontend/batch.html')
|
return send_file(os.path.join(frontend_path, 'batch.html'))
|
||||||
|
|
||||||
@app.route('/static/<path:filename>')
|
@app.route('/static/<path:filename>')
|
||||||
def serve_static(filename):
|
def serve_static(filename):
|
||||||
return send_file(os.path.join('frontend/static', filename))
|
return send_file(os.path.join(frontend_path, 'static', filename))
|
||||||
|
|
||||||
@app.route('/pdf-files')
|
@app.route('/pdf-files')
|
||||||
def pdf_files():
|
def pdf_files():
|
||||||
|
|
@ -123,19 +132,22 @@ def run_command(task_id, command):
|
||||||
current_dir = os.getcwd()
|
current_dir = os.getcwd()
|
||||||
logger.info(f"Current working directory: {current_dir}")
|
logger.info(f"Current working directory: {current_dir}")
|
||||||
|
|
||||||
# Check if the script exists
|
# The command already contains the full path, so we just need to verify it exists
|
||||||
script_name = command.split()[1]
|
script_parts = command.split()
|
||||||
if not os.path.exists(script_name):
|
script_path = script_parts[1]
|
||||||
logger.error(f"Script not found: {script_name} in {current_dir}")
|
|
||||||
|
if not os.path.exists(script_path):
|
||||||
|
logger.error(f"Script not found: {script_path} in {current_dir}")
|
||||||
task_results[task_id] = {
|
task_results[task_id] = {
|
||||||
'success': False,
|
'success': False,
|
||||||
'error': f"Script not found: {script_name}. Make sure all scripts are in the same directory as app.py.",
|
'error': f"Script not found: {script_path}. Make sure all scripts are in the correct directory.",
|
||||||
'done': True
|
'done': True
|
||||||
}
|
}
|
||||||
return
|
return
|
||||||
|
|
||||||
# Run the command with more detailed error capture
|
# Run the command with more detailed error capture
|
||||||
# Important: Use pipe input to automatically answer "n" to the prompt
|
# Important: Use pipe input to automatically answer "n" to the prompt
|
||||||
|
# Make sure we run from the project root directory
|
||||||
process = subprocess.Popen(
|
process = subprocess.Popen(
|
||||||
command,
|
command,
|
||||||
shell=True,
|
shell=True,
|
||||||
|
|
@ -143,7 +155,8 @@ def run_command(task_id, command):
|
||||||
stdout=subprocess.PIPE,
|
stdout=subprocess.PIPE,
|
||||||
stderr=subprocess.PIPE,
|
stderr=subprocess.PIPE,
|
||||||
text=True,
|
text=True,
|
||||||
universal_newlines=True
|
universal_newlines=True,
|
||||||
|
cwd=os.getcwd() # Explicitly set working directory to current directory
|
||||||
)
|
)
|
||||||
|
|
||||||
# Send "n" to the process to bypass the "process all pages" prompt
|
# Send "n" to the process to bypass the "process all pages" prompt
|
||||||
|
|
@ -198,13 +211,14 @@ def run_analyzer():
|
||||||
logger.error(f"PDF file not found: {pdf_file}")
|
logger.error(f"PDF file not found: {pdf_file}")
|
||||||
return jsonify({'success': False, 'error': f'PDF file not found: {pdf_file}'}), 400
|
return jsonify({'success': False, 'error': f'PDF file not found: {pdf_file}'}), 400
|
||||||
|
|
||||||
# Check if analyzer.py exists
|
# Check if analyzer.py exists in the new location
|
||||||
if not os.path.exists('analyzer.py'):
|
analyzer_path = os.path.join('backend', 'page_treatment', 'analyzer.py')
|
||||||
logger.error("analyzer.py not found in current directory")
|
if not os.path.exists(analyzer_path):
|
||||||
return jsonify({'success': False, 'error': 'analyzer.py not found in current directory'}), 500
|
logger.error(f"analyzer.py not found at: {analyzer_path}")
|
||||||
|
return jsonify({'success': False, 'error': 'analyzer.py not found in backend/page_treatment directory'}), 500
|
||||||
|
|
||||||
# Add the --all-pages=false flag or explicitly specify the page to avoid the prompt
|
# Add the --all-pages=false flag or explicitly specify the page to avoid the prompt
|
||||||
command = f"python analyzer.py --image {pdf_file} --page {page_num} --start-page {page_num} --end-page {page_num}"
|
command = f"python backend/page_treatment/analyzer.py --image {pdf_file} --page {page_num} --start-page {page_num} --end-page {page_num}"
|
||||||
|
|
||||||
# Generate a task ID
|
# Generate a task ID
|
||||||
task_id = f"analyzer_{int(time.time())}"
|
task_id = f"analyzer_{int(time.time())}"
|
||||||
|
|
@ -238,15 +252,40 @@ def task_status(task_id):
|
||||||
if task_id not in task_results:
|
if task_id not in task_results:
|
||||||
return jsonify({'success': False, 'error': 'Task not found'}), 404
|
return jsonify({'success': False, 'error': 'Task not found'}), 404
|
||||||
|
|
||||||
result = task_results[task_id]
|
result = task_results[task_id].copy() # Make a copy to avoid modifying the original
|
||||||
|
|
||||||
# If task is completed, add file paths
|
# If task is completed, add file paths and verify they exist
|
||||||
if result['done'] and result['success']:
|
if result['done'] and result['success']:
|
||||||
if task_id.startswith('analyzer_'):
|
if task_id.startswith('analyzer_'):
|
||||||
result['doctags_file'] = f"results/output.doctags.txt"
|
doctags_path = Path("results") / "output.doctags.txt"
|
||||||
|
if doctags_path.exists():
|
||||||
|
result['doctags_file'] = "results/output.doctags.txt"
|
||||||
|
else:
|
||||||
|
logger.warning(f"DocTags file not found: {doctags_path}")
|
||||||
|
|
||||||
elif task_id.startswith('visualizer_'):
|
elif task_id.startswith('visualizer_'):
|
||||||
page_num = task_id.split('_')[-1] if len(task_id.split('_')) > 2 else "1"
|
page_num = task_id.split('_')[-1] if len(task_id.split('_')) > 2 else "1"
|
||||||
result['image_file'] = f"results/visualization_page_{page_num}.png"
|
viz_filename = f"visualization_page_{page_num}.png"
|
||||||
|
|
||||||
|
# Check multiple possible locations
|
||||||
|
possible_paths = [
|
||||||
|
Path("results") / viz_filename,
|
||||||
|
Path.cwd() / "results" / viz_filename,
|
||||||
|
Path(__file__).parent.parent / "results" / viz_filename
|
||||||
|
]
|
||||||
|
|
||||||
|
for path in possible_paths:
|
||||||
|
if path.exists():
|
||||||
|
result['image_file'] = f"results/{viz_filename}"
|
||||||
|
logger.info(f"Found visualization at: {path}")
|
||||||
|
break
|
||||||
|
else:
|
||||||
|
logger.error(f"Visualization file not found in any location for page {page_num}")
|
||||||
|
# Log what files exist in results
|
||||||
|
results_dir = Path("results")
|
||||||
|
if results_dir.exists():
|
||||||
|
files = list(results_dir.glob("*.png"))
|
||||||
|
logger.info(f"PNG files in results: {[f.name for f in files[:5]]}")
|
||||||
|
|
||||||
return jsonify(result)
|
return jsonify(result)
|
||||||
|
|
||||||
|
|
@ -260,7 +299,7 @@ def run_visualizer():
|
||||||
if not pdf_file:
|
if not pdf_file:
|
||||||
return jsonify({'success': False, 'error': 'PDF file not specified'}), 400
|
return jsonify({'success': False, 'error': 'PDF file not specified'}), 400
|
||||||
|
|
||||||
command = f"python visualizer.py --doctags results/output.doctags.txt --pdf {pdf_file} --page {page_num}"
|
command = f"python backend/page_treatment/visualizer.py --doctags results/output.doctags.txt --pdf {pdf_file} --page {page_num}"
|
||||||
if adjust:
|
if adjust:
|
||||||
command += " --adjust"
|
command += " --adjust"
|
||||||
|
|
||||||
|
|
@ -298,7 +337,7 @@ def run_extractor():
|
||||||
if not pdf_file:
|
if not pdf_file:
|
||||||
return jsonify({'success': False, 'error': 'PDF file not specified'}), 400
|
return jsonify({'success': False, 'error': 'PDF file not specified'}), 400
|
||||||
|
|
||||||
command = f"python picture_extractor.py --doctags results/output.doctags.txt --pdf {pdf_file} --page {page_num}"
|
command = f"python backend/page_treatment/picture_extractor.py --doctags results/output.doctags.txt --pdf {pdf_file} --page {page_num}"
|
||||||
if adjust:
|
if adjust:
|
||||||
command += " --adjust"
|
command += " --adjust"
|
||||||
|
|
||||||
|
|
@ -329,14 +368,28 @@ def run_extractor():
|
||||||
@app.route('/results/<path:filename>')
|
@app.route('/results/<path:filename>')
|
||||||
def results(filename):
|
def results(filename):
|
||||||
try:
|
try:
|
||||||
file_path = os.path.join('results', filename)
|
# Always use the results directory relative to current working directory
|
||||||
if os.path.exists(file_path):
|
results_dir = Path.cwd() / "results"
|
||||||
return send_file(file_path)
|
file_path = results_dir / filename
|
||||||
|
|
||||||
|
logger.info(f"Requested file: {filename}")
|
||||||
|
logger.info(f"Looking for file at: {file_path.absolute()}")
|
||||||
|
logger.info(f"File exists: {file_path.exists()}")
|
||||||
|
|
||||||
|
if file_path.exists() and file_path.is_file():
|
||||||
|
logger.info(f"Serving file: {file_path}")
|
||||||
|
return send_file(str(file_path.absolute()))
|
||||||
else:
|
else:
|
||||||
logger.error(f"File not found: {file_path}")
|
logger.error(f"File not found: {file_path}")
|
||||||
|
# List what files ARE in the results directory
|
||||||
|
if results_dir.exists():
|
||||||
|
files = list(results_dir.glob("*"))
|
||||||
|
logger.info(f"Files in results directory: {[f.name for f in files[:10]]}")
|
||||||
return jsonify({'error': f"File not found: {filename}"}), 404
|
return jsonify({'error': f"File not found: {filename}"}), 404
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"Error serving file {filename}: {str(e)}")
|
logger.error(f"Error serving file {filename}: {str(e)}")
|
||||||
|
import traceback
|
||||||
|
logger.error(traceback.format_exc())
|
||||||
return jsonify({'error': f"Error serving file: {filename}"}), 500
|
return jsonify({'error': f"Error serving file: {filename}"}), 500
|
||||||
|
|
||||||
@app.route('/pdf-preview/<pdf_file>/<int:page_num>')
|
@app.route('/pdf-preview/<pdf_file>/<int:page_num>')
|
||||||
|
|
@ -405,9 +458,15 @@ def check_environment():
|
||||||
# List all files in directory
|
# List all files in directory
|
||||||
files = os.listdir('.')
|
files = os.listdir('.')
|
||||||
|
|
||||||
# Check for required scripts
|
# Check for required scripts in new location
|
||||||
|
backend_page_dir = os.path.join('backend', 'page_treatment')
|
||||||
required_scripts = ['analyzer.py', 'visualizer.py', 'picture_extractor.py']
|
required_scripts = ['analyzer.py', 'visualizer.py', 'picture_extractor.py']
|
||||||
missing_scripts = [script for script in required_scripts if script not in files]
|
missing_scripts = []
|
||||||
|
|
||||||
|
for script in required_scripts:
|
||||||
|
script_path = os.path.join(backend_page_dir, script)
|
||||||
|
if not os.path.exists(script_path):
|
||||||
|
missing_scripts.append(script)
|
||||||
|
|
||||||
# Check for PDFs
|
# Check for PDFs
|
||||||
pdf_files = [f for f in files if f.endswith('.pdf')]
|
pdf_files = [f for f in files if f.endswith('.pdf')]
|
||||||
|
|
@ -415,6 +474,7 @@ def check_environment():
|
||||||
# Check for results directory
|
# Check for results directory
|
||||||
results_dir = Path("results")
|
results_dir = Path("results")
|
||||||
results_dir_exists = results_dir.exists()
|
results_dir_exists = results_dir.exists()
|
||||||
|
results_files = []
|
||||||
|
|
||||||
# Try to check Python version and installed packages
|
# Try to check Python version and installed packages
|
||||||
python_info = subprocess.run(['python', '--version'], capture_output=True, text=True)
|
python_info = subprocess.run(['python', '--version'], capture_output=True, text=True)
|
||||||
|
|
@ -429,6 +489,9 @@ def check_environment():
|
||||||
f.write("test")
|
f.write("test")
|
||||||
os.remove(test_file)
|
os.remove(test_file)
|
||||||
results_writable = True
|
results_writable = True
|
||||||
|
|
||||||
|
# List files in results directory
|
||||||
|
results_files = [f.name for f in results_dir.iterdir() if f.is_file()][:20] # Limit to 20 files
|
||||||
except:
|
except:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
|
@ -439,6 +502,7 @@ def check_environment():
|
||||||
'pdf_files': pdf_files,
|
'pdf_files': pdf_files,
|
||||||
'results_dir_exists': results_dir_exists,
|
'results_dir_exists': results_dir_exists,
|
||||||
'results_dir_writable': results_writable,
|
'results_dir_writable': results_writable,
|
||||||
|
'results_files': results_files,
|
||||||
'python_version': python_version,
|
'python_version': python_version,
|
||||||
'batch_processing_available': batch_processing_available
|
'batch_processing_available': batch_processing_available
|
||||||
})
|
})
|
||||||
|
|
@ -460,6 +524,12 @@ def run_manual_command():
|
||||||
|
|
||||||
logger.info(f"Running manual command: {command}")
|
logger.info(f"Running manual command: {command}")
|
||||||
|
|
||||||
|
# Update paths in manual commands to use backend directory only if not already present
|
||||||
|
if 'backend/page_treatment/' not in command:
|
||||||
|
command = command.replace('analyzer.py', 'backend/page_treatment/analyzer.py')
|
||||||
|
command = command.replace('visualizer.py', 'backend/page_treatment/visualizer.py')
|
||||||
|
command = command.replace('picture_extractor.py', 'backend/page_treatment/picture_extractor.py')
|
||||||
|
|
||||||
try:
|
try:
|
||||||
# Run the command synchronously for immediate feedback
|
# Run the command synchronously for immediate feedback
|
||||||
process = subprocess.Popen(
|
process = subprocess.Popen(
|
||||||
|
|
@ -766,6 +836,45 @@ def open_results_folder():
|
||||||
logger.error(f"Error opening results folder: {str(e)}")
|
logger.error(f"Error opening results folder: {str(e)}")
|
||||||
return jsonify({'success': False, 'error': str(e)}), 500
|
return jsonify({'success': False, 'error': str(e)}), 500
|
||||||
|
|
||||||
|
@app.route('/debug-results')
|
||||||
|
def debug_results():
|
||||||
|
"""Debug endpoint to check results directory"""
|
||||||
|
try:
|
||||||
|
results_info = {
|
||||||
|
'cwd': os.getcwd(),
|
||||||
|
'results_paths_checked': []
|
||||||
|
}
|
||||||
|
|
||||||
|
# Check multiple possible results locations
|
||||||
|
possible_results_dirs = [
|
||||||
|
Path("results"),
|
||||||
|
Path.cwd() / "results",
|
||||||
|
Path(__file__).parent.parent / "results"
|
||||||
|
]
|
||||||
|
|
||||||
|
for results_dir in possible_results_dirs:
|
||||||
|
dir_info = {
|
||||||
|
'path': str(results_dir),
|
||||||
|
'absolute_path': str(results_dir.absolute()),
|
||||||
|
'exists': results_dir.exists(),
|
||||||
|
'files': []
|
||||||
|
}
|
||||||
|
|
||||||
|
if results_dir.exists():
|
||||||
|
try:
|
||||||
|
# List all files in the directory
|
||||||
|
files = list(results_dir.glob("*"))
|
||||||
|
dir_info['files'] = [f.name for f in files if f.is_file()][:20] # Limit to 20 files
|
||||||
|
except Exception as e:
|
||||||
|
dir_info['error'] = str(e)
|
||||||
|
|
||||||
|
results_info['results_paths_checked'].append(dir_info)
|
||||||
|
|
||||||
|
return jsonify(results_info)
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
return jsonify({'error': str(e)}), 500
|
||||||
|
|
||||||
# Cleanup task for batch processors
|
# Cleanup task for batch processors
|
||||||
def cleanup_task():
|
def cleanup_task():
|
||||||
"""Periodic cleanup of old batch processors"""
|
"""Periodic cleanup of old batch processors"""
|
||||||
0
backend/batch_treatment/__init__.py
Normal file
0
backend/batch_treatment/__init__.py
Normal file
|
|
@ -5,6 +5,7 @@ Handles batch processing of PDF documents with parallel processing support
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import os
|
import os
|
||||||
|
import sys
|
||||||
import json
|
import json
|
||||||
import time
|
import time
|
||||||
import threading
|
import threading
|
||||||
|
|
@ -17,6 +18,9 @@ import subprocess
|
||||||
import shutil
|
import shutil
|
||||||
import zipfile
|
import zipfile
|
||||||
|
|
||||||
|
# Add the parent directory to Python path to allow imports
|
||||||
|
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
|
||||||
|
|
||||||
# Configure logging
|
# Configure logging
|
||||||
logging.basicConfig(
|
logging.basicConfig(
|
||||||
level=logging.INFO,
|
level=logging.INFO,
|
||||||
|
|
@ -172,8 +176,9 @@ class BatchProcessor:
|
||||||
# Use standard results directory for analyzer output
|
# Use standard results directory for analyzer output
|
||||||
output_base = Path("results") / f"output"
|
output_base = Path("results") / f"output"
|
||||||
|
|
||||||
|
# Update path to use backend directory
|
||||||
command = [
|
command = [
|
||||||
"python", "analyzer.py",
|
"python", "backend/page_treatment/analyzer.py",
|
||||||
"--image", self.pdf_file,
|
"--image", self.pdf_file,
|
||||||
"--page", str(page_num),
|
"--page", str(page_num),
|
||||||
"--output", str(output_base),
|
"--output", str(output_base),
|
||||||
|
|
@ -224,8 +229,9 @@ class BatchProcessor:
|
||||||
if page_doctags.exists():
|
if page_doctags.exists():
|
||||||
shutil.copy2(page_doctags, doctags_path)
|
shutil.copy2(page_doctags, doctags_path)
|
||||||
|
|
||||||
|
# Update path to use backend directory
|
||||||
command = [
|
command = [
|
||||||
"python", "visualizer.py",
|
"python", "backend/page_treatment/visualizer.py",
|
||||||
"--doctags", str(doctags_path),
|
"--doctags", str(doctags_path),
|
||||||
"--pdf", self.pdf_file,
|
"--pdf", self.pdf_file,
|
||||||
"--page", str(page_num)
|
"--page", str(page_num)
|
||||||
|
|
@ -272,9 +278,9 @@ class BatchProcessor:
|
||||||
if page_doctags.exists():
|
if page_doctags.exists():
|
||||||
shutil.copy2(page_doctags, doctags_path)
|
shutil.copy2(page_doctags, doctags_path)
|
||||||
|
|
||||||
# Let extractor create files in its default location first
|
# Update path to use backend directory
|
||||||
command = [
|
command = [
|
||||||
"python", "picture_extractor.py",
|
"python", "backend/page_treatment/picture_extractor.py",
|
||||||
"--doctags", str(doctags_path),
|
"--doctags", str(doctags_path),
|
||||||
"--pdf", self.pdf_file,
|
"--pdf", self.pdf_file,
|
||||||
"--page", str(page_num)
|
"--page", str(page_num)
|
||||||
0
backend/page_treatment/__init__.py
Normal file
0
backend/page_treatment/__init__.py
Normal file
|
|
@ -28,10 +28,17 @@ from docling_core.types.doc.document import DocTagsDocument, DoclingDocument
|
||||||
|
|
||||||
def ensure_results_folder():
|
def ensure_results_folder():
|
||||||
"""Create the results folder if it doesn't exist."""
|
"""Create the results folder if it doesn't exist."""
|
||||||
results_dir = Path("results")
|
# Get the project root directory (where the script is called from)
|
||||||
|
# Since we're in backend/page_treatment/, we need to go up to the root
|
||||||
|
script_dir = Path(__file__).parent
|
||||||
|
project_root = script_dir.parent.parent # Go up two levels from backend/page_treatment/
|
||||||
|
results_dir = project_root / "results"
|
||||||
|
|
||||||
if not results_dir.exists():
|
if not results_dir.exists():
|
||||||
results_dir.mkdir()
|
results_dir.mkdir(parents=True)
|
||||||
print(f"Created results directory: {results_dir}")
|
print(f"Created results directory: {results_dir}")
|
||||||
|
|
||||||
|
print(f"Using results directory: {results_dir.absolute()}")
|
||||||
return results_dir
|
return results_dir
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -23,11 +23,17 @@ def ensure_results_folder(custom_path=None):
|
||||||
if custom_path:
|
if custom_path:
|
||||||
results_dir = Path(custom_path)
|
results_dir = Path(custom_path)
|
||||||
else:
|
else:
|
||||||
results_dir = Path("results")
|
# Get the project root directory (where the script is called from)
|
||||||
|
# Since we're in backend/page_treatment/, we need to go up to the root
|
||||||
|
script_dir = Path(__file__).parent
|
||||||
|
project_root = script_dir.parent.parent # Go up two levels from backend/page_treatment/
|
||||||
|
results_dir = project_root / "results"
|
||||||
|
|
||||||
if not results_dir.exists():
|
if not results_dir.exists():
|
||||||
results_dir.mkdir(parents=True)
|
results_dir.mkdir(parents=True)
|
||||||
print(f"Created directory: {results_dir}")
|
print(f"Created directory: {results_dir}")
|
||||||
|
|
||||||
|
print(f"Using results directory: {results_dir.absolute()}")
|
||||||
return results_dir
|
return results_dir
|
||||||
|
|
||||||
def parse_arguments():
|
def parse_arguments():
|
||||||
|
|
@ -4,7 +4,7 @@ DocTags Zone Visualizer - Simple script to visualize zones identified in DocTags
|
||||||
PNG-only version: Creates debug images with rectangles around zones.
|
PNG-only version: Creates debug images with rectangles around zones.
|
||||||
|
|
||||||
Usage:
|
Usage:
|
||||||
python visualizer_png_only.py --doctags output.doctags.txt --pdf document.pdf --page 8
|
python visualizer.py --doctags output.doctags.txt --pdf document.pdf --page 8
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import argparse
|
import argparse
|
||||||
|
|
@ -20,10 +20,17 @@ LOC_PATTERN = r'<loc_(\d+)><loc_(\d+)><loc_(\d+)><loc_(\d+)>'
|
||||||
|
|
||||||
def ensure_results_folder():
|
def ensure_results_folder():
|
||||||
"""Create the results folder if it doesn't exist."""
|
"""Create the results folder if it doesn't exist."""
|
||||||
results_dir = Path("results")
|
# Get the project root directory (where the script is called from)
|
||||||
|
# Since we're in backend/page_treatment/, we need to go up to the root
|
||||||
|
script_dir = Path(__file__).parent
|
||||||
|
project_root = script_dir.parent.parent # Go up two levels from backend/page_treatment/
|
||||||
|
results_dir = project_root / "results"
|
||||||
|
|
||||||
if not results_dir.exists():
|
if not results_dir.exists():
|
||||||
results_dir.mkdir()
|
results_dir.mkdir(parents=True)
|
||||||
print(f"Created results directory: {results_dir}")
|
print(f"Created results directory: {results_dir}")
|
||||||
|
|
||||||
|
print(f"Using results directory: {results_dir.absolute()}")
|
||||||
return results_dir
|
return results_dir
|
||||||
|
|
||||||
def parse_arguments():
|
def parse_arguments():
|
||||||
|
|
@ -227,6 +234,7 @@ def create_debug_image(image, zones, page_num, output_path):
|
||||||
# Save the debug image
|
# Save the debug image
|
||||||
debug_img.save(output_path)
|
debug_img.save(output_path)
|
||||||
print(f"Debug image saved to: {output_path}")
|
print(f"Debug image saved to: {output_path}")
|
||||||
|
print(f"Absolute path: {output_path.absolute()}")
|
||||||
|
|
||||||
return debug_img
|
return debug_img
|
||||||
|
|
||||||
|
|
@ -270,6 +278,9 @@ def process_page(pdf_path, page_num, doctags_path, output_path, dpi=200, scale=1
|
||||||
output_name = f"visualization_page_{page_num}.png"
|
output_name = f"visualization_page_{page_num}.png"
|
||||||
output_path = results_dir / output_name
|
output_path = results_dir / output_name
|
||||||
|
|
||||||
|
# Make sure output_path is a Path object
|
||||||
|
output_path = Path(output_path)
|
||||||
|
|
||||||
# Load the page image
|
# Load the page image
|
||||||
try:
|
try:
|
||||||
image = load_image_from_pdf(pdf_path, page_num, dpi)
|
image = load_image_from_pdf(pdf_path, page_num, dpi)
|
||||||
|
|
@ -372,9 +383,6 @@ def process_all_pages(pdf_path, doctags_path, output_base, dpi=200, scale=1.0, s
|
||||||
return True
|
return True
|
||||||
|
|
||||||
def main():
|
def main():
|
||||||
# Ensure results folder exists
|
|
||||||
ensure_results_folder()
|
|
||||||
|
|
||||||
# Parse arguments
|
# Parse arguments
|
||||||
args = parse_arguments()
|
args = parse_arguments()
|
||||||
|
|
||||||
|
|
@ -3,12 +3,26 @@
|
||||||
<head>
|
<head>
|
||||||
<meta charset="UTF-8">
|
<meta charset="UTF-8">
|
||||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||||
<title>DocTags Analyzer & Visualizer</title>
|
<title>SCUB-Doc Intelligence Suite - Professional Document Analysis</title>
|
||||||
<link rel="stylesheet" href="static/styles.css">
|
<link rel="stylesheet" href="static/styles.css">
|
||||||
</head>
|
</head>
|
||||||
<body>
|
<body>
|
||||||
<div class="container">
|
<div class="container">
|
||||||
<h1>DocTags Analyzer & Visualizer</h1>
|
<!-- Professional Enterprise-Style Title -->
|
||||||
|
<div class="title-enterprise">
|
||||||
|
<div class="brand-icon">
|
||||||
|
<svg viewBox="0 0 24 24">
|
||||||
|
<path d="M6,2A2,2 0 0,0 4,4V20A2,2 0 0,0 6,22H18A2,2 0 0,0 20,20V8L14,2H6M6,4H13V9H18V20H6V4M8,12V14H16V12H8M8,16V18H13V16H8Z"/>
|
||||||
|
</svg>
|
||||||
|
</div>
|
||||||
|
<div class="brand-text">
|
||||||
|
<h1>
|
||||||
|
DocTags Intelligence Suite
|
||||||
|
<span class="pro-badge">AI powered</span>
|
||||||
|
</h1>
|
||||||
|
<p class="tagline">Enterprise-grade document processing and analysis</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
<!-- Settings Panel -->
|
<!-- Settings Panel -->
|
||||||
<div class="settings-panel">
|
<div class="settings-panel">
|
||||||
|
|
|
||||||
|
|
@ -327,6 +327,11 @@ function checkEnvironment() {
|
||||||
} else {
|
} else {
|
||||||
html += '<li class="env-error">✗ Results directory is not writable</li>';
|
html += '<li class="env-error">✗ Results directory is not writable</li>';
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Show files in results directory
|
||||||
|
if (data.results_files && data.results_files.length > 0) {
|
||||||
|
html += '<li>Files in results directory: <code>' + data.results_files.join(', ') + '</code></li>';
|
||||||
|
}
|
||||||
} else {
|
} else {
|
||||||
html += '<li class="env-error">✗ Results directory does not exist</li>';
|
html += '<li class="env-error">✗ Results directory does not exist</li>';
|
||||||
}
|
}
|
||||||
|
|
@ -338,6 +343,18 @@ function checkEnvironment() {
|
||||||
data.files.join('\n') + '</pre></details>';
|
data.files.join('\n') + '</pre></details>';
|
||||||
|
|
||||||
envDetails.innerHTML = html;
|
envDetails.innerHTML = html;
|
||||||
|
|
||||||
|
// Also check debug-results endpoint
|
||||||
|
fetch('/debug-results')
|
||||||
|
.then(response => response.json())
|
||||||
|
.then(debugData => {
|
||||||
|
html += '<details><summary>Results Directory Debug Info</summary><pre>' +
|
||||||
|
JSON.stringify(debugData, null, 2) + '</pre></details>';
|
||||||
|
envDetails.innerHTML = html;
|
||||||
|
})
|
||||||
|
.catch(error => {
|
||||||
|
console.error('Error getting debug info:', error);
|
||||||
|
});
|
||||||
})
|
})
|
||||||
.catch(error => {
|
.catch(error => {
|
||||||
envDetails.innerHTML = '<div class="env-error">Error checking environment: ' + error.message + '</div>';
|
envDetails.innerHTML = '<div class="env-error">Error checking environment: ' + error.message + '</div>';
|
||||||
|
|
@ -346,7 +363,7 @@ function checkEnvironment() {
|
||||||
|
|
||||||
// Manual command execution for debugging
|
// Manual command execution for debugging
|
||||||
function manuallyRunScript() {
|
function manuallyRunScript() {
|
||||||
const command = prompt("Enter command to run (e.g., 'python analyzer.py --image document.pdf --page 1')");
|
const command = prompt("Enter command to run (e.g., 'python backend/page_treatment/analyzer.py --image document.pdf --page 1')");
|
||||||
if (!command) return;
|
if (!command) return;
|
||||||
|
|
||||||
const outputDiv = document.getElementById('output');
|
const outputDiv = document.getElementById('output');
|
||||||
|
|
@ -423,8 +440,11 @@ function pollTasks() {
|
||||||
|
|
||||||
// Show image if available for visualizer
|
// Show image if available for visualizer
|
||||||
if (taskInfo.type === 'visualizer' && data.image_file) {
|
if (taskInfo.type === 'visualizer' && data.image_file) {
|
||||||
|
// Fix: Ensure we're using the correct path format
|
||||||
generatedOutputs.visualizer = '/' + data.image_file;
|
generatedOutputs.visualizer = '/' + data.image_file;
|
||||||
document.getElementById('result-image').src = generatedOutputs.visualizer + '?t=' + new Date().getTime();
|
const imagePath = generatedOutputs.visualizer + '?t=' + new Date().getTime();
|
||||||
|
console.log('Loading visualization from:', imagePath);
|
||||||
|
document.getElementById('result-image').src = imagePath;
|
||||||
document.getElementById('image-container').classList.remove('hidden');
|
document.getElementById('image-container').classList.remove('hidden');
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -31,6 +31,128 @@
|
||||||
--spacing-lg: 24px;
|
--spacing-lg: 24px;
|
||||||
--spacing-xl: 32px;
|
--spacing-xl: 32px;
|
||||||
}
|
}
|
||||||
|
/* Professional Enterprise Title Styles - Add this after the :root variables in styles.css */
|
||||||
|
|
||||||
|
/* Enterprise Style Title */
|
||||||
|
.title-enterprise {
|
||||||
|
display: flex;
|
||||||
|
align-items: center;
|
||||||
|
justify-content: center;
|
||||||
|
gap: 24px;
|
||||||
|
margin-bottom: 48px;
|
||||||
|
padding-top: 16px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.title-enterprise .brand-icon {
|
||||||
|
display: flex;
|
||||||
|
align-items: center;
|
||||||
|
justify-content: center;
|
||||||
|
width: 64px;
|
||||||
|
height: 64px;
|
||||||
|
background: var(--surface-white);
|
||||||
|
border-radius: 16px;
|
||||||
|
box-shadow: var(--shadow-lg);
|
||||||
|
position: relative;
|
||||||
|
overflow: hidden;
|
||||||
|
flex-shrink: 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
.title-enterprise .brand-icon::before {
|
||||||
|
content: '';
|
||||||
|
position: absolute;
|
||||||
|
top: -50%;
|
||||||
|
left: -50%;
|
||||||
|
width: 200%;
|
||||||
|
height: 200%;
|
||||||
|
background: linear-gradient(45deg, var(--accent-mint) 0%, transparent 70%);
|
||||||
|
transform: rotate(45deg);
|
||||||
|
}
|
||||||
|
|
||||||
|
.title-enterprise .brand-icon svg {
|
||||||
|
width: 36px;
|
||||||
|
height: 36px;
|
||||||
|
fill: var(--primary-dark);
|
||||||
|
position: relative;
|
||||||
|
z-index: 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
.title-enterprise .brand-text {
|
||||||
|
text-align: left;
|
||||||
|
}
|
||||||
|
|
||||||
|
.title-enterprise h1 {
|
||||||
|
font-size: 2.25rem;
|
||||||
|
font-weight: 600;
|
||||||
|
color: var(--primary-dark);
|
||||||
|
margin: 0 0 4px 0;
|
||||||
|
letter-spacing: -0.025em;
|
||||||
|
display: flex;
|
||||||
|
align-items: baseline;
|
||||||
|
gap: 12px;
|
||||||
|
line-height: 1.2;
|
||||||
|
}
|
||||||
|
|
||||||
|
.title-enterprise .pro-badge {
|
||||||
|
display: inline-flex;
|
||||||
|
align-items: center;
|
||||||
|
padding: 4px 12px;
|
||||||
|
background: var(--accent-amber);
|
||||||
|
color: var(--primary-dark);
|
||||||
|
font-size: 0.75rem;
|
||||||
|
font-weight: 700;
|
||||||
|
letter-spacing: 0.05em;
|
||||||
|
border-radius: 6px;
|
||||||
|
text-transform: uppercase;
|
||||||
|
line-height: 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
.title-enterprise .tagline {
|
||||||
|
font-size: 1rem;
|
||||||
|
color: var(--text-medium);
|
||||||
|
font-weight: 400;
|
||||||
|
margin: 0;
|
||||||
|
letter-spacing: 0.01em;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Update the old h1 styles to remove conflicts */
|
||||||
|
body > .container > h1 {
|
||||||
|
display: none; /* Hide if there's still an old h1 */
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Responsive adjustments for the title */
|
||||||
|
@media (max-width: 768px) {
|
||||||
|
.title-enterprise {
|
||||||
|
flex-direction: column;
|
||||||
|
text-align: center;
|
||||||
|
gap: 16px;
|
||||||
|
margin-bottom: 32px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.title-enterprise .brand-text {
|
||||||
|
text-align: center;
|
||||||
|
}
|
||||||
|
|
||||||
|
.title-enterprise h1 {
|
||||||
|
font-size: 1.75rem;
|
||||||
|
flex-direction: column;
|
||||||
|
gap: 8px;
|
||||||
|
align-items: center;
|
||||||
|
}
|
||||||
|
|
||||||
|
.title-enterprise .brand-icon {
|
||||||
|
width: 56px;
|
||||||
|
height: 56px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.title-enterprise .brand-icon svg {
|
||||||
|
width: 32px;
|
||||||
|
height: 32px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.title-enterprise .tagline {
|
||||||
|
font-size: 0.9rem;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
* {
|
* {
|
||||||
box-sizing: border-box;
|
box-sizing: border-box;
|
||||||
|
|
@ -844,4 +966,5 @@ button:disabled {
|
||||||
font-size: 0.8rem;
|
font-size: 0.8rem;
|
||||||
padding: var(--spacing-xs) var(--spacing-sm);
|
padding: var(--spacing-xs) var(--spacing-sm);
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
20
run_app.sh
Normal file → Executable file
20
run_app.sh
Normal file → Executable file
|
|
@ -0,0 +1,20 @@
|
||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
echo "Starting DocTags Application..."
|
||||||
|
|
||||||
|
# Check if backend directory exists
|
||||||
|
if [ ! -d "backend" ]; then
|
||||||
|
echo "Error: backend directory not found!"
|
||||||
|
echo "Please run the reorganization script first."
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Check if app.py exists in backend
|
||||||
|
if [ ! -f "backend/app.py" ]; then
|
||||||
|
echo "Error: backend/app.py not found!"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Run the Flask application
|
||||||
|
echo "Running Flask application..."
|
||||||
|
python backend/app.py
|
||||||
Loading…
Reference in a new issue