danvics/docling-studio
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

Update project structure to isolate backend files in order to preprare containerisation properly

Browse source
This commit is contained in:
pjmalandrino 2025-05-30 10:18:08 +02:00
parent 4d998251a9
commit 9945a29a8e
12 changed files with 360 additions and 47 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

0
backend/__init__.py Normal file
View file

169
app.py → backend/app.py
View file

@ -1,22 +1,27 @@
from flask import Flask, request, send_file, jsonify, Response
import subprocess
import os
import sys
import time
import threading
import logging
from pathlib import Path
import sys
import uuid
import pdf2image
# Add the parent directory to Python path to allow imports
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
# Configure logging
logging.basicConfig(level=logging.INFO,
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
logger = logging.getLogger(__name__)
# Initialize Flask app with frontend folder structure
# Frontend is in the parent directory
frontend_path = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), 'frontend')
app = Flask(__name__,
static_folder='frontend/static',
static_folder=os.path.join(frontend_path, 'static'),
static_url_path='/static')
# Dictionary to store background task results
@ -24,7 +29,7 @@ task_results = {}
# Import batch processor if available
try:
from batch_processor import start_batch_processing, get_batch_processor, cleanup_old_batches
from backend.batch_treatment.batch_processor import start_batch_processing, get_batch_processor, cleanup_old_batches
batch_processing_available = True
except ImportError:
logger.warning("batch_processor.py not found. Batch processing features will be disabled.")
@ -32,14 +37,18 @@ except ImportError:
# Ensure results folder exists
def ensure_results_folder():
results_dir = Path("results")
# Always create results folder relative to where app.py is run from
results_dir = Path.cwd() / "results"
if not results_dir.exists():
results_dir.mkdir()
logger.info(f"Created results directory at: {results_dir.absolute()}")
else:
logger.info(f"Results directory exists at: {results_dir.absolute()}")
return results_dir
# Ensure frontend folders exist
def ensure_frontend_folders():
frontend_dir = Path("frontend")
frontend_dir = Path(frontend_path)
if not frontend_dir.exists():
frontend_dir.mkdir()
logger.info(f"Created frontend directory: {frontend_dir}")
@ -53,16 +62,16 @@ def ensure_frontend_folders():
@app.route('/')
def index():
return send_file('frontend/index.html')
return send_file(os.path.join(frontend_path, 'index.html'))
@app.route('/batch')
def batch_interface():
"""Serve the batch processing interface"""
return send_file('frontend/batch.html')
return send_file(os.path.join(frontend_path, 'batch.html'))
@app.route('/static/<path:filename>')
def serve_static(filename):
return send_file(os.path.join('frontend/static', filename))
return send_file(os.path.join(frontend_path, 'static', filename))
@app.route('/pdf-files')
def pdf_files():
@ -123,19 +132,22 @@ def run_command(task_id, command):
current_dir = os.getcwd()
logger.info(f"Current working directory: {current_dir}")
# Check if the script exists
script_name = command.split()[1]
if not os.path.exists(script_name):
logger.error(f"Script not found: {script_name} in {current_dir}")
# The command already contains the full path, so we just need to verify it exists
script_parts = command.split()
script_path = script_parts[1]
if not os.path.exists(script_path):
logger.error(f"Script not found: {script_path} in {current_dir}")
task_results[task_id] = {
'success': False,
'error': f"Script not found: {script_name}. Make sure all scripts are in the same directory as app.py.",
'error': f"Script not found: {script_path}. Make sure all scripts are in the correct directory.",
'done': True
}
return
# Run the command with more detailed error capture
# Important: Use pipe input to automatically answer "n" to the prompt
# Make sure we run from the project root directory
process = subprocess.Popen(
command,
shell=True,
@ -143,7 +155,8 @@ def run_command(task_id, command):
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True,
universal_newlines=True
universal_newlines=True,
cwd=os.getcwd() # Explicitly set working directory to current directory
)
# Send "n" to the process to bypass the "process all pages" prompt
@ -198,13 +211,14 @@ def run_analyzer():
logger.error(f"PDF file not found: {pdf_file}")
return jsonify({'success': False, 'error': f'PDF file not found: {pdf_file}'}), 400
# Check if analyzer.py exists
if not os.path.exists('analyzer.py'):
logger.error("analyzer.py not found in current directory")
return jsonify({'success': False, 'error': 'analyzer.py not found in current directory'}), 500
# Check if analyzer.py exists in the new location
analyzer_path = os.path.join('backend', 'page_treatment', 'analyzer.py')
if not os.path.exists(analyzer_path):
logger.error(f"analyzer.py not found at: {analyzer_path}")
return jsonify({'success': False, 'error': 'analyzer.py not found in backend/page_treatment directory'}), 500
# Add the --all-pages=false flag or explicitly specify the page to avoid the prompt
command = f"python analyzer.py --image {pdf_file} --page {page_num} --start-page {page_num} --end-page {page_num}"
command = f"python backend/page_treatment/analyzer.py --image {pdf_file} --page {page_num} --start-page {page_num} --end-page {page_num}"
# Generate a task ID
task_id = f"analyzer_{int(time.time())}"
@ -238,15 +252,40 @@ def task_status(task_id):
if task_id not in task_results:
return jsonify({'success': False, 'error': 'Task not found'}), 404
result = task_results[task_id]
result = task_results[task_id].copy() # Make a copy to avoid modifying the original
# If task is completed, add file paths
# If task is completed, add file paths and verify they exist
if result['done'] and result['success']:
if task_id.startswith('analyzer_'):
result['doctags_file'] = f"results/output.doctags.txt"
doctags_path = Path("results") / "output.doctags.txt"
if doctags_path.exists():
result['doctags_file'] = "results/output.doctags.txt"
else:
logger.warning(f"DocTags file not found: {doctags_path}")
elif task_id.startswith('visualizer_'):
page_num = task_id.split('_')[-1] if len(task_id.split('_')) > 2 else "1"
result['image_file'] = f"results/visualization_page_{page_num}.png"
viz_filename = f"visualization_page_{page_num}.png"
# Check multiple possible locations
possible_paths = [
Path("results") / viz_filename,
Path.cwd() / "results" / viz_filename,
Path(__file__).parent.parent / "results" / viz_filename
]
for path in possible_paths:
if path.exists():
result['image_file'] = f"results/{viz_filename}"
logger.info(f"Found visualization at: {path}")
break
else:
logger.error(f"Visualization file not found in any location for page {page_num}")
# Log what files exist in results
results_dir = Path("results")
if results_dir.exists():
files = list(results_dir.glob("*.png"))
logger.info(f"PNG files in results: {[f.name for f in files[:5]]}")
return jsonify(result)
@ -260,7 +299,7 @@ def run_visualizer():
if not pdf_file:
return jsonify({'success': False, 'error': 'PDF file not specified'}), 400
command = f"python visualizer.py --doctags results/output.doctags.txt --pdf {pdf_file} --page {page_num}"
command = f"python backend/page_treatment/visualizer.py --doctags results/output.doctags.txt --pdf {pdf_file} --page {page_num}"
if adjust:
command += " --adjust"
@ -298,7 +337,7 @@ def run_extractor():
if not pdf_file:
return jsonify({'success': False, 'error': 'PDF file not specified'}), 400
command = f"python picture_extractor.py --doctags results/output.doctags.txt --pdf {pdf_file} --page {page_num}"
command = f"python backend/page_treatment/picture_extractor.py --doctags results/output.doctags.txt --pdf {pdf_file} --page {page_num}"
if adjust:
command += " --adjust"
@ -329,14 +368,28 @@ def run_extractor():
@app.route('/results/<path:filename>')
def results(filename):
try:
file_path = os.path.join('results', filename)
if os.path.exists(file_path):
return send_file(file_path)
# Always use the results directory relative to current working directory
results_dir = Path.cwd() / "results"
file_path = results_dir / filename
logger.info(f"Requested file: {filename}")
logger.info(f"Looking for file at: {file_path.absolute()}")
logger.info(f"File exists: {file_path.exists()}")
if file_path.exists() and file_path.is_file():
logger.info(f"Serving file: {file_path}")
return send_file(str(file_path.absolute()))
else:
logger.error(f"File not found: {file_path}")
# List what files ARE in the results directory
if results_dir.exists():
files = list(results_dir.glob("*"))
logger.info(f"Files in results directory: {[f.name for f in files[:10]]}")
return jsonify({'error': f"File not found: {filename}"}), 404
except Exception as e:
logger.error(f"Error serving file {filename}: {str(e)}")
import traceback
logger.error(traceback.format_exc())
return jsonify({'error': f"Error serving file: {filename}"}), 500
@app.route('/pdf-preview/<pdf_file>/<int:page_num>')
@ -405,9 +458,15 @@ def check_environment():
# List all files in directory
files = os.listdir('.')
# Check for required scripts
# Check for required scripts in new location
backend_page_dir = os.path.join('backend', 'page_treatment')
required_scripts = ['analyzer.py', 'visualizer.py', 'picture_extractor.py']
missing_scripts = [script for script in required_scripts if script not in files]
missing_scripts = []
for script in required_scripts:
script_path = os.path.join(backend_page_dir, script)
if not os.path.exists(script_path):
missing_scripts.append(script)
# Check for PDFs
pdf_files = [f for f in files if f.endswith('.pdf')]
@ -415,6 +474,7 @@ def check_environment():
# Check for results directory
results_dir = Path("results")
results_dir_exists = results_dir.exists()
results_files = []
# Try to check Python version and installed packages
python_info = subprocess.run(['python', '--version'], capture_output=True, text=True)
@ -429,6 +489,9 @@ def check_environment():
f.write("test")
os.remove(test_file)
results_writable = True
# List files in results directory
results_files = [f.name for f in results_dir.iterdir() if f.is_file()][:20] # Limit to 20 files
except:
pass
@ -439,6 +502,7 @@ def check_environment():
'pdf_files': pdf_files,
'results_dir_exists': results_dir_exists,
'results_dir_writable': results_writable,
'results_files': results_files,
'python_version': python_version,
'batch_processing_available': batch_processing_available
})
@ -460,6 +524,12 @@ def run_manual_command():
logger.info(f"Running manual command: {command}")
# Update paths in manual commands to use backend directory only if not already present
if 'backend/page_treatment/' not in command:
command = command.replace('analyzer.py', 'backend/page_treatment/analyzer.py')
command = command.replace('visualizer.py', 'backend/page_treatment/visualizer.py')
command = command.replace('picture_extractor.py', 'backend/page_treatment/picture_extractor.py')
try:
# Run the command synchronously for immediate feedback
process = subprocess.Popen(
@ -766,6 +836,45 @@ def open_results_folder():
logger.error(f"Error opening results folder: {str(e)}")
return jsonify({'success': False, 'error': str(e)}), 500
@app.route('/debug-results')
def debug_results():
"""Debug endpoint to check results directory"""
try:
results_info = {
'cwd': os.getcwd(),
'results_paths_checked': []
}
# Check multiple possible results locations
possible_results_dirs = [
Path("results"),
Path.cwd() / "results",
Path(__file__).parent.parent / "results"
]
for results_dir in possible_results_dirs:
dir_info = {
'path': str(results_dir),
'absolute_path': str(results_dir.absolute()),
'exists': results_dir.exists(),
'files': []
}
if results_dir.exists():
try:
# List all files in the directory
files = list(results_dir.glob("*"))
dir_info['files'] = [f.name for f in files if f.is_file()][:20] # Limit to 20 files
except Exception as e:
dir_info['error'] = str(e)
results_info['results_paths_checked'].append(dir_info)
return jsonify(results_info)
except Exception as e:
return jsonify({'error': str(e)}), 500
# Cleanup task for batch processors
def cleanup_task():
"""Periodic cleanup of old batch processors"""

0
backend/batch_treatment/__init__.py Normal file
View file

14
batch_processor.py → backend/batch_treatment/batch_processor.py
View file

@ -5,6 +5,7 @@ Handles batch processing of PDF documents with parallel processing support
"""
import os
import sys
import json
import time
import threading
@ -17,6 +18,9 @@ import subprocess
import shutil
import zipfile
# Add the parent directory to Python path to allow imports
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
# Configure logging
logging.basicConfig(
level=logging.INFO,
@ -172,8 +176,9 @@ class BatchProcessor:
# Use standard results directory for analyzer output
output_base = Path("results") / f"output"
# Update path to use backend directory
command = [
"python", "analyzer.py",
"python", "backend/page_treatment/analyzer.py",
"--image", self.pdf_file,
"--page", str(page_num),
"--output", str(output_base),
@ -224,8 +229,9 @@ class BatchProcessor:
if page_doctags.exists():
shutil.copy2(page_doctags, doctags_path)
# Update path to use backend directory
command = [
"python", "visualizer.py",
"python", "backend/page_treatment/visualizer.py",
"--doctags", str(doctags_path),
"--pdf", self.pdf_file,
"--page", str(page_num)
@ -272,9 +278,9 @@ class BatchProcessor:
if page_doctags.exists():
shutil.copy2(page_doctags, doctags_path)
# Let extractor create files in its default location first
# Update path to use backend directory
command = [
"python", "picture_extractor.py",
"python", "backend/page_treatment/picture_extractor.py",
"--doctags", str(doctags_path),
"--pdf", self.pdf_file,
"--page", str(page_num)

0
backend/page_treatment/__init__.py Normal file
View file

11
analyzer.py → backend/page_treatment/analyzer.py
View file

@ -28,10 +28,17 @@ from docling_core.types.doc.document import DocTagsDocument, DoclingDocument
def ensure_results_folder():
"""Create the results folder if it doesn't exist."""
results_dir = Path("results")
# Get the project root directory (where the script is called from)
# Since we're in backend/page_treatment/, we need to go up to the root
script_dir = Path(__file__).parent
project_root = script_dir.parent.parent # Go up two levels from backend/page_treatment/
results_dir = project_root / "results"
if not results_dir.exists():
results_dir.mkdir()
results_dir.mkdir(parents=True)
print(f"Created results directory: {results_dir}")
print(f"Using results directory: {results_dir.absolute()}")
return results_dir

8
picture_extractor.py → backend/page_treatment/picture_extractor.py
View file

@ -23,11 +23,17 @@ def ensure_results_folder(custom_path=None):
if custom_path:
results_dir = Path(custom_path)
else:
results_dir = Path("results")
# Get the project root directory (where the script is called from)
# Since we're in backend/page_treatment/, we need to go up to the root
script_dir = Path(__file__).parent
project_root = script_dir.parent.parent # Go up two levels from backend/page_treatment/
results_dir = project_root / "results"
if not results_dir.exists():
results_dir.mkdir(parents=True)
print(f"Created directory: {results_dir}")
print(f"Using results directory: {results_dir.absolute()}")
return results_dir
def parse_arguments():

20
visualizer.py → backend/page_treatment/visualizer.py
View file

@ -4,7 +4,7 @@ DocTags Zone Visualizer - Simple script to visualize zones identified in DocTags
PNG-only version: Creates debug images with rectangles around zones.
Usage:
python visualizer_png_only.py --doctags output.doctags.txt --pdf document.pdf --page 8
python visualizer.py --doctags output.doctags.txt --pdf document.pdf --page 8
"""
import argparse
@ -20,10 +20,17 @@ LOC_PATTERN = r'<loc_(\d+)><loc_(\d+)><loc_(\d+)><loc_(\d+)>'
def ensure_results_folder():
"""Create the results folder if it doesn't exist."""
results_dir = Path("results")
# Get the project root directory (where the script is called from)
# Since we're in backend/page_treatment/, we need to go up to the root
script_dir = Path(__file__).parent
project_root = script_dir.parent.parent # Go up two levels from backend/page_treatment/
results_dir = project_root / "results"
if not results_dir.exists():
results_dir.mkdir()
results_dir.mkdir(parents=True)
print(f"Created results directory: {results_dir}")
print(f"Using results directory: {results_dir.absolute()}")
return results_dir
def parse_arguments():
@ -227,6 +234,7 @@ def create_debug_image(image, zones, page_num, output_path):
# Save the debug image
debug_img.save(output_path)
print(f"Debug image saved to: {output_path}")
print(f"Absolute path: {output_path.absolute()}")
return debug_img
@ -270,6 +278,9 @@ def process_page(pdf_path, page_num, doctags_path, output_path, dpi=200, scale=1
output_name = f"visualization_page_{page_num}.png"
output_path = results_dir / output_name
# Make sure output_path is a Path object
output_path = Path(output_path)
# Load the page image
try:
image = load_image_from_pdf(pdf_path, page_num, dpi)
@ -372,9 +383,6 @@ def process_all_pages(pdf_path, doctags_path, output_base, dpi=200, scale=1.0, s
return True
def main():
# Ensure results folder exists
ensure_results_folder()
# Parse arguments
args = parse_arguments()

18
frontend/index.html
View file

@ -3,12 +3,26 @@
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>DocTags Analyzer & Visualizer</title>
<title>SCUB-Doc Intelligence Suite - Professional Document Analysis</title>
<link rel="stylesheet" href="static/styles.css">
</head>
<body>
<div class="container">
<h1>DocTags Analyzer & Visualizer</h1>
<!-- Professional Enterprise-Style Title -->
<div class="title-enterprise">
<div class="brand-icon">
<svg viewBox="0 0 24 24">
<path d="M6,2A2,2 0 0,0 4,4V20A2,2 0 0,0 6,22H18A2,2 0 0,0 20,20V8L14,2H6M6,4H13V9H18V20H6V4M8,12V14H16V12H8M8,16V18H13V16H8Z"/>
</svg>
</div>
<div class="brand-text">
<h1>
DocTags Intelligence Suite
<span class="pro-badge">AI powered</span>
</h1>
<p class="tagline">Enterprise-grade document processing and analysis</p>
</div>
</div>
<!-- Settings Panel -->
<div class="settings-panel">

24
frontend/static/app.js
View file

@ -327,6 +327,11 @@ function checkEnvironment() {
} else {
html += '<li class="env-error">✗ Results directory is not writable</li>';
}
// Show files in results directory
if (data.results_files && data.results_files.length > 0) {
html += '<li>Files in results directory: <code>' + data.results_files.join(', ') + '</code></li>';
}
} else {
html += '<li class="env-error">✗ Results directory does not exist</li>';
}
@ -338,6 +343,18 @@ function checkEnvironment() {
data.files.join('\n') + '</pre></details>';
envDetails.innerHTML = html;
// Also check debug-results endpoint
fetch('/debug-results')
.then(response => response.json())
.then(debugData => {
html += '<details><summary>Results Directory Debug Info</summary><pre>' +
JSON.stringify(debugData, null, 2) + '</pre></details>';
envDetails.innerHTML = html;
})
.catch(error => {
console.error('Error getting debug info:', error);
});
})
.catch(error => {
envDetails.innerHTML = '<div class="env-error">Error checking environment: ' + error.message + '</div>';
@ -346,7 +363,7 @@ function checkEnvironment() {
// Manual command execution for debugging
function manuallyRunScript() {
const command = prompt("Enter command to run (e.g., 'python analyzer.py --image document.pdf --page 1')");
const command = prompt("Enter command to run (e.g., 'python backend/page_treatment/analyzer.py --image document.pdf --page 1')");
if (!command) return;
const outputDiv = document.getElementById('output');
@ -423,8 +440,11 @@ function pollTasks() {
// Show image if available for visualizer
if (taskInfo.type === 'visualizer' && data.image_file) {
// Fix: Ensure we're using the correct path format
generatedOutputs.visualizer = '/' + data.image_file;
document.getElementById('result-image').src = generatedOutputs.visualizer + '?t=' + new Date().getTime();
const imagePath = generatedOutputs.visualizer + '?t=' + new Date().getTime();
console.log('Loading visualization from:', imagePath);
document.getElementById('result-image').src = imagePath;
document.getElementById('image-container').classList.remove('hidden');
}

123
frontend/static/styles.css
View file

@ -31,6 +31,128 @@
--spacing-lg: 24px;
--spacing-xl: 32px;
}
/* Professional Enterprise Title Styles - Add this after the :root variables in styles.css */
/* Enterprise Style Title */
.title-enterprise {
display: flex;
align-items: center;
justify-content: center;
gap: 24px;
margin-bottom: 48px;
padding-top: 16px;
}
.title-enterprise .brand-icon {
display: flex;
align-items: center;
justify-content: center;
width: 64px;
height: 64px;
background: var(--surface-white);
border-radius: 16px;
box-shadow: var(--shadow-lg);
position: relative;
overflow: hidden;
flex-shrink: 0;
}
.title-enterprise .brand-icon::before {
content: '';
position: absolute;
top: -50%;
left: -50%;
width: 200%;
height: 200%;
background: linear-gradient(45deg, var(--accent-mint) 0%, transparent 70%);
transform: rotate(45deg);
}
.title-enterprise .brand-icon svg {
width: 36px;
height: 36px;
fill: var(--primary-dark);
position: relative;
z-index: 1;
}
.title-enterprise .brand-text {
text-align: left;
}
.title-enterprise h1 {
font-size: 2.25rem;
font-weight: 600;
color: var(--primary-dark);
margin: 0 0 4px 0;
letter-spacing: -0.025em;
display: flex;
align-items: baseline;
gap: 12px;
line-height: 1.2;
}
.title-enterprise .pro-badge {
display: inline-flex;
align-items: center;
padding: 4px 12px;
background: var(--accent-amber);
color: var(--primary-dark);
font-size: 0.75rem;
font-weight: 700;
letter-spacing: 0.05em;
border-radius: 6px;
text-transform: uppercase;
line-height: 1;
}
.title-enterprise .tagline {
font-size: 1rem;
color: var(--text-medium);
font-weight: 400;
margin: 0;
letter-spacing: 0.01em;
}
/* Update the old h1 styles to remove conflicts */
body > .container > h1 {
display: none; /* Hide if there's still an old h1 */
}
/* Responsive adjustments for the title */
@media (max-width: 768px) {
.title-enterprise {
flex-direction: column;
text-align: center;
gap: 16px;
margin-bottom: 32px;
}
.title-enterprise .brand-text {
text-align: center;
}
.title-enterprise h1 {
font-size: 1.75rem;
flex-direction: column;
gap: 8px;
align-items: center;
}
.title-enterprise .brand-icon {
width: 56px;
height: 56px;
}
.title-enterprise .brand-icon svg {
width: 32px;
height: 32px;
}
.title-enterprise .tagline {
font-size: 0.9rem;
}
}
* {
box-sizing: border-box;
@ -844,4 +966,5 @@ button:disabled {
font-size: 0.8rem;
padding: var(--spacing-xs) var(--spacing-sm);
}
}

20
run_app.sh Normal file → Executable file
View file

@ -0,0 +1,20 @@
#!/bin/bash
echo "Starting DocTags Application..."
# Check if backend directory exists
if [ ! -d "backend" ]; then
echo "Error: backend directory not found!"
echo "Please run the reorganization script first."
exit 1
fi
# Check if app.py exists in backend
if [ ! -f "backend/app.py" ]; then
echo "Error: backend/app.py not found!"
exit 1
fi
# Run the Flask application
echo "Running Flask application..."
python backend/app.py