Fix for scaling issues and clean of useless files
This commit is contained in:
pjmalandrino
parent
98e4afe491
commit
14c532b138
7 changed files with 43 additions and 2080 deletions
19
README.md
19
README.md
|
|
@ -1,18 +1 @@
|
|||
pip install --upgrade pip
|
||||
pip install pdf2image
|
||||
|
||||
|
||||
# Generate DocTags for all pages and run the visualizer
|
||||
python doctags_gen.py --pdf document.pdf --output results/ && python create_index.py --directory results/
|
||||
|
||||
python analyzer.py --image document.pdf --page 13
|
||||
|
||||
python visualizer.py --doctags output.doctags.txt --pdf document.pdf --page 13 --adjust --show
|
||||
|
||||
|
||||
python analyzer.py --image document.pdf --page 13 && python visualizer.py --doctags output.doctags.txt --pdf document.pdf --page 13 --adjust --show
|
||||
|
||||
|
||||
|
||||
## Run full with scaling fix
|
||||
python analyzer.py --image document.pdf --page 7 && python fix_scaling.py --doctags output.doctags.txt --output fixed_output.doctags.txt --x-factor 0.7 --y-factor 0.7 && python visualizer.py --doctags fixed_output.doctags.txt --pdf document.pdf --page 7 --adjust --show
|
||||
python analyzer.py --image document.pdf --page 8 && python visualizer.py --doctags output.doctags.txt --pdf document.pdf --page 8 --adjust --show
|
||||
|
|
@ -1,213 +0,0 @@
|
|||
#!/bin/bash
|
||||
# find_optimal_scale_fixed.sh
|
||||
# This script helps find the optimal scaling factors for a PDF page with improved error handling
|
||||
|
||||
# Default values
|
||||
PDF_FILE=""
|
||||
PAGE=1
|
||||
MIN_X=0.1
|
||||
MAX_X=1.5
|
||||
MIN_Y=0.1
|
||||
MAX_Y=1.5
|
||||
STEPS=5
|
||||
|
||||
# Function to display usage information
|
||||
function show_usage {
|
||||
echo "Usage: $0 -f PDF_FILE [options]"
|
||||
echo ""
|
||||
echo "Required:"
|
||||
echo " -f, --file FILE Path to the PDF file"
|
||||
echo ""
|
||||
echo "Options:"
|
||||
echo " -p, --page PAGE Page number to analyze (default: 1)"
|
||||
echo " --min-x VALUE Minimum X scaling factor (default: 0.1)"
|
||||
echo " --max-x VALUE Maximum X scaling factor (default: 1.5)"
|
||||
echo " --min-y VALUE Minimum Y scaling factor (default: 0.1)"
|
||||
echo " --max-y VALUE Maximum Y scaling factor (default: 1.5)"
|
||||
echo " --steps NUMBER Number of steps between min and max (default: 5)"
|
||||
echo " -h, --help Display this help message"
|
||||
echo ""
|
||||
echo "Example:"
|
||||
echo " $0 -f document.pdf -p 3 --min-x 0.5 --max-x 1.0 --steps 6"
|
||||
}
|
||||
|
||||
# Parse command line arguments
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
-f|--file)
|
||||
PDF_FILE="$2"
|
||||
shift 2
|
||||
;;
|
||||
-p|--page)
|
||||
PAGE="$2"
|
||||
shift 2
|
||||
;;
|
||||
--min-x)
|
||||
MIN_X="$2"
|
||||
shift 2
|
||||
;;
|
||||
--max-x)
|
||||
MAX_X="$2"
|
||||
shift 2
|
||||
;;
|
||||
--min-y)
|
||||
MIN_Y="$2"
|
||||
shift 2
|
||||
;;
|
||||
--max-y)
|
||||
MAX_Y="$2"
|
||||
shift 2
|
||||
;;
|
||||
--steps)
|
||||
STEPS="$2"
|
||||
shift 2
|
||||
;;
|
||||
-h|--help)
|
||||
show_usage
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
echo "Error: Unknown option $1"
|
||||
show_usage
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
# Check if PDF file is provided
|
||||
if [ -z "$PDF_FILE" ]; then
|
||||
echo "Error: PDF file is required"
|
||||
show_usage
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Check if PDF file exists
|
||||
if [ ! -f "$PDF_FILE" ]; then
|
||||
echo "Error: PDF file '$PDF_FILE' not found"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Get PDF basename for output files
|
||||
PDF_BASENAME=$(basename "$PDF_FILE" .pdf)
|
||||
OUTPUT_DIR="${PDF_BASENAME}_scale_test"
|
||||
mkdir -p "$OUTPUT_DIR"
|
||||
|
||||
# Step 1: Run analyzer once to get the DocTags
|
||||
echo "Step 1: Analyzing page $PAGE to generate DocTags..."
|
||||
DOCTAGS_FILE="$OUTPUT_DIR/page_${PAGE}.doctags.txt"
|
||||
python analyzer.py --image "$PDF_FILE" --page "$PAGE" --output "$DOCTAGS_FILE"
|
||||
|
||||
# Check for DocTags file with different extensions
|
||||
if [ ! -f "$DOCTAGS_FILE" ]; then
|
||||
# Try alternative extensions
|
||||
ALTERNATIVE_FILE="$OUTPUT_DIR/page_${PAGE}.doctags.doctags.txt"
|
||||
if [ -f "$ALTERNATIVE_FILE" ]; then
|
||||
echo "Found DocTags at alternative location: $ALTERNATIVE_FILE"
|
||||
DOCTAGS_FILE="$ALTERNATIVE_FILE"
|
||||
else
|
||||
# Try to find any doctags file in the output directory
|
||||
FOUND_FILE=$(find "$OUTPUT_DIR" -name "*.doctags.txt" -print -quit)
|
||||
if [ -n "$FOUND_FILE" ]; then
|
||||
echo "Found DocTags at location: $FOUND_FILE"
|
||||
DOCTAGS_FILE="$FOUND_FILE"
|
||||
else
|
||||
echo "Error: Failed to generate DocTags for page $PAGE"
|
||||
echo "Could not find any doctags files in $OUTPUT_DIR"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
echo "Using DocTags file: $DOCTAGS_FILE"
|
||||
|
||||
# Create HTML index to easily compare results
|
||||
INDEX_FILE="$OUTPUT_DIR/index.html"
|
||||
echo "<!DOCTYPE html>
|
||||
<html>
|
||||
<head>
|
||||
<meta charset='UTF-8'>
|
||||
<title>Scale Factor Test - Page $PAGE</title>
|
||||
<style>
|
||||
body { font-family: Arial, sans-serif; margin: 20px; }
|
||||
.grid { display: grid; grid-template-columns: repeat(auto-fill, minmax(300px, 1fr)); gap: 20px; }
|
||||
.card { border: 1px solid #ccc; border-radius: 5px; padding: 10px; }
|
||||
.card h3 { margin-top: 0; text-align: center; }
|
||||
.card img { max-width: 100%; border: 1px solid #eee; }
|
||||
.card a { display: block; text-align: center; margin-top: 10px; }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<h1>Scale Factor Test - Page $PAGE of $PDF_BASENAME.pdf</h1>
|
||||
<p>Click on any thumbnail to open the full visualization.</p>
|
||||
<div class='grid'>" > "$INDEX_FILE"
|
||||
|
||||
# Step 2: Try different combinations of scaling factors
|
||||
echo "Step 2: Testing different scaling factors..."
|
||||
|
||||
# Calculate step sizes
|
||||
X_STEP=$(echo "scale=6; ($MAX_X - $MIN_X) / $STEPS" | bc)
|
||||
Y_STEP=$(echo "scale=6; ($MAX_Y - $MIN_Y) / $STEPS" | bc)
|
||||
|
||||
for i in $(seq 0 $STEPS); do
|
||||
X_FACTOR=$(echo "scale=2; $MIN_X + ($i * $X_STEP)" | bc)
|
||||
|
||||
for j in $(seq 0 $STEPS); do
|
||||
Y_FACTOR=$(echo "scale=2; $MIN_Y + ($j * $Y_STEP)" | bc)
|
||||
|
||||
echo "Testing X=$X_FACTOR, Y=$Y_FACTOR"
|
||||
|
||||
# Generate fixed DocTags
|
||||
FIXED_DOCTAGS="$OUTPUT_DIR/page_${PAGE}_x${X_FACTOR}_y${Y_FACTOR}.doctags.txt"
|
||||
python fix_scaling.py --doctags "$DOCTAGS_FILE" --output "$FIXED_DOCTAGS" --x-factor "$X_FACTOR" --y-factor "$Y_FACTOR" > /dev/null
|
||||
|
||||
# Generate visualization
|
||||
HTML_OUTPUT="$OUTPUT_DIR/page_${PAGE}_x${X_FACTOR}_y${Y_FACTOR}.html"
|
||||
python visualizer.py --doctags "$FIXED_DOCTAGS" --pdf "$PDF_FILE" --page "$PAGE" --output "$HTML_OUTPUT" --adjust > /dev/null
|
||||
|
||||
# Generate debug image for thumbnail
|
||||
DEBUG_IMG="$OUTPUT_DIR/page_${PAGE}_x${X_FACTOR}_y${Y_FACTOR}.debug.png"
|
||||
|
||||
# Check if the debug image exists
|
||||
if [ ! -f "$DEBUG_IMG" ]; then
|
||||
echo "Warning: Debug image not found, trying alternative name..."
|
||||
ALT_DEBUG_IMG=$(find "$OUTPUT_DIR" -name "page_${PAGE}_x${X_FACTOR}_y${Y_FACTOR}*.png" -print -quit)
|
||||
if [ -n "$ALT_DEBUG_IMG" ]; then
|
||||
DEBUG_IMG="$ALT_DEBUG_IMG"
|
||||
fi
|
||||
fi
|
||||
|
||||
# Add to index only if files exist
|
||||
if [ -f "$HTML_OUTPUT" ] && [ -f "$DEBUG_IMG" ]; then
|
||||
echo " <div class='card'>
|
||||
<h3>X=$X_FACTOR, Y=$Y_FACTOR</h3>
|
||||
<a href='$(basename "$HTML_OUTPUT")'>
|
||||
<img src='$(basename "$DEBUG_IMG")' alt='X=$X_FACTOR, Y=$Y_FACTOR'>
|
||||
</a>
|
||||
<a href='$(basename "$HTML_OUTPUT")'>Open Visualization</a>
|
||||
</div>" >> "$INDEX_FILE"
|
||||
else
|
||||
echo "Warning: Skipping this combination as files not created successfully."
|
||||
fi
|
||||
done
|
||||
done
|
||||
|
||||
# Finish HTML index
|
||||
echo " </div>
|
||||
</body>
|
||||
</html>" >> "$INDEX_FILE"
|
||||
|
||||
echo "
|
||||
Scale factor testing complete!
|
||||
Open $INDEX_FILE in a web browser to compare results.
|
||||
"
|
||||
|
||||
# Try to open the index file in a browser
|
||||
if command -v xdg-open &> /dev/null; then
|
||||
xdg-open "$INDEX_FILE" &
|
||||
elif command -v open &> /dev/null; then
|
||||
open "$INDEX_FILE" &
|
||||
elif command -v explorer &> /dev/null; then
|
||||
explorer "$INDEX_FILE" &
|
||||
fi
|
||||
|
||||
exit 0
|
||||
196
fix_scaling.py
196
fix_scaling.py
|
|
@ -1,196 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
"""
|
||||
DocTags Scaling Fix - Script to fix scaling issues in the DocTags visualizer.
|
||||
|
||||
Usage:
|
||||
python fix_scaling.py --doctags output.doctags.txt --output fixed_output.doctags.txt
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
import json
|
||||
|
||||
# Regular expression to extract location data
|
||||
LOC_PATTERN = r'<loc_(\d+)><loc_(\d+)><loc_(\d+)><loc_(\d+)>'
|
||||
|
||||
def parse_arguments():
|
||||
"""Parse command line arguments."""
|
||||
parser = argparse.ArgumentParser(description='Fix scaling issues in DocTags output')
|
||||
parser.add_argument('--doctags', '-d', type=str, required=True,
|
||||
help='Path to original DocTags file')
|
||||
parser.add_argument('--output', '-o', type=str, required=True,
|
||||
help='Path to save the fixed DocTags file')
|
||||
parser.add_argument('--x-factor', '-x', type=float, default=0.7,
|
||||
help='X-axis scaling factor (default: 0.7)')
|
||||
parser.add_argument('--y-factor', '-y', type=float, default=0.7,
|
||||
help='Y-axis scaling factor (default: 0.7)')
|
||||
parser.add_argument('--x-offset', type=int, default=0,
|
||||
help='X-axis offset (default: 0)')
|
||||
parser.add_argument('--y-offset', type=int, default=0,
|
||||
help='Y-axis offset (default: 0)')
|
||||
return parser.parse_args()
|
||||
|
||||
def read_doctags_file(doctags_path):
|
||||
"""Read the DocTags file."""
|
||||
with open(doctags_path, 'r', encoding='utf-8') as f:
|
||||
return f.read()
|
||||
|
||||
def write_doctags_file(content, output_path):
|
||||
"""Write the fixed DocTags file."""
|
||||
with open(output_path, 'w', encoding='utf-8') as f:
|
||||
f.write(content)
|
||||
print(f"Fixed DocTags saved to: {output_path}")
|
||||
|
||||
def fix_scaling(doctags_content, x_factor, y_factor, x_offset, y_offset):
|
||||
"""Apply scaling factors to all location tags in the DocTags content."""
|
||||
def apply_scaling(match):
|
||||
x1, y1, x2, y2 = map(int, match.groups())
|
||||
|
||||
# Apply scaling and offset
|
||||
new_x1 = int(x1 * x_factor) + x_offset
|
||||
new_y1 = int(y1 * y_factor) + y_offset
|
||||
new_x2 = int(x2 * x_factor) + x_offset
|
||||
new_y2 = int(y2 * y_factor) + y_offset
|
||||
|
||||
# Ensure coordinates are positive
|
||||
new_x1 = max(0, new_x1)
|
||||
new_y1 = max(0, new_y1)
|
||||
new_x2 = max(0, new_x2)
|
||||
new_y2 = max(0, new_y2)
|
||||
|
||||
return f"<loc_{new_x1}><loc_{new_y1}><loc_{new_x2}><loc_{new_y2}>"
|
||||
|
||||
# Apply the scaling to all location tags
|
||||
fixed_content = re.sub(LOC_PATTERN, apply_scaling, doctags_content)
|
||||
|
||||
# Count how many replacements were made
|
||||
original_matches = re.findall(LOC_PATTERN, doctags_content)
|
||||
fixed_matches = re.findall(LOC_PATTERN, fixed_content)
|
||||
|
||||
print(f"Modified {len(original_matches)} location tags")
|
||||
|
||||
# Print before/after sample for the first few zones
|
||||
if original_matches and fixed_matches:
|
||||
print("\nBefore/After comparison (first 3 zones):")
|
||||
for i, (orig, fixed) in enumerate(zip(original_matches, fixed_matches)):
|
||||
if i >= 3:
|
||||
break
|
||||
orig_x1, orig_y1, orig_x2, orig_y2 = map(int, orig)
|
||||
fixed_x1, fixed_y1, fixed_x2, fixed_y2 = map(int, fixed_matches[i])
|
||||
print(f"Zone {i+1}: ({orig_x1},{orig_y1},{orig_x2},{orig_y2}) → ({fixed_x1},{fixed_y1},{fixed_x2},{fixed_y2})")
|
||||
|
||||
return fixed_content
|
||||
|
||||
def analyze_doctags(doctags_content):
|
||||
"""Analyze the DocTags content to suggest scaling factors."""
|
||||
# Find all location tags
|
||||
locations = re.findall(LOC_PATTERN, doctags_content)
|
||||
|
||||
if not locations:
|
||||
print("No location tags found in the DocTags file.")
|
||||
return
|
||||
|
||||
# Extract coordinates
|
||||
coords = [(int(x1), int(y1), int(x2), int(y2)) for x1, y1, x2, y2 in locations]
|
||||
|
||||
# Find the boundaries
|
||||
min_x = min(min(x1, x2) for x1, y1, x2, y2 in coords)
|
||||
min_y = min(min(y1, y2) for x1, y1, x2, y2 in coords)
|
||||
max_x = max(max(x1, x2) for x1, y1, x2, y2 in coords)
|
||||
max_y = max(max(y1, y2) for x1, y1, x2, y2 in coords)
|
||||
|
||||
# Calculate average zone size
|
||||
avg_width = sum(abs(x2 - x1) for x1, y1, x2, y2 in coords) / len(coords)
|
||||
avg_height = sum(abs(y2 - y1) for x1, y1, x2, y2 in coords) / len(coords)
|
||||
|
||||
print("\nDocTags Analysis:")
|
||||
print(f"Found {len(locations)} zones with coordinates")
|
||||
print(f"Coordinate boundaries: X({min_x}-{max_x}), Y({min_y}-{max_y})")
|
||||
print(f"Average zone size: {avg_width:.1f} x {avg_height:.1f}")
|
||||
|
||||
# Suggest appropriate scaling for standard page sizes
|
||||
a4_width, a4_height = 595, 842 # A4 in points
|
||||
|
||||
# Calculate suggested scaling to fit A4
|
||||
x_factor = a4_width / max_x if max_x > 0 else 1.0
|
||||
y_factor = a4_height / max_y if max_y > 0 else 1.0
|
||||
|
||||
# Apply some heuristics for common scaling issues
|
||||
if max_x > 1000 and max_y > 1000:
|
||||
print("\nDetected large coordinates - typical of high-resolution scans or OCR")
|
||||
print("Suggested scaling factors for standard page:")
|
||||
print(f"X-factor: {x_factor:.3f} (to fit width to A4)")
|
||||
print(f"Y-factor: {y_factor:.3f} (to fit height to A4)")
|
||||
elif max_x < 300 and max_y < 300:
|
||||
print("\nDetected small coordinates - might be normalized values")
|
||||
print("Suggested scaling factors for standard page:")
|
||||
print(f"X-factor: {a4_width/300:.3f} (to expand to A4 width)")
|
||||
print(f"Y-factor: {a4_height/300:.3f} (to expand to A4 height)")
|
||||
|
||||
# Check for pattern inconsistencies (possible corruption or bad parsing)
|
||||
widths = [abs(x2 - x1) for x1, y1, x2, y2 in coords]
|
||||
heights = [abs(y2 - y1) for x1, y1, x2, y2 in coords]
|
||||
width_std_dev = (sum((w - avg_width) ** 2 for w in widths) / len(widths)) ** 0.5
|
||||
height_std_dev = (sum((h - avg_height) ** 2 for h in heights) / len(heights)) ** 0.5
|
||||
|
||||
if width_std_dev > avg_width * 1.5 or height_std_dev > avg_height * 1.5:
|
||||
print("\nWarning: High variance in zone sizes detected!")
|
||||
print("This might indicate inconsistent scaling or parsing issues.")
|
||||
|
||||
return {
|
||||
'min_x': min_x, 'max_x': max_x, 'min_y': min_y, 'max_y': max_y,
|
||||
'avg_width': avg_width, 'avg_height': avg_height,
|
||||
'suggested_x_factor': x_factor, 'suggested_y_factor': y_factor
|
||||
}
|
||||
|
||||
def main():
|
||||
args = parse_arguments()
|
||||
|
||||
# Read the original DocTags file
|
||||
try:
|
||||
doctags_content = read_doctags_file(args.doctags)
|
||||
print(f"Read DocTags file: {args.doctags} ({len(doctags_content)} bytes)")
|
||||
except Exception as e:
|
||||
print(f"Error reading DocTags file: {e}")
|
||||
return 1
|
||||
|
||||
# Analyze the DocTags content
|
||||
analysis = analyze_doctags(doctags_content)
|
||||
|
||||
# Confirm with user
|
||||
if not analysis:
|
||||
print("No analysis could be performed. Using specified scaling factors.")
|
||||
else:
|
||||
print(f"\nYou specified x-factor={args.x_factor}, y-factor={args.y_factor}")
|
||||
print(f"Analysis suggests x-factor={analysis.get('suggested_x_factor', 1.0):.3f}, "
|
||||
f"y-factor={analysis.get('suggested_y_factor', 1.0):.3f}")
|
||||
|
||||
use_suggested = input("\nUse suggested scaling factors instead? [y/N]: ").lower()
|
||||
if use_suggested.startswith('y'):
|
||||
args.x_factor = analysis.get('suggested_x_factor', args.x_factor)
|
||||
args.y_factor = analysis.get('suggested_y_factor', args.y_factor)
|
||||
print(f"Using suggested factors: x={args.x_factor:.3f}, y={args.y_factor:.3f}")
|
||||
|
||||
# Apply the scaling fix
|
||||
try:
|
||||
fixed_content = fix_scaling(doctags_content, args.x_factor, args.y_factor,
|
||||
args.x_offset, args.y_offset)
|
||||
|
||||
# Save the fixed DocTags file
|
||||
write_doctags_file(fixed_content, args.output)
|
||||
|
||||
print("\nScaling fix complete!")
|
||||
print("Run visualizer with the fixed DocTags file:")
|
||||
print(f"python visualizer.py --doctags {args.output} --pdf your_document.pdf --page 1 --show")
|
||||
|
||||
return 0
|
||||
except Exception as e:
|
||||
print(f"Error applying scaling fix: {e}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
return 1
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
1306
index.html
1306
index.html
File diff suppressed because it is too large
Load diff
|
|
@ -1,172 +0,0 @@
|
|||
#!/bin/bash
|
||||
# process_full_document.sh
|
||||
# This script processes all pages in a PDF document using the DocTags workflow
|
||||
|
||||
# Default values
|
||||
PDF_FILE=""
|
||||
START_PAGE=1
|
||||
END_PAGE=0
|
||||
X_FACTOR=0.7
|
||||
Y_FACTOR=0.7
|
||||
DPI=200
|
||||
SHOW_OUTPUT=false
|
||||
|
||||
# Function to display usage information
|
||||
function show_usage {
|
||||
echo "Usage: $0 -f PDF_FILE [options]"
|
||||
echo ""
|
||||
echo "Required:"
|
||||
echo " -f, --file FILE Path to the PDF file"
|
||||
echo ""
|
||||
echo "Options:"
|
||||
echo " -s, --start PAGE Starting page number (default: 1)"
|
||||
echo " -e, --end PAGE Ending page number (default: last page)"
|
||||
echo " -x, --x-factor VALUE X-axis scaling factor (default: 0.7)"
|
||||
echo " -y, --y-factor VALUE Y-axis scaling factor (default: 0.7)"
|
||||
echo " -d, --dpi VALUE DPI for PDF rendering (default: 200)"
|
||||
echo " --show Open visualization in browser (default: false)"
|
||||
echo " -h, --help Display this help message"
|
||||
echo ""
|
||||
echo "Example:"
|
||||
echo " $0 -f document.pdf -s 1 -e 10 -x 0.8 -y 0.8 --show"
|
||||
}
|
||||
|
||||
# Parse command line arguments
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
-f|--file)
|
||||
PDF_FILE="$2"
|
||||
shift 2
|
||||
;;
|
||||
-s|--start)
|
||||
START_PAGE="$2"
|
||||
shift 2
|
||||
;;
|
||||
-e|--end)
|
||||
END_PAGE="$2"
|
||||
shift 2
|
||||
;;
|
||||
-x|--x-factor)
|
||||
X_FACTOR="$2"
|
||||
shift 2
|
||||
;;
|
||||
-y|--y-factor)
|
||||
Y_FACTOR="$2"
|
||||
shift 2
|
||||
;;
|
||||
-d|--dpi)
|
||||
DPI="$2"
|
||||
shift 2
|
||||
;;
|
||||
--show)
|
||||
SHOW_OUTPUT=true
|
||||
shift
|
||||
;;
|
||||
-h|--help)
|
||||
show_usage
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
echo "Error: Unknown option $1"
|
||||
show_usage
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
# Check if PDF file is provided
|
||||
if [ -z "$PDF_FILE" ]; then
|
||||
echo "Error: PDF file is required"
|
||||
show_usage
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Check if PDF file exists
|
||||
if [ ! -f "$PDF_FILE" ]; then
|
||||
echo "Error: PDF file '$PDF_FILE' not found"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Get the PDF filename without extension
|
||||
PDF_BASENAME=$(basename "$PDF_FILE" .pdf)
|
||||
|
||||
# If end page is not specified, get the total number of pages
|
||||
if [ "$END_PAGE" -eq 0 ]; then
|
||||
echo "Determining total number of pages..."
|
||||
# Use the visualizer's --page-count feature to get the number of pages
|
||||
TOTAL_PAGES=$(python visualizer.py --pdf "$PDF_FILE" --page-count 2>&1 | grep "The PDF has" | awk '{print $4}')
|
||||
|
||||
if [ -z "$TOTAL_PAGES" ]; then
|
||||
echo "Error: Could not determine the total number of pages"
|
||||
echo "Please specify the end page with -e"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
END_PAGE=$TOTAL_PAGES
|
||||
echo "Total pages detected: $TOTAL_PAGES"
|
||||
fi
|
||||
|
||||
echo "Processing PDF: $PDF_FILE (pages $START_PAGE to $END_PAGE)"
|
||||
echo "Scaling factors: X=$X_FACTOR, Y=$Y_FACTOR"
|
||||
echo ""
|
||||
|
||||
# Create output directory
|
||||
OUTPUT_DIR="${PDF_BASENAME}_output"
|
||||
mkdir -p "$OUTPUT_DIR"
|
||||
|
||||
# Process each page
|
||||
for ((PAGE=$START_PAGE; PAGE<=$END_PAGE; PAGE++)); do
|
||||
echo "===================================================="
|
||||
echo "Processing page $PAGE of $END_PAGE"
|
||||
echo "===================================================="
|
||||
|
||||
# Set output filenames
|
||||
DOCTAGS_FILE="$OUTPUT_DIR/page_${PAGE}.doctags.txt"
|
||||
FIXED_DOCTAGS_FILE="$OUTPUT_DIR/page_${PAGE}.fixed.doctags.txt"
|
||||
HTML_OUTPUT="$OUTPUT_DIR/page_${PAGE}.html"
|
||||
|
||||
# Step 1: Analyze the page
|
||||
echo "Step 1: Analyzing page $PAGE..."
|
||||
python analyzer.py --image "$PDF_FILE" --page "$PAGE" --output "$DOCTAGS_FILE" --dpi "$DPI"
|
||||
|
||||
if [ ! -f "$DOCTAGS_FILE" ]; then
|
||||
echo "Warning: Failed to generate DocTags for page $PAGE, skipping to next page"
|
||||
continue
|
||||
fi
|
||||
|
||||
# Step 2: Fix scaling issues
|
||||
echo "Step 2: Fixing scaling for page $PAGE..."
|
||||
python fix_scaling.py --doctags "$DOCTAGS_FILE" --output "$FIXED_DOCTAGS_FILE" --x-factor "$X_FACTOR" --y-factor "$Y_FACTOR"
|
||||
|
||||
if [ ! -f "$FIXED_DOCTAGS_FILE" ]; then
|
||||
echo "Warning: Failed to fix scaling for page $PAGE, using original DocTags"
|
||||
FIXED_DOCTAGS_FILE="$DOCTAGS_FILE"
|
||||
fi
|
||||
|
||||
# Step 3: Visualize the page
|
||||
echo "Step 3: Generating visualization for page $PAGE..."
|
||||
SHOW_FLAG=""
|
||||
if [ "$SHOW_OUTPUT" = true ] && [ "$PAGE" -eq "$END_PAGE" ]; then
|
||||
# Only show the last page in browser to avoid opening too many windows
|
||||
SHOW_FLAG="--show"
|
||||
fi
|
||||
|
||||
python visualizer.py --doctags "$FIXED_DOCTAGS_FILE" --pdf "$PDF_FILE" --page "$PAGE" --output "$HTML_OUTPUT" --adjust $SHOW_FLAG
|
||||
|
||||
echo "Page $PAGE processing complete. Output saved to $HTML_OUTPUT"
|
||||
echo ""
|
||||
done
|
||||
|
||||
echo "All pages processed successfully!"
|
||||
echo "Output files are saved in: $OUTPUT_DIR"
|
||||
|
||||
# Open the output directory
|
||||
if command -v xdg-open &> /dev/null; then
|
||||
xdg-open "$OUTPUT_DIR" &
|
||||
elif command -v open &> /dev/null; then
|
||||
open "$OUTPUT_DIR" &
|
||||
elif command -v explorer &> /dev/null; then
|
||||
explorer "$OUTPUT_DIR" &
|
||||
fi
|
||||
|
||||
exit 0
|
||||
|
|
@ -1,165 +0,0 @@
|
|||
#!/bin/bash
|
||||
# process_single_page_fixed.sh
|
||||
# This script processes a single page in a PDF document using the DocTags workflow with improved error handling
|
||||
|
||||
# Default values
|
||||
PDF_FILE=""
|
||||
PAGE=0
|
||||
X_FACTOR=0.7
|
||||
Y_FACTOR=0.7
|
||||
DPI=200
|
||||
|
||||
# Function to display usage information
|
||||
function show_usage {
|
||||
echo "Usage: $0 -f PDF_FILE [options]"
|
||||
echo ""
|
||||
echo "Required:"
|
||||
echo " -f, --file FILE Path to the PDF file"
|
||||
echo ""
|
||||
echo "Options:"
|
||||
echo " -p, --page PAGE Page number (if not provided, will prompt)"
|
||||
echo " -x, --x-factor VALUE X-axis scaling factor (default: 0.7)"
|
||||
echo " -y, --y-factor VALUE Y-axis scaling factor (default: 0.7)"
|
||||
echo " -d, --dpi VALUE DPI for PDF rendering (default: 200)"
|
||||
echo " -h, --help Display this help message"
|
||||
echo ""
|
||||
echo "Example:"
|
||||
echo " $0 -f document.pdf -p 7 -x 0.8 -y 0.8"
|
||||
}
|
||||
|
||||
# Parse command line arguments
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
-f|--file)
|
||||
PDF_FILE="$2"
|
||||
shift 2
|
||||
;;
|
||||
-p|--page)
|
||||
PAGE="$2"
|
||||
shift 2
|
||||
;;
|
||||
-x|--x-factor)
|
||||
X_FACTOR="$2"
|
||||
shift 2
|
||||
;;
|
||||
-y|--y-factor)
|
||||
Y_FACTOR="$2"
|
||||
shift 2
|
||||
;;
|
||||
-d|--dpi)
|
||||
DPI="$2"
|
||||
shift 2
|
||||
;;
|
||||
-h|--help)
|
||||
show_usage
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
echo "Error: Unknown option $1"
|
||||
show_usage
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
# Check if PDF file is provided
|
||||
if [ -z "$PDF_FILE" ]; then
|
||||
echo "Error: PDF file is required"
|
||||
show_usage
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Check if PDF file exists
|
||||
if [ ! -f "$PDF_FILE" ]; then
|
||||
echo "Error: PDF file '$PDF_FILE' not found"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Get the PDF filename without extension
|
||||
PDF_BASENAME=$(basename "$PDF_FILE" .pdf)
|
||||
|
||||
# If page is not specified, determine the total pages and prompt user
|
||||
if [ "$PAGE" -eq 0 ]; then
|
||||
echo "Determining total number of pages..."
|
||||
# Use the visualizer's --page-count feature to get the number of pages
|
||||
TOTAL_PAGES=$(python visualizer.py --pdf "$PDF_FILE" --page-count 2>&1 | grep "The PDF has" | awk '{print $4}')
|
||||
|
||||
if [ -z "$TOTAL_PAGES" ]; then
|
||||
echo "Error: Could not determine the total number of pages"
|
||||
echo "Please specify the page with -p"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "The PDF has $TOTAL_PAGES pages."
|
||||
|
||||
# Prompt for page number
|
||||
while true; do
|
||||
read -p "Enter the page number to process (1-$TOTAL_PAGES): " PAGE
|
||||
if [[ "$PAGE" =~ ^[0-9]+$ ]] && [ "$PAGE" -ge 1 ] && [ "$PAGE" -le "$TOTAL_PAGES" ]; then
|
||||
break
|
||||
else
|
||||
echo "Invalid page number. Please enter a number between 1 and $TOTAL_PAGES."
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
# Create output directory
|
||||
OUTPUT_DIR="${PDF_BASENAME}_output"
|
||||
mkdir -p "$OUTPUT_DIR"
|
||||
|
||||
# Set output filenames - explicit paths
|
||||
DOCTAGS_FILE="$OUTPUT_DIR/page_${PAGE}.doctags.txt"
|
||||
FIXED_DOCTAGS_FILE="$OUTPUT_DIR/page_${PAGE}.fixed.doctags.txt"
|
||||
HTML_OUTPUT="$OUTPUT_DIR/page_${PAGE}.html"
|
||||
|
||||
echo "Processing PDF: $PDF_FILE (page $PAGE)"
|
||||
echo "Scaling factors: X=$X_FACTOR, Y=$Y_FACTOR"
|
||||
echo ""
|
||||
|
||||
# Step 1: Analyze the page
|
||||
echo "Step 1: Analyzing page $PAGE..."
|
||||
python analyzer.py --image "$PDF_FILE" --page "$PAGE" --output "$DOCTAGS_FILE" --dpi "$DPI"
|
||||
|
||||
# Check for DocTags file with different extensions
|
||||
if [ ! -f "$DOCTAGS_FILE" ]; then
|
||||
# Try alternative extensions
|
||||
ALTERNATIVE_FILE="$OUTPUT_DIR/page_${PAGE}.doctags.doctags.txt"
|
||||
if [ -f "$ALTERNATIVE_FILE" ]; then
|
||||
echo "Found DocTags at alternative location: $ALTERNATIVE_FILE"
|
||||
DOCTAGS_FILE="$ALTERNATIVE_FILE"
|
||||
else
|
||||
# Try to find any doctags file in the output directory
|
||||
FOUND_FILE=$(find "$OUTPUT_DIR" -name "*.doctags.txt" -print -quit)
|
||||
if [ -n "$FOUND_FILE" ]; then
|
||||
echo "Found DocTags at location: $FOUND_FILE"
|
||||
DOCTAGS_FILE="$FOUND_FILE"
|
||||
else
|
||||
echo "Error: Failed to generate DocTags for page $PAGE"
|
||||
echo "Could not find any doctags files in $OUTPUT_DIR"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
# Step 2: Fix scaling issues
|
||||
echo "Step 2: Fixing scaling for page $PAGE..."
|
||||
python fix_scaling.py --doctags "$DOCTAGS_FILE" --output "$FIXED_DOCTAGS_FILE" --x-factor "$X_FACTOR" --y-factor "$Y_FACTOR"
|
||||
|
||||
if [ ! -f "$FIXED_DOCTAGS_FILE" ]; then
|
||||
echo "Warning: Failed to fix scaling for page $PAGE, using original DocTags"
|
||||
FIXED_DOCTAGS_FILE="$DOCTAGS_FILE"
|
||||
fi
|
||||
|
||||
# Step 3: Visualize the page
|
||||
echo "Step 3: Generating visualization for page $PAGE..."
|
||||
python visualizer.py --doctags "$FIXED_DOCTAGS_FILE" --pdf "$PDF_FILE" --page "$PAGE" --output "$HTML_OUTPUT" --adjust --show
|
||||
|
||||
echo "Page $PAGE processing complete."
|
||||
echo "Output saved to: $HTML_OUTPUT"
|
||||
|
||||
# Show the command used to process this page
|
||||
echo ""
|
||||
echo "Command used:"
|
||||
echo "python analyzer.py --image \"$PDF_FILE\" --page $PAGE && python fix_scaling.py --doctags \"$DOCTAGS_FILE\" --output \"$FIXED_DOCTAGS_FILE\" --x-factor $X_FACTOR --y-factor $Y_FACTOR && python visualizer.py --doctags \"$FIXED_DOCTAGS_FILE\" --pdf \"$PDF_FILE\" --page $PAGE --adjust --show"
|
||||
|
||||
exit 0
|
||||
|
|
@ -426,6 +426,36 @@ def create_visualization(image, zones, pdf_path, page_num, total_pages, output_p
|
|||
|
||||
return output_path
|
||||
|
||||
def normalize_coordinates(zones, image_width, image_height, grid_size=500):
|
||||
"""
|
||||
Normalize coordinates from the DocTags grid (0-500) to actual image dimensions.
|
||||
|
||||
Args:
|
||||
zones: List of zone dictionaries with x1, y1, x2, y2 coordinates
|
||||
image_width: Width of the PDF page image in pixels
|
||||
image_height: Height of the PDF page image in pixels
|
||||
grid_size: The grid size used in DocTags (default 500)
|
||||
|
||||
Returns:
|
||||
The same zones list with updated coordinates
|
||||
"""
|
||||
# Create a copy of the zones to avoid modifying the original
|
||||
normalized_zones = []
|
||||
|
||||
for zone in zones:
|
||||
# Clone the zone
|
||||
new_zone = zone.copy()
|
||||
|
||||
# Convert from grid coordinates to actual page dimensions
|
||||
new_zone['x1'] = int(zone['x1'] * image_width / grid_size)
|
||||
new_zone['y1'] = int(zone['y1'] * image_height / grid_size)
|
||||
new_zone['x2'] = int(zone['x2'] * image_width / grid_size)
|
||||
new_zone['y2'] = int(zone['y2'] * image_height / grid_size)
|
||||
|
||||
normalized_zones.append(new_zone)
|
||||
|
||||
return normalized_zones
|
||||
|
||||
def create_debug_image(image, zones, page_num, output_path):
|
||||
"""Create a debug image with rectangles around zones."""
|
||||
# Create a copy of the input image
|
||||
|
|
@ -511,20 +541,22 @@ def process_page(pdf_path, page_num, doctags_path, output_base, dpi=200, show=Fa
|
|||
print(f"Found {len(zones)} zones in DocTags")
|
||||
|
||||
# Debug output to understand scaling issues
|
||||
# After parsing the zones from DocTags
|
||||
if zones:
|
||||
img_width, img_height = image.size
|
||||
print(f"Image dimensions: {img_width}x{img_height}")
|
||||
|
||||
# Check if we need to normalize grid coordinates
|
||||
max_x = max([zone['x2'] for zone in zones])
|
||||
max_y = max([zone['y2'] for zone in zones])
|
||||
min_x = min([zone['x1'] for zone in zones])
|
||||
min_y = min([zone['y1'] for zone in zones])
|
||||
img_width, img_height = image.size
|
||||
|
||||
print(f"Image dimensions: {img_width}x{img_height}")
|
||||
print(f"Zone boundaries: X({min_x}-{max_x}), Y({min_y}-{max_y})")
|
||||
print(f"Suggested scale factors: X={img_width/max_x:.3f}, Y={img_height/max_y:.3f}")
|
||||
|
||||
# Always apply automatic scaling (making adjust=True by default)
|
||||
# This solves the common scaling issue between DocTags coordinates and image dimensions
|
||||
if adjust:
|
||||
# If coordinates seem to be in a normalized grid (0-500 range)
|
||||
if max_x <= 500 and max_y <= 500:
|
||||
print(f"Detected normalized coordinates (0-500 grid)")
|
||||
zones = normalize_coordinates(zones, img_width, img_height)
|
||||
print(f"Applied automatic grid normalization")
|
||||
# If auto-adjust is enabled and coordinates are not in normalized grid
|
||||
elif adjust:
|
||||
width, height = image.size
|
||||
|
||||
# Calculate appropriate scaling factors with better heuristics
|
||||
|
|
|
|||
Loading…
Reference in a new issue