Add basic automation

This commit is contained in:
pjmalandrino 2025-05-15 15:04:23 +02:00
parent c163dd5cd7
commit ab4e3c06db
3 changed files with 550 additions and 0 deletions

213
find_optimal_scale.sh Normal file → Executable file
View file

@ -0,0 +1,213 @@
#!/bin/bash
# find_optimal_scale_fixed.sh
# This script helps find the optimal scaling factors for a PDF page with improved error handling
# Default values
PDF_FILE=""
PAGE=1
MIN_X=0.1
MAX_X=1.5
MIN_Y=0.1
MAX_Y=1.5
STEPS=5
# Function to display usage information
function show_usage {
echo "Usage: $0 -f PDF_FILE [options]"
echo ""
echo "Required:"
echo " -f, --file FILE Path to the PDF file"
echo ""
echo "Options:"
echo " -p, --page PAGE Page number to analyze (default: 1)"
echo " --min-x VALUE Minimum X scaling factor (default: 0.1)"
echo " --max-x VALUE Maximum X scaling factor (default: 1.5)"
echo " --min-y VALUE Minimum Y scaling factor (default: 0.1)"
echo " --max-y VALUE Maximum Y scaling factor (default: 1.5)"
echo " --steps NUMBER Number of steps between min and max (default: 5)"
echo " -h, --help Display this help message"
echo ""
echo "Example:"
echo " $0 -f document.pdf -p 3 --min-x 0.5 --max-x 1.0 --steps 6"
}
# Parse command line arguments
while [[ $# -gt 0 ]]; do
case "$1" in
-f|--file)
PDF_FILE="$2"
shift 2
;;
-p|--page)
PAGE="$2"
shift 2
;;
--min-x)
MIN_X="$2"
shift 2
;;
--max-x)
MAX_X="$2"
shift 2
;;
--min-y)
MIN_Y="$2"
shift 2
;;
--max-y)
MAX_Y="$2"
shift 2
;;
--steps)
STEPS="$2"
shift 2
;;
-h|--help)
show_usage
exit 0
;;
*)
echo "Error: Unknown option $1"
show_usage
exit 1
;;
esac
done
# Check if PDF file is provided
if [ -z "$PDF_FILE" ]; then
echo "Error: PDF file is required"
show_usage
exit 1
fi
# Check if PDF file exists
if [ ! -f "$PDF_FILE" ]; then
echo "Error: PDF file '$PDF_FILE' not found"
exit 1
fi
# Get PDF basename for output files
PDF_BASENAME=$(basename "$PDF_FILE" .pdf)
OUTPUT_DIR="${PDF_BASENAME}_scale_test"
mkdir -p "$OUTPUT_DIR"
# Step 1: Run analyzer once to get the DocTags
echo "Step 1: Analyzing page $PAGE to generate DocTags..."
DOCTAGS_FILE="$OUTPUT_DIR/page_${PAGE}.doctags.txt"
python analyzer.py --image "$PDF_FILE" --page "$PAGE" --output "$DOCTAGS_FILE"
# Check for DocTags file with different extensions
if [ ! -f "$DOCTAGS_FILE" ]; then
# Try alternative extensions
ALTERNATIVE_FILE="$OUTPUT_DIR/page_${PAGE}.doctags.doctags.txt"
if [ -f "$ALTERNATIVE_FILE" ]; then
echo "Found DocTags at alternative location: $ALTERNATIVE_FILE"
DOCTAGS_FILE="$ALTERNATIVE_FILE"
else
# Try to find any doctags file in the output directory
FOUND_FILE=$(find "$OUTPUT_DIR" -name "*.doctags.txt" -print -quit)
if [ -n "$FOUND_FILE" ]; then
echo "Found DocTags at location: $FOUND_FILE"
DOCTAGS_FILE="$FOUND_FILE"
else
echo "Error: Failed to generate DocTags for page $PAGE"
echo "Could not find any doctags files in $OUTPUT_DIR"
exit 1
fi
fi
fi
echo "Using DocTags file: $DOCTAGS_FILE"
# Create HTML index to easily compare results
INDEX_FILE="$OUTPUT_DIR/index.html"
echo "<!DOCTYPE html>
<html>
<head>
<meta charset='UTF-8'>
<title>Scale Factor Test - Page $PAGE</title>
<style>
body { font-family: Arial, sans-serif; margin: 20px; }
.grid { display: grid; grid-template-columns: repeat(auto-fill, minmax(300px, 1fr)); gap: 20px; }
.card { border: 1px solid #ccc; border-radius: 5px; padding: 10px; }
.card h3 { margin-top: 0; text-align: center; }
.card img { max-width: 100%; border: 1px solid #eee; }
.card a { display: block; text-align: center; margin-top: 10px; }
</style>
</head>
<body>
<h1>Scale Factor Test - Page $PAGE of $PDF_BASENAME.pdf</h1>
<p>Click on any thumbnail to open the full visualization.</p>
<div class='grid'>" > "$INDEX_FILE"
# Step 2: Try different combinations of scaling factors
echo "Step 2: Testing different scaling factors..."
# Calculate step sizes
X_STEP=$(echo "scale=6; ($MAX_X - $MIN_X) / $STEPS" | bc)
Y_STEP=$(echo "scale=6; ($MAX_Y - $MIN_Y) / $STEPS" | bc)
for i in $(seq 0 $STEPS); do
X_FACTOR=$(echo "scale=2; $MIN_X + ($i * $X_STEP)" | bc)
for j in $(seq 0 $STEPS); do
Y_FACTOR=$(echo "scale=2; $MIN_Y + ($j * $Y_STEP)" | bc)
echo "Testing X=$X_FACTOR, Y=$Y_FACTOR"
# Generate fixed DocTags
FIXED_DOCTAGS="$OUTPUT_DIR/page_${PAGE}_x${X_FACTOR}_y${Y_FACTOR}.doctags.txt"
python fix_scaling.py --doctags "$DOCTAGS_FILE" --output "$FIXED_DOCTAGS" --x-factor "$X_FACTOR" --y-factor "$Y_FACTOR" > /dev/null
# Generate visualization
HTML_OUTPUT="$OUTPUT_DIR/page_${PAGE}_x${X_FACTOR}_y${Y_FACTOR}.html"
python visualizer.py --doctags "$FIXED_DOCTAGS" --pdf "$PDF_FILE" --page "$PAGE" --output "$HTML_OUTPUT" --adjust > /dev/null
# Generate debug image for thumbnail
DEBUG_IMG="$OUTPUT_DIR/page_${PAGE}_x${X_FACTOR}_y${Y_FACTOR}.debug.png"
# Check if the debug image exists
if [ ! -f "$DEBUG_IMG" ]; then
echo "Warning: Debug image not found, trying alternative name..."
ALT_DEBUG_IMG=$(find "$OUTPUT_DIR" -name "page_${PAGE}_x${X_FACTOR}_y${Y_FACTOR}*.png" -print -quit)
if [ -n "$ALT_DEBUG_IMG" ]; then
DEBUG_IMG="$ALT_DEBUG_IMG"
fi
fi
# Add to index only if files exist
if [ -f "$HTML_OUTPUT" ] && [ -f "$DEBUG_IMG" ]; then
echo " <div class='card'>
<h3>X=$X_FACTOR, Y=$Y_FACTOR</h3>
<a href='$(basename "$HTML_OUTPUT")'>
<img src='$(basename "$DEBUG_IMG")' alt='X=$X_FACTOR, Y=$Y_FACTOR'>
</a>
<a href='$(basename "$HTML_OUTPUT")'>Open Visualization</a>
</div>" >> "$INDEX_FILE"
else
echo "Warning: Skipping this combination as files not created successfully."
fi
done
done
# Finish HTML index
echo " </div>
</body>
</html>" >> "$INDEX_FILE"
echo "
Scale factor testing complete!
Open $INDEX_FILE in a web browser to compare results.
"
# Try to open the index file in a browser
if command -v xdg-open &> /dev/null; then
xdg-open "$INDEX_FILE" &
elif command -v open &> /dev/null; then
open "$INDEX_FILE" &
elif command -v explorer &> /dev/null; then
explorer "$INDEX_FILE" &
fi
exit 0

View file

@ -0,0 +1,172 @@
#!/bin/bash
# process_full_document.sh
# This script processes all pages in a PDF document using the DocTags workflow
# Default values
PDF_FILE=""
START_PAGE=1
END_PAGE=0
X_FACTOR=0.7
Y_FACTOR=0.7
DPI=200
SHOW_OUTPUT=false
# Function to display usage information
function show_usage {
echo "Usage: $0 -f PDF_FILE [options]"
echo ""
echo "Required:"
echo " -f, --file FILE Path to the PDF file"
echo ""
echo "Options:"
echo " -s, --start PAGE Starting page number (default: 1)"
echo " -e, --end PAGE Ending page number (default: last page)"
echo " -x, --x-factor VALUE X-axis scaling factor (default: 0.7)"
echo " -y, --y-factor VALUE Y-axis scaling factor (default: 0.7)"
echo " -d, --dpi VALUE DPI for PDF rendering (default: 200)"
echo " --show Open visualization in browser (default: false)"
echo " -h, --help Display this help message"
echo ""
echo "Example:"
echo " $0 -f document.pdf -s 1 -e 10 -x 0.8 -y 0.8 --show"
}
# Parse command line arguments
while [[ $# -gt 0 ]]; do
case "$1" in
-f|--file)
PDF_FILE="$2"
shift 2
;;
-s|--start)
START_PAGE="$2"
shift 2
;;
-e|--end)
END_PAGE="$2"
shift 2
;;
-x|--x-factor)
X_FACTOR="$2"
shift 2
;;
-y|--y-factor)
Y_FACTOR="$2"
shift 2
;;
-d|--dpi)
DPI="$2"
shift 2
;;
--show)
SHOW_OUTPUT=true
shift
;;
-h|--help)
show_usage
exit 0
;;
*)
echo "Error: Unknown option $1"
show_usage
exit 1
;;
esac
done
# Check if PDF file is provided
if [ -z "$PDF_FILE" ]; then
echo "Error: PDF file is required"
show_usage
exit 1
fi
# Check if PDF file exists
if [ ! -f "$PDF_FILE" ]; then
echo "Error: PDF file '$PDF_FILE' not found"
exit 1
fi
# Get the PDF filename without extension
PDF_BASENAME=$(basename "$PDF_FILE" .pdf)
# If end page is not specified, get the total number of pages
if [ "$END_PAGE" -eq 0 ]; then
echo "Determining total number of pages..."
# Use the visualizer's --page-count feature to get the number of pages
TOTAL_PAGES=$(python visualizer.py --pdf "$PDF_FILE" --page-count 2>&1 | grep "The PDF has" | awk '{print $4}')
if [ -z "$TOTAL_PAGES" ]; then
echo "Error: Could not determine the total number of pages"
echo "Please specify the end page with -e"
exit 1
fi
END_PAGE=$TOTAL_PAGES
echo "Total pages detected: $TOTAL_PAGES"
fi
echo "Processing PDF: $PDF_FILE (pages $START_PAGE to $END_PAGE)"
echo "Scaling factors: X=$X_FACTOR, Y=$Y_FACTOR"
echo ""
# Create output directory
OUTPUT_DIR="${PDF_BASENAME}_output"
mkdir -p "$OUTPUT_DIR"
# Process each page
for ((PAGE=$START_PAGE; PAGE<=$END_PAGE; PAGE++)); do
echo "===================================================="
echo "Processing page $PAGE of $END_PAGE"
echo "===================================================="
# Set output filenames
DOCTAGS_FILE="$OUTPUT_DIR/page_${PAGE}.doctags.txt"
FIXED_DOCTAGS_FILE="$OUTPUT_DIR/page_${PAGE}.fixed.doctags.txt"
HTML_OUTPUT="$OUTPUT_DIR/page_${PAGE}.html"
# Step 1: Analyze the page
echo "Step 1: Analyzing page $PAGE..."
python analyzer.py --image "$PDF_FILE" --page "$PAGE" --output "$DOCTAGS_FILE" --dpi "$DPI"
if [ ! -f "$DOCTAGS_FILE" ]; then
echo "Warning: Failed to generate DocTags for page $PAGE, skipping to next page"
continue
fi
# Step 2: Fix scaling issues
echo "Step 2: Fixing scaling for page $PAGE..."
python fix_scaling.py --doctags "$DOCTAGS_FILE" --output "$FIXED_DOCTAGS_FILE" --x-factor "$X_FACTOR" --y-factor "$Y_FACTOR"
if [ ! -f "$FIXED_DOCTAGS_FILE" ]; then
echo "Warning: Failed to fix scaling for page $PAGE, using original DocTags"
FIXED_DOCTAGS_FILE="$DOCTAGS_FILE"
fi
# Step 3: Visualize the page
echo "Step 3: Generating visualization for page $PAGE..."
SHOW_FLAG=""
if [ "$SHOW_OUTPUT" = true ] && [ "$PAGE" -eq "$END_PAGE" ]; then
# Only show the last page in browser to avoid opening too many windows
SHOW_FLAG="--show"
fi
python visualizer.py --doctags "$FIXED_DOCTAGS_FILE" --pdf "$PDF_FILE" --page "$PAGE" --output "$HTML_OUTPUT" --adjust $SHOW_FLAG
echo "Page $PAGE processing complete. Output saved to $HTML_OUTPUT"
echo ""
done
echo "All pages processed successfully!"
echo "Output files are saved in: $OUTPUT_DIR"
# Open the output directory
if command -v xdg-open &> /dev/null; then
xdg-open "$OUTPUT_DIR" &
elif command -v open &> /dev/null; then
open "$OUTPUT_DIR" &
elif command -v explorer &> /dev/null; then
explorer "$OUTPUT_DIR" &
fi
exit 0

165
process_single_page.sh Normal file → Executable file
View file

@ -0,0 +1,165 @@
#!/bin/bash
# process_single_page_fixed.sh
# This script processes a single page in a PDF document using the DocTags workflow with improved error handling
# Default values
PDF_FILE=""
PAGE=0
X_FACTOR=0.7
Y_FACTOR=0.7
DPI=200
# Function to display usage information
function show_usage {
echo "Usage: $0 -f PDF_FILE [options]"
echo ""
echo "Required:"
echo " -f, --file FILE Path to the PDF file"
echo ""
echo "Options:"
echo " -p, --page PAGE Page number (if not provided, will prompt)"
echo " -x, --x-factor VALUE X-axis scaling factor (default: 0.7)"
echo " -y, --y-factor VALUE Y-axis scaling factor (default: 0.7)"
echo " -d, --dpi VALUE DPI for PDF rendering (default: 200)"
echo " -h, --help Display this help message"
echo ""
echo "Example:"
echo " $0 -f document.pdf -p 7 -x 0.8 -y 0.8"
}
# Parse command line arguments
while [[ $# -gt 0 ]]; do
case "$1" in
-f|--file)
PDF_FILE="$2"
shift 2
;;
-p|--page)
PAGE="$2"
shift 2
;;
-x|--x-factor)
X_FACTOR="$2"
shift 2
;;
-y|--y-factor)
Y_FACTOR="$2"
shift 2
;;
-d|--dpi)
DPI="$2"
shift 2
;;
-h|--help)
show_usage
exit 0
;;
*)
echo "Error: Unknown option $1"
show_usage
exit 1
;;
esac
done
# Check if PDF file is provided
if [ -z "$PDF_FILE" ]; then
echo "Error: PDF file is required"
show_usage
exit 1
fi
# Check if PDF file exists
if [ ! -f "$PDF_FILE" ]; then
echo "Error: PDF file '$PDF_FILE' not found"
exit 1
fi
# Get the PDF filename without extension
PDF_BASENAME=$(basename "$PDF_FILE" .pdf)
# If page is not specified, determine the total pages and prompt user
if [ "$PAGE" -eq 0 ]; then
echo "Determining total number of pages..."
# Use the visualizer's --page-count feature to get the number of pages
TOTAL_PAGES=$(python visualizer.py --pdf "$PDF_FILE" --page-count 2>&1 | grep "The PDF has" | awk '{print $4}')
if [ -z "$TOTAL_PAGES" ]; then
echo "Error: Could not determine the total number of pages"
echo "Please specify the page with -p"
exit 1
fi
echo "The PDF has $TOTAL_PAGES pages."
# Prompt for page number
while true; do
read -p "Enter the page number to process (1-$TOTAL_PAGES): " PAGE
if [[ "$PAGE" =~ ^[0-9]+$ ]] && [ "$PAGE" -ge 1 ] && [ "$PAGE" -le "$TOTAL_PAGES" ]; then
break
else
echo "Invalid page number. Please enter a number between 1 and $TOTAL_PAGES."
fi
done
fi
# Create output directory
OUTPUT_DIR="${PDF_BASENAME}_output"
mkdir -p "$OUTPUT_DIR"
# Set output filenames - explicit paths
DOCTAGS_FILE="$OUTPUT_DIR/page_${PAGE}.doctags.txt"
FIXED_DOCTAGS_FILE="$OUTPUT_DIR/page_${PAGE}.fixed.doctags.txt"
HTML_OUTPUT="$OUTPUT_DIR/page_${PAGE}.html"
echo "Processing PDF: $PDF_FILE (page $PAGE)"
echo "Scaling factors: X=$X_FACTOR, Y=$Y_FACTOR"
echo ""
# Step 1: Analyze the page
echo "Step 1: Analyzing page $PAGE..."
python analyzer.py --image "$PDF_FILE" --page "$PAGE" --output "$DOCTAGS_FILE" --dpi "$DPI"
# Check for DocTags file with different extensions
if [ ! -f "$DOCTAGS_FILE" ]; then
# Try alternative extensions
ALTERNATIVE_FILE="$OUTPUT_DIR/page_${PAGE}.doctags.doctags.txt"
if [ -f "$ALTERNATIVE_FILE" ]; then
echo "Found DocTags at alternative location: $ALTERNATIVE_FILE"
DOCTAGS_FILE="$ALTERNATIVE_FILE"
else
# Try to find any doctags file in the output directory
FOUND_FILE=$(find "$OUTPUT_DIR" -name "*.doctags.txt" -print -quit)
if [ -n "$FOUND_FILE" ]; then
echo "Found DocTags at location: $FOUND_FILE"
DOCTAGS_FILE="$FOUND_FILE"
else
echo "Error: Failed to generate DocTags for page $PAGE"
echo "Could not find any doctags files in $OUTPUT_DIR"
exit 1
fi
fi
fi
# Step 2: Fix scaling issues
echo "Step 2: Fixing scaling for page $PAGE..."
python fix_scaling.py --doctags "$DOCTAGS_FILE" --output "$FIXED_DOCTAGS_FILE" --x-factor "$X_FACTOR" --y-factor "$Y_FACTOR"
if [ ! -f "$FIXED_DOCTAGS_FILE" ]; then
echo "Warning: Failed to fix scaling for page $PAGE, using original DocTags"
FIXED_DOCTAGS_FILE="$DOCTAGS_FILE"
fi
# Step 3: Visualize the page
echo "Step 3: Generating visualization for page $PAGE..."
python visualizer.py --doctags "$FIXED_DOCTAGS_FILE" --pdf "$PDF_FILE" --page "$PAGE" --output "$HTML_OUTPUT" --adjust --show
echo "Page $PAGE processing complete."
echo "Output saved to: $HTML_OUTPUT"
# Show the command used to process this page
echo ""
echo "Command used:"
echo "python analyzer.py --image \"$PDF_FILE\" --page $PAGE && python fix_scaling.py --doctags \"$DOCTAGS_FILE\" --output \"$FIXED_DOCTAGS_FILE\" --x-factor $X_FACTOR --y-factor $Y_FACTOR && python visualizer.py --doctags \"$FIXED_DOCTAGS_FILE\" --pdf \"$PDF_FILE\" --page $PAGE --adjust --show"
exit 0