diff --git a/README.md b/README.md index 4001b9e..d96d108 100644 --- a/README.md +++ b/README.md @@ -1,18 +1 @@ -pip install --upgrade pip -pip install pdf2image - - -# Generate DocTags for all pages and run the visualizer -python doctags_gen.py --pdf document.pdf --output results/ && python create_index.py --directory results/ - -python analyzer.py --image document.pdf --page 13 - -python visualizer.py --doctags output.doctags.txt --pdf document.pdf --page 13 --adjust --show - - -python analyzer.py --image document.pdf --page 13 && python visualizer.py --doctags output.doctags.txt --pdf document.pdf --page 13 --adjust --show - - - -## Run full with scaling fix -python analyzer.py --image document.pdf --page 7 && python fix_scaling.py --doctags output.doctags.txt --output fixed_output.doctags.txt --x-factor 0.7 --y-factor 0.7 && python visualizer.py --doctags fixed_output.doctags.txt --pdf document.pdf --page 7 --adjust --show \ No newline at end of file +python analyzer.py --image document.pdf --page 8 && python visualizer.py --doctags output.doctags.txt --pdf document.pdf --page 8 --adjust --show \ No newline at end of file diff --git a/find_optimal_scale.sh b/find_optimal_scale.sh deleted file mode 100755 index f17dca2..0000000 --- a/find_optimal_scale.sh +++ /dev/null @@ -1,213 +0,0 @@ -#!/bin/bash -# find_optimal_scale_fixed.sh -# This script helps find the optimal scaling factors for a PDF page with improved error handling - -# Default values -PDF_FILE="" -PAGE=1 -MIN_X=0.1 -MAX_X=1.5 -MIN_Y=0.1 -MAX_Y=1.5 -STEPS=5 - -# Function to display usage information -function show_usage { - echo "Usage: $0 -f PDF_FILE [options]" - echo "" - echo "Required:" - echo " -f, --file FILE Path to the PDF file" - echo "" - echo "Options:" - echo " -p, --page PAGE Page number to analyze (default: 1)" - echo " --min-x VALUE Minimum X scaling factor (default: 0.1)" - echo " --max-x VALUE Maximum X scaling factor (default: 1.5)" - echo " --min-y VALUE Minimum Y scaling factor (default: 0.1)" - echo " --max-y VALUE Maximum Y scaling factor (default: 1.5)" - echo " --steps NUMBER Number of steps between min and max (default: 5)" - echo " -h, --help Display this help message" - echo "" - echo "Example:" - echo " $0 -f document.pdf -p 3 --min-x 0.5 --max-x 1.0 --steps 6" -} - -# Parse command line arguments -while [[ $# -gt 0 ]]; do - case "$1" in - -f|--file) - PDF_FILE="$2" - shift 2 - ;; - -p|--page) - PAGE="$2" - shift 2 - ;; - --min-x) - MIN_X="$2" - shift 2 - ;; - --max-x) - MAX_X="$2" - shift 2 - ;; - --min-y) - MIN_Y="$2" - shift 2 - ;; - --max-y) - MAX_Y="$2" - shift 2 - ;; - --steps) - STEPS="$2" - shift 2 - ;; - -h|--help) - show_usage - exit 0 - ;; - *) - echo "Error: Unknown option $1" - show_usage - exit 1 - ;; - esac -done - -# Check if PDF file is provided -if [ -z "$PDF_FILE" ]; then - echo "Error: PDF file is required" - show_usage - exit 1 -fi - -# Check if PDF file exists -if [ ! -f "$PDF_FILE" ]; then - echo "Error: PDF file '$PDF_FILE' not found" - exit 1 -fi - -# Get PDF basename for output files -PDF_BASENAME=$(basename "$PDF_FILE" .pdf) -OUTPUT_DIR="${PDF_BASENAME}_scale_test" -mkdir -p "$OUTPUT_DIR" - -# Step 1: Run analyzer once to get the DocTags -echo "Step 1: Analyzing page $PAGE to generate DocTags..." -DOCTAGS_FILE="$OUTPUT_DIR/page_${PAGE}.doctags.txt" -python analyzer.py --image "$PDF_FILE" --page "$PAGE" --output "$DOCTAGS_FILE" - -# Check for DocTags file with different extensions -if [ ! -f "$DOCTAGS_FILE" ]; then - # Try alternative extensions - ALTERNATIVE_FILE="$OUTPUT_DIR/page_${PAGE}.doctags.doctags.txt" - if [ -f "$ALTERNATIVE_FILE" ]; then - echo "Found DocTags at alternative location: $ALTERNATIVE_FILE" - DOCTAGS_FILE="$ALTERNATIVE_FILE" - else - # Try to find any doctags file in the output directory - FOUND_FILE=$(find "$OUTPUT_DIR" -name "*.doctags.txt" -print -quit) - if [ -n "$FOUND_FILE" ]; then - echo "Found DocTags at location: $FOUND_FILE" - DOCTAGS_FILE="$FOUND_FILE" - else - echo "Error: Failed to generate DocTags for page $PAGE" - echo "Could not find any doctags files in $OUTPUT_DIR" - exit 1 - fi - fi -fi - -echo "Using DocTags file: $DOCTAGS_FILE" - -# Create HTML index to easily compare results -INDEX_FILE="$OUTPUT_DIR/index.html" -echo " - - - - Scale Factor Test - Page $PAGE - - - -

Scale Factor Test - Page $PAGE of $PDF_BASENAME.pdf

-

Click on any thumbnail to open the full visualization.

-
" > "$INDEX_FILE" - -# Step 2: Try different combinations of scaling factors -echo "Step 2: Testing different scaling factors..." - -# Calculate step sizes -X_STEP=$(echo "scale=6; ($MAX_X - $MIN_X) / $STEPS" | bc) -Y_STEP=$(echo "scale=6; ($MAX_Y - $MIN_Y) / $STEPS" | bc) - -for i in $(seq 0 $STEPS); do - X_FACTOR=$(echo "scale=2; $MIN_X + ($i * $X_STEP)" | bc) - - for j in $(seq 0 $STEPS); do - Y_FACTOR=$(echo "scale=2; $MIN_Y + ($j * $Y_STEP)" | bc) - - echo "Testing X=$X_FACTOR, Y=$Y_FACTOR" - - # Generate fixed DocTags - FIXED_DOCTAGS="$OUTPUT_DIR/page_${PAGE}_x${X_FACTOR}_y${Y_FACTOR}.doctags.txt" - python fix_scaling.py --doctags "$DOCTAGS_FILE" --output "$FIXED_DOCTAGS" --x-factor "$X_FACTOR" --y-factor "$Y_FACTOR" > /dev/null - - # Generate visualization - HTML_OUTPUT="$OUTPUT_DIR/page_${PAGE}_x${X_FACTOR}_y${Y_FACTOR}.html" - python visualizer.py --doctags "$FIXED_DOCTAGS" --pdf "$PDF_FILE" --page "$PAGE" --output "$HTML_OUTPUT" --adjust > /dev/null - - # Generate debug image for thumbnail - DEBUG_IMG="$OUTPUT_DIR/page_${PAGE}_x${X_FACTOR}_y${Y_FACTOR}.debug.png" - - # Check if the debug image exists - if [ ! -f "$DEBUG_IMG" ]; then - echo "Warning: Debug image not found, trying alternative name..." - ALT_DEBUG_IMG=$(find "$OUTPUT_DIR" -name "page_${PAGE}_x${X_FACTOR}_y${Y_FACTOR}*.png" -print -quit) - if [ -n "$ALT_DEBUG_IMG" ]; then - DEBUG_IMG="$ALT_DEBUG_IMG" - fi - fi - - # Add to index only if files exist - if [ -f "$HTML_OUTPUT" ] && [ -f "$DEBUG_IMG" ]; then - echo "
-

X=$X_FACTOR, Y=$Y_FACTOR

- - X=$X_FACTOR, Y=$Y_FACTOR - - Open Visualization -
" >> "$INDEX_FILE" - else - echo "Warning: Skipping this combination as files not created successfully." - fi - done -done - -# Finish HTML index -echo "
- -" >> "$INDEX_FILE" - -echo " -Scale factor testing complete! -Open $INDEX_FILE in a web browser to compare results. -" - -# Try to open the index file in a browser -if command -v xdg-open &> /dev/null; then - xdg-open "$INDEX_FILE" & -elif command -v open &> /dev/null; then - open "$INDEX_FILE" & -elif command -v explorer &> /dev/null; then - explorer "$INDEX_FILE" & -fi - -exit 0 diff --git a/fix_scaling.py b/fix_scaling.py deleted file mode 100644 index fc4c1de..0000000 --- a/fix_scaling.py +++ /dev/null @@ -1,196 +0,0 @@ -#!/usr/bin/env python3 -""" -DocTags Scaling Fix - Script to fix scaling issues in the DocTags visualizer. - -Usage: - python fix_scaling.py --doctags output.doctags.txt --output fixed_output.doctags.txt -""" - -import argparse -import re -import sys -from pathlib import Path -import json - -# Regular expression to extract location data -LOC_PATTERN = r'' - -def parse_arguments(): - """Parse command line arguments.""" - parser = argparse.ArgumentParser(description='Fix scaling issues in DocTags output') - parser.add_argument('--doctags', '-d', type=str, required=True, - help='Path to original DocTags file') - parser.add_argument('--output', '-o', type=str, required=True, - help='Path to save the fixed DocTags file') - parser.add_argument('--x-factor', '-x', type=float, default=0.7, - help='X-axis scaling factor (default: 0.7)') - parser.add_argument('--y-factor', '-y', type=float, default=0.7, - help='Y-axis scaling factor (default: 0.7)') - parser.add_argument('--x-offset', type=int, default=0, - help='X-axis offset (default: 0)') - parser.add_argument('--y-offset', type=int, default=0, - help='Y-axis offset (default: 0)') - return parser.parse_args() - -def read_doctags_file(doctags_path): - """Read the DocTags file.""" - with open(doctags_path, 'r', encoding='utf-8') as f: - return f.read() - -def write_doctags_file(content, output_path): - """Write the fixed DocTags file.""" - with open(output_path, 'w', encoding='utf-8') as f: - f.write(content) - print(f"Fixed DocTags saved to: {output_path}") - -def fix_scaling(doctags_content, x_factor, y_factor, x_offset, y_offset): - """Apply scaling factors to all location tags in the DocTags content.""" - def apply_scaling(match): - x1, y1, x2, y2 = map(int, match.groups()) - - # Apply scaling and offset - new_x1 = int(x1 * x_factor) + x_offset - new_y1 = int(y1 * y_factor) + y_offset - new_x2 = int(x2 * x_factor) + x_offset - new_y2 = int(y2 * y_factor) + y_offset - - # Ensure coordinates are positive - new_x1 = max(0, new_x1) - new_y1 = max(0, new_y1) - new_x2 = max(0, new_x2) - new_y2 = max(0, new_y2) - - return f"" - - # Apply the scaling to all location tags - fixed_content = re.sub(LOC_PATTERN, apply_scaling, doctags_content) - - # Count how many replacements were made - original_matches = re.findall(LOC_PATTERN, doctags_content) - fixed_matches = re.findall(LOC_PATTERN, fixed_content) - - print(f"Modified {len(original_matches)} location tags") - - # Print before/after sample for the first few zones - if original_matches and fixed_matches: - print("\nBefore/After comparison (first 3 zones):") - for i, (orig, fixed) in enumerate(zip(original_matches, fixed_matches)): - if i >= 3: - break - orig_x1, orig_y1, orig_x2, orig_y2 = map(int, orig) - fixed_x1, fixed_y1, fixed_x2, fixed_y2 = map(int, fixed_matches[i]) - print(f"Zone {i+1}: ({orig_x1},{orig_y1},{orig_x2},{orig_y2}) → ({fixed_x1},{fixed_y1},{fixed_x2},{fixed_y2})") - - return fixed_content - -def analyze_doctags(doctags_content): - """Analyze the DocTags content to suggest scaling factors.""" - # Find all location tags - locations = re.findall(LOC_PATTERN, doctags_content) - - if not locations: - print("No location tags found in the DocTags file.") - return - - # Extract coordinates - coords = [(int(x1), int(y1), int(x2), int(y2)) for x1, y1, x2, y2 in locations] - - # Find the boundaries - min_x = min(min(x1, x2) for x1, y1, x2, y2 in coords) - min_y = min(min(y1, y2) for x1, y1, x2, y2 in coords) - max_x = max(max(x1, x2) for x1, y1, x2, y2 in coords) - max_y = max(max(y1, y2) for x1, y1, x2, y2 in coords) - - # Calculate average zone size - avg_width = sum(abs(x2 - x1) for x1, y1, x2, y2 in coords) / len(coords) - avg_height = sum(abs(y2 - y1) for x1, y1, x2, y2 in coords) / len(coords) - - print("\nDocTags Analysis:") - print(f"Found {len(locations)} zones with coordinates") - print(f"Coordinate boundaries: X({min_x}-{max_x}), Y({min_y}-{max_y})") - print(f"Average zone size: {avg_width:.1f} x {avg_height:.1f}") - - # Suggest appropriate scaling for standard page sizes - a4_width, a4_height = 595, 842 # A4 in points - - # Calculate suggested scaling to fit A4 - x_factor = a4_width / max_x if max_x > 0 else 1.0 - y_factor = a4_height / max_y if max_y > 0 else 1.0 - - # Apply some heuristics for common scaling issues - if max_x > 1000 and max_y > 1000: - print("\nDetected large coordinates - typical of high-resolution scans or OCR") - print("Suggested scaling factors for standard page:") - print(f"X-factor: {x_factor:.3f} (to fit width to A4)") - print(f"Y-factor: {y_factor:.3f} (to fit height to A4)") - elif max_x < 300 and max_y < 300: - print("\nDetected small coordinates - might be normalized values") - print("Suggested scaling factors for standard page:") - print(f"X-factor: {a4_width/300:.3f} (to expand to A4 width)") - print(f"Y-factor: {a4_height/300:.3f} (to expand to A4 height)") - - # Check for pattern inconsistencies (possible corruption or bad parsing) - widths = [abs(x2 - x1) for x1, y1, x2, y2 in coords] - heights = [abs(y2 - y1) for x1, y1, x2, y2 in coords] - width_std_dev = (sum((w - avg_width) ** 2 for w in widths) / len(widths)) ** 0.5 - height_std_dev = (sum((h - avg_height) ** 2 for h in heights) / len(heights)) ** 0.5 - - if width_std_dev > avg_width * 1.5 or height_std_dev > avg_height * 1.5: - print("\nWarning: High variance in zone sizes detected!") - print("This might indicate inconsistent scaling or parsing issues.") - - return { - 'min_x': min_x, 'max_x': max_x, 'min_y': min_y, 'max_y': max_y, - 'avg_width': avg_width, 'avg_height': avg_height, - 'suggested_x_factor': x_factor, 'suggested_y_factor': y_factor - } - -def main(): - args = parse_arguments() - - # Read the original DocTags file - try: - doctags_content = read_doctags_file(args.doctags) - print(f"Read DocTags file: {args.doctags} ({len(doctags_content)} bytes)") - except Exception as e: - print(f"Error reading DocTags file: {e}") - return 1 - - # Analyze the DocTags content - analysis = analyze_doctags(doctags_content) - - # Confirm with user - if not analysis: - print("No analysis could be performed. Using specified scaling factors.") - else: - print(f"\nYou specified x-factor={args.x_factor}, y-factor={args.y_factor}") - print(f"Analysis suggests x-factor={analysis.get('suggested_x_factor', 1.0):.3f}, " - f"y-factor={analysis.get('suggested_y_factor', 1.0):.3f}") - - use_suggested = input("\nUse suggested scaling factors instead? [y/N]: ").lower() - if use_suggested.startswith('y'): - args.x_factor = analysis.get('suggested_x_factor', args.x_factor) - args.y_factor = analysis.get('suggested_y_factor', args.y_factor) - print(f"Using suggested factors: x={args.x_factor:.3f}, y={args.y_factor:.3f}") - - # Apply the scaling fix - try: - fixed_content = fix_scaling(doctags_content, args.x_factor, args.y_factor, - args.x_offset, args.y_offset) - - # Save the fixed DocTags file - write_doctags_file(fixed_content, args.output) - - print("\nScaling fix complete!") - print("Run visualizer with the fixed DocTags file:") - print(f"python visualizer.py --doctags {args.output} --pdf your_document.pdf --page 1 --show") - - return 0 - except Exception as e: - print(f"Error applying scaling fix: {e}") - import traceback - traceback.print_exc() - return 1 - -if __name__ == "__main__": - sys.exit(main()) \ No newline at end of file diff --git a/index.html b/index.html deleted file mode 100644 index 23134b9..0000000 --- a/index.html +++ /dev/null @@ -1,1306 +0,0 @@ - - - - - - PDF DocTags Analyzer - - - - - - - -
-

PDF DocTags Analyzer

-
- - - -
-
- -
-

Processing Settings

-
-
- - -
-
- - -
-
- - -
-
-
-
- - -
-
- - -
-
- - -
-
- - -
-
-
- -
- -
- - - - - - - -

Upload a PDF to Get Started

-

- The DocTags Analyzer will process your PDF document to identify and visualize - structured elements like headers, text blocks, tables, and images. -

- - -
-
-

- - - - - - - - - - Visualize Content -

-

- See how your document is structured with visual overlays showing text blocks, - headers, images and tables. -

-
-
-

- - - - - Extract DocTags -

-

- Generate structured DocTags markup that identifies document elements - with semantic information. -

-
-
-

- - - - - - Optimize Scale -

-

- Find the optimal scaling factors to perfectly align the detected elements - with the original document. -

-
-
-

- - - - - - Export Results -

-

- Save visualizations as HTML and extract the processed content in multiple - formats including Markdown and HTML. -

-
-
-
- - - - - -
- - - - \ No newline at end of file diff --git a/process_full_document.sh b/process_full_document.sh deleted file mode 100644 index ad84657..0000000 --- a/process_full_document.sh +++ /dev/null @@ -1,172 +0,0 @@ -#!/bin/bash -# process_full_document.sh -# This script processes all pages in a PDF document using the DocTags workflow - -# Default values -PDF_FILE="" -START_PAGE=1 -END_PAGE=0 -X_FACTOR=0.7 -Y_FACTOR=0.7 -DPI=200 -SHOW_OUTPUT=false - -# Function to display usage information -function show_usage { - echo "Usage: $0 -f PDF_FILE [options]" - echo "" - echo "Required:" - echo " -f, --file FILE Path to the PDF file" - echo "" - echo "Options:" - echo " -s, --start PAGE Starting page number (default: 1)" - echo " -e, --end PAGE Ending page number (default: last page)" - echo " -x, --x-factor VALUE X-axis scaling factor (default: 0.7)" - echo " -y, --y-factor VALUE Y-axis scaling factor (default: 0.7)" - echo " -d, --dpi VALUE DPI for PDF rendering (default: 200)" - echo " --show Open visualization in browser (default: false)" - echo " -h, --help Display this help message" - echo "" - echo "Example:" - echo " $0 -f document.pdf -s 1 -e 10 -x 0.8 -y 0.8 --show" -} - -# Parse command line arguments -while [[ $# -gt 0 ]]; do - case "$1" in - -f|--file) - PDF_FILE="$2" - shift 2 - ;; - -s|--start) - START_PAGE="$2" - shift 2 - ;; - -e|--end) - END_PAGE="$2" - shift 2 - ;; - -x|--x-factor) - X_FACTOR="$2" - shift 2 - ;; - -y|--y-factor) - Y_FACTOR="$2" - shift 2 - ;; - -d|--dpi) - DPI="$2" - shift 2 - ;; - --show) - SHOW_OUTPUT=true - shift - ;; - -h|--help) - show_usage - exit 0 - ;; - *) - echo "Error: Unknown option $1" - show_usage - exit 1 - ;; - esac -done - -# Check if PDF file is provided -if [ -z "$PDF_FILE" ]; then - echo "Error: PDF file is required" - show_usage - exit 1 -fi - -# Check if PDF file exists -if [ ! -f "$PDF_FILE" ]; then - echo "Error: PDF file '$PDF_FILE' not found" - exit 1 -fi - -# Get the PDF filename without extension -PDF_BASENAME=$(basename "$PDF_FILE" .pdf) - -# If end page is not specified, get the total number of pages -if [ "$END_PAGE" -eq 0 ]; then - echo "Determining total number of pages..." - # Use the visualizer's --page-count feature to get the number of pages - TOTAL_PAGES=$(python visualizer.py --pdf "$PDF_FILE" --page-count 2>&1 | grep "The PDF has" | awk '{print $4}') - - if [ -z "$TOTAL_PAGES" ]; then - echo "Error: Could not determine the total number of pages" - echo "Please specify the end page with -e" - exit 1 - fi - - END_PAGE=$TOTAL_PAGES - echo "Total pages detected: $TOTAL_PAGES" -fi - -echo "Processing PDF: $PDF_FILE (pages $START_PAGE to $END_PAGE)" -echo "Scaling factors: X=$X_FACTOR, Y=$Y_FACTOR" -echo "" - -# Create output directory -OUTPUT_DIR="${PDF_BASENAME}_output" -mkdir -p "$OUTPUT_DIR" - -# Process each page -for ((PAGE=$START_PAGE; PAGE<=$END_PAGE; PAGE++)); do - echo "====================================================" - echo "Processing page $PAGE of $END_PAGE" - echo "====================================================" - - # Set output filenames - DOCTAGS_FILE="$OUTPUT_DIR/page_${PAGE}.doctags.txt" - FIXED_DOCTAGS_FILE="$OUTPUT_DIR/page_${PAGE}.fixed.doctags.txt" - HTML_OUTPUT="$OUTPUT_DIR/page_${PAGE}.html" - - # Step 1: Analyze the page - echo "Step 1: Analyzing page $PAGE..." - python analyzer.py --image "$PDF_FILE" --page "$PAGE" --output "$DOCTAGS_FILE" --dpi "$DPI" - - if [ ! -f "$DOCTAGS_FILE" ]; then - echo "Warning: Failed to generate DocTags for page $PAGE, skipping to next page" - continue - fi - - # Step 2: Fix scaling issues - echo "Step 2: Fixing scaling for page $PAGE..." - python fix_scaling.py --doctags "$DOCTAGS_FILE" --output "$FIXED_DOCTAGS_FILE" --x-factor "$X_FACTOR" --y-factor "$Y_FACTOR" - - if [ ! -f "$FIXED_DOCTAGS_FILE" ]; then - echo "Warning: Failed to fix scaling for page $PAGE, using original DocTags" - FIXED_DOCTAGS_FILE="$DOCTAGS_FILE" - fi - - # Step 3: Visualize the page - echo "Step 3: Generating visualization for page $PAGE..." - SHOW_FLAG="" - if [ "$SHOW_OUTPUT" = true ] && [ "$PAGE" -eq "$END_PAGE" ]; then - # Only show the last page in browser to avoid opening too many windows - SHOW_FLAG="--show" - fi - - python visualizer.py --doctags "$FIXED_DOCTAGS_FILE" --pdf "$PDF_FILE" --page "$PAGE" --output "$HTML_OUTPUT" --adjust $SHOW_FLAG - - echo "Page $PAGE processing complete. Output saved to $HTML_OUTPUT" - echo "" -done - -echo "All pages processed successfully!" -echo "Output files are saved in: $OUTPUT_DIR" - -# Open the output directory -if command -v xdg-open &> /dev/null; then - xdg-open "$OUTPUT_DIR" & -elif command -v open &> /dev/null; then - open "$OUTPUT_DIR" & -elif command -v explorer &> /dev/null; then - explorer "$OUTPUT_DIR" & -fi - -exit 0 \ No newline at end of file diff --git a/process_single_page.sh b/process_single_page.sh deleted file mode 100755 index fe91987..0000000 --- a/process_single_page.sh +++ /dev/null @@ -1,165 +0,0 @@ -#!/bin/bash -# process_single_page_fixed.sh -# This script processes a single page in a PDF document using the DocTags workflow with improved error handling - -# Default values -PDF_FILE="" -PAGE=0 -X_FACTOR=0.7 -Y_FACTOR=0.7 -DPI=200 - -# Function to display usage information -function show_usage { - echo "Usage: $0 -f PDF_FILE [options]" - echo "" - echo "Required:" - echo " -f, --file FILE Path to the PDF file" - echo "" - echo "Options:" - echo " -p, --page PAGE Page number (if not provided, will prompt)" - echo " -x, --x-factor VALUE X-axis scaling factor (default: 0.7)" - echo " -y, --y-factor VALUE Y-axis scaling factor (default: 0.7)" - echo " -d, --dpi VALUE DPI for PDF rendering (default: 200)" - echo " -h, --help Display this help message" - echo "" - echo "Example:" - echo " $0 -f document.pdf -p 7 -x 0.8 -y 0.8" -} - -# Parse command line arguments -while [[ $# -gt 0 ]]; do - case "$1" in - -f|--file) - PDF_FILE="$2" - shift 2 - ;; - -p|--page) - PAGE="$2" - shift 2 - ;; - -x|--x-factor) - X_FACTOR="$2" - shift 2 - ;; - -y|--y-factor) - Y_FACTOR="$2" - shift 2 - ;; - -d|--dpi) - DPI="$2" - shift 2 - ;; - -h|--help) - show_usage - exit 0 - ;; - *) - echo "Error: Unknown option $1" - show_usage - exit 1 - ;; - esac -done - -# Check if PDF file is provided -if [ -z "$PDF_FILE" ]; then - echo "Error: PDF file is required" - show_usage - exit 1 -fi - -# Check if PDF file exists -if [ ! -f "$PDF_FILE" ]; then - echo "Error: PDF file '$PDF_FILE' not found" - exit 1 -fi - -# Get the PDF filename without extension -PDF_BASENAME=$(basename "$PDF_FILE" .pdf) - -# If page is not specified, determine the total pages and prompt user -if [ "$PAGE" -eq 0 ]; then - echo "Determining total number of pages..." - # Use the visualizer's --page-count feature to get the number of pages - TOTAL_PAGES=$(python visualizer.py --pdf "$PDF_FILE" --page-count 2>&1 | grep "The PDF has" | awk '{print $4}') - - if [ -z "$TOTAL_PAGES" ]; then - echo "Error: Could not determine the total number of pages" - echo "Please specify the page with -p" - exit 1 - fi - - echo "The PDF has $TOTAL_PAGES pages." - - # Prompt for page number - while true; do - read -p "Enter the page number to process (1-$TOTAL_PAGES): " PAGE - if [[ "$PAGE" =~ ^[0-9]+$ ]] && [ "$PAGE" -ge 1 ] && [ "$PAGE" -le "$TOTAL_PAGES" ]; then - break - else - echo "Invalid page number. Please enter a number between 1 and $TOTAL_PAGES." - fi - done -fi - -# Create output directory -OUTPUT_DIR="${PDF_BASENAME}_output" -mkdir -p "$OUTPUT_DIR" - -# Set output filenames - explicit paths -DOCTAGS_FILE="$OUTPUT_DIR/page_${PAGE}.doctags.txt" -FIXED_DOCTAGS_FILE="$OUTPUT_DIR/page_${PAGE}.fixed.doctags.txt" -HTML_OUTPUT="$OUTPUT_DIR/page_${PAGE}.html" - -echo "Processing PDF: $PDF_FILE (page $PAGE)" -echo "Scaling factors: X=$X_FACTOR, Y=$Y_FACTOR" -echo "" - -# Step 1: Analyze the page -echo "Step 1: Analyzing page $PAGE..." -python analyzer.py --image "$PDF_FILE" --page "$PAGE" --output "$DOCTAGS_FILE" --dpi "$DPI" - -# Check for DocTags file with different extensions -if [ ! -f "$DOCTAGS_FILE" ]; then - # Try alternative extensions - ALTERNATIVE_FILE="$OUTPUT_DIR/page_${PAGE}.doctags.doctags.txt" - if [ -f "$ALTERNATIVE_FILE" ]; then - echo "Found DocTags at alternative location: $ALTERNATIVE_FILE" - DOCTAGS_FILE="$ALTERNATIVE_FILE" - else - # Try to find any doctags file in the output directory - FOUND_FILE=$(find "$OUTPUT_DIR" -name "*.doctags.txt" -print -quit) - if [ -n "$FOUND_FILE" ]; then - echo "Found DocTags at location: $FOUND_FILE" - DOCTAGS_FILE="$FOUND_FILE" - else - echo "Error: Failed to generate DocTags for page $PAGE" - echo "Could not find any doctags files in $OUTPUT_DIR" - exit 1 - fi - fi -fi - -# Step 2: Fix scaling issues -echo "Step 2: Fixing scaling for page $PAGE..." -python fix_scaling.py --doctags "$DOCTAGS_FILE" --output "$FIXED_DOCTAGS_FILE" --x-factor "$X_FACTOR" --y-factor "$Y_FACTOR" - -if [ ! -f "$FIXED_DOCTAGS_FILE" ]; then - echo "Warning: Failed to fix scaling for page $PAGE, using original DocTags" - FIXED_DOCTAGS_FILE="$DOCTAGS_FILE" -fi - -# Step 3: Visualize the page -echo "Step 3: Generating visualization for page $PAGE..." -python visualizer.py --doctags "$FIXED_DOCTAGS_FILE" --pdf "$PDF_FILE" --page "$PAGE" --output "$HTML_OUTPUT" --adjust --show - -echo "Page $PAGE processing complete." -echo "Output saved to: $HTML_OUTPUT" - -# Show the command used to process this page -echo "" -echo "Command used:" -echo "python analyzer.py --image \"$PDF_FILE\" --page $PAGE && python fix_scaling.py --doctags \"$DOCTAGS_FILE\" --output \"$FIXED_DOCTAGS_FILE\" --x-factor $X_FACTOR --y-factor $Y_FACTOR && python visualizer.py --doctags \"$FIXED_DOCTAGS_FILE\" --pdf \"$PDF_FILE\" --page $PAGE --adjust --show" - -exit 0 \ No newline at end of file diff --git a/visualizer.py b/visualizer.py index 90221c8..d987dd2 100644 --- a/visualizer.py +++ b/visualizer.py @@ -426,6 +426,36 @@ def create_visualization(image, zones, pdf_path, page_num, total_pages, output_p return output_path +def normalize_coordinates(zones, image_width, image_height, grid_size=500): + """ + Normalize coordinates from the DocTags grid (0-500) to actual image dimensions. + + Args: + zones: List of zone dictionaries with x1, y1, x2, y2 coordinates + image_width: Width of the PDF page image in pixels + image_height: Height of the PDF page image in pixels + grid_size: The grid size used in DocTags (default 500) + + Returns: + The same zones list with updated coordinates + """ + # Create a copy of the zones to avoid modifying the original + normalized_zones = [] + + for zone in zones: + # Clone the zone + new_zone = zone.copy() + + # Convert from grid coordinates to actual page dimensions + new_zone['x1'] = int(zone['x1'] * image_width / grid_size) + new_zone['y1'] = int(zone['y1'] * image_height / grid_size) + new_zone['x2'] = int(zone['x2'] * image_width / grid_size) + new_zone['y2'] = int(zone['y2'] * image_height / grid_size) + + normalized_zones.append(new_zone) + + return normalized_zones + def create_debug_image(image, zones, page_num, output_path): """Create a debug image with rectangles around zones.""" # Create a copy of the input image @@ -511,20 +541,22 @@ def process_page(pdf_path, page_num, doctags_path, output_base, dpi=200, show=Fa print(f"Found {len(zones)} zones in DocTags") # Debug output to understand scaling issues + # After parsing the zones from DocTags if zones: + img_width, img_height = image.size + print(f"Image dimensions: {img_width}x{img_height}") + + # Check if we need to normalize grid coordinates max_x = max([zone['x2'] for zone in zones]) max_y = max([zone['y2'] for zone in zones]) - min_x = min([zone['x1'] for zone in zones]) - min_y = min([zone['y1'] for zone in zones]) - img_width, img_height = image.size - print(f"Image dimensions: {img_width}x{img_height}") - print(f"Zone boundaries: X({min_x}-{max_x}), Y({min_y}-{max_y})") - print(f"Suggested scale factors: X={img_width/max_x:.3f}, Y={img_height/max_y:.3f}") - - # Always apply automatic scaling (making adjust=True by default) - # This solves the common scaling issue between DocTags coordinates and image dimensions - if adjust: + # If coordinates seem to be in a normalized grid (0-500 range) + if max_x <= 500 and max_y <= 500: + print(f"Detected normalized coordinates (0-500 grid)") + zones = normalize_coordinates(zones, img_width, img_height) + print(f"Applied automatic grid normalization") + # If auto-adjust is enabled and coordinates are not in normalized grid + elif adjust: width, height = image.size # Calculate appropriate scaling factors with better heuristics