Remove useless code from analyzer
This commit is contained in:
pjmalandrino
parent
9bc8ba9424
commit
46ac5adca2
1 changed files with 0 additions and 180 deletions
180
analyzer.py
180
analyzer.py
|
|
@ -261,161 +261,6 @@ def debug_doctags(doctags_text, debug_mode=False):
|
||||||
|
|
||||||
return doctags_text
|
return doctags_text
|
||||||
|
|
||||||
|
|
||||||
def create_html_document(image_path, doctags_text, pil_image, args):
|
|
||||||
"""Create HTML directly from DocTags and image."""
|
|
||||||
print("Creating HTML document directly...")
|
|
||||||
|
|
||||||
# Get filename for document title
|
|
||||||
doc_name = Path(image_path).stem
|
|
||||||
|
|
||||||
# Extract content from doctags
|
|
||||||
title_match = re.search(r'<section_header_level_1>.*?>(.*?)</section_header_level_1>', doctags_text)
|
|
||||||
title = title_match.group(1) if title_match else doc_name
|
|
||||||
|
|
||||||
# Clean title by removing any location tags
|
|
||||||
title = re.sub(r'<loc_\d+>', '', title)
|
|
||||||
|
|
||||||
# Extract text content
|
|
||||||
text_matches = re.findall(r'<text>.*?>(.*?)</text>', doctags_text)
|
|
||||||
paragraphs = []
|
|
||||||
for text in text_matches:
|
|
||||||
# Clean text by removing any location tags
|
|
||||||
cleaned_text = re.sub(r'<loc_\d+>', '', text)
|
|
||||||
paragraphs.append(cleaned_text)
|
|
||||||
|
|
||||||
# Extract list items
|
|
||||||
list_items = []
|
|
||||||
item_matches = re.findall(r'<list_item>.*?>(.*?)</list_item>', doctags_text)
|
|
||||||
for item in item_matches:
|
|
||||||
# Clean item by removing any location tags
|
|
||||||
cleaned_item = re.sub(r'<loc_\d+>', '', item)
|
|
||||||
list_items.append(cleaned_item)
|
|
||||||
|
|
||||||
# Extract footer
|
|
||||||
footer_match = re.search(r'<page_footer>.*?>(.*?)</page_footer>', doctags_text)
|
|
||||||
footer_text = ""
|
|
||||||
if footer_match:
|
|
||||||
footer_text = re.sub(r'<loc_\d+>', '', footer_match.group(1))
|
|
||||||
|
|
||||||
# Create image data URI
|
|
||||||
max_dimension = 1200
|
|
||||||
img_for_embedding = pil_image
|
|
||||||
if max(pil_image.size) > max_dimension:
|
|
||||||
ratio = min(max_dimension / pil_image.size[0], max_dimension / pil_image.size[1])
|
|
||||||
new_size = (int(pil_image.size[0] * ratio), int(pil_image.size[1] * ratio))
|
|
||||||
img_for_embedding = pil_image.resize(new_size, Image.LANCZOS)
|
|
||||||
|
|
||||||
buffered = BytesIO()
|
|
||||||
img_for_embedding.save(buffered, format="PNG", optimize=True, quality=90)
|
|
||||||
img_str = base64.b64encode(buffered.getvalue()).decode()
|
|
||||||
img_data_uri = f"data:image/png;base64,{img_str}"
|
|
||||||
|
|
||||||
# Build HTML paragraphs
|
|
||||||
paragraphs_html = "".join([f"<p>{text}</p>\n" for text in paragraphs]) if paragraphs else ""
|
|
||||||
|
|
||||||
# Build list HTML
|
|
||||||
list_html = ""
|
|
||||||
if list_items:
|
|
||||||
list_html = "<ul>\n"
|
|
||||||
for item in list_items:
|
|
||||||
list_html += f"<li>{item}</li>\n"
|
|
||||||
list_html += "</ul>\n"
|
|
||||||
|
|
||||||
# Create footer HTML
|
|
||||||
footer_html = f"<div class='page-footer'>{footer_text}</div>\n" if footer_text else ""
|
|
||||||
|
|
||||||
# Create HTML document
|
|
||||||
html_content = f'''<!DOCTYPE html>
|
|
||||||
<html>
|
|
||||||
<head>
|
|
||||||
<meta charset="UTF-8">
|
|
||||||
<title>{title}</title>
|
|
||||||
<style>
|
|
||||||
body {{ font-family: Arial, sans-serif; margin: 0 auto; max-width: 800px; padding: 20px; line-height: 1.6; }}
|
|
||||||
img {{ max-width: 100%; height: auto; }}
|
|
||||||
h1 {{ color: #333; border-bottom: 1px solid #eee; padding-bottom: 10px; }}
|
|
||||||
figure {{ margin: 20px 0; text-align: center; }}
|
|
||||||
figcaption {{ color: #666; font-style: italic; margin-top: 5px; }}
|
|
||||||
ul {{ margin-left: 20px; padding-left: 20px; }}
|
|
||||||
li {{ margin-bottom: 10px; }}
|
|
||||||
.page-footer {{ color: #666; margin-top: 20px; border-top: 1px solid #eee; padding-top: 5px; text-align: center; }}
|
|
||||||
</style>
|
|
||||||
</head>
|
|
||||||
<body>
|
|
||||||
<div class='page'>
|
|
||||||
<h1>{title}</h1>
|
|
||||||
<figure>
|
|
||||||
<img src="{img_data_uri}" alt="Document Image"/>
|
|
||||||
<figcaption>Page {args.page}</figcaption>
|
|
||||||
</figure>
|
|
||||||
{paragraphs_html}
|
|
||||||
{list_html}
|
|
||||||
{footer_html}
|
|
||||||
</div>
|
|
||||||
</body>
|
|
||||||
</html>'''
|
|
||||||
|
|
||||||
return html_content
|
|
||||||
|
|
||||||
|
|
||||||
def create_markdown_document(image_path, doctags_text, args):
|
|
||||||
"""Create Markdown document from DocTags."""
|
|
||||||
print("Creating Markdown document...")
|
|
||||||
|
|
||||||
# Get filename for document title
|
|
||||||
doc_name = Path(image_path).stem
|
|
||||||
|
|
||||||
# Extract content from doctags
|
|
||||||
title_match = re.search(r'<section_header_level_1>.*?>(.*?)</section_header_level_1>', doctags_text)
|
|
||||||
title = title_match.group(1) if title_match else doc_name
|
|
||||||
|
|
||||||
# Clean title by removing any location tags
|
|
||||||
title = re.sub(r'<loc_\d+>', '', title)
|
|
||||||
|
|
||||||
# Extract text content
|
|
||||||
text_matches = re.findall(r'<text>.*?>(.*?)</text>', doctags_text)
|
|
||||||
paragraphs = []
|
|
||||||
for text in text_matches:
|
|
||||||
# Clean text by removing any location tags
|
|
||||||
cleaned_text = re.sub(r'<loc_\d+>', '', text)
|
|
||||||
paragraphs.append(cleaned_text)
|
|
||||||
|
|
||||||
# Extract list items
|
|
||||||
list_items = []
|
|
||||||
item_matches = re.findall(r'<list_item>.*?>(.*?)</list_item>', doctags_text)
|
|
||||||
for item in item_matches:
|
|
||||||
# Clean item by removing any location tags
|
|
||||||
cleaned_item = re.sub(r'<loc_\d+>', '', item)
|
|
||||||
list_items.append(cleaned_item)
|
|
||||||
|
|
||||||
# Extract footer
|
|
||||||
footer_match = re.search(r'<page_footer>.*?>(.*?)</page_footer>', doctags_text)
|
|
||||||
footer_text = ""
|
|
||||||
if footer_match:
|
|
||||||
footer_text = re.sub(r'<loc_\d+>', '', footer_match.group(1))
|
|
||||||
|
|
||||||
# Build markdown content
|
|
||||||
markdown = f"# {title}\n\n"
|
|
||||||
markdown += f"*Page {args.page}*\n\n"
|
|
||||||
|
|
||||||
# Add paragraphs
|
|
||||||
for p in paragraphs:
|
|
||||||
markdown += f"{p}\n\n"
|
|
||||||
|
|
||||||
# Add list
|
|
||||||
if list_items:
|
|
||||||
for item in list_items:
|
|
||||||
markdown += f"* {item}\n"
|
|
||||||
markdown += "\n"
|
|
||||||
|
|
||||||
# Add footer
|
|
||||||
if footer_text:
|
|
||||||
markdown += f"---\n{footer_text}\n"
|
|
||||||
|
|
||||||
return markdown
|
|
||||||
|
|
||||||
|
|
||||||
def process_page(args, model, processor, config, image_path, pil_image, page_num=1):
|
def process_page(args, model, processor, config, image_path, pil_image, page_num=1):
|
||||||
"""Process a single page from a PDF or image file."""
|
"""Process a single page from a PDF or image file."""
|
||||||
from mlx_vlm.prompt_utils import apply_chat_template
|
from mlx_vlm.prompt_utils import apply_chat_template
|
||||||
|
|
@ -487,31 +332,6 @@ def process_page(args, model, processor, config, image_path, pil_image, page_num
|
||||||
f.write(tags_analysis)
|
f.write(tags_analysis)
|
||||||
print(f"DocTags analysis saved to: {tags_path}")
|
print(f"DocTags analysis saved to: {tags_path}")
|
||||||
|
|
||||||
# If not in DocTags-only mode, create documents
|
|
||||||
if not args.doctags_only:
|
|
||||||
# Create markdown document
|
|
||||||
md_content = create_markdown_document(image_path, output, args)
|
|
||||||
md_path = results_dir / f"{output_path.stem}.md"
|
|
||||||
with open(md_path, 'w', encoding='utf-8') as f:
|
|
||||||
f.write(md_content)
|
|
||||||
print(f"Markdown saved to: {md_path}")
|
|
||||||
|
|
||||||
# Create HTML document
|
|
||||||
html_content = create_html_document(image_path, output, pil_image, args)
|
|
||||||
html_path = results_dir / f"{output_path.stem}.html"
|
|
||||||
with open(html_path, 'w', encoding='utf-8') as f:
|
|
||||||
f.write(html_content)
|
|
||||||
print(f"HTML saved to: {html_path}")
|
|
||||||
|
|
||||||
# Calculate final file size
|
|
||||||
html_size = os.path.getsize(html_path)
|
|
||||||
print(f"Final HTML file size: {html_size} bytes")
|
|
||||||
|
|
||||||
# Open in browser if requested (only for the first page or single pages)
|
|
||||||
if args.show and html_size > 100 and page_num <= 1:
|
|
||||||
import webbrowser
|
|
||||||
webbrowser.open(f"file:///{str(html_path.resolve())}")
|
|
||||||
|
|
||||||
# Save a copy of the processed image for reference if in debug mode
|
# Save a copy of the processed image for reference if in debug mode
|
||||||
if args.debug:
|
if args.debug:
|
||||||
img_debug_path = results_dir / f"{output_path.stem}.debug.png"
|
img_debug_path = results_dir / f"{output_path.stem}.debug.png"
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue