Remove useless code from analyzer
This commit is contained in:
pjmalandrino
parent
9bc8ba9424
commit
46ac5adca2
1 changed files with 0 additions and 180 deletions
180
analyzer.py
180
analyzer.py
|
|
@ -261,161 +261,6 @@ def debug_doctags(doctags_text, debug_mode=False):
|
|||
|
||||
return doctags_text
|
||||
|
||||
|
||||
def create_html_document(image_path, doctags_text, pil_image, args):
|
||||
"""Create HTML directly from DocTags and image."""
|
||||
print("Creating HTML document directly...")
|
||||
|
||||
# Get filename for document title
|
||||
doc_name = Path(image_path).stem
|
||||
|
||||
# Extract content from doctags
|
||||
title_match = re.search(r'<section_header_level_1>.*?>(.*?)</section_header_level_1>', doctags_text)
|
||||
title = title_match.group(1) if title_match else doc_name
|
||||
|
||||
# Clean title by removing any location tags
|
||||
title = re.sub(r'<loc_\d+>', '', title)
|
||||
|
||||
# Extract text content
|
||||
text_matches = re.findall(r'<text>.*?>(.*?)</text>', doctags_text)
|
||||
paragraphs = []
|
||||
for text in text_matches:
|
||||
# Clean text by removing any location tags
|
||||
cleaned_text = re.sub(r'<loc_\d+>', '', text)
|
||||
paragraphs.append(cleaned_text)
|
||||
|
||||
# Extract list items
|
||||
list_items = []
|
||||
item_matches = re.findall(r'<list_item>.*?>(.*?)</list_item>', doctags_text)
|
||||
for item in item_matches:
|
||||
# Clean item by removing any location tags
|
||||
cleaned_item = re.sub(r'<loc_\d+>', '', item)
|
||||
list_items.append(cleaned_item)
|
||||
|
||||
# Extract footer
|
||||
footer_match = re.search(r'<page_footer>.*?>(.*?)</page_footer>', doctags_text)
|
||||
footer_text = ""
|
||||
if footer_match:
|
||||
footer_text = re.sub(r'<loc_\d+>', '', footer_match.group(1))
|
||||
|
||||
# Create image data URI
|
||||
max_dimension = 1200
|
||||
img_for_embedding = pil_image
|
||||
if max(pil_image.size) > max_dimension:
|
||||
ratio = min(max_dimension / pil_image.size[0], max_dimension / pil_image.size[1])
|
||||
new_size = (int(pil_image.size[0] * ratio), int(pil_image.size[1] * ratio))
|
||||
img_for_embedding = pil_image.resize(new_size, Image.LANCZOS)
|
||||
|
||||
buffered = BytesIO()
|
||||
img_for_embedding.save(buffered, format="PNG", optimize=True, quality=90)
|
||||
img_str = base64.b64encode(buffered.getvalue()).decode()
|
||||
img_data_uri = f"data:image/png;base64,{img_str}"
|
||||
|
||||
# Build HTML paragraphs
|
||||
paragraphs_html = "".join([f"<p>{text}</p>\n" for text in paragraphs]) if paragraphs else ""
|
||||
|
||||
# Build list HTML
|
||||
list_html = ""
|
||||
if list_items:
|
||||
list_html = "<ul>\n"
|
||||
for item in list_items:
|
||||
list_html += f"<li>{item}</li>\n"
|
||||
list_html += "</ul>\n"
|
||||
|
||||
# Create footer HTML
|
||||
footer_html = f"<div class='page-footer'>{footer_text}</div>\n" if footer_text else ""
|
||||
|
||||
# Create HTML document
|
||||
html_content = f'''<!DOCTYPE html>
|
||||
<html>
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<title>{title}</title>
|
||||
<style>
|
||||
body {{ font-family: Arial, sans-serif; margin: 0 auto; max-width: 800px; padding: 20px; line-height: 1.6; }}
|
||||
img {{ max-width: 100%; height: auto; }}
|
||||
h1 {{ color: #333; border-bottom: 1px solid #eee; padding-bottom: 10px; }}
|
||||
figure {{ margin: 20px 0; text-align: center; }}
|
||||
figcaption {{ color: #666; font-style: italic; margin-top: 5px; }}
|
||||
ul {{ margin-left: 20px; padding-left: 20px; }}
|
||||
li {{ margin-bottom: 10px; }}
|
||||
.page-footer {{ color: #666; margin-top: 20px; border-top: 1px solid #eee; padding-top: 5px; text-align: center; }}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div class='page'>
|
||||
<h1>{title}</h1>
|
||||
<figure>
|
||||
<img src="{img_data_uri}" alt="Document Image"/>
|
||||
<figcaption>Page {args.page}</figcaption>
|
||||
</figure>
|
||||
{paragraphs_html}
|
||||
{list_html}
|
||||
{footer_html}
|
||||
</div>
|
||||
</body>
|
||||
</html>'''
|
||||
|
||||
return html_content
|
||||
|
||||
|
||||
def create_markdown_document(image_path, doctags_text, args):
|
||||
"""Create Markdown document from DocTags."""
|
||||
print("Creating Markdown document...")
|
||||
|
||||
# Get filename for document title
|
||||
doc_name = Path(image_path).stem
|
||||
|
||||
# Extract content from doctags
|
||||
title_match = re.search(r'<section_header_level_1>.*?>(.*?)</section_header_level_1>', doctags_text)
|
||||
title = title_match.group(1) if title_match else doc_name
|
||||
|
||||
# Clean title by removing any location tags
|
||||
title = re.sub(r'<loc_\d+>', '', title)
|
||||
|
||||
# Extract text content
|
||||
text_matches = re.findall(r'<text>.*?>(.*?)</text>', doctags_text)
|
||||
paragraphs = []
|
||||
for text in text_matches:
|
||||
# Clean text by removing any location tags
|
||||
cleaned_text = re.sub(r'<loc_\d+>', '', text)
|
||||
paragraphs.append(cleaned_text)
|
||||
|
||||
# Extract list items
|
||||
list_items = []
|
||||
item_matches = re.findall(r'<list_item>.*?>(.*?)</list_item>', doctags_text)
|
||||
for item in item_matches:
|
||||
# Clean item by removing any location tags
|
||||
cleaned_item = re.sub(r'<loc_\d+>', '', item)
|
||||
list_items.append(cleaned_item)
|
||||
|
||||
# Extract footer
|
||||
footer_match = re.search(r'<page_footer>.*?>(.*?)</page_footer>', doctags_text)
|
||||
footer_text = ""
|
||||
if footer_match:
|
||||
footer_text = re.sub(r'<loc_\d+>', '', footer_match.group(1))
|
||||
|
||||
# Build markdown content
|
||||
markdown = f"# {title}\n\n"
|
||||
markdown += f"*Page {args.page}*\n\n"
|
||||
|
||||
# Add paragraphs
|
||||
for p in paragraphs:
|
||||
markdown += f"{p}\n\n"
|
||||
|
||||
# Add list
|
||||
if list_items:
|
||||
for item in list_items:
|
||||
markdown += f"* {item}\n"
|
||||
markdown += "\n"
|
||||
|
||||
# Add footer
|
||||
if footer_text:
|
||||
markdown += f"---\n{footer_text}\n"
|
||||
|
||||
return markdown
|
||||
|
||||
|
||||
def process_page(args, model, processor, config, image_path, pil_image, page_num=1):
|
||||
"""Process a single page from a PDF or image file."""
|
||||
from mlx_vlm.prompt_utils import apply_chat_template
|
||||
|
|
@ -487,31 +332,6 @@ def process_page(args, model, processor, config, image_path, pil_image, page_num
|
|||
f.write(tags_analysis)
|
||||
print(f"DocTags analysis saved to: {tags_path}")
|
||||
|
||||
# If not in DocTags-only mode, create documents
|
||||
if not args.doctags_only:
|
||||
# Create markdown document
|
||||
md_content = create_markdown_document(image_path, output, args)
|
||||
md_path = results_dir / f"{output_path.stem}.md"
|
||||
with open(md_path, 'w', encoding='utf-8') as f:
|
||||
f.write(md_content)
|
||||
print(f"Markdown saved to: {md_path}")
|
||||
|
||||
# Create HTML document
|
||||
html_content = create_html_document(image_path, output, pil_image, args)
|
||||
html_path = results_dir / f"{output_path.stem}.html"
|
||||
with open(html_path, 'w', encoding='utf-8') as f:
|
||||
f.write(html_content)
|
||||
print(f"HTML saved to: {html_path}")
|
||||
|
||||
# Calculate final file size
|
||||
html_size = os.path.getsize(html_path)
|
||||
print(f"Final HTML file size: {html_size} bytes")
|
||||
|
||||
# Open in browser if requested (only for the first page or single pages)
|
||||
if args.show and html_size > 100 and page_num <= 1:
|
||||
import webbrowser
|
||||
webbrowser.open(f"file:///{str(html_path.resolve())}")
|
||||
|
||||
# Save a copy of the processed image for reference if in debug mode
|
||||
if args.debug:
|
||||
img_debug_path = results_dir / f"{output_path.stem}.debug.png"
|
||||
|
|
|
|||
Loading…
Reference in a new issue