danvics/reddit-universal-scraper
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix: improved regex parsing for real-time metrics extraction

Browse source
This commit is contained in:
Sanjeev Kumar 2025-12-14 07:37:22 +05:30
parent 9703075561
commit 5f0a5ba0db
1 changed files with 53 additions and 12 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

65
dashboard/app.py
View file

@ -394,37 +394,78 @@ def main():
output_lines.append(line) output_lines.append(line)
# Parse metrics from output # Parse metrics from output
if "Progress:" in line: # Match: 📊 Progress: 5/100 posts OR Progress: 5/100
if "Progress:" in line and "/" in line:
try: try:
parts = line.split("/") # Extract numbers around the /
posts_count = int(parts[0].split()[-1]) import re
total = int(parts[1].split()[0]) match = re.search(r'(\d+)\s*/\s*(\d+)', line)
progress_bar.progress(min(posts_count / total, 1.0)) if match:
posts_metric.metric("📊 Posts", f"{posts_count}/{total}") posts_count = int(match.group(1))
total = int(match.group(2))
progress_bar.progress(min(posts_count / total, 1.0))
posts_metric.metric("📊 Posts", f"{posts_count}/{total}")
except: except:
pass pass
# Match: Saved X new posts
if "Saved" in line and "posts" in line:
try:
import re
match = re.search(r'Saved (\d+)', line)
if match:
posts_count += int(match.group(1))
posts_metric.metric("📊 Posts", str(posts_count))
except:
pass
# Match: 💬 Comments: X OR Comments: X
if "Comments:" in line: if "Comments:" in line:
try: try:
comments_count = int(line.split("Comments:")[-1].strip().split()[0]) import re
comments_metric.metric("💬 Comments", str(comments_count)) match = re.search(r'Comments:\s*(\d+)', line)
if match:
comments_count = int(match.group(1))
comments_metric.metric("💬 Comments", str(comments_count))
except: except:
pass pass
# Count comment fetching lines
if "Fetching comments for:" in line:
comments_count += 1
comments_metric.metric("💬 Comments", f"~{comments_count}")
# Match: 🖼️ Images: X OR Images: X
if "Images:" in line: if "Images:" in line:
try: try:
images_count = int(line.split("Images:")[-1].strip().split()[0]) import re
images_metric.metric("🖼️ Images", str(images_count)) match = re.search(r'Images:\s*(\d+)', line)
if match:
images_count = int(match.group(1))
images_metric.metric("🖼️ Images", str(images_count))
except: except:
pass pass
# Match: 🎬 Videos: X OR Videos: X
if "Videos:" in line: if "Videos:" in line:
try: try:
videos_count = int(line.split("Videos:")[-1].strip().split()[0]) import re
videos_metric.metric("🎬 Videos", str(videos_count)) match = re.search(r'Videos:\s*(\d+)', line)
if match:
videos_count = int(match.group(1))
videos_metric.metric("🎬 Videos", str(videos_count))
except: except:
pass pass
# Match: Found X posts in this batch
if "Found" in line and "posts" in line:
try:
import re
match = re.search(r'Found (\d+) posts', line)
if match:
batch = int(match.group(1))
status_container.info(f"⏱️ Found {batch} posts in batch...")
# Update status # Update status
elapsed = time.time() - start_time elapsed = time.time() - start_time
rate = posts_count / elapsed if elapsed > 0 else 0 rate = posts_count / elapsed if elapsed > 0 else 0