danvics/reddit-universal-scraper
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat: real-time progress for posts processing and comments

Browse source
This commit is contained in:
Sanjeev Kumar 2025-12-14 07:47:51 +05:30
parent 3c6f5415cb
commit 8ffb53eae3
2 changed files with 47 additions and 13 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

54
dashboard/app.py
View file

@ -356,35 +356,63 @@ def main():
lines = f.readlines() lines = f.readlines()
# Parse metrics from lines # Parse metrics from lines
posts_count = 0 posts_saved = 0
comments_count = 0 comments_count = 0
images_count = 0 images_count = 0
videos_count = 0 videos_count = 0
found_posts = 0
processed_posts = 0
for line in lines: for line in lines:
import re import re
# Progress: X/Y # Progress: X/Y (Saved posts)
m = re.search(r'Progress: (\d+)/(\d+)', line) m = re.search(r'Progress: (\d+)/(\d+)', line)
if m: posts_count = int(m.group(1)) if m: posts_saved = int(m.group(1))
# Saved X posts # Saved X posts
m = re.search(r'Saved (\d+)', line) m = re.search(r'Saved (\d+)', line)
if m: posts_count += int(m.group(1)) if m: posts_saved += int(m.group(1))
# Comments: X # Found X posts
m = re.search(r'Found (\d+) posts', line)
if m: found_posts += int(m.group(1))
# Processed posts (Fetching comments)
if "Fetching comments for:" in line:
processed_posts += 1
# Comments: X (Summary)
m = re.search(r'Comments:\s*(\d+)', line) m = re.search(r'Comments:\s*(\d+)', line)
if m: comments_count = int(m.group(1)) if m:
if "Fetching comments for:" in line: comments_count += 1 comments_count = int(m.group(1))
else:
# Incremental comments
m = re.search(r'\+ Scraped (\d+) comments', line)
if m: comments_count += int(m.group(1))
# Images/Videos # Images/Videos (Summary line)
m = re.search(r'Images:\s*(\d+)', line) m = re.search(r'Images:\s*(\d+).*Videos:\s*(\d+)', line)
if m: images_count = int(m.group(1)) if m:
m = re.search(r'Videos:\s*(\d+)', line) images_count = int(m.group(1))
if m: videos_count = int(m.group(1)) videos_count = int(m.group(2))
# Images/Videos (Real-time line)
m = re.search(r'\+ Downloaded: (\d+) images, (\d+) videos', line)
if m:
images_count += int(m.group(1))
videos_count += int(m.group(2))
# Display Metrics # Display Metrics
col1, col2, col3, col4 = st.columns(4) col1, col2, col3, col4 = st.columns(4)
col1.metric("📊 Posts", posts_count)
# Posts Metric Logic
if posts_saved > 0:
col1.metric("📊 Posts", f"{posts_saved} (Found {found_posts})")
elif found_posts > 0:
col1.metric("📊 Posts", f"Processing: {processed_posts}/{found_posts}")
else:
col1.metric("📊 Posts", "0")
col2.metric("💬 Comments", comments_count) col2.metric("💬 Comments", comments_count)
col3.metric("🖼️ Images", images_count) col3.metric("🖼️ Images", images_count)
col4.metric("🎬 Videos", videos_count) col4.metric("🎬 Videos", videos_count)

6
main.py
View file

@ -313,6 +313,9 @@ def scrape_comments(permalink, max_depth=3):
except Exception as e: except Exception as e:
pass pass
if len(comments) > 0:
print(f" + Scraped {len(comments)} comments")
return comments return comments
def parse_comments(comment_list, post_permalink, depth=0, max_depth=3): def parse_comments(comment_list, post_permalink, depth=0, max_depth=3):
@ -481,6 +484,9 @@ def run_full_history(target, limit, is_user=False, download_media_flag=True,
post['media_downloaded'] = downloaded['images'] > 0 or downloaded['videos'] > 0 post['media_downloaded'] = downloaded['images'] > 0 or downloaded['videos'] > 0
total_media['images'] += downloaded['images'] total_media['images'] += downloaded['images']
total_media['videos'] += downloaded['videos'] total_media['videos'] += downloaded['videos']
if downloaded['images'] > 0 or downloaded['videos'] > 0:
print(f" + Downloaded: {downloaded['images']} images, {downloaded['videos']} videos")
posts.append(post) posts.append(post)