danvics/reddit-universal-scraper
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix: unbuffered subprocess output for real-time dashboard updates

Browse source
This commit is contained in:
Sanjeev Kumar 2025-12-14 07:34:47 +05:30
parent 4c4b8cb04f
commit 9703075561
1 changed files with 11 additions and 8 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

19
dashboard/app.py
View file

@ -365,17 +365,18 @@ def main():
import os import os
env = os.environ.copy() env = os.environ.copy()
env['PYTHONIOENCODING'] = 'utf-8' env['PYTHONIOENCODING'] = 'utf-8'
env['PYTHONUNBUFFERED'] = '1' # Force unbuffered output
# Use -u flag for unbuffered Python output
cmd_with_unbuffered = ["python", "-u"] + cmd[1:] # Replace python with python -u
process = subprocess.Popen( process = subprocess.Popen(
cmd, cmd_with_unbuffered,
stdout=subprocess.PIPE, stdout=subprocess.PIPE,
stderr=subprocess.STDOUT, stderr=subprocess.STDOUT,
text=True, bufsize=0, # Unbuffered
bufsize=1,
cwd=str(Path(__file__).parent.parent), cwd=str(Path(__file__).parent.parent),
env=env, env=env
encoding='utf-8',
errors='replace'
) )
output_lines = [] output_lines = []
@ -385,9 +386,11 @@ def main():
images_count = 0 images_count = 0
videos_count = 0 videos_count = 0
for line in iter(process.stdout.readline, ''): for raw_line in iter(process.stdout.readline, b''):
if not line: if not raw_line:
break break
# Decode binary to text
line = raw_line.decode('utf-8', errors='replace')
output_lines.append(line) output_lines.append(line)
# Parse metrics from output # Parse metrics from output