danvics/ytptube
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

try to differentiate yt-dlp logs for each download.

Browse source
This commit is contained in:
ArabCoders 2025-03-15 20:10:28 +03:00
parent d94f004eed
commit 1b57e41e1b
2 changed files with 51 additions and 27 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
.vscode/settings.json vendored
View file

@ -21,6 +21,7 @@
"finaldir", "finaldir",
"getpid", "getpid",
"httpx", "httpx",
"levelno",
"libcurl", "libcurl",
"libx", "libx",
"matroska", "matroska",

77
app/library/Download.py
View file

@ -3,6 +3,7 @@ import hashlib
import logging import logging
import multiprocessing import multiprocessing
import os import os
import re
import shutil import shutil
import time import time
from email.utils import formatdate from email.utils import formatdate
@ -16,7 +17,22 @@ from .ffprobe import ffprobe
from .ItemDTO import ItemDTO from .ItemDTO import ItemDTO
from .YTDLPOpts import YTDLPOpts from .YTDLPOpts import YTDLPOpts
LOG = logging.getLogger("download")
class DebugPreProcessor(logging.Filter):
def filter(self, record: logging.LogRecord):
if record.levelno != logging.DEBUG:
return True
if record.__dict__.get("processed", None):
return True
logging.getLogger(record.name).log(
level=logging.DEBUG if record.msg.startswith("") else logging.INFO,
msg=re.sub(r"^\[.*\] ", "", record.msg),
extra={"processed": True},
)
return False
class Download: class Download:
@ -89,6 +105,9 @@ class Download:
self.is_live = bool(info.is_live) or info.live_in is not None self.is_live = bool(info.is_live) or info.live_in is not None
self.is_manifestless = "is_manifestless" in self.info.options and self.info.options["is_manifestless"] is True self.is_manifestless = "is_manifestless" in self.info.options and self.info.options["is_manifestless"] is True
self.info_dict = info_dict self.info_dict = info_dict
self.logger = logging.getLogger(f"Download.{info.id if info.id else info._id}")
self.ytdlp_logger = logging.getLogger(f"ytdlp.{info.id if info.id else info._id}")
self.ytdlp_logger.addFilter(DebugPreProcessor())
def _progress_hook(self, data: dict): def _progress_hook(self, data: dict):
dataDict = {k: v for k, v in data.items() if k in self._ytdlp_fields} dataDict = {k: v for k, v in data.items() if k in self._ytdlp_fields}
@ -105,7 +124,7 @@ class Download:
return return
if self.debug: if self.debug:
LOG.debug(f"Postprocessor hook: {data}") self.logger.debug(f"Postprocessor hook: {data}")
if "__finaldir" in data["info_dict"]: if "__finaldir" in data["info_dict"]:
filename = os.path.join(data["info_dict"]["__finaldir"], os.path.basename(data["info_dict"]["filepath"])) filename = os.path.join(data["info_dict"]["__finaldir"], os.path.basename(data["info_dict"]["filepath"]))
@ -154,12 +173,14 @@ class Download:
if self.info.cookies: if self.info.cookies:
try: try:
cookie_file = os.path.join(self.temp_path, f"cookie_{self.info._id}.txt") cookie_file = os.path.join(self.temp_path, f"cookie_{self.info._id}.txt")
LOG.debug(f"Creating cookie file for '{self.info.id}: {self.info.title}' - '{cookie_file}'.") self.logger.debug(
f"Creating cookie file for '{self.info.id}: {self.info.title}' - '{cookie_file}'."
)
with open(cookie_file, "w") as f: with open(cookie_file, "w") as f:
f.write(self.info.cookies) f.write(self.info.cookies)
params["cookiefile"] = f.name params["cookiefile"] = f.name
except ValueError as e: except ValueError as e:
LOG.error(f"Failed to create cookie file for '{self.info.id}: {self.info.title}'. '{e!s}'.") self.logger.error(f"Failed to create cookie file for '{self.info.id}: {self.info.title}'. '{e!s}'.")
if self.is_live or self.is_manifestless: if self.is_live or self.is_manifestless:
hasDeletedOptions = False hasDeletedOptions = False
@ -171,37 +192,39 @@ class Download:
deletedOpts.append(opt) deletedOpts.append(opt)
if hasDeletedOptions: if hasDeletedOptions:
LOG.warning( self.logger.warning(
f"Live stream detected for '{self.info.title}', The following opts '{deletedOpts=}' have been deleted which are known to cause issues with live stream and post stream manifestless mode." f"Live stream detected for '{self.info.title}', The following opts '{deletedOpts=}' have been deleted which are known to cause issues with live stream and post stream manifestless mode."
) )
LOG.info( self.logger.info(
f'Task id="{self.info.id}" PID="{os.getpid()}" title="{self.info.title}" preset="{self.preset}" started.' f'Task id="{self.info.id}" PID="{os.getpid()}" title="{self.info.title}" preset="{self.preset}" started.'
) )
LOG.debug("Params before passing to yt-dlp.", extra=params) self.logger.debug("Params before passing to yt-dlp.", extra=params)
if "impersonate" in params: if "impersonate" in params:
from yt_dlp.networking.impersonate import ImpersonateTarget from yt_dlp.networking.impersonate import ImpersonateTarget
params["impersonate"] = ImpersonateTarget.from_str(params["impersonate"]) params["impersonate"] = ImpersonateTarget.from_str(params["impersonate"])
params["logger"] = self.ytdlp_logger
cls = yt_dlp.YoutubeDL(params=params) cls = yt_dlp.YoutubeDL(params=params)
if isinstance(self.info_dict, dict) and len(self.info_dict) > 1: if isinstance(self.info_dict, dict) and len(self.info_dict) > 1:
LOG.debug(f"Downloading '{self.info.url}' using pre-info.") self.logger.debug(f"Downloading '{self.info.url}' using pre-info.")
cls.process_ie_result(self.info_dict, download=True) cls.process_ie_result(self.info_dict, download=True)
ret = cls._download_retcode ret = cls._download_retcode
else: else:
LOG.debug(f"Downloading using url: {self.info.url}") self.logger.debug(f"Downloading using url: {self.info.url}")
ret = cls.download([self.info.url]) ret = cls.download([self.info.url])
self.status_queue.put({"id": self.id, "status": "finished" if ret == 0 else "error"}) self.status_queue.put({"id": self.id, "status": "finished" if ret == 0 else "error"})
except Exception as exc: except Exception as exc:
LOG.exception(exc) self.logger.exception(exc)
self.status_queue.put({"id": self.id, "status": "error", "msg": str(exc), "error": str(exc)}) self.status_queue.put({"id": self.id, "status": "error", "msg": str(exc), "error": str(exc)})
LOG.info(f'Task id="{self.info.id}" PID="{os.getpid()}" title="{self.info.title}" completed.') self.logger.info(f'Task id="{self.info.id}" PID="{os.getpid()}" title="{self.info.title}" completed.')
async def start(self, emitter: Emitter): async def start(self, emitter: Emitter):
self.emitter = emitter self.emitter = emitter
@ -244,28 +267,28 @@ class Download:
self.cancel_in_progress = True self.cancel_in_progress = True
procId = self.proc.ident procId = self.proc.ident
LOG.info(f"Closing PID='{procId}' download process.") self.logger.info(f"Closing PID='{procId}' download process.")
try: try:
try: try:
if "update_task" in self.__dict__ and self.update_task.done() is False: if "update_task" in self.__dict__ and self.update_task.done() is False:
self.update_task.cancel() self.update_task.cancel()
except Exception as e: except Exception as e:
LOG.error(f"Failed to close status queue: '{procId}'. {e}") self.logger.error(f"Failed to close status queue: '{procId}'. {e}")
self.kill() self.kill()
loop = asyncio.get_running_loop() loop = asyncio.get_running_loop()
if self.proc.is_alive(): if self.proc.is_alive():
LOG.debug(f"Waiting for PID='{procId}' to close.") self.logger.debug(f"Waiting for PID='{procId}' to close.")
await loop.run_in_executor(None, self.proc.join) await loop.run_in_executor(None, self.proc.join)
LOG.debug(f"PID='{procId}' closed.") self.logger.debug(f"PID='{procId}' closed.")
if self.status_queue: if self.status_queue:
LOG.debug(f"Closing status queue for PID='{procId}'.") self.logger.debug(f"Closing status queue for PID='{procId}'.")
self.status_queue.put(Terminator()) self.status_queue.put(Terminator())
LOG.debug(f"Closed status queue for PID='{procId}'.") self.logger.debug(f"Closed status queue for PID='{procId}'.")
if self.proc: if self.proc:
self.proc.close() self.proc.close()
@ -273,11 +296,11 @@ class Download:
self.delete_temp() self.delete_temp()
LOG.debug(f"Closed PID='{procId}' download process.") self.logger.debug(f"Closed PID='{procId}' download process.")
return True return True
except Exception as e: except Exception as e:
LOG.error(f"Failed to close process: '{procId}'. {e}") self.logger.error(f"Failed to close process: '{procId}'. {e}")
return False return False
@ -295,11 +318,11 @@ class Download:
return False return False
try: try:
LOG.info(f"Killing download process: '{self.proc.ident}'.") self.logger.info(f"Killing download process: '{self.proc.ident}'.")
self.proc.kill() self.proc.kill()
return True return True
except Exception as e: except Exception as e:
LOG.error(f"Failed to kill process: '{self.proc.ident}'. {e}") self.logger.error(f"Failed to kill process: '{self.proc.ident}'. {e}")
return False return False
@ -308,7 +331,7 @@ class Download:
return return
if "finished" != self.info.status and self.is_live: if "finished" != self.info.status and self.is_live:
LOG.warning( self.logger.warning(
f"Keeping live temp folder '{self.temp_path}', as the reported status is not finished '{self.info.status}'." f"Keeping live temp folder '{self.temp_path}', as the reported status is not finished '{self.info.status}'."
) )
return return
@ -317,12 +340,12 @@ class Download:
return return
if self.temp_path == self.temp_dir: if self.temp_path == self.temp_dir:
LOG.warning( self.logger.warning(
f"Attempted to delete video temp folder: {self.temp_path}, but it is the same as main temp folder." f"Attempted to delete video temp folder: {self.temp_path}, but it is the same as main temp folder."
) )
return return
LOG.info(f"Deleting Temp folder '{self.temp_path}'.") self.logger.info(f"Deleting Temp folder '{self.temp_path}'.")
shutil.rmtree(self.temp_path, ignore_errors=True) shutil.rmtree(self.temp_path, ignore_errors=True)
async def progress_update(self): async def progress_update(self):
@ -343,7 +366,7 @@ class Download:
continue continue
if self.debug: if self.debug:
LOG.debug(f"Status Update: {self.info._id=} {status=}") self.logger.debug(f"Status Update: {self.info._id=} {status=}")
if isinstance(status, str): if isinstance(status, str):
asyncio.create_task(self.emitter.updated(dl=self.info), name=f"emitter-u-{self.id}") asyncio.create_task(self.emitter.updated(dl=self.info), name=f"emitter-u-{self.id}")
@ -400,7 +423,7 @@ class Download:
except Exception as e: except Exception as e:
self.info.extras["is_video"] = True self.info.extras["is_video"] = True
self.info.extras["is_audio"] = True self.info.extras["is_audio"] = True
LOG.exception(e) self.logger.exception(e)
LOG.error(f"Failed to ffprobe: {status.get}. {e}") self.logger.error(f"Failed to ffprobe: {status.get}. {e}")
asyncio.create_task(self.emitter.updated(dl=self.info), name=f"emitter-u-{self.id}") asyncio.create_task(self.emitter.updated(dl=self.info), name=f"emitter-u-{self.id}")