From cc65b4b3aee15cc7e949061017249f9ff645d842 Mon Sep 17 00:00:00 2001 From: arabcoders Date: Thu, 1 May 2025 18:28:12 +0300 Subject: [PATCH] Communicate more error logs to user when yt-dlp throws them. --- app/library/Download.py | 30 +++++++++++++++++++++++++++--- app/library/DownloadQueue.py | 30 +++++++++++++++++++++++------- app/library/Utils.py | 29 +++++++++++++++++++++++++++++ 3 files changed, 79 insertions(+), 10 deletions(-) diff --git a/app/library/Download.py b/app/library/Download.py index 7b59098e..fc705f93 100644 --- a/app/library/Download.py +++ b/app/library/Download.py @@ -16,7 +16,7 @@ from .config import Config from .Events import EventBus, Events from .ffprobe import ffprobe from .ItemDTO import ItemDTO -from .Utils import extract_info, load_cookies +from .Utils import extract_info, extract_ytdlp_logs, load_cookies from .YTDLPOpts import YTDLPOpts @@ -81,7 +81,19 @@ class Download: temp_keep: bool = False "Keep temp folder after download." - def __init__(self, info: ItemDTO, info_dict: dict = None): + logs: list = [] + "Logs from yt-dlp." + + def __init__(self, info: ItemDTO, info_dict: dict = None, logs: list | None = None): + """ + Initialize download task. + + Args: + info (ItemDTO): ItemDTO object. + info_dict (dict): yt-dlp metadata dict. + logs (list): Logs from yt-dlp. + + """ config = Config.get_instance() self.download_dir = info.download_dir @@ -106,6 +118,7 @@ class Download: self.logger = logging.getLogger(f"Download.{info.id if info.id else info._id}") self.started_time = 0 self.queue_time = datetime.now(tz=UTC) + self.logs = logs if logs else [] def _progress_hook(self, data: dict): dataDict = {k: v for k, v in data.items() if k in self._ytdlp_fields} @@ -193,6 +206,14 @@ class Download: if not self.info_dict: self.logger.info(f"Extracting info for '{self.info.url}'.") + self.logs = [] + ie_params = params.copy() + ie_params["callback"] = { + "func": lambda _, msg: self.logs.append(msg), + "level": logging.WARNING, + "name": "callback-logger", + } + info = extract_info( config=params, url=self.info.url, @@ -219,7 +240,10 @@ class Download: ) if isinstance(self.info_dict, dict) and len(self.info_dict.get("formats", [])) < 1: - msg = f"Failed to extract formats for '{self.info.url}'. The extracted info dict is empty." + msg = f"Failed to extract any formats for '{self.info.url}'." + if filtered_logs := extract_ytdlp_logs(self.logs): + msg += f" Logs: {', '.join(filtered_logs)}" + raise ValueError(msg) # noqa: TRY301 self.logger.info( diff --git a/app/library/DownloadQueue.py b/app/library/DownloadQueue.py index 6edc4add..db3769b7 100644 --- a/app/library/DownloadQueue.py +++ b/app/library/DownloadQueue.py @@ -24,7 +24,15 @@ from .ItemDTO import Item, ItemDTO from .Presets import Presets from .Scheduler import Scheduler from .Singleton import Singleton -from .Utils import arg_converter, calc_download_path, dt_delta, extract_info, is_downloaded, load_cookies +from .Utils import ( + arg_converter, + calc_download_path, + dt_delta, + extract_info, + extract_ytdlp_logs, + is_downloaded, + load_cookies, +) from .YTDLPOpts import YTDLPOpts LOG = logging.getLogger("DownloadQueue") @@ -184,7 +192,7 @@ class DownloadQueue(metaclass=Singleton): except Exception as e: LOG.error(f"Failed to cancel downloads. {e!s}") - async def __add_entry(self, entry: dict, item: Item, already=None): + async def __add_entry(self, entry: dict, item: Item, already=None, logs: list | None = None): """ Add an entry to the download queue. @@ -192,11 +200,15 @@ class DownloadQueue(metaclass=Singleton): entry (dict): The entry to add to the download queue. item (Item): The item to add to the download queue. already (set): The set of already downloaded items. + logs (list): The list of logs generated during information extraction. Returns: dict: The status of the operation. """ + if not logs: + logs = [] + if item.has_extras(): for key in item.extras.copy(): if not self.keep_extra_key(key): @@ -320,12 +332,15 @@ class DownloadQueue(metaclass=Singleton): ) try: - dlInfo: Download = Download(info=dl, info_dict=entry) + dlInfo: Download = Download(info=dl, info_dict=entry, logs=logs) + text_logs = "" + if filtered_logs := extract_ytdlp_logs(logs): + text_logs = f" Logs: {', '.join(filtered_logs)}" if live_in or "is_upcoming" == entry.get("live_status"): NotifyEvent = Events.COMPLETED dlInfo.info.status = "not_live" - dlInfo.info.msg = "Stream is not live yet." + dlInfo.info.msg = "Stream is not live yet." + text_logs itemDownload = self.done.put(dlInfo) elif len(entry.get("formats", [])) < 1: availability = entry.get("availability", "public") @@ -333,14 +348,15 @@ class DownloadQueue(metaclass=Singleton): if availability and availability not in ("public",): msg += f" Availability is set for '{availability}'." + dlInfo.info.error = msg + text_logs dlInfo.info.status = "error" - dlInfo.info.error = msg itemDownload = self.done.put(dlInfo) NotifyEvent = Events.COMPLETED await self._notify.emit(Events.LOG_WARNING, data=event_warning(msg)) elif self.config.allow_manifestless is False and is_manifestless is True: dlInfo.info.status = "error" - dlInfo.info.error = "Video is in post-live manifestless mode." + dlInfo.info.error = "Video is in post-live manifestless mode." + text_logs + itemDownload = self.done.put(dlInfo) NotifyEvent = Events.COMPLETED else: @@ -485,7 +501,7 @@ class DownloadQueue(metaclass=Singleton): except Exception as e: LOG.error(f"Failed to remove cookie file '{yt_conf['cookiefile']}'. {e!s}") - return await self.__add_entry(entry=entry, item=item, already=already) + return await self.__add_entry(entry=entry, item=item, already=already, logs=logs) async def cancel(self, ids: list[str]) -> dict[str, str]: """ diff --git a/app/library/Utils.py b/app/library/Utils.py index 0f5ee00f..837e5d39 100644 --- a/app/library/Utils.py +++ b/app/library/Utils.py @@ -1069,3 +1069,32 @@ def dt_delta(delta: timedelta) -> str: parts.append("<1s") return " ".join(parts) + + +def extract_ytdlp_logs(logs: list[str], filters: list[str | re.Pattern] = None) -> list[str]: + """ + Extract yt-dlp log lines matching built-in filters plus any extras. + + Args: + logs (list): log strings. + filters (list[str|Re.Pattern]): Optional extra filters of strings and/or regex. + + Returns: + (list): List of matching log lines. + + """ + all_patterns: list[str | re.Pattern] = [ + "This live event will begin", + "Video unavailable. This video is private", + "This video is available to this channel", + "Private video. Sign in if you've been granted access to this video", + ] + (filters or []) + + compiled: list[re.Pattern] = [ + p if isinstance(p, re.Pattern) else re.compile(re.escape(p), re.IGNORECASE) for p in all_patterns + ] + + matched: list[str] = [] + matched.extend(line for line in logs if line and any(p.search(line) for p in compiled)) + + return list(dict.fromkeys(matched))