From 2d86aaefacdf09a91cc12fa7a3804540da7184b8 Mon Sep 17 00:00:00 2001 From: arabcoders Date: Sun, 27 Apr 2025 19:44:44 +0300 Subject: [PATCH 1/3] attempt at fixing duplicate logging --- app/library/DownloadQueue.py | 2 +- app/library/LogWrapper.py | 68 +++++++++++++++++++++++++++++++++--- app/library/Utils.py | 45 ++++++++++++++---------- 3 files changed, 92 insertions(+), 23 deletions(-) diff --git a/app/library/DownloadQueue.py b/app/library/DownloadQueue.py index a062983e..afeabb02 100644 --- a/app/library/DownloadQueue.py +++ b/app/library/DownloadQueue.py @@ -761,7 +761,7 @@ class DownloadQueue(metaclass=Singleton): """ Monitor the queue and pool for stale downloads and cancel them if needed. """ - if self.is_paused() or self.queue.empty(): + if self.is_paused(): return if not self.queue.empty(): diff --git a/app/library/LogWrapper.py b/app/library/LogWrapper.py index 3a8df8f1..d27e3155 100644 --- a/app/library/LogWrapper.py +++ b/app/library/LogWrapper.py @@ -1,4 +1,6 @@ import logging +import weakref +from collections.abc import Callable from dataclasses import dataclass @@ -8,6 +10,7 @@ class LogTarget: A data class that represents a logging target with its level and type. Attributes: + name (str): The name of the logging target. target: The logging target, which can be a logging.Logger instance or a callable. level (int): The logging level for the target. logger (bool): True if the target is a logging.Logger instance, False otherwise. @@ -15,7 +18,8 @@ class LogTarget: """ - target: "logging.Logger|callable" + name: str | None = None + target: logging.Logger|Callable level: int logger: bool @@ -55,17 +59,73 @@ class LogWrapper: """ targets: list[LogTarget] = [] + """A list of dictionaries where each dictionary represents a logging target with its level and type.""" - def add_target(self, target: "logging.Logger|callable", level: int = logging.DEBUG): + def __init__(self): + self.targets: list[LogTarget] = [] + weakref.finalize(self, LogWrapper.cleanup, self) + + def add_target(self, target: logging.Logger | Callable, level: int = logging.DEBUG): """ Adds a new logging target with the specified logging level. Args: - target (logging.Logger|callable): The logging target, which can be a logging.Logger instance or a callable. + target (logging.Logger|Callable): The logging target, which can be a logging.Logger instance or a Callable. level (int): The logging level for the target. Defaults to logging.DEBUG. """ - self.targets.append(LogTarget(target=target, level=level, logger=isinstance(target, logging.Logger))) + if not isinstance(target, logging.Logger | Callable): + msg = "Target must be a logging.Logger instance or a callable." + raise TypeError(msg) + + self.targets.append( + LogTarget( + name=target.name if isinstance(target, logging.Logger) else None, + target=target, + level=level, + logger=isinstance(target, logging.Logger), + ) + ) + + def cleanup(self, name: str = "yt-dlp"): + """ + Remove automatic handlers + + Args: + name (str): The name of the logger to clean up. Defaults to "ytdlp". + + """ + mgr = logging.root.manager + to_drop = [] + + for tgt in list(self.targets): + # only consider real Logger targets whose name + if tgt.logger and tgt.name and tgt.name.startswith(name): + name = tgt.name + logger = tgt.target + + # 1) detach all handlers. + for h in list(logger.handlers): + logger.removeHandler(h) + + # 2) restore default logger settings + logger.propagate = True + logger.setLevel(logging.NOTSET) + + # 3) purge logger & its children from the internal registry + for key in [k for k in mgr.loggerDict if k == name or k.startswith(name + ".")]: + mgr.loggerDict.pop(key, None) + + to_drop.append(tgt) + + # drop those targets from our wrapper + self.targets = [t for t in self.targets if t not in to_drop] + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, tb): + self.cleanup() def has_targets(self): """ diff --git a/app/library/Utils.py b/app/library/Utils.py index 084d3460..78d5e5a9 100644 --- a/app/library/Utils.py +++ b/app/library/Utils.py @@ -109,6 +109,7 @@ def extract_info( no_archive: bool = False, follow_redirect: bool = False, sanitize_info: bool = False, + **kwargs, ) -> dict: """ Extracts video information from the given URL. @@ -120,12 +121,14 @@ def extract_info( no_archive (bool): Disable download archive. follow_redirect (bool): Follow URL redirects. sanitize_info (bool): Sanitize the extracted information + **kwargs: Additional arguments. Returns: dict: Video information. """ - log_wrapper = LogWrapper() + is_recursive = bool(kwargs.get("log_wrapper")) + log_wrapper = kwargs.get("log_wrapper") or LogWrapper() params: dict = { **config, @@ -141,32 +144,37 @@ def extract_info( for key in keys_to_remove: params.pop(key, None) - id = None - idDict = get_archive_id(url=url) - if idDict.get("id"): - id = f".{idDict['id']}" - - log_wrapper.add_target(target=logging.getLogger(f"yt-dlp{id}"), level=logging.DEBUG if debug else logging.WARNING) - if debug: params["verbose"] = True else: params["quiet"] = True - if "callback" in params: - if isinstance(params["callback"], dict): - log_wrapper.add_target( - target=params["callback"]["func"], - level=params["callback"]["level"] or logging.ERROR, - ) - else: - log_wrapper.add_target(target=params["callback"], level=logging.ERROR) - del params["callback"] + if not is_recursive: + archive_id = None + idDict = get_archive_id(url=url) + if idDict.get("id"): + archive_id = f".{idDict['id']}" + + _logger = logging.getLogger(f"yt-dlp{archive_id}") + _logger.propagate = False + log_wrapper.add_target(target=_logger, level=logging.DEBUG if debug else logging.WARNING) + + if "callback" in params: + if isinstance(params["callback"], dict): + log_wrapper.add_target( + target=params["callback"]["func"], + level=params["callback"]["level"] or logging.ERROR, + ) + else: + log_wrapper.add_target(target=params["callback"], level=logging.ERROR) if log_wrapper.has_targets(): - if "logger" in params: + if "logger" in params and not is_recursive: log_wrapper.add_target(target=params["logger"], level=logging.DEBUG) + if "callback" in params: + del params["callback"] + params["logger"] = log_wrapper if no_archive and "download_archive" in params: @@ -182,6 +190,7 @@ def extract_info( no_archive=no_archive, follow_redirect=follow_redirect, sanitize_info=sanitize_info, + log_wrapper=log_wrapper, ) if not data: From d4ca763adbd78ce6c83bf937556432de30022869 Mon Sep 17 00:00:00 2001 From: arabcoders Date: Mon, 28 Apr 2025 18:15:11 +0300 Subject: [PATCH 2/3] reverted some changes to logging --- app/library/DownloadQueue.py | 6 ++-- app/library/LogWrapper.py | 54 ++++++------------------------------ app/library/Utils.py | 46 ++++++++++++++---------------- 3 files changed, 33 insertions(+), 73 deletions(-) diff --git a/app/library/DownloadQueue.py b/app/library/DownloadQueue.py index afeabb02..6edc4add 100644 --- a/app/library/DownloadQueue.py +++ b/app/library/DownloadQueue.py @@ -410,6 +410,7 @@ class DownloadQueue(metaclass=Singleton): "callback": { "func": lambda _, msg: logs.append(msg), "level": logging.WARNING, + "name": "callback-logger", }, **YTDLPOpts.get_instance() .preset(name=item.preset, with_cookies=not item.cookies) @@ -462,9 +463,8 @@ class DownloadQueue(metaclass=Singleton): if not entry: return {"status": "error", "msg": "Unable to extract info." + "\n".join(logs)} - LOG.debug( - f"extract_info: for 'URL: {item.url}' is done in '{time.perf_counter() - started}'. Length: '{len(entry)}/keys'." - ) + end_time = time.perf_counter() - started + LOG.debug(f"extract_info: for 'URL: {item.url}' is done in '{end_time:.3f}'. Length: '{len(entry)}/keys'.") except yt_dlp.utils.ExistingVideoReached as exc: LOG.error(f"Video has been downloaded already and recorded in archive.log file. '{exc!s}'.") return {"status": "error", "msg": "Video has been downloaded already and recorded in archive.log file."} diff --git a/app/library/LogWrapper.py b/app/library/LogWrapper.py index d27e3155..2f19a9fb 100644 --- a/app/library/LogWrapper.py +++ b/app/library/LogWrapper.py @@ -1,8 +1,9 @@ import logging -import weakref from collections.abc import Callable from dataclasses import dataclass +LOG = logging.getLogger(__name__) + @dataclass(kw_only=True) class LogTarget: @@ -19,7 +20,7 @@ class LogTarget: """ name: str | None = None - target: logging.Logger|Callable + target: logging.Logger | Callable level: int logger: bool @@ -63,70 +64,33 @@ class LogWrapper: def __init__(self): self.targets: list[LogTarget] = [] - weakref.finalize(self, LogWrapper.cleanup, self) - def add_target(self, target: logging.Logger | Callable, level: int = logging.DEBUG): + def add_target(self, target: logging.Logger | Callable, level: int = logging.DEBUG, name: str | None = None): """ Adds a new logging target with the specified logging level. Args: target (logging.Logger|Callable): The logging target, which can be a logging.Logger instance or a Callable. level (int): The logging level for the target. Defaults to logging.DEBUG. + name (str|None): The name of the logging target. Defaults to None. """ if not isinstance(target, logging.Logger | Callable): msg = "Target must be a logging.Logger instance or a callable." raise TypeError(msg) + if name is None: + name = target.name if isinstance(target, logging.Logger) else target.__name__ + self.targets.append( LogTarget( - name=target.name if isinstance(target, logging.Logger) else None, + name=name, target=target, level=level, logger=isinstance(target, logging.Logger), ) ) - def cleanup(self, name: str = "yt-dlp"): - """ - Remove automatic handlers - - Args: - name (str): The name of the logger to clean up. Defaults to "ytdlp". - - """ - mgr = logging.root.manager - to_drop = [] - - for tgt in list(self.targets): - # only consider real Logger targets whose name - if tgt.logger and tgt.name and tgt.name.startswith(name): - name = tgt.name - logger = tgt.target - - # 1) detach all handlers. - for h in list(logger.handlers): - logger.removeHandler(h) - - # 2) restore default logger settings - logger.propagate = True - logger.setLevel(logging.NOTSET) - - # 3) purge logger & its children from the internal registry - for key in [k for k in mgr.loggerDict if k == name or k.startswith(name + ".")]: - mgr.loggerDict.pop(key, None) - - to_drop.append(tgt) - - # drop those targets from our wrapper - self.targets = [t for t in self.targets if t not in to_drop] - - def __enter__(self): - return self - - def __exit__(self, exc_type, exc, tb): - self.cleanup() - def has_targets(self): """ Checks if there are any logging targets added. diff --git a/app/library/Utils.py b/app/library/Utils.py index 78d5e5a9..0f5ee00f 100644 --- a/app/library/Utils.py +++ b/app/library/Utils.py @@ -109,7 +109,7 @@ def extract_info( no_archive: bool = False, follow_redirect: bool = False, sanitize_info: bool = False, - **kwargs, + **kwargs, # noqa: ARG001 ) -> dict: """ Extracts video information from the given URL. @@ -127,9 +127,6 @@ def extract_info( dict: Video information. """ - is_recursive = bool(kwargs.get("log_wrapper")) - log_wrapper = kwargs.get("log_wrapper") or LogWrapper() - params: dict = { **config, "color": "no_color", @@ -149,32 +146,32 @@ def extract_info( else: params["quiet"] = True - if not is_recursive: - archive_id = None - idDict = get_archive_id(url=url) - if idDict.get("id"): - archive_id = f".{idDict['id']}" + log_wrapper = LogWrapper() + idDict = get_archive_id(url=url) + archive_id = f".{idDict['id']}" if idDict.get("id") else None - _logger = logging.getLogger(f"yt-dlp{archive_id}") - _logger.propagate = False - log_wrapper.add_target(target=_logger, level=logging.DEBUG if debug else logging.WARNING) + log_wrapper.add_target( + target=logging.getLogger(f"yt-dlp{archive_id}"), + level=logging.DEBUG if debug else logging.WARNING, + ) - if "callback" in params: - if isinstance(params["callback"], dict): - log_wrapper.add_target( - target=params["callback"]["func"], - level=params["callback"]["level"] or logging.ERROR, - ) - else: - log_wrapper.add_target(target=params["callback"], level=logging.ERROR) + if "callback" in params: + if isinstance(params["callback"], dict): + log_wrapper.add_target( + target=params["callback"]["func"], + level=params["callback"]["level"] or logging.ERROR, + name=params["callback"]["name"] or "callback", + ) + else: + log_wrapper.add_target(target=params["callback"], level=logging.ERROR, name="callback") + + if "callback" in params: + del params["callback"] if log_wrapper.has_targets(): - if "logger" in params and not is_recursive: + if "logger" in params: log_wrapper.add_target(target=params["logger"], level=logging.DEBUG) - if "callback" in params: - del params["callback"] - params["logger"] = log_wrapper if no_archive and "download_archive" in params: @@ -190,7 +187,6 @@ def extract_info( no_archive=no_archive, follow_redirect=follow_redirect, sanitize_info=sanitize_info, - log_wrapper=log_wrapper, ) if not data: From d89590434b84d2dc8be9ad78141eaeec2f634e30 Mon Sep 17 00:00:00 2001 From: arabcoders Date: Mon, 28 Apr 2025 18:40:23 +0300 Subject: [PATCH 3/3] Switched newDownload form to use HttpAPI for easier unlocking of download button. --- app/library/HttpAPI.py | 15 +++++---- ui/components/NewDownload.vue | 61 ++++++++++++++++------------------- 2 files changed, 35 insertions(+), 41 deletions(-) diff --git a/app/library/HttpAPI.py b/app/library/HttpAPI.py index 8e1c5912..6a611938 100644 --- a/app/library/HttpAPI.py +++ b/app/library/HttpAPI.py @@ -722,14 +722,15 @@ class HttpAPI(Common): except ValueError as e: return web.json_response(data={"error": str(e), "data": item}, status=web.HTTPBadRequest.status_code) - return web.json_response( - data=await asyncio.wait_for( - fut=asyncio.gather(*[self.add(item=item) for item in items]), - timeout=None, - ), - status=web.HTTPOk.status_code, - dumps=self.encoder.encode, + status = await asyncio.wait_for( + fut=asyncio.gather(*[self.add(item=item) for item in items]), + timeout=None, ) + response = [] + for i, item in enumerate(items): + response.append({"item": item, "status": status[i].get("status") == "ok", "msg": status[i].get("msg")}) + + return web.json_response(data=response, status=web.HTTPOk.status_code, dumps=self.encoder.encode) @route("GET", "api/logs") async def logs(self, request: Request) -> Response: diff --git a/ui/components/NewDownload.vue b/ui/components/NewDownload.vue index 58084dd5..de488b08 100644 --- a/ui/components/NewDownload.vue +++ b/ui/components/NewDownload.vue @@ -206,12 +206,13 @@ const addDownload = async () => { } } - addInProgress.value = true + const request_data = [] - form.value.url.split(',').forEach(url => { + form.value.url.split(',').forEach(async (url) => { if (!url.trim()) { return } + const data = { url: url, preset: config.app.basic_mode ? config.app.default_preset : form.value.preset, @@ -225,8 +226,31 @@ const addDownload = async () => { data.extras = form.value.extras } - socket.emit('add_url', data) + request_data.push(data) }) + + try { + addInProgress.value = true + const response = await request('/api/history', { + credentials: 'include', + method: 'POST', + body: JSON.stringify(request_data), + }) + + if (!response.ok) { + const data = await response.json() + throw new Error(data.error) + } + + form.value.url = '' + emitter('clear_form') + } + catch (e) { + console.error(e) + toast.error(`Error: ${e.message}`) + } finally { + addInProgress.value = false + } } const resetConfig = () => { @@ -250,29 +274,6 @@ const resetConfig = () => { toast.success('Local configuration has been reset.') } -const statusHandler = async stream => { - const { status, msg } = JSON.parse(stream) - - addInProgress.value = false - - if ('error' === status) { - toast.error(msg) - return - } - - form.value.url = '' -} - -const unlockDownload = async stream => { - const json = JSON.parse(stream) - if (!json?.data) { - return - } - if ("unlock" in json.data && true === json.data.unlock) { - addInProgress.value = false - } -} - const convertOptions = async args => { try { const response = await convertCliOptions(args) @@ -294,9 +295,6 @@ const convertOptions = async args => { } onMounted(async () => { - socket.on('status', statusHandler) - socket.on('error', unlockDownload) - await nextTick() if ('' === form.value?.preset) { @@ -317,11 +315,6 @@ onMounted(async () => { } }) -onUnmounted(() => { - socket.off('status', statusHandler) - socket.off('error', unlockDownload) -}) - const hasFormatInConfig = computed(() => { if (!form?.value?.value) { return false