From 1b57e41e1b5b4bd4bc59251f2073514f037c229f Mon Sep 17 00:00:00 2001 From: ArabCoders Date: Sat, 15 Mar 2025 20:10:28 +0300 Subject: [PATCH 1/5] try to differentiate yt-dlp logs for each download. --- .vscode/settings.json | 1 + app/library/Download.py | 77 ++++++++++++++++++++++++++--------------- 2 files changed, 51 insertions(+), 27 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index 8a57380c..cb2fc994 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -21,6 +21,7 @@ "finaldir", "getpid", "httpx", + "levelno", "libcurl", "libx", "matroska", diff --git a/app/library/Download.py b/app/library/Download.py index 7f794e89..15af8d9f 100644 --- a/app/library/Download.py +++ b/app/library/Download.py @@ -3,6 +3,7 @@ import hashlib import logging import multiprocessing import os +import re import shutil import time from email.utils import formatdate @@ -16,7 +17,22 @@ from .ffprobe import ffprobe from .ItemDTO import ItemDTO from .YTDLPOpts import YTDLPOpts -LOG = logging.getLogger("download") + +class DebugPreProcessor(logging.Filter): + def filter(self, record: logging.LogRecord): + if record.levelno != logging.DEBUG: + return True + + if record.__dict__.get("processed", None): + return True + + logging.getLogger(record.name).log( + level=logging.DEBUG if record.msg.startswith("") else logging.INFO, + msg=re.sub(r"^\[.*\] ", "", record.msg), + extra={"processed": True}, + ) + + return False class Download: @@ -89,6 +105,9 @@ class Download: self.is_live = bool(info.is_live) or info.live_in is not None self.is_manifestless = "is_manifestless" in self.info.options and self.info.options["is_manifestless"] is True self.info_dict = info_dict + self.logger = logging.getLogger(f"Download.{info.id if info.id else info._id}") + self.ytdlp_logger = logging.getLogger(f"ytdlp.{info.id if info.id else info._id}") + self.ytdlp_logger.addFilter(DebugPreProcessor()) def _progress_hook(self, data: dict): dataDict = {k: v for k, v in data.items() if k in self._ytdlp_fields} @@ -105,7 +124,7 @@ class Download: return if self.debug: - LOG.debug(f"Postprocessor hook: {data}") + self.logger.debug(f"Postprocessor hook: {data}") if "__finaldir" in data["info_dict"]: filename = os.path.join(data["info_dict"]["__finaldir"], os.path.basename(data["info_dict"]["filepath"])) @@ -154,12 +173,14 @@ class Download: if self.info.cookies: try: cookie_file = os.path.join(self.temp_path, f"cookie_{self.info._id}.txt") - LOG.debug(f"Creating cookie file for '{self.info.id}: {self.info.title}' - '{cookie_file}'.") + self.logger.debug( + f"Creating cookie file for '{self.info.id}: {self.info.title}' - '{cookie_file}'." + ) with open(cookie_file, "w") as f: f.write(self.info.cookies) params["cookiefile"] = f.name except ValueError as e: - LOG.error(f"Failed to create cookie file for '{self.info.id}: {self.info.title}'. '{e!s}'.") + self.logger.error(f"Failed to create cookie file for '{self.info.id}: {self.info.title}'. '{e!s}'.") if self.is_live or self.is_manifestless: hasDeletedOptions = False @@ -171,37 +192,39 @@ class Download: deletedOpts.append(opt) if hasDeletedOptions: - LOG.warning( + self.logger.warning( f"Live stream detected for '{self.info.title}', The following opts '{deletedOpts=}' have been deleted which are known to cause issues with live stream and post stream manifestless mode." ) - LOG.info( + self.logger.info( f'Task id="{self.info.id}" PID="{os.getpid()}" title="{self.info.title}" preset="{self.preset}" started.' ) - LOG.debug("Params before passing to yt-dlp.", extra=params) + self.logger.debug("Params before passing to yt-dlp.", extra=params) if "impersonate" in params: from yt_dlp.networking.impersonate import ImpersonateTarget params["impersonate"] = ImpersonateTarget.from_str(params["impersonate"]) + params["logger"] = self.ytdlp_logger + cls = yt_dlp.YoutubeDL(params=params) if isinstance(self.info_dict, dict) and len(self.info_dict) > 1: - LOG.debug(f"Downloading '{self.info.url}' using pre-info.") + self.logger.debug(f"Downloading '{self.info.url}' using pre-info.") cls.process_ie_result(self.info_dict, download=True) ret = cls._download_retcode else: - LOG.debug(f"Downloading using url: {self.info.url}") + self.logger.debug(f"Downloading using url: {self.info.url}") ret = cls.download([self.info.url]) self.status_queue.put({"id": self.id, "status": "finished" if ret == 0 else "error"}) except Exception as exc: - LOG.exception(exc) + self.logger.exception(exc) self.status_queue.put({"id": self.id, "status": "error", "msg": str(exc), "error": str(exc)}) - LOG.info(f'Task id="{self.info.id}" PID="{os.getpid()}" title="{self.info.title}" completed.') + self.logger.info(f'Task id="{self.info.id}" PID="{os.getpid()}" title="{self.info.title}" completed.') async def start(self, emitter: Emitter): self.emitter = emitter @@ -244,28 +267,28 @@ class Download: self.cancel_in_progress = True procId = self.proc.ident - LOG.info(f"Closing PID='{procId}' download process.") + self.logger.info(f"Closing PID='{procId}' download process.") try: try: if "update_task" in self.__dict__ and self.update_task.done() is False: self.update_task.cancel() except Exception as e: - LOG.error(f"Failed to close status queue: '{procId}'. {e}") + self.logger.error(f"Failed to close status queue: '{procId}'. {e}") self.kill() loop = asyncio.get_running_loop() if self.proc.is_alive(): - LOG.debug(f"Waiting for PID='{procId}' to close.") + self.logger.debug(f"Waiting for PID='{procId}' to close.") await loop.run_in_executor(None, self.proc.join) - LOG.debug(f"PID='{procId}' closed.") + self.logger.debug(f"PID='{procId}' closed.") if self.status_queue: - LOG.debug(f"Closing status queue for PID='{procId}'.") + self.logger.debug(f"Closing status queue for PID='{procId}'.") self.status_queue.put(Terminator()) - LOG.debug(f"Closed status queue for PID='{procId}'.") + self.logger.debug(f"Closed status queue for PID='{procId}'.") if self.proc: self.proc.close() @@ -273,11 +296,11 @@ class Download: self.delete_temp() - LOG.debug(f"Closed PID='{procId}' download process.") + self.logger.debug(f"Closed PID='{procId}' download process.") return True except Exception as e: - LOG.error(f"Failed to close process: '{procId}'. {e}") + self.logger.error(f"Failed to close process: '{procId}'. {e}") return False @@ -295,11 +318,11 @@ class Download: return False try: - LOG.info(f"Killing download process: '{self.proc.ident}'.") + self.logger.info(f"Killing download process: '{self.proc.ident}'.") self.proc.kill() return True except Exception as e: - LOG.error(f"Failed to kill process: '{self.proc.ident}'. {e}") + self.logger.error(f"Failed to kill process: '{self.proc.ident}'. {e}") return False @@ -308,7 +331,7 @@ class Download: return if "finished" != self.info.status and self.is_live: - LOG.warning( + self.logger.warning( f"Keeping live temp folder '{self.temp_path}', as the reported status is not finished '{self.info.status}'." ) return @@ -317,12 +340,12 @@ class Download: return if self.temp_path == self.temp_dir: - LOG.warning( + self.logger.warning( f"Attempted to delete video temp folder: {self.temp_path}, but it is the same as main temp folder." ) return - LOG.info(f"Deleting Temp folder '{self.temp_path}'.") + self.logger.info(f"Deleting Temp folder '{self.temp_path}'.") shutil.rmtree(self.temp_path, ignore_errors=True) async def progress_update(self): @@ -343,7 +366,7 @@ class Download: continue if self.debug: - LOG.debug(f"Status Update: {self.info._id=} {status=}") + self.logger.debug(f"Status Update: {self.info._id=} {status=}") if isinstance(status, str): asyncio.create_task(self.emitter.updated(dl=self.info), name=f"emitter-u-{self.id}") @@ -400,7 +423,7 @@ class Download: except Exception as e: self.info.extras["is_video"] = True self.info.extras["is_audio"] = True - LOG.exception(e) - LOG.error(f"Failed to ffprobe: {status.get}. {e}") + self.logger.exception(e) + self.logger.error(f"Failed to ffprobe: {status.get}. {e}") asyncio.create_task(self.emitter.updated(dl=self.info), name=f"emitter-u-{self.id}") From 095bca9e88849734d8a737c654846322f9d3a92c Mon Sep 17 00:00:00 2001 From: ArabCoders Date: Sat, 15 Mar 2025 22:29:05 +0300 Subject: [PATCH 2/5] more fit approch to handle yt-dlp logging messages. --- .vscode/launch.json | 7 +----- app/library/Download.py | 43 ++++++++++++++++++------------------ app/library/DownloadQueue.py | 2 +- 3 files changed, 24 insertions(+), 28 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index 892e5664..23a44740 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -35,12 +35,7 @@ "YTP_DOWNLOAD_PATH": "${workspaceFolder}/var/downloads", "YTP_TEMP_PATH": "${workspaceFolder}/var/tmp", "PYDEVD_DISABLE_FILE_VALIDATION": "1", - "YTP_IGNORE_UI": "true", - "YTP_PIP_IGNORE_UPDATES": "true", - "YTP_LOG_LEVEL": "DEBUG", - "YTP_DEBUG": "true", - "YTP_YTDL_DEBUG": "false", - "YTP_ACCESS_LOG": "true", + "YTP_IGNORE_UI": "true" } }, { diff --git a/app/library/Download.py b/app/library/Download.py index 15af8d9f..4e2ca691 100644 --- a/app/library/Download.py +++ b/app/library/Download.py @@ -17,22 +17,24 @@ from .ffprobe import ffprobe from .ItemDTO import ItemDTO from .YTDLPOpts import YTDLPOpts +LOG = logging.getLogger("Download") -class DebugPreProcessor(logging.Filter): - def filter(self, record: logging.LogRecord): - if record.levelno != logging.DEBUG: - return True - if record.__dict__.get("processed", None): - return True +class NestedLogger: + debug_messages = ["[debug] ", "[download] "] - logging.getLogger(record.name).log( - level=logging.DEBUG if record.msg.startswith("") else logging.INFO, - msg=re.sub(r"^\[.*\] ", "", record.msg), - extra={"processed": True}, - ) + def __init__(self, logger: logging.Logger): + self.logger = logger - return False + def debug(self, msg: str): + levelno = logging.DEBUG if any(msg.startswith(x) for x in self.debug_messages) else logging.INFO + self.logger.log(level=levelno, msg=re.sub(r"^\[(debug|info)\] ", "", msg, flags=re.IGNORECASE)) + + def error(self, msg): + self.logger.error(msg) + + def warning(self, msg): + self.logger.warning(msg) class Download: @@ -82,7 +84,7 @@ class Download: temp_keep: bool = False "Keep temp folder after download." - def __init__(self, info: ItemDTO, info_dict: dict = None, debug: bool = False): + def __init__(self, info: ItemDTO, info_dict: dict = None): config = Config.get_instance() self.download_dir = info.download_dir @@ -94,7 +96,8 @@ class Download: self.info = info self.id = info._id self.default_ytdl_opts = config.ytdl_options - self.debug = debug + self.debug = bool(config.debug) + self.debug_ytdl = bool(config.ytdl_debug) self.cancelled = False self.tmpfilename = None self.status_queue = None @@ -106,8 +109,6 @@ class Download: self.is_manifestless = "is_manifestless" in self.info.options and self.info.options["is_manifestless"] is True self.info_dict = info_dict self.logger = logging.getLogger(f"Download.{info.id if info.id else info._id}") - self.ytdlp_logger = logging.getLogger(f"ytdlp.{info.id if info.id else info._id}") - self.ytdlp_logger.addFilter(DebugPreProcessor()) def _progress_hook(self, data: dict): dataDict = {k: v for k, v in data.items() if k in self._ytdlp_fields} @@ -118,14 +119,14 @@ class Download: self.status_queue.put({"id": self.id, **dataDict}) def _postprocessor_hook(self, data: dict): + if self.debug: + self.logger.debug(f"Postprocessor hook: {data}") + if "MoveFiles" != data.get("postprocessor") or "finished" != data.get("status"): dataDict = {k: v for k, v in data.items() if k in self._ytdlp_fields} self.status_queue.put({"id": self.id, **dataDict, "status": "postprocessing"}) return - if self.debug: - self.logger.debug(f"Postprocessor hook: {data}") - if "__finaldir" in data["info_dict"]: filename = os.path.join(data["info_dict"]["__finaldir"], os.path.basename(data["info_dict"]["filepath"])) else: @@ -166,7 +167,7 @@ class Download: } ) - if self.debug: + if self.debug_ytdl: params["verbose"] = True params["noprogress"] = False @@ -207,7 +208,7 @@ class Download: params["impersonate"] = ImpersonateTarget.from_str(params["impersonate"]) - params["logger"] = self.ytdlp_logger + params["logger"] = NestedLogger(self.logger) cls = yt_dlp.YoutubeDL(params=params) diff --git a/app/library/DownloadQueue.py b/app/library/DownloadQueue.py index c58d7233..c6f491ce 100644 --- a/app/library/DownloadQueue.py +++ b/app/library/DownloadQueue.py @@ -310,7 +310,7 @@ class DownloadQueue(metaclass=Singleton): if property.startswith("playlist"): dl.template = str(dl.template).replace(f"%({property})s", str(value)) - dlInfo: Download = Download(info=dl, info_dict=entry, debug=bool(self.config.ytdl_debug)) + dlInfo: Download = Download(info=dl, info_dict=entry) if dlInfo.info.live_in or "is_upcoming" == entry.get("live_status"): dlInfo.info.status = "not_live" From 2e9c76a1f7f83da15f699d0bfa2c317732dbc4ea Mon Sep 17 00:00:00 2001 From: ArabCoders Date: Sun, 16 Mar 2025 02:03:43 +0300 Subject: [PATCH 3/5] bundle mp4box into the image --- .vscode/settings.json | 1 + Dockerfile | 13 +++++++++++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index cb2fc994..cdf1ad9f 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -20,6 +20,7 @@ "dotenv", "finaldir", "getpid", + "gpac", "httpx", "levelno", "libcurl", diff --git a/Dockerfile b/Dockerfile index 767b716e..0bcaa4f3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,7 +13,7 @@ ENV PYTHONFAULTHANDLER=1 # Use sed to strip carriage-return characters from the entrypoint script (in case building on Windows) # Install dependencies -RUN apk add --update coreutils curl gcc g++ musl-dev libffi-dev openssl-dev && pip install pipenv +RUN apk add --update coreutils curl gcc g++ musl-dev libffi-dev openssl-dev curl make && pip install pipenv WORKDIR /app @@ -21,6 +21,13 @@ ARG PIPENV_FLAGS="--deploy" COPY ./Pipfile* . RUN PIPENV_VENV_IN_PROJECT=1 pipenv install ${PIPENV_FLAGS} +# Bundle mp4box into the image +WORKDIR /tmp +RUN mkdir /gpac-master && curl -k -L https://github.com/gpac/gpac/archive/refs/tags/v2.4.0.zip -o /tmp/gpac.zip && \ + unzip /tmp/gpac.zip && mv /tmp/gpac-*/* /gpac-master +WORKDIR /gpac-master +RUN ./configure --static-mp4box --use-zlib=no && make -j4 + FROM python:3.11-alpine ARG TZ=UTC @@ -46,13 +53,15 @@ RUN sed -i 's/\r$//g' /entrypoint.sh && chmod +x /entrypoint.sh COPY --chown=app:app ./app /app/app COPY --chown=app:app --from=node_builder /app/exported /app/ui/exported COPY --chown=app:app --from=python_builder /app/.venv /opt/python +COPY --chown=app:app --from=python_builder /gpac-master/bin/gcc/MP4Box /usr/bin/mp4box COPY --chown=app:app ./healthcheck.sh /usr/local/bin/healthcheck ENV PATH="/opt/python/bin:$PATH" RUN chown -R app:app /config /downloads && chmod +x /usr/local/bin/healthcheck && \ sed -i 's$#!\/app\/\.venv\/bin\/python$#!/opt/python/bin/python$' /opt/python/bin/* && \ - sed -i "s%'\/app\/\.venv'%'/opt/python'%" /opt/python/bin/activate* + sed -i "s%'\/app\/\.venv'%'/opt/python'%" /opt/python/bin/activate* && \ + chmod +x /usr/bin/mp4box VOLUME /config VOLUME /downloads From b116d48da61f482cf9f862e7c2b4e569bd656754 Mon Sep 17 00:00:00 2001 From: ArabCoders Date: Sun, 16 Mar 2025 20:12:18 +0300 Subject: [PATCH 4/5] added converter for date range objects param --- .vscode/settings.json | 1 + app/library/Utils.py | 5 ++++- app/library/YTDLPOpts.py | 5 +++++ app/library/encoder.py | 9 +++++++++ 4 files changed, 19 insertions(+), 1 deletion(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index cdf1ad9f..e407602f 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -17,6 +17,7 @@ "aiocron", "arrowless", "copyts", + "daterange", "dotenv", "finaldir", "getpid", diff --git a/app/library/Utils.py b/app/library/Utils.py index 59ae1c50..07d56261 100644 --- a/app/library/Utils.py +++ b/app/library/Utils.py @@ -424,6 +424,7 @@ def arg_converter(args: str) -> dict: """ import yt_dlp.options + from yt_dlp.utils import DateRange create_parser = yt_dlp.options.create_parser @@ -443,7 +444,9 @@ def arg_converter(args: str) -> dict: if "postprocessors" in diff: diff["postprocessors"] = [pp for pp in diff["postprocessors"] if pp not in default_opts["postprocessors"]] - return json.loads(json.dumps(diff)) + from .encoder import Encoder + + return json.loads(json.dumps(diff, cls=Encoder)) def validate_uuid(uuid_str: str, version: int = 4) -> bool: diff --git a/app/library/YTDLPOpts.py b/app/library/YTDLPOpts.py index d853e5bd..a6c7726c 100644 --- a/app/library/YTDLPOpts.py +++ b/app/library/YTDLPOpts.py @@ -132,4 +132,9 @@ class YTDLPOpts(metaclass=Singleton): if "format" in data and data["format"] in ["not_set", "default"]: data["format"] = None + if "daterange" in data and isinstance(data["daterange"], dict): + from yt_dlp.utils import DateRange + + data["daterange"] = DateRange(data["daterange"]["start"], data["daterange"]["end"]) + return data diff --git a/app/library/encoder.py b/app/library/encoder.py index 0ed61c57..70ccd127 100644 --- a/app/library/encoder.py +++ b/app/library/encoder.py @@ -1,6 +1,9 @@ import json +from datetime import date from pathlib import Path +from yt_dlp.utils import DateRange + class Encoder(json.JSONEncoder): """ @@ -13,6 +16,12 @@ class Encoder(json.JSONEncoder): if isinstance(o, Path): return str(o) + if isinstance(o, DateRange): + return {"start": str(o.start).replace("-", ""), "end": str(o.end).replace("-", "")} + + if isinstance(o, date): + return str(o) + if isinstance(o, object): if hasattr(o, "serialize"): return o.serialize() From 1e794a62cdfecd44ff819ca4e33f1c07a5380cfa Mon Sep 17 00:00:00 2001 From: ArabCoders Date: Sun, 16 Mar 2025 21:06:07 +0300 Subject: [PATCH 5/5] updated README --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 42daa75b..cf8ebd32 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,7 @@ YTPTube started as a fork of [meTube](https://github.com/alexta69/metube), Since * Basic Authentication support. * Support for curl_cffi, see [yt-dlp documentation](https://github.com/yt-dlp/yt-dlp?tab=readme-ov-file#impersonation) * Support for both advanced and basic mode for WebUI. +* Bundled tools in container: curl-cffi, ffmpeg, ffprobe, aria2, rtmpdump, mkvtoolsnix, mp4box. For more API endpoints, please refer to the [API documentation](API.md).