diff --git a/.vscode/settings.json b/.vscode/settings.json index d961a26d..3696f9c5 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -19,11 +19,16 @@ "attl", "autonumber", "changeslog", + "consoletitle", + "cookiesfrombrowser", "copyts", "daterange", "dotenv", "finaldir", "flac", + "forcejson", + "forceprint", + "fribidi", "getpid", "gpac", "httpx", @@ -41,6 +46,7 @@ "postprocessor", "preferredcodec", "preferredquality", + "printtraffic", "quicktime", "tmpfilename", "urandom", @@ -56,7 +62,6 @@ "en" ], "spellright.documentTypes": [ - "latex", - "plaintext" + "latex" ], } diff --git a/Dockerfile b/Dockerfile index 6d45a974..2c1d4dc8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -35,7 +35,7 @@ ENV XDG_CONFIG_HOME=/config ENV XDG_CACHE_HOME=/tmp RUN mkdir /config /downloads && ln -snf /usr/share/zoneinfo/${TZ} /etc/localtime && echo ${TZ} > /etc/timezone && \ - apk add --update --no-cache bash mkvtoolnix patch aria2 coreutils curl shadow sqlite tzdata libmagic ffmpeg rtmpdump && \ + apk add --update --no-cache bash mkvtoolnix patch aria2 coreutils curl shadow sqlite tzdata libmagic ffmpeg rtmpdump fribidi && \ useradd -u ${USER_ID:-1000} -U -d /app -s /bin/bash app && \ rm -rf /var/cache/apk/* diff --git a/app/library/Download.py b/app/library/Download.py index cfeacb72..a3fe0c7d 100644 --- a/app/library/Download.py +++ b/app/library/Download.py @@ -114,6 +114,8 @@ class Download: def _progress_hook(self, data: dict): dataDict = {k: v for k, v in data.items() if k in self._ytdlp_fields} + LOG.debug("Status: %s", data) + if "finished" == data.get("status") and data.get("info_dict", {}).get("filename", None): dataDict["filename"] = data["info_dict"]["filename"] diff --git a/app/library/Events.py b/app/library/Events.py index 7fd52b47..156b3a65 100644 --- a/app/library/Events.py +++ b/app/library/Events.py @@ -90,7 +90,6 @@ class Events: LOG_SUCCESS = "log_success" INITIAL_DATA = "initial_data" - YTDLP_CONVERT = "ytdlp_convert" ITEM_DELETE = "item_delete" ITEM_CANCEL = "item_cancel" STATUS = "status" diff --git a/app/library/HttpAPI.py b/app/library/HttpAPI.py index 268c12f6..98e91349 100644 --- a/app/library/HttpAPI.py +++ b/app/library/HttpAPI.py @@ -35,7 +35,7 @@ from .Segments import Segments from .Subtitle import Subtitle from .Tasks import Task, Tasks from .Utils import ( - IGNORED_KEYS, + REMOVE_KEYS, StreamingError, arg_converter, decrypt_data, @@ -445,7 +445,7 @@ class HttpAPI(Common): try: response = {"opts": {}, "output_template": None, "download_path": None} - data = arg_converter(args) + data = arg_converter(args, dumps=True) if "outtmpl" in data and "default" in data["outtmpl"]: response["output_template"] = data["outtmpl"]["default"] @@ -456,12 +456,19 @@ class HttpAPI(Common): if "format" in data: response["format"] = data["format"] + bad_options = {k: v for d in REMOVE_KEYS for k, v in d.items()} + removed_options = [] + for key in data: - if key in IGNORED_KEYS: + if key in bad_options.items(): + removed_options.append(bad_options[key]) continue if not key.startswith("_"): response["opts"][key] = data[key] + if len(removed_options) > 0: + response["opts"]["removed"] = ", ".join(removed_options) + return web.json_response(data=response, status=web.HTTPOk.status_code) except Exception as e: err = str(e).strip() diff --git a/app/library/HttpSocket.py b/app/library/HttpSocket.py index 12d379cc..5d3d22b4 100644 --- a/app/library/HttpSocket.py +++ b/app/library/HttpSocket.py @@ -18,7 +18,7 @@ from .DownloadQueue import DownloadQueue from .encoder import Encoder from .Events import Event, EventBus, Events, error from .Presets import Presets -from .Utils import arg_converter, is_downloaded +from .Utils import is_downloaded LOG = logging.getLogger("socket_api") @@ -285,25 +285,3 @@ class HttpSocket(Common): async def resume(self, *_, **__): self.queue.resume() await self._notify.emit(Events.PAUSED, data={"paused": False, "at": time.time()}) - - @ws_event - async def ytdlp_convert(self, sid: str, data: dict): - if not isinstance(data, dict) or "args" not in data: - await self._notify.emit(Events.ERROR, data=error("Invalid request or no options were given."), to=sid) - return - - args: str | None = data.get("args") - - if not args: - await self._notify.emit(Events.ERROR, data=error("no options were given."), to=sid) - return - - try: - await self._notify.emit(Events.YTDLP_CONVERT, data=arg_converter(args), to=sid) - except Exception as e: - err = str(e).strip() - err = err.split("\n")[-1] if "\n" in err else err - LOG.error(f"Failed to convert args. '{err}'.") - await self._notify.emit(Events.ERROR, data=error(f"Failed to convert options. '{err}'."), to=sid) - - return diff --git a/app/library/Utils.py b/app/library/Utils.py index 0bddc15b..5f8acdcd 100644 --- a/app/library/Utils.py +++ b/app/library/Utils.py @@ -21,7 +21,39 @@ from .LogWrapper import LogWrapper LOG = logging.getLogger("Utils") -IGNORED_KEYS: tuple[str] = ("paths", "outtmpl", "progress_hooks", "postprocessor_hooks", "download_archive") +REMOVE_KEYS: list = [ + { + "paths": "-P, --paths", + "outtmpl": "-o, --output", + "progress_hooks": "--progress_hooks", + "postprocessor_hooks": "--postprocessor_hooks", + "download_archive": "--download_archive", + }, + { + "quiet": "-q, --quiet", + "no_warnings": "--no-warnings", + "skip_download": "--skip-download", + "forceprint": "-O, --print", + "simulate": "--simulate", + "noprogress": "--no-progress", + "wait_for_video": "--wait-for-video", + "mark_watched": "--mark-watched", + "color": "--color", + "verbose": "-v, --verbose", + "debug_printtraffic": "--print-traffic", + "write_pages": "--write-pages", + "dump_intermediate_pages": "--dump-pages", + "progress_delta": " --progress-delta", + "progress_template": "--progress-template", + "consoletitle": "--console-title", + "progress_with_newline": "--newline", + "forcejson": "-j, --dump-json", + "print_to_file": "--print-to-file", + "cookiesfrombrowser": "--cookies-from-browser", + "cookiefile": "--cookies", + }, +] + YTDLP_INFO_CLS: yt_dlp.YoutubeDL = None ALLOWED_SUBS_EXTENSIONS: tuple[str] = (".srt", ".vtt", ".ass") @@ -434,13 +466,20 @@ def validate_url(url: str) -> bool: return True -def arg_converter(args: str, remove_options: bool = False) -> dict: +def arg_converter( + args: str, + level: int | bool | None = None, + dumps: bool = False, + removed_options: list | None = None, +) -> dict: """ Convert yt-dlp options to a dictionary. Args: args (str): yt-dlp options string. - remove_options (bool): Remove default options. + level (int|bool|None): Level of options to remove, True for all. + dumps (bool): Dump options as JSON. + removed_options (list|None): List of removed options. Returns: dict: yt-dlp options dictionary. @@ -473,17 +512,33 @@ def arg_converter(args: str, remove_options: bool = False) -> dict: if isinstance(matchFilter, set): diff["match_filter"] = {"filters": list(matchFilter)} - if remove_options: - for key in diff.copy(): - if key in IGNORED_KEYS: - diff.pop(key, None) - if "_warnings" in diff: diff.pop("_warnings", None) - from .encoder import Encoder + if level: + bad_options = {} + level = len(REMOVE_KEYS) if not isinstance(level, int) else level - return json.loads(json.dumps(diff, cls=Encoder)) + for i, item in enumerate(REMOVE_KEYS): + if i > level: + break + bad_options.update(item.items()) + + for key in diff.copy(): + if key not in bad_options: + continue + + if isinstance(removed_options, list): + removed_options.append(bad_options[key]) + + diff.pop(key, None) + + if dumps is True: + from .encoder import Encoder + + return json.loads(json.dumps(diff, cls=Encoder)) + + return diff def validate_uuid(uuid_str: str, version: int = 4) -> bool: diff --git a/app/library/YTDLPOpts.py b/app/library/YTDLPOpts.py index 18673a71..a85ab6ce 100644 --- a/app/library/YTDLPOpts.py +++ b/app/library/YTDLPOpts.py @@ -4,7 +4,7 @@ from pathlib import Path from .config import Config from .Presets import Presets from .Singleton import Singleton -from .Utils import IGNORED_KEYS, arg_converter, calc_download_path, merge_dict +from .Utils import REMOVE_KEYS, arg_converter, calc_download_path, merge_dict LOG = logging.getLogger("YTDLPOpts") @@ -48,11 +48,22 @@ class YTDLPOpts(metaclass=Singleton): YTDLPOpts: The instance of the class """ + bad_options = {} + if from_user: + bad_options = {k: v for d in REMOVE_KEYS for k, v in d.items()} + + removed_options = [] + for key, value in config.items(): - if key in IGNORED_KEYS and from_user: + if from_user and key in bad_options: + removed_options.append(bad_options[key]) continue + self._item_opts[key] = value + if len(removed_options) > 0: + LOG.warning("Removed the following options: '%s'.", ", ".join(removed_options)) + return self def preset(self, name: str, with_cookies: bool = False) -> "YTDLPOpts": @@ -73,7 +84,13 @@ class YTDLPOpts(metaclass=Singleton): if preset.cli: try: - self._preset_opts = arg_converter(args=preset.cli, remove_options=True) + removed_options = [] + self._preset_opts = arg_converter(args=preset.cli, level=True, removed_options=removed_options) + if len(removed_options) > 0: + LOG.warning( + "Removed the following options '%s' from preset '%s'.", ", ".join(removed_options), preset.name + ) + except Exception as e: msg = f"Invalid cli options for preset '{preset.name}'. '{e!s}'." raise ValueError(msg) from e @@ -106,11 +123,19 @@ class YTDLPOpts(metaclass=Singleton): self._preset_opts["postprocessors"] = preset.postprocessors if preset.args and isinstance(preset.args, dict) and len(preset.args) > 0: + bad_options = {k: v for d in REMOVE_KEYS for k, v in d.items()} + removed_options = [] for key, value in preset.args.items(): - if key in IGNORED_KEYS: + if key in bad_options: + removed_options.append(bad_options[key]) continue self._preset_opts[key] = value + if len(removed_options) > 0: + LOG.warning( + "Removed the following options '%s' from '%s' args.", ", ".join(removed_options), preset.name + ) + return self def get_all(self, keep: bool = False) -> dict: diff --git a/app/library/common.py b/app/library/common.py index a89d3815..1fb524fd 100644 --- a/app/library/common.py +++ b/app/library/common.py @@ -98,7 +98,11 @@ class Common: cli = item.get("cli") if cli and len(cli) > 1: try: - config = arg_converter(args=cli, remove_options=True) + removed_options = [] + config = arg_converter(args=cli, level=True, removed_options=removed_options) + if len(removed_options) > 0: + LOG.warning("Removed the following options '%s'.", ", ".join(removed_options)) + except Exception as e: msg = f"Failed to parse yt-dlp cli options. {e!s}" raise ValueError(msg) from e diff --git a/app/library/config.py b/app/library/config.py index 00e013af..91b837ed 100644 --- a/app/library/config.py +++ b/app/library/config.py @@ -12,7 +12,7 @@ import coloredlogs from dotenv import load_dotenv from yt_dlp.version import __version__ as YTDLP_VERSION -from .Utils import IGNORED_KEYS, load_file, merge_dict +from .Utils import REMOVE_KEYS, arg_converter, load_file, merge_dict from .version import APP_VERSION @@ -218,7 +218,7 @@ class Config: "instance_title", "sentry_dsn", "console_enabled", - "browser_enabled" + "browser_enabled", ) "The variables that are relevant to the frontend." @@ -321,8 +321,38 @@ class Config: except Exception as e: LOG.error(f"Error starting debugpy server at '0.0.0.0:{self.debugpy_port}'. {e}") + ytdlp_cli: str = os.path.join(self.config_path, "ytdlp.cli") optsFile: str = os.path.join(self.config_path, "ytdlp.json") - if os.path.exists(optsFile) and os.path.getsize(optsFile) > 5: + if os.path.exists(ytdlp_cli) and os.path.getsize(ytdlp_cli) > 2: + LOG.info(f"Loading yt-dlp custom options from '{ytdlp_cli}'.") + with open(ytdlp_cli) as f: + ytdlp_cli_opts = f.read().strip() + if ytdlp_cli_opts: + try: + removed_options = [] + self.ytdl_options = arg_converter( + args=ytdlp_cli_opts, + level=True, + removed_options=removed_options, + ) + + try: + LOG.debug("Parsed yt-dlp cli options '%s'.", self.ytdl_options) + except Exception: + pass + + if len(removed_options) > 0: + LOG.warning( + "Removed the following options: '%s' from '%s'", ", ".join(removed_options), ytdlp_cli + ) + except Exception as e: + msg = f"Failed to parse yt-dlp cli options from '{ytdlp_cli}'. '{e!s}'." + raise ValueError(msg) from e + else: + LOG.warning(f"Empty yt-dlp custom options file '{ytdlp_cli}'.") + # @Deprecated - To be removed in future versions. + elif os.path.exists(optsFile) and os.path.getsize(optsFile) > 5: + LOG.warning("The JSON ytdlp.json options file is deprecated, please use 'ytdlp.cli' file instead.") LOG.info(f"Loading yt-dlp custom options from '{optsFile}'.") (opts, status, error) = load_file(optsFile, dict) @@ -330,16 +360,24 @@ class Config: LOG.error(f"Could not load yt-dlp custom options from '{optsFile}'. {error}") sys.exit(1) if isinstance(opts, dict): - for key in IGNORED_KEYS: - if key in opts: - LOG.error(f"Key '{key}' is not allowed to be loaded via 'ytdlp.json' file.") - del opts[key] + bad_options = {k: v for d in REMOVE_KEYS for k, v in d.items()} + removed_options = [] + for key in opts.copy(): + if key not in bad_options: + continue + removed_options.append(bad_options[key]) + opts.pop(key, None) + + if len(removed_options) > 0: + LOG.warning( + "Removed the following options: '%s' from 'ytdlp.json' file.", ", ".join(removed_options) + ) self.ytdl_options = merge_dict(self.ytdl_options, opts) else: LOG.error(f"Invalid yt-dlp custom options file '{optsFile}'.") else: - LOG.info(f"No yt-dlp custom options found at '{optsFile}'.") + LOG.info(f"No yt-dlp custom options found at '{ytdlp_cli}'.") self.ytdl_options["socket_timeout"] = self.socket_timeout @@ -373,7 +411,6 @@ class Config: key_file: str = os.path.join(self.config_path, "secret.key") - # save key as bytes. if os.path.exists(key_file) and os.path.getsize(key_file) > 5: with open(key_file, "rb") as f: self.secret_key = f.read().strip() diff --git a/ui/components/History.vue b/ui/components/History.vue index 965ea74c..147b4cd3 100644 --- a/ui/components/History.vue +++ b/ui/components/History.vue @@ -141,17 +141,17 @@
-
-
-
+