diff --git a/.vscode/settings.json b/.vscode/settings.json index d961a26d..3696f9c5 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -19,11 +19,16 @@ "attl", "autonumber", "changeslog", + "consoletitle", + "cookiesfrombrowser", "copyts", "daterange", "dotenv", "finaldir", "flac", + "forcejson", + "forceprint", + "fribidi", "getpid", "gpac", "httpx", @@ -41,6 +46,7 @@ "postprocessor", "preferredcodec", "preferredquality", + "printtraffic", "quicktime", "tmpfilename", "urandom", @@ -56,7 +62,6 @@ "en" ], "spellright.documentTypes": [ - "latex", - "plaintext" + "latex" ], } diff --git a/Dockerfile b/Dockerfile index 6d45a974..2c1d4dc8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -35,7 +35,7 @@ ENV XDG_CONFIG_HOME=/config ENV XDG_CACHE_HOME=/tmp RUN mkdir /config /downloads && ln -snf /usr/share/zoneinfo/${TZ} /etc/localtime && echo ${TZ} > /etc/timezone && \ - apk add --update --no-cache bash mkvtoolnix patch aria2 coreutils curl shadow sqlite tzdata libmagic ffmpeg rtmpdump && \ + apk add --update --no-cache bash mkvtoolnix patch aria2 coreutils curl shadow sqlite tzdata libmagic ffmpeg rtmpdump fribidi && \ useradd -u ${USER_ID:-1000} -U -d /app -s /bin/bash app && \ rm -rf /var/cache/apk/* diff --git a/README.md b/README.md index c59b6db7..7eb3c710 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ Web GUI for [yt-dlp](https://github.com/yt-dlp/yt-dlp) with playlist & channel s * Can Handle live streams. * Scheduler to queue channels or playlists to be downloaded automatically at a specified time. * Send notification to targets based on selected events. -* Support per link `yt-dlp JSON config or cli options`, `cookies` & `output format`. +* Support per link `cli options` & `cookies` * Queue multiple URLs separated by comma. * Simple file browser. `Disabled by default` * A built in video player that can play any video file regardless of the format. **With support for sidecar external subtitles**. @@ -25,22 +25,47 @@ Web GUI for [yt-dlp](https://github.com/yt-dlp/yt-dlp) with playlist & channel s * Support for curl_cffi, see [yt-dlp documentation](https://github.com/yt-dlp/yt-dlp?tab=readme-ov-file#impersonation) * Support for both advanced and basic mode for WebUI. * Bundled tools in container: curl-cffi, ffmpeg, ffprobe, aria2, rtmpdump, mkvtoolsnix, mp4box. - -# Recommended basic `ytdlp.json` file settings -Your `/config/ytdlp.json` config should include the following basic options for optimal working conditions. +# Breaking changes -```json -{ - "windowsfilenames": true, - "continue_dl": true, - "live_from_start": true, - "format_sort": [ "codec:avc:m4a" ], -} -``` +Starting with versions tagged `*-20250330-*` I have deprecated the JSON yt-dlp config and all of it's related settings. -> [!NOTE] -> Note, the `format_sort`, forces YouTube to use x264 instead of vp9 codec, you can ignore it if you want. i prefer the media in x264. +To be frank, it was pain to manage and hard for users to understand how to map cli options to json options. So, I have +decided to just use the cli options directly. This means that you can now use the same options you would use in the command line +directly in the WebUI, presets, and tasks. + +## `ytdlp.json` vs `ytdlp.cli` + +I have also added a new `ytdlp.cli` file that will be used to store the global options for yt-dlp. This file is located in the `/config` directory +and will be used to store the global options for yt-dlp. This file is not required and presets can fill the gap for most of the +use cases. But, if you want to use the same options for all your downloads, you can use this file to store the global options. + +So, if you have `ytdlp.cli` file it will take priority over the `ytdlp.json` file if both exists. As mitigation, I have implemented +fallback to `ytdlp.json` file if `ytdlp.cli` file is not found. + +## Presets + +I have deprecated the `JSON yt-dlp config` and `JSON yt-dlp Post-Processors` fields, and added new `Command arguments for yt-dlp` +field to the presets. This field will be used to store the command line options for yt-dlp. I have also have added fallback, +if you have the `JSON yt-dlp config/Post-Processors` field set, the logic is, if `Command arguments for yt-dlp` is set +it will take priority over the `JSON yt-dlp config/Post-Processors` field. If not set, it will fallback to the `JSON yt-dlp config/Post-Processors` field. + +If you are adding new presets, the deprecated fields will not show up anymore, they will only show up if actually have content in them +and editing old preset. So, Please migrate your presets to the new format. + +## Tasks + +I have also removed the `JSON yt-dlp config` and replaced it with `Command arguments for yt-dlp` field. Sadly, there is +no fallback, and once you upgrade to any version after `*-20250330-*` your `tasks.json` file will be updated to remove the +`config` key. so, please make sure to backup your `tasks.json` file before upgrading. + +## closing statement + +I know it's a painful breaking change, but for the sake of maintainability and ease of use, I have decided to +to make it happen sooner than later, the `JSON yt-dlp config`, was hard to manage and some features weren't really working +as expected, for example `--match-filter` and `--date` arguments etc. + +So, Starting with `*-2025040*-*` tagged versions, the all the fallbacks and backwards compatibility will be removed. # Run using docker command @@ -240,25 +265,19 @@ A Docker image can be built locally (it will build the UI too): docker build . -t ytptube ``` -# ytdlp.json file +# ytdlp.cli file -The `config/ytdlp.json`, is a json file which can be used to alter the default `yt-dlp` config settings globally. +The `config/ytdlp.cli`, is a command line options file for `yt-dlp` it will be globally applied to all downloads. -We recommend not use this file for options that aren't **truly global**, everything that can be done via the `ytdlp.json` file -can be done via a preset, which only effects the download that uses it. Example of good basic `ytdlp.json` file. +We strongly recommend not use this file for options that aren't **truly global**, everything that can be done via the file +can also be done via the presets which is dynamic can be altered per download. Example of good global options are to be +used for all downloads are: -```json -{ - "windowsfilenames": true, - "continue_dl": true, - "live_from_start": true, - "format_sort": [ "codec:avc:m4a" ], -} +```bash +--continue --windows-filenames --live-from-start ``` -Everything else can be done via the presets, and it's more flexible and easier to manage. You can convert your -own yt-dlp command arguments into a preset using the box found in the presets add page. For reference, The options can be found -at [yt-dlp YoutubeDL.py](https://github.com/yt-dlp/yt-dlp/blob/master/yt_dlp/YoutubeDL.py#L214) file. And for the postprocessors at [yt-dlp postprocessor](https://github.com/yt-dlp/yt-dlp/tree/master/yt_dlp/postprocessor). +Everything else can be done via the presets, and it's more flexible and easier to manage. # Authentication @@ -275,7 +294,7 @@ What does the basic mode do? it hides the the following features from the WebUI. ### Header -It disables everything except the `theme switcher` and `reload` button. +It disables everything except the `settings button` and `reload` button. ### Add form diff --git a/app/library/Download.py b/app/library/Download.py index 68e33bf9..f05eda47 100644 --- a/app/library/Download.py +++ b/app/library/Download.py @@ -135,6 +135,12 @@ class Download: self.status_queue.put({"id": self.id, "status": "finished", "filename": filename}) + def post_hooks(self, filename: str | None = None): + if not filename: + return + + self.status_queue.put({"id": self.id, "filename": filename}) + def _download(self): try: params = ( @@ -178,6 +184,7 @@ class Download: { "progress_hooks": [self._progress_hook], "postprocessor_hooks": [self._postprocessor_hook], + "post_hooks": [self.post_hooks], } ) @@ -215,7 +222,7 @@ class Download: f'Task id="{self.info.id}" PID="{os.getpid()}" title="{self.info.title}" preset="{self.preset}" started.' ) - self.logger.debug("Params before passing to yt-dlp.", extra=params) + self.logger.debug(f"Params before passing to yt-dlp. {params}") params["logger"] = NestedLogger(self.logger) diff --git a/app/library/DownloadQueue.py b/app/library/DownloadQueue.py index 7aac9fba..491910ff 100644 --- a/app/library/DownloadQueue.py +++ b/app/library/DownloadQueue.py @@ -21,7 +21,7 @@ from .Events import EventBus, Events from .ItemDTO import ItemDTO from .Presets import Presets from .Singleton import Singleton -from .Utils import calc_download_path, extract_info, is_downloaded +from .Utils import arg_converter, calc_download_path, extract_info, is_downloaded from .YTDLPOpts import YTDLPOpts LOG = logging.getLogger("DownloadQueue") @@ -177,6 +177,7 @@ class DownloadQueue(metaclass=Singleton): cookies: str = "", template: str = "", extras: dict | None = None, + cli: str = "", already=None, ): """ @@ -190,9 +191,9 @@ class DownloadQueue(metaclass=Singleton): cookies (str): The cookies to use for the download. template (str): The output template to use for the download. extras (dict): The extra information to add to the download. + cli (str): The yt-dlp command line options to use for the download. already (set): The set of already downloaded items. - Returns: dict: The status of the operation. @@ -244,6 +245,7 @@ class DownloadQueue(metaclass=Singleton): cookies=cookies, template=template, extras=extras, + cli=cli, already=already, ) ) @@ -320,6 +322,7 @@ class DownloadQueue(metaclass=Singleton): is_live=is_live, live_in=live_in, options=options, + cli=cli, extras=extras, ) @@ -352,6 +355,7 @@ class DownloadQueue(metaclass=Singleton): cookies=cookies, template=template, extras=extras, + cli=cli, already=already, ) @@ -365,12 +369,39 @@ class DownloadQueue(metaclass=Singleton): config: dict | None = None, cookies: str = "", template: str = "", + cli: str = "", extras: dict | None = None, already=None, ): + """ + Add an item to the download queue. + + Args: + url (str): The url to be added to the queue. + preset (str): The preset to be used for the download. + folder (str): The folder to save the download to. + config (dict): The yt-dlp config to be used for the download. + cookies (str): The cookies to be used for the download. + template (str): The template to be used for the download. + cli (str): The yt-dlp cli options to be used for the download. + extras (dict): Extra data to be added to the download + already (set): Set of already downloaded items. + + Returns: + dict[str, str]: The status of the download. + { "status": "text" } + + """ _preset = Presets.get_instance().get(name=preset) config = config if config else {} + if cli: + try: + config = arg_converter(args=cli, level=True) + except Exception as e: + LOG.error(f"Invalid cli options '{cli}'. {e!s}") + return {"status": "error", "msg": f"Invalid cli options '{cli}'. {e!s}"} + folder = str(folder) if folder else "" if not extras: extras = {} @@ -390,7 +421,7 @@ class DownloadQueue(metaclass=Singleton): cookie_file = os.path.join(self.config.temp_path, f"c_{uuid.uuid4().hex}.txt") LOG.info( - f"Adding 'URL: {url}' to 'Folder: {filePath}' with 'Preset: {preset}' 'Naming: {template}', 'Cookies: {len(cookies)}/chars' 'YTConfig: {config}' 'Extras: {extras}'." + f"Adding 'URL: {url}' to 'Folder: {filePath}' with 'Preset: {preset}' 'Naming: {template}', 'Cookies: {len(cookies)}/chars' 'CLI: {cli}' 'Extras: {extras}'." ) if isinstance(config, str): @@ -486,6 +517,7 @@ class DownloadQueue(metaclass=Singleton): template=template, already=already, extras=extras, + cli=cli, ) async def cancel(self, ids: list[str]) -> dict[str, str]: diff --git a/app/library/Events.py b/app/library/Events.py index 7fd52b47..156b3a65 100644 --- a/app/library/Events.py +++ b/app/library/Events.py @@ -90,7 +90,6 @@ class Events: LOG_SUCCESS = "log_success" INITIAL_DATA = "initial_data" - YTDLP_CONVERT = "ytdlp_convert" ITEM_DELETE = "item_delete" ITEM_CANCEL = "item_cancel" STATUS = "status" diff --git a/app/library/HttpAPI.py b/app/library/HttpAPI.py index 74177928..74387c55 100644 --- a/app/library/HttpAPI.py +++ b/app/library/HttpAPI.py @@ -35,7 +35,7 @@ from .Segments import Segments from .Subtitle import Subtitle from .Tasks import Task, Tasks from .Utils import ( - IGNORED_KEYS, + REMOVE_KEYS, StreamingError, arg_converter, decrypt_data, @@ -445,7 +445,7 @@ class HttpAPI(Common): try: response = {"opts": {}, "output_template": None, "download_path": None} - data = arg_converter(args) + data = arg_converter(args, dumps=True) if "outtmpl" in data and "default" in data["outtmpl"]: response["output_template"] = data["outtmpl"]["default"] @@ -453,20 +453,29 @@ class HttpAPI(Common): if "paths" in data and "home" in data["paths"]: response["download_path"] = data["paths"]["home"] + if "format" in data: + response["format"] = data["format"] + + bad_options = {k: v for d in REMOVE_KEYS for k, v in d.items()} + removed_options = [] + for key in data: - if key in IGNORED_KEYS: + if key in bad_options.items(): + removed_options.append(bad_options[key]) continue if not key.startswith("_"): response["opts"][key] = data[key] + if len(removed_options) > 0: + response["opts"]["removed"] = ", ".join(removed_options) + return web.json_response(data=response, status=web.HTTPOk.status_code) except Exception as e: err = str(e).strip() err = err.split("\n")[-1] if "\n" in err else err - LOG.error(f"Failed to convert args. '{err}'.") - LOG.exception(e) + err = err.replace("main.py: error: ", "").strip().capitalize() return web.json_response( - data={"error": f"Failed to convert args. '{err}'."}, status=web.HTTPBadRequest.status_code + data={"error": f"Failed to command line arguments for yt-dlp. '{err}'."}, status=web.HTTPBadRequest.status_code ) @route("GET", "api/yt-dlp/url/info") @@ -872,15 +881,12 @@ class HttpAPI(Common): if not item.get("timer", None) or str(item.get("timer")).strip() == "": item["timer"] = f"{random.randint(1,59)} */1 * * *" # noqa: S311 - if not item.get("cookies", None): - item["cookies"] = "" - - if not item.get("config", None) or str(item.get("config")).strip() == "": - item["config"] = {} - if not item.get("template", None): item["template"] = "" + if not item.get("cli", None): + item["cli"] = "" + try: ins.validate(item) except ValueError as e: diff --git a/app/library/HttpSocket.py b/app/library/HttpSocket.py index 12d379cc..5d3d22b4 100644 --- a/app/library/HttpSocket.py +++ b/app/library/HttpSocket.py @@ -18,7 +18,7 @@ from .DownloadQueue import DownloadQueue from .encoder import Encoder from .Events import Event, EventBus, Events, error from .Presets import Presets -from .Utils import arg_converter, is_downloaded +from .Utils import is_downloaded LOG = logging.getLogger("socket_api") @@ -285,25 +285,3 @@ class HttpSocket(Common): async def resume(self, *_, **__): self.queue.resume() await self._notify.emit(Events.PAUSED, data={"paused": False, "at": time.time()}) - - @ws_event - async def ytdlp_convert(self, sid: str, data: dict): - if not isinstance(data, dict) or "args" not in data: - await self._notify.emit(Events.ERROR, data=error("Invalid request or no options were given."), to=sid) - return - - args: str | None = data.get("args") - - if not args: - await self._notify.emit(Events.ERROR, data=error("no options were given."), to=sid) - return - - try: - await self._notify.emit(Events.YTDLP_CONVERT, data=arg_converter(args), to=sid) - except Exception as e: - err = str(e).strip() - err = err.split("\n")[-1] if "\n" in err else err - LOG.error(f"Failed to convert args. '{err}'.") - await self._notify.emit(Events.ERROR, data=error(f"Failed to convert options. '{err}'."), to=sid) - - return diff --git a/app/library/ItemDTO.py b/app/library/ItemDTO.py index a3020545..36bcd3c0 100644 --- a/app/library/ItemDTO.py +++ b/app/library/ItemDTO.py @@ -27,7 +27,6 @@ class ItemDTO: temp_dir: str | None = None status: str | None = None cookies: str | None = None - config: dict = field(default_factory=dict) template: str | None = None template_chapter: str | None = None timestamp: float = field(default_factory=lambda: time.time_ns()) @@ -37,6 +36,7 @@ class ItemDTO: file_size: int | None = None options: dict = field(default_factory=dict) extras: dict = field(default_factory=dict) + cli: str = "" # yt-dlp injected fields. tmpfilename: str | None = None @@ -49,12 +49,13 @@ class ItemDTO: speed: str | None = None eta: str | None = None - # DEPRECATED: These fields are deprecated and will be removed in the future. + # @Deprecated: These fields are deprecated and will be removed in the future. thumbnail: str | None = None ytdlp_cookies: str | None = None ytdlp_config: dict = field(default_factory=dict) output_template: str | None = None output_template_chapter: str | None = None + config: dict = field(default_factory=dict) def serialize(self) -> dict: deprecated: tuple = ( @@ -65,6 +66,7 @@ class ItemDTO: "ytdlp_config", "output_template", "output_template_chapter", + "config", ) if self.thumbnail and "thumbnail" not in self.extras: diff --git a/app/library/Presets.py b/app/library/Presets.py index 39ae334e..83f65e16 100644 --- a/app/library/Presets.py +++ b/app/library/Presets.py @@ -41,6 +41,9 @@ class Preset: cookies: str = "" """The default cookies to use if non is given.""" + cli: str = "" + """yt-dlp cli command line arguments.""" + default: bool = False def serialize(self) -> dict: @@ -220,6 +223,15 @@ class Presets(metaclass=Singleton): msg = "Invalid postprocessors type. expected list." raise ValueError(msg) + if preset.get("cli"): + try: + from .Utils import arg_converter + + arg_converter(args=preset.get("cli")) + except Exception as e: + msg = f"Invalid cli options. '{e!s}'." + raise ValueError(msg) from e + return True def save(self, presets: list[Preset | dict]) -> "Presets": diff --git a/app/library/Tasks.py b/app/library/Tasks.py index b3ac7a33..dc695114 100644 --- a/app/library/Tasks.py +++ b/app/library/Tasks.py @@ -3,7 +3,7 @@ import json import logging import os import time -from dataclasses import dataclass, field +from dataclasses import dataclass from datetime import UTC, datetime from typing import Any @@ -28,8 +28,7 @@ class Task: preset: str = "" timer: str = "" template: str = "" - cookies: str = "" - config: dict[str, str] = field(default_factory=dict) + cli: str = "" def serialize(self) -> dict: return self.__dict__ @@ -134,16 +133,20 @@ class Tasks(metaclass=Singleton): with open(self._file) as f: tasks = json.load(f) except Exception as e: - LOG.error(f"Failed to parse tasks from '{self._file}'. '{e}'.") + LOG.error(f"Failed to parse tasks from '{self._file}'. '{e!s}'.") return self if not tasks or len(tasks) < 1: LOG.info(f"No tasks were defined in '{self._file}'.") return self + need_update = False for i, task in enumerate(tasks): try: + task, task_status = self.clean_task(task) task = Task(**task) + if task_status: + need_update = True except Exception as e: LOG.error(f"Failed to parse task at list position '{i}'. '{e!s}'.") continue @@ -157,6 +160,10 @@ class Tasks(metaclass=Singleton): LOG.exception(e) LOG.error(f"Failed to queue task '{i}: {task.name}'. '{e!s}'.") + if need_update: + LOG.info("Updating tasks file to remove old keys.") + self.save(self.get_all()) + return self def clear(self, shutdown: bool = False) -> "Tasks": @@ -213,17 +220,13 @@ class Tasks(metaclass=Singleton): msg = "No URL found." raise ValueError(msg) - if not isinstance(task.get("cookies"), str): - msg = "Invalid cookies type." - raise ValueError(msg) # noqa: TRY004 - - if not isinstance(task.get("config"), dict): - msg = "Invalid config type." - raise ValueError(msg) # noqa: TRY004 - - if not isinstance(task.get("template"), str): - msg = "Invalid template type." - raise ValueError(msg) # noqa: TRY004 + if task.get("cli"): + try: + from .Utils import arg_converter + arg_converter(args=task.get("cli")) + except Exception as e: + msg = f"Invalid cli options. '{e!s}'." + raise ValueError(msg) from e return True @@ -263,6 +266,27 @@ class Tasks(metaclass=Singleton): return self + def clean_task(self, task: dict) -> tuple[dict, bool]: + """ + Clean the task from old keys. + + Args: + task (dict): The task to clean. + + Returns: + tuple[dict, bool]: The cleaned task and a status if the task was cleaned. + + """ + status = False + removedKeys = ["cookies", "config"] + + for key in removedKeys: + if key in task: + status = True + task.pop(key) + + return task, status + async def _runner(self, task: Task): """ Run the task. @@ -283,16 +307,8 @@ class Tasks(metaclass=Singleton): preset: str = str(task.preset or self._default_preset) folder: str = task.folder if task.folder else "" - cookies: str = str(task.cookies) if task.cookies else "" template: str = task.template if task.template else "" - - config = task.config if task.config else {} - if isinstance(config, str) and config: - try: - config = json.loads(config) - except Exception as e: - LOG.error(f"Failed to parse json yt-dlp config for '{task.name}'. {e!s}") - return + cli: str = task.cli if task.cli else "" LOG.info(f"Task '{task.id}: {task.name}' dispatched at '{timeNow}'.") @@ -307,9 +323,8 @@ class Tasks(metaclass=Singleton): "url": task.url, "preset": preset, "folder": folder, - "cookies": cookies, - "config": config, "template": template, + "cli": cli, }, id=task.id, ), diff --git a/app/library/Utils.py b/app/library/Utils.py index 4251bb5e..155f1ceb 100644 --- a/app/library/Utils.py +++ b/app/library/Utils.py @@ -21,7 +21,42 @@ from .LogWrapper import LogWrapper LOG = logging.getLogger("Utils") -IGNORED_KEYS: tuple[str] = ("paths", "outtmpl", "progress_hooks", "postprocessor_hooks", "download_archive") +REMOVE_KEYS: list = [ + { + "paths": "-P, --paths", + "outtmpl": "-o, --output", + "progress_hooks": "--progress_hooks", + "postprocessor_hooks": "--postprocessor_hooks", + "post_hooks": "--post_hooks", + "download_archive": "--download_archive", + }, + { + "quiet": "-q, --quiet", + "no_warnings": "--no-warnings", + "skip_download": "--skip-download", + "forceprint": "-O, --print", + "simulate": "--simulate", + "noprogress": "--no-progress", + "wait_for_video": "--wait-for-video", + "mark_watched": "--mark-watched", + "color": "--color", + "verbose": "-v, --verbose", + "debug_printtraffic": "--print-traffic", + "write_pages": "--write-pages", + "dump_intermediate_pages": "--dump-pages", + "progress_delta": " --progress-delta", + "progress_template": "--progress-template", + "consoletitle": "--console-title", + "progress_with_newline": "--newline", + "forcejson": "-j, --dump-json", + "print_to_file": "--print-to-file", + "cookiesfrombrowser": "--cookies-from-browser", + }, + { + "cookiefile": "--cookies", + }, +] + YTDLP_INFO_CLS: yt_dlp.YoutubeDL = None ALLOWED_SUBS_EXTENSIONS: tuple[str] = (".srt", ".vtt", ".ass") @@ -434,12 +469,20 @@ def validate_url(url: str) -> bool: return True -def arg_converter(args: str) -> dict: +def arg_converter( + args: str, + level: int | bool | None = None, + dumps: bool = False, + removed_options: list | None = None, +) -> dict: """ Convert yt-dlp options to a dictionary. Args: args (str): yt-dlp options string. + level (int|bool|None): Level of options to remove, True for all. + dumps (bool): Dump options as JSON. + removed_options (list|None): List of removed options. Returns: dict: yt-dlp options dictionary. @@ -465,16 +508,44 @@ def arg_converter(args: str) -> dict: if "postprocessors" in diff: diff["postprocessors"] = [pp for pp in diff["postprocessors"] if pp not in default_opts["postprocessors"]] - if "match_filter" in diff: - import inspect + if "_warnings" in diff: + diff.pop("_warnings", None) - matchFilter = inspect.getclosurevars(diff["match_filter"].func).nonlocals["filters"] - if isinstance(matchFilter, set): - diff["match_filter"] = {"filters": list(matchFilter)} + if level is True or isinstance(level, int): + bad_options = {} + if isinstance(level, bool) or not isinstance(level, int): + level = len(REMOVE_KEYS) - from .encoder import Encoder + for i, item in enumerate(REMOVE_KEYS): + if i > level: + break - return json.loads(json.dumps(diff, cls=Encoder)) + bad_options.update(item.items()) + + LOG.debug("Removed %i the following options: '%s'.", level, ", ".join(bad_options.values())) + + for key in diff.copy(): + if key not in bad_options: + continue + + if isinstance(removed_options, list): + removed_options.append(bad_options[key]) + + diff.pop(key, None) + + if dumps is True: + from .encoder import Encoder + + if "match_filter" in diff: + import inspect + + matchFilter = inspect.getclosurevars(diff["match_filter"].func).nonlocals["filters"] + if isinstance(matchFilter, set): + diff["match_filter"] = {"filters": list(matchFilter)} + + return json.loads(json.dumps(diff, cls=Encoder)) + + return diff def validate_uuid(uuid_str: str, version: int = 4) -> bool: diff --git a/app/library/YTDLPOpts.py b/app/library/YTDLPOpts.py index 1a5fd206..2a87ca91 100644 --- a/app/library/YTDLPOpts.py +++ b/app/library/YTDLPOpts.py @@ -4,7 +4,7 @@ from pathlib import Path from .config import Config from .Presets import Presets from .Singleton import Singleton -from .Utils import IGNORED_KEYS, calc_download_path, merge_dict +from .Utils import REMOVE_KEYS, arg_converter, calc_download_path, merge_dict LOG = logging.getLogger("YTDLPOpts") @@ -48,11 +48,22 @@ class YTDLPOpts(metaclass=Singleton): YTDLPOpts: The instance of the class """ + bad_options = {} + if from_user: + bad_options = {k: v for d in REMOVE_KEYS for k, v in d.items()} + + removed_options = [] + for key, value in config.items(): - if key in IGNORED_KEYS and from_user: + if from_user and key in bad_options: + removed_options.append(bad_options[key]) continue + self._item_opts[key] = value + if len(removed_options) > 0: + LOG.warning("Removed the following options: '%s'.", ", ".join(removed_options)) + return self def preset(self, name: str, with_cookies: bool = False) -> "YTDLPOpts": @@ -71,6 +82,19 @@ class YTDLPOpts(metaclass=Singleton): if not preset or "default" == name: return self + if preset.cli: + try: + removed_options = [] + self._preset_opts = arg_converter(args=preset.cli, level=True, removed_options=removed_options) + if len(removed_options) > 0: + LOG.warning( + "Removed the following options '%s' from preset '%s'.", ", ".join(removed_options), preset.name + ) + + except Exception as e: + msg = f"Invalid cli options for preset '{preset.name}'. '{e!s}'." + raise ValueError(msg) from e + if preset.cookies and with_cookies: file = Path(self._config.config_path, "cookies", f"{preset.id}.txt") @@ -94,14 +118,24 @@ class YTDLPOpts(metaclass=Singleton): "temp": self._config.temp_path, } - if preset.postprocessors and isinstance(preset.postprocessors, list) and len(preset.postprocessors) > 0: - self._preset_opts["postprocessors"] = preset.postprocessors + # @Deprecated - To be removed in future versions. + if not preset.cli: + if preset.postprocessors and isinstance(preset.postprocessors, list) and len(preset.postprocessors) > 0: + self._preset_opts["postprocessors"] = preset.postprocessors - if preset.args and isinstance(preset.args, dict) and len(preset.args) > 0: - for key, value in preset.args.items(): - if key in IGNORED_KEYS: - continue - self._preset_opts[key] = value + if preset.args and isinstance(preset.args, dict) and len(preset.args) > 0: + bad_options = {k: v for d in REMOVE_KEYS for k, v in d.items()} + removed_options = [] + for key, value in preset.args.items(): + if key in bad_options: + removed_options.append(bad_options[key]) + continue + self._preset_opts[key] = value + + if len(removed_options) > 0: + LOG.warning( + "Removed the following options '%s' from '%s' args.", ", ".join(removed_options), preset.name + ) return self @@ -132,6 +166,7 @@ class YTDLPOpts(metaclass=Singleton): if "format" in data and data["format"] in ["not_set", "default"]: data["format"] = None + # @Deprecated - To be removed in future versions, All the checks below this line are deprecated. if "daterange" in data and isinstance(data["daterange"], dict): from yt_dlp.utils import DateRange diff --git a/app/library/common.py b/app/library/common.py index 371ca22a..2f26b4a9 100644 --- a/app/library/common.py +++ b/app/library/common.py @@ -4,6 +4,7 @@ import logging from .config import Config from .DownloadQueue import DownloadQueue from .encoder import Encoder +from .Utils import arg_converter LOG = logging.getLogger("common") @@ -27,7 +28,16 @@ class Common: self.default_preset = config.default_preset async def add( - self, url: str, preset: str, folder: str, cookies: str, config: dict, template: str, extras: dict | None = None + self, + url: str, + preset: str, + folder: str, + cookies: str, + # @deprecated: config: dict, + config: dict, + template: str, + extras: dict | None = None, + cli: str = "", ) -> dict[str, str]: """ Add an item to the download queue. @@ -40,6 +50,7 @@ class Common: config (dict): The yt-dlp config to be used for the download. template (str): The template to be used for the download. extras (dict): Extra data to be added to the download + cli (str): The yt-dlp cli options to be used for the download. Returns: dict[str, str]: The status of the download. @@ -56,6 +67,7 @@ class Common: cookies=cookies, config=config if isinstance(config, dict) else {}, template=template, + cli=cli, extras=extras, ) @@ -94,6 +106,19 @@ class Common: msg = f"Failed to parse json yt-dlp config for '{url}'. {e!s}" raise ValueError(msg) from e + cli = item.get("cli") + if cli and len(cli) > 2: + try: + removed_options = [] + arg_converter(args=cli, level=True, removed_options=removed_options) + if len(removed_options) > 0: + LOG.warning("Removed the following options '%s'.", ", ".join(removed_options)) + + config = {} + except Exception as e: + msg = f"Failed to parse yt-dlp cli options. {e!s}" + raise ValueError(msg) from e + return { "url": url, "preset": preset, @@ -101,5 +126,6 @@ class Common: "cookies": cookies, "config": config if isinstance(config, dict) else {}, "template": template, + "cli": cli if isinstance(cli, str) else "", "extras": extras if isinstance(extras, dict) else {}, } diff --git a/app/library/config.py b/app/library/config.py index 00e013af..fb4c0ab3 100644 --- a/app/library/config.py +++ b/app/library/config.py @@ -12,7 +12,7 @@ import coloredlogs from dotenv import load_dotenv from yt_dlp.version import __version__ as YTDLP_VERSION -from .Utils import IGNORED_KEYS, load_file, merge_dict +from .Utils import REMOVE_KEYS, arg_converter, load_file, merge_dict from .version import APP_VERSION @@ -218,7 +218,7 @@ class Config: "instance_title", "sentry_dsn", "console_enabled", - "browser_enabled" + "browser_enabled", ) "The variables that are relevant to the frontend." @@ -321,8 +321,38 @@ class Config: except Exception as e: LOG.error(f"Error starting debugpy server at '0.0.0.0:{self.debugpy_port}'. {e}") + ytdlp_cli: str = os.path.join(self.config_path, "ytdlp.cli") optsFile: str = os.path.join(self.config_path, "ytdlp.json") - if os.path.exists(optsFile) and os.path.getsize(optsFile) > 5: + if os.path.exists(ytdlp_cli) and os.path.getsize(ytdlp_cli) > 2: + LOG.info(f"Loading yt-dlp custom options from '{ytdlp_cli}'.") + with open(ytdlp_cli) as f: + ytdlp_cli_opts = f.read().strip() + if ytdlp_cli_opts: + try: + removed_options = [] + self.ytdl_options = arg_converter( + args=ytdlp_cli_opts, + level=1, + removed_options=removed_options, + ) + + try: + LOG.debug("Parsed yt-dlp cli options '%s'.", self.ytdl_options) + except Exception: + pass + + if len(removed_options) > 0: + LOG.warning( + "Removed the following options: '%s' from '%s'", ", ".join(removed_options), ytdlp_cli + ) + except Exception as e: + msg = f"Failed to parse yt-dlp cli options from '{ytdlp_cli}'. '{e!s}'." + raise ValueError(msg) from e + else: + LOG.warning(f"Empty yt-dlp custom options file '{ytdlp_cli}'.") + # @Deprecated - To be removed in future versions. + elif os.path.exists(optsFile) and os.path.getsize(optsFile) > 5: + LOG.warning("The JSON ytdlp.json options file is deprecated, please use 'ytdlp.cli' file instead.") LOG.info(f"Loading yt-dlp custom options from '{optsFile}'.") (opts, status, error) = load_file(optsFile, dict) @@ -330,16 +360,24 @@ class Config: LOG.error(f"Could not load yt-dlp custom options from '{optsFile}'. {error}") sys.exit(1) if isinstance(opts, dict): - for key in IGNORED_KEYS: - if key in opts: - LOG.error(f"Key '{key}' is not allowed to be loaded via 'ytdlp.json' file.") - del opts[key] + bad_options = {k: v for d in REMOVE_KEYS for k, v in d.items()} + removed_options = [] + for key in opts.copy(): + if key not in bad_options: + continue + removed_options.append(bad_options[key]) + opts.pop(key, None) + + if len(removed_options) > 0: + LOG.warning( + "Removed the following options: '%s' from 'ytdlp.json' file.", ", ".join(removed_options) + ) self.ytdl_options = merge_dict(self.ytdl_options, opts) else: LOG.error(f"Invalid yt-dlp custom options file '{optsFile}'.") else: - LOG.info(f"No yt-dlp custom options found at '{optsFile}'.") + LOG.info(f"No yt-dlp custom options found at '{ytdlp_cli}'.") self.ytdl_options["socket_timeout"] = self.socket_timeout @@ -373,7 +411,6 @@ class Config: key_file: str = os.path.join(self.config_path, "secret.key") - # save key as bytes. if os.path.exists(key_file) and os.path.getsize(key_file) > 5: with open(key_file, "rb") as f: self.secret_key = f.read().strip() diff --git a/app/library/presets.json b/app/library/presets.json index 52e0eae7..86c6492c 100644 --- a/app/library/presets.json +++ b/app/library/presets.json @@ -6,6 +6,7 @@ "folder": "", "template": "", "cookies": "", + "cli": "", "default": true }, { @@ -15,69 +16,37 @@ "folder": "", "template": "", "cookies": "", + "cli": "", "default": true }, { "id": "441675ed-b739-40f0-a0b0-1ecfcb9dc48b", "name": "1080p H264/m4a or best available", "format": "bv[height<=1080][ext=mp4]+ba[ext=m4a]/b[ext=mp4]/b[ext=webm]", - "args": { - "format_sort": [ - "vcodec:h264" - ] - }, "folder": "", "template": "", "cookies": "", + "cli": "-S vcodec:h264", "default": true }, { "id": "9719fcc3-4cf2-4d88-b1e4-74dff3dba00e", "name": "720p h264/m4a or best available", "format": "bv[height<=720][ext=mp4]+ba[ext=m4a]/b[ext=mp4]/b[ext=webm]", - "args": { - "format_sort": [ - "vcodec:h264" - ] - }, "folder": "", "template": "", "cookies": "", + "cli": "-S vcodec:h264", "default": true }, { "id": "a6fd4b25-2b3e-458d-bb57-b75e41cc4330", "name": "Audio only", "format": "bestaudio/best", - "args": { - "writethumbnail": true - }, - "postprocessors": [ - { - "key": "FFmpegExtractAudio", - "preferredcodec": "best", - "preferredquality": "5", - "nopostoverwrites": false - }, - { - "key": "FFmpegMetadata", - "add_chapters": true, - "add_metadata": true, - "add_infojson": "if_exists" - }, - { - "key": "EmbedThumbnail", - "already_have_thumbnail": false - }, - { - "key": "FFmpegConcat", - "only_multi_video": true, - "when": "playlist" - } - ], "folder": "", "template": "", "cookies": "", + "cli": "--extract-audio --add-chapters --embed-metadata --embed-thumbnail", "default": true } ] diff --git a/ui/components/History.vue b/ui/components/History.vue index 965ea74c..e14b655b 100644 --- a/ui/components/History.vue +++ b/ui/components/History.vue @@ -141,17 +141,17 @@
-
-
-
+