From cc21a93cb5eebfea70f81a19322dbe539a03eccc Mon Sep 17 00:00:00 2001 From: ArabCoders Date: Fri, 7 Mar 2025 02:14:10 +0300 Subject: [PATCH] Make presets more powerful --- app/library/Download.py | 43 +++--- app/library/DownloadQueue.py | 18 ++- app/library/HttpAPI.py | 2 +- app/library/Presets.py | 13 +- app/library/Utils.py | 202 +++++++++++--------------- app/library/YTDLPOpts.py | 102 +++++++++++++ app/library/presets.json | 15 ++ ui/assets/css/style.css | 7 +- ui/components/NewDownload.vue | 4 +- ui/components/PresetForm.vue | 264 ++++++++++++++++++++++------------ ui/components/TaskForm.vue | 2 +- ui/pages/presets.vue | 2 +- 12 files changed, 439 insertions(+), 235 deletions(-) create mode 100644 app/library/YTDLPOpts.py diff --git a/app/library/Download.py b/app/library/Download.py index 6747ead6..cb7c0cc2 100644 --- a/app/library/Download.py +++ b/app/library/Download.py @@ -14,7 +14,7 @@ from .config import Config from .Emitter import Emitter from .ffprobe import ffprobe from .ItemDTO import ItemDTO -from .Utils import get_opts, merge_config +from .YTDLPOpts import YTDLPOpts LOG = logging.getLogger("download") @@ -114,22 +114,30 @@ class Download: def _download(self): try: - params: dict = get_opts(self.preset, merge_config(self.default_ytdl_opts, self.ytdl_opts)) - params.update( - { - "color": "no_color", - "paths": {"home": self.download_dir, "temp": self.temp_path}, - "outtmpl": {"default": self.template, "chapter": self.template_chapter}, - "noprogress": True, - "break_on_existing": True, - "progress_hooks": [self._progress_hook], - "postprocessor_hooks": [self._postprocessor_hook], - "ignoreerrors": False, - } + params = ( + YTDLPOpts.get_instance() + .preset(self.preset) + .add(self.ytdl_opts, from_user=True) + .add( + { + "color": "no_color", + "paths": {"home": self.download_dir, "temp": self.temp_path}, + "outtmpl": {"default": self.template, "chapter": self.template_chapter}, + "noprogress": True, + "break_on_existing": True, + "ignoreerrors": False, + }, + from_user=False, + ) + .get_all() ) - if "format" not in params and self.default_ytdl_opts.get("format", None): - params["format"] = "best" + params.update( + { + "progress_hooks": [self._progress_hook], + "postprocessor_hooks": [self._postprocessor_hook], + } + ) if self.debug: params["verbose"] = True @@ -137,7 +145,9 @@ class Download: if self.info.cookies: try: - with open(os.path.join(self.temp_path, f"cookie_{self.info._id}.txt"), "w") as f: + cookie_file = os.path.join(self.temp_path, f"cookie_{self.info._id}.txt") + LOG.debug(f"Creating cookie file for '{self.info.id}: {self.info.title}' - '{cookie_file}'.") + with open(cookie_file, "w") as f: f.write(self.info.cookies) params["cookiefile"] = f.name except ValueError as e: @@ -175,6 +185,7 @@ class Download: self.status_queue.put({"id": self.id, "status": "finished" if ret == 0 else "error"}) except Exception as exc: + LOG.exception(exc) self.status_queue.put({"id": self.id, "status": "error", "msg": str(exc), "error": str(exc)}) LOG.info(f'Task id="{self.info.id}" PID="{os.getpid()}" title="{self.info.title}" completed.') diff --git a/app/library/DownloadQueue.py b/app/library/DownloadQueue.py index fc718055..2da63433 100644 --- a/app/library/DownloadQueue.py +++ b/app/library/DownloadQueue.py @@ -19,8 +19,10 @@ from .Download import Download from .Emitter import Emitter from .EventsSubscriber import Events from .ItemDTO import ItemDTO +from .Presets import Presets from .Singleton import Singleton -from .Utils import calc_download_path, extract_info, get_opts, is_downloaded, merge_config +from .Utils import calc_download_path, extract_info, is_downloaded +from .YTDLPOpts import YTDLPOpts LOG = logging.getLogger("DownloadQueue") @@ -352,9 +354,21 @@ class DownloadQueue(metaclass=Singleton): template: str = "", already=None, ): + _preset = Presets.get_instance().get(name=preset) + config = config if config else {} folder = str(folder) if folder else "" + if _preset: + if _preset.folder and not folder: + folder = _preset.folder + + if _preset.template and not template: + template = _preset.template + + if _preset.cookies and not cookies: + cookies = _preset.cookies + filePath = calc_download_path(base_path=self.config.download_path, folder=folder) yt_conf = {} cookie_file = os.path.join(self.config.temp_path, f"c_{uuid.uuid4().hex}.txt") @@ -395,7 +409,7 @@ class DownloadQueue(metaclass=Singleton): "func": lambda _, msg: logs.append(msg), "level": logging.WARNING, }, - **get_opts(preset, merge_config(self.config.ytdl_options, config)), + **YTDLPOpts.get_instance().preset(name=preset).add(config=config, from_user=True).get_all(), } if cookies: diff --git a/app/library/HttpAPI.py b/app/library/HttpAPI.py index b07cc69d..fb77f979 100644 --- a/app/library/HttpAPI.py +++ b/app/library/HttpAPI.py @@ -708,7 +708,7 @@ class HttpAPI(Common): item["id"] = str(uuid.uuid4()) if not item.get("args", None) or str(item.get("args")).strip() == "": - item["config"] = {} + item["args"] = {} if item.get("args", None) and isinstance(item.get("args"), str): item["args"] = json.loads(item.get("args")) diff --git a/app/library/Presets.py b/app/library/Presets.py index 5e45859a..e2f8a271 100644 --- a/app/library/Presets.py +++ b/app/library/Presets.py @@ -27,12 +27,21 @@ class Preset: format: str """The format of the preset.""" - args: dict[str, list[str] | bool] | None = field(default_factory=dict) + args: dict[str, list[str] | bool] = field(default_factory=dict) """The arguments of the preset.""" - postprocessors: list | None = field(default_factory=list) + postprocessors: list = field(default_factory=list) """The postprocessors of the preset.""" + folder: str = "" + """The default download folder to use if non is given.""" + + template: str = "" + """The default template to use if non is given.""" + + cookies: str = "" + """The default cookies to use if non is given.""" + default: bool = False def serialize(self) -> dict: diff --git a/app/library/Utils.py b/app/library/Utils.py index 61b7eab9..d056c4c3 100644 --- a/app/library/Utils.py +++ b/app/library/Utils.py @@ -31,52 +31,7 @@ class StreamingError(Exception): """Raised when an error occurs during streaming.""" -def get_opts(preset: str, ytdl_opts: dict) -> dict: - """ - Returns ytdlp options download options - - Args: - preset (str): the name of the preset selected. - ytdl_opts (dict): current options selected - - Returns: - ytdl extra options - - """ - if "format" in ytdl_opts and len(ytdl_opts["format"]) > 2: - format = ytdl_opts["format"] - LOG.info(f"Format '{format}' was given via yt-dlp options. Therefore, the preset will be ignored.") - return ytdl_opts - - opts = copy.deepcopy(ytdl_opts) - - if "default" == preset: - LOG.debug("Using default preset.") - return opts - - from .Presets import Presets - - p = Presets.get_instance().get(name=preset) - if not p: - LOG.error(f"Preset '{preset}' is not defined as preset.") - return opts - - opts["format"] = p.get("format") - - postprocessors = p.get("postprocessors", []) - if isinstance(postprocessors, list) and len(postprocessors) > 0: - opts["postprocessors"] = postprocessors - - args = p.get("args", {}) - if isinstance(args, dict) and len(args) > 0: - for key, value in args.items(): - opts[key] = value - - LOG.debug(f"Using preset '{preset}', altered options: {opts}") - return opts - - -def get_video_info(url: str, ytdlp_opts: dict = None, no_archive: bool = True) -> Any | dict[str, Any] | None: +def get_video_info(url: str, ytdlp_opts: dict | None = None, no_archive: bool = True) -> Any | dict[str, Any] | None: """ Extracts video information from the given URL. @@ -111,6 +66,11 @@ def calc_download_path(base_path: str, folder: str | None = None, create_path: b """ Calculates download path and prevents folder traversal. + Args: + base_path (str): Base download path. + folder (str): Folder to add to the base path. + create_path (bool): Create the path if it does not exist. + Returns: Download path with base folder factored in. @@ -135,29 +95,33 @@ def calc_download_path(base_path: str, folder: str | None = None, create_path: b def extract_info(config: dict, url: str, debug: bool = False) -> dict: + """ + Extracts video information from the given URL. + + Args: + config (dict): Configuration options. + url (str): URL to extract information from. + debug (bool): Enable debug logging. + + Returns: + dict: Video information. + + """ log_wrapper = LogWrapper() params: dict = { + **config, "color": "no_color", "extract_flat": True, "skip_download": True, "ignoreerrors": True, "ignore_no_formats_error": True, - **config, } # Remove keys that are not needed for info extraction. - keys: list = [ - "writeinfojson", - "writethumbnail", - "writedescription", - "writeautomaticsub", - "postprocessors", - ] - - for key in keys: - if key in params: - params.pop(key) + keys_to_remove = [key for key in params if str(key).startswith("write") or key in ["postprocessors"]] + for key in keys_to_remove: + params.pop(key, None) log_wrapper.add_target(target=logging.getLogger("yt-dlp"), level=logging.DEBUG if debug else logging.WARNING) if debug: @@ -166,7 +130,6 @@ def extract_info(config: dict, url: str, debug: bool = False) -> dict: params["quiet"] = True if "callback" in params: - # callback can be a function or dict with {level: level, func: target} if isinstance(params["callback"], dict): log_wrapper.add_target( target=params["callback"]["func"], @@ -186,47 +149,56 @@ def extract_info(config: dict, url: str, debug: bool = False) -> dict: def merge_dict(source: dict, destination: dict) -> dict: - """Merge data from source into destination""" + """ + Merge data from source into destination safely. + + Args: + source (dict): Source data + destination (dict): Destination data + + Returns: + dict: The merged dictionary + + """ + if not isinstance(source, dict) or not isinstance(destination, dict): + msg = "Both source and destination must be dictionaries." + raise TypeError(msg) + destination_copy = copy.deepcopy(destination) for key, value in source.items(): - destination_key_value = destination_copy.get(key) - if isinstance(value, dict) and isinstance(destination_key_value, dict): - destination_copy[key] = merge_dict(source=value, destination=destination_copy.setdefault(key, {})) - elif isinstance(value, list) and isinstance(destination_key_value, list): - destination_copy[key] = destination_key_value + value + if key in {"__class__", "__dict__", "__globals__", "__builtins__"}: + continue + + destination_value = destination_copy.get(key) + + # Recursively merge dictionaries + if isinstance(value, dict) and isinstance(destination_value, dict): + destination_copy[key] = merge_dict(value, destination_value) + + # Safely extend lists without reference issues + elif isinstance(value, list) and isinstance(destination_value, list): + destination_copy[key] = copy.deepcopy(destination_value) + copy.deepcopy(value) + else: - destination_copy[key] = value + destination_copy[key] = copy.deepcopy(value) return destination_copy -def merge_config(config: dict, new_config: dict) -> dict: +def is_downloaded(archive_file: str, url: str) -> tuple[bool, dict[str | None, str | None, str | None]]: """ - Merge user provided config into default config + Check if the video is already downloaded. Args: - config (dict): Default config - new_config (dict): User provided config + archive_file (str): Archive file path. + url (str): URL to check. Returns: - dict: Merged config + bool: True if the video is already downloaded. + dict: Video information. """ - for key in IGNORED_KEYS: - if key in new_config: - LOG.error(f"Key '{key}' is not allowed to be manually set via config.") - del new_config[key] - - conf = merge_dict(new_config, config) - - if "impersonate" in conf: - conf["impersonate"] = ImpersonateTarget.from_str(conf["impersonate"]) - - return conf - - -def is_downloaded(archive_file: str, url: str) -> tuple[bool, dict[str | None, str | None, str | None]]: global YTDLP_INFO_CLS # noqa: PLW0603 idDict = { @@ -301,11 +273,7 @@ def load_file(file: str, check_type=None) -> tuple[dict | list, bool, str]: if check_type: assert isinstance(opts, check_type) # noqa: S101 - return ( - opts, - True, - "", - ) + return (opts, True, "") except Exception: with open(file) as json_data: from pyjson5 import load as json5_load @@ -316,23 +284,11 @@ def load_file(file: str, check_type=None) -> tuple[dict | list, bool, str]: if check_type: assert isinstance(opts, check_type) # noqa: S101 - return ( - opts, - True, - "", - ) + return (opts, True, "") except AssertionError: - return ( - {}, - False, - f"Failed to assert that the contents '{type(opts)}' are of type '{check_type}'.", - ) + return ({}, False, f"Failed to assert that the contents '{type(opts)}' are of type '{check_type}'.") except Exception as e: - return ( - {}, - False, - f"{e}", - ) + return ({}, False, f"{e}") def check_id(file: pathlib.Path) -> bool | str: @@ -340,10 +296,12 @@ def check_id(file: pathlib.Path) -> bool | str: Check if we are able to get an id from the file name. if so check if any video file with the same id exists. - :param basePath: Base path to strip. - :param file: File to check. + Args: + file (pathlib.Path): File to check. + + Returns: + bool|str: False if no file found, else the file path. - :return: False if no id found, otherwise the id. """ match = re.search(r"(?<=\[)(?:youtube-)?(?P[a-zA-Z0-9\-_]{11})(?=\])", file.stem, re.IGNORECASE) if not match: @@ -371,14 +329,18 @@ def ag(array: dict | list, path: list[str | int] | str | int, default: Any = Non """ dict/array getter: Retrieve a value from a nested dict or object using a path. - :param array_or_object: dict-like or object from which to retrieve values - :param path: string, list, or None. Represents the path to retrieve: - - If None or empty string, returns the entire structure. - - If list, tries each path and returns the first found. - - If string, navigates through nested dict keys separated by `separator`. - :param default: Value (or callable) returned if nothing is found. - :param separator: Separator for nested paths in strings. - :return: The found value or the default if not found. + Args: + array (dict|list): dict-like or object from which to retrieve values. + path (list|str|int): Represents the path to retrieve: + - If None or empty string, returns the entire structure. + - If list, tries each path and returns the first found. + - If string, navigates through nested dict keys separated by `separator`. + default (Any): Value (or callable) returned if nothing is found. + separator (str): Separator for nested paths in strings. + + Returns: + Any: The found value or the default if not found. + """ if path is None or path == "": return array @@ -523,8 +485,12 @@ def get_sidecar_subtitles(file: pathlib.Path) -> list[dict]: """ Get sidecar files for the given file. - :param file: File to get sidecar files for. - :return: List of sidecar files. + Args: + file (pathlib.Path): The video file. + + Returns: + list: List of sidecar files. + """ files = [] diff --git a/app/library/YTDLPOpts.py b/app/library/YTDLPOpts.py new file mode 100644 index 00000000..8eddf274 --- /dev/null +++ b/app/library/YTDLPOpts.py @@ -0,0 +1,102 @@ +from pathlib import Path + +from .config import Config +from .Presets import Presets +from .Singleton import Singleton +from .Utils import IGNORED_KEYS, calc_download_path, merge_dict + + +class YTDLPOpts(metaclass=Singleton): + item_opts: dict = {} + preset_opts: dict = {} + + _instance = None + """The instance of the class.""" + + def __init__(self): + self._config = Config.get_instance() + + @staticmethod + def get_instance() -> "YTDLPOpts": + """ + Get the instance of the class. + + Returns: + Presets: The instance of the class + + """ + if not YTDLPOpts._instance: + YTDLPOpts._instance = YTDLPOpts() + + return YTDLPOpts._instance + + def add(self, config: dict, from_user: bool = False): + for key, value in config.items(): + if key in IGNORED_KEYS and from_user: + continue + self.item_opts[key] = value + + return self + + def preset(self, name: str, with_cookies: bool = False) -> "YTDLPOpts": + preset = Presets.get_instance().get(name=name) + if not preset or "default" == name: + return self + + if preset.cookies and with_cookies: + file = Path(self._config.config_path, "cookies", f"{preset.id}.txt") + + if not file.parent.exists(): + file.parent.mkdir(parents=True) + + with open(file, "w") as f: + f.write(preset.cookies) + + self.preset_opts["cookiefile"] = str(file) + + if preset.format: + self.preset_opts["format"] = preset.format + + if preset.template: + self.preset_opts["outtmpl"] = {"default": preset.template, "chapter": self._config.output_template_chapter} + + if preset.folder: + self.preset_opts["paths"] = { + "home": calc_download_path(base_path=self._config.download_path, folder=preset.folder), + "temp": self._config.temp_path, + } + + if preset.postprocessors and isinstance(preset.postprocessors, list) and len(preset.postprocessors) > 0: + self.preset_opts["postprocessors"] = preset.postprocessors + + if preset.args and isinstance(preset.args, dict) and len(preset.args) > 0: + for key, value in preset.args.items(): + if key in IGNORED_KEYS: + continue + self.preset_opts[key] = value + + return self + + def get_all(self, keep: bool = False) -> dict: + default_opts = self._config.ytdl_options + default_opts["paths"] = {"home": self._config.download_path, "temp": self._config.temp_path} + default_opts["outtmpl"] = { + "default": self._config.output_template, + "chapter": self._config.output_template_chapter, + } + + if "format" in default_opts or "format" in self.item_opts: + return merge_dict(default_opts, self.item_opts) + + data = merge_dict(merge_dict(self.preset_opts, default_opts), self.item_opts) + + if not keep: + self.presets_opts = {} + self.item_opts = {} + + if "impersonate" in data: + from yt_dlp.networking.impersonate import ImpersonateTarget + + data["impersonate"] = ImpersonateTarget.from_str(data["impersonate"]) + + return data diff --git a/app/library/presets.json b/app/library/presets.json index dcc33cc3..52e0eae7 100644 --- a/app/library/presets.json +++ b/app/library/presets.json @@ -3,12 +3,18 @@ "id": "3e163c6c-64eb-4448-924f-814b629b3810", "name": "default", "format": "default", + "folder": "", + "template": "", + "cookies": "", "default": true }, { "id": "5bf9c42b-8852-468a-99f5-915622dfba25", "name": "Best video and audio", "format": "bv+ba/b", + "folder": "", + "template": "", + "cookies": "", "default": true }, { @@ -20,6 +26,9 @@ "vcodec:h264" ] }, + "folder": "", + "template": "", + "cookies": "", "default": true }, { @@ -31,6 +40,9 @@ "vcodec:h264" ] }, + "folder": "", + "template": "", + "cookies": "", "default": true }, { @@ -63,6 +75,9 @@ "when": "playlist" } ], + "folder": "", + "template": "", + "cookies": "", "default": true } ] diff --git a/ui/assets/css/style.css b/ui/assets/css/style.css index 83dfe4ea..f166f27f 100644 --- a/ui/assets/css/style.css +++ b/ui/assets/css/style.css @@ -221,7 +221,8 @@ hr { padding-top: 0.5em; } -.play-overlay, .is-pointer { +.play-overlay, +.is-pointer { cursor: pointer; } @@ -254,3 +255,7 @@ hr { object-fit: cover; object-position: top; } + +.is-pre { + white-space: pre; +} diff --git a/ui/components/NewDownload.vue b/ui/components/NewDownload.vue index 8cf879ee..18dcf896 100644 --- a/ui/components/NewDownload.vue +++ b/ui/components/NewDownload.vue @@ -107,8 +107,8 @@ Cookies
- + +