From be0f6f2d2d1a0eee185870f225422c7d66967b99 Mon Sep 17 00:00:00 2001 From: Jesse Bannon Date: Tue, 9 Jan 2024 14:35:19 -0800 Subject: [PATCH 1/7] [BUGFIX] Parent entry is not a dict (#888) Fixes an AttributeError bug --- src/ytdl_sub/downloaders/ytdlp.py | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/src/ytdl_sub/downloaders/ytdlp.py b/src/ytdl_sub/downloaders/ytdlp.py index 0f43b606..8fa7f10e 100644 --- a/src/ytdl_sub/downloaders/ytdlp.py +++ b/src/ytdl_sub/downloaders/ytdlp.py @@ -221,6 +221,7 @@ class YTDLP: with cls._listen_and_log_downloaded_info_json( working_directory=working_directory, log_prefix=log_prefix_on_info_json_dl ): + # TODO: Fetch parent_dict if it breaks parent_dict = cls.extract_info( ytdl_options_overrides=ytdl_options_overrides, **kwargs ) @@ -239,17 +240,21 @@ class YTDLP: # For YouTube playlists in particular, channel metadata is not fetched. Attempt to get # channel metadata via grabbing uploader_url info json a max of 3 times - current_iter = 0 - url = kwargs.get("url") - uploader_url = parent_dict.get("uploader_url") - while current_iter < 3 and uploader_url and url != uploader_url: - cls.logger.debug("Attempting to get parent metadata from URL %s", uploader_url) - parent_dict = cls.extract_info( - ytdl_options_overrides=ytdl_options_overrides | {"playlist_items": "0:0"}, - url=uploader_url, - ) - current_iter += 1 - url = uploader_url + if isinstance(parent_dict, dict): + current_iter = 0 + url = kwargs.get("url") uploader_url = parent_dict.get("uploader_url") + while current_iter < 3 and uploader_url and url != uploader_url: + cls.logger.debug("Attempting to get parent metadata from URL %s", uploader_url) + parent_dict = cls.extract_info( + ytdl_options_overrides=ytdl_options_overrides | {"playlist_items": "0:0"}, + url=uploader_url, + ) + + if not isinstance(parent_dict, dict): + break + current_iter += 1 + url = uploader_url + uploader_url = parent_dict.get("uploader_url") return cls._get_entry_dicts_from_info_json_files(working_directory=working_directory) From 3b36ffd05051590064f487d8503f3adf95f5323a Mon Sep 17 00:00:00 2001 From: Jesse Bannon Date: Tue, 9 Jan 2024 19:25:09 -0800 Subject: [PATCH 2/7] [BUGFIX] Fix source metadata getting fetched when yt-dlp breaks (#889) Prior PR only got channel metadata for a playlist if it never 'broke' (i.e. MaxDownloads, BreakOnExisting). Will now fetch it regardless. --- src/ytdl_sub/downloaders/ytdlp.py | 45 +++++++++++-------- .../youtube/test_channel_full.json | 2 +- 2 files changed, 27 insertions(+), 20 deletions(-) diff --git a/src/ytdl_sub/downloaders/ytdlp.py b/src/ytdl_sub/downloaders/ytdlp.py index 8fa7f10e..970acb1f 100644 --- a/src/ytdl_sub/downloaders/ytdlp.py +++ b/src/ytdl_sub/downloaders/ytdlp.py @@ -216,15 +216,11 @@ class YTDLP: **kwargs arguments passed directory to YoutubeDL extract_info """ - parent_dict: Dict = {} try: with cls._listen_and_log_downloaded_info_json( working_directory=working_directory, log_prefix=log_prefix_on_info_json_dl ): - # TODO: Fetch parent_dict if it breaks - parent_dict = cls.extract_info( - ytdl_options_overrides=ytdl_options_overrides, **kwargs - ) + cls.extract_info(ytdl_options_overrides=ytdl_options_overrides, **kwargs) except RejectedVideoReached: cls.logger.debug( "RejectedVideoReached, stopping additional downloads " @@ -238,23 +234,34 @@ class YTDLP: except MaxDownloadsReached: cls.logger.info("MaxDownloadsReached, stopping additional downloads.") - # For YouTube playlists in particular, channel metadata is not fetched. Attempt to get - # channel metadata via grabbing uploader_url info json a max of 3 times - if isinstance(parent_dict, dict): - current_iter = 0 - url = kwargs.get("url") - uploader_url = parent_dict.get("uploader_url") - while current_iter < 3 and uploader_url and url != uploader_url: - cls.logger.debug("Attempting to get parent metadata from URL %s", uploader_url) + parent_dicts: List[Dict] = [] + entry_dicts = cls._get_entry_dicts_from_info_json_files(working_directory=working_directory) + entry_ids = {entry_dict.get("id") for entry_dict in entry_dicts} + + # Try to get additional uploader (source) metadata that yt-dlp does not fetch + # in a single request + for entry_dict in entry_dicts: + if not (uploader_id := entry_dict.get("uploader_id")): + continue + + if uploader_id in entry_ids or not (uploader_url := entry_dict.get("uploader_url")): + continue + + cls.logger.debug("Attempting to get parent metadata from URL %s", uploader_url) + try: parent_dict = cls.extract_info( ytdl_options_overrides=ytdl_options_overrides | {"playlist_items": "0:0"}, url=uploader_url, ) + except Exception: # pylint: disable=broad-except + # Do not try this uploader_id again + entry_ids.add(uploader_id) + break - if not isinstance(parent_dict, dict): - break - current_iter += 1 - url = uploader_url - uploader_url = parent_dict.get("uploader_url") + if isinstance(parent_dict, dict): + parent_id = parent_dict.get("id") + parent_dicts.append(parent_dict) + entry_ids |= {uploader_id, parent_id} + cls.logger.debug("Adding parent metadata with ids [%s, %s]", uploader_id, parent_id) - return cls._get_entry_dicts_from_info_json_files(working_directory=working_directory) + return entry_dicts + parent_dicts diff --git a/tests/resources/expected_downloads_summaries/youtube/test_channel_full.json b/tests/resources/expected_downloads_summaries/youtube/test_channel_full.json index 74e55e90..38361867 100644 --- a/tests/resources/expected_downloads_summaries/youtube/test_channel_full.json +++ b/tests/resources/expected_downloads_summaries/youtube/test_channel_full.json @@ -30,7 +30,7 @@ "Project ⧸ Zombie/Season 2011/s2011.e063001 - Project Zombie ｜Fin｜.nfo": "54ea4a48116aa98480a79495036c25e9", "Project ⧸ Zombie/Season 2011/s2011.e112101 - Skyrim 'Ultra HD w⧸Mods' [PC]-thumb.jpg": "1718599d5189c65f7d8cf6acfa5ea851", "Project ⧸ Zombie/Season 2011/s2011.e112101 - Skyrim 'Ultra HD w⧸Mods' [PC].info.json": "INFO_JSON", - "Project ⧸ Zombie/Season 2011/s2011.e112101 - Skyrim 'Ultra HD w⧸Mods' [PC].mp4": "2a18cc4baa198edecdc5154082f73eff", + "Project ⧸ Zombie/Season 2011/s2011.e112101 - Skyrim 'Ultra HD w⧸Mods' [PC].mp4": "554d46311112228c22ca9532007803cd", "Project ⧸ Zombie/Season 2011/s2011.e112101 - Skyrim 'Ultra HD w⧸Mods' [PC].nfo": "04f6aad56d85b1f65b81b6b8000f6479", "Project ⧸ Zombie/Season 2012/s2012.e012301 - Project Zombie ｜Map Trailer｜-thumb.jpg": "54ebe9df801b278fdd17b21afa8373a6", "Project ⧸ Zombie/Season 2012/s2012.e012301 - Project Zombie ｜Map Trailer｜.info.json": "INFO_JSON", From 305d084a6d60064af147f299168938591a062ff8 Mon Sep 17 00:00:00 2001 From: Jesse Bannon Date: Tue, 9 Jan 2024 20:50:31 -0800 Subject: [PATCH 3/7] [BUGFIX] Only apply --dl-override to subscriptions in --match (#890) As title, to not accidently add breaking changes for subscriptions that are not going to run --- src/ytdl_sub/cli/entrypoint.py | 7 +----- src/ytdl_sub/subscriptions/subscription.py | 26 +++++++++++++++++----- 2 files changed, 21 insertions(+), 12 deletions(-) diff --git a/src/ytdl_sub/cli/entrypoint.py b/src/ytdl_sub/cli/entrypoint.py index 1d6a19af..86a98e57 100644 --- a/src/ytdl_sub/cli/entrypoint.py +++ b/src/ytdl_sub/cli/entrypoint.py @@ -107,15 +107,10 @@ def _download_subscriptions_from_yaml_files( subscriptions += Subscription.from_file_path( config=config, subscription_path=path, + subscription_matches=subscription_matches, subscription_override_dict=subscription_override_dict, ) - if subscriptions and subscription_matches: - logger.info("Filtering subscriptions by name based on --match arguments") - subscriptions = [ - sub for sub in subscriptions if any(match in sub.name for match in subscription_matches) - ] - for subscription in subscriptions: with subscription.exception_handling(): logger.info( diff --git a/src/ytdl_sub/subscriptions/subscription.py b/src/ytdl_sub/subscriptions/subscription.py index 37d0168f..e40c2663 100644 --- a/src/ytdl_sub/subscriptions/subscription.py +++ b/src/ytdl_sub/subscriptions/subscription.py @@ -75,6 +75,7 @@ class Subscription(SubscriptionDownload): cls, config: ConfigFile, subscription_path: str | Path, + subscription_matches: Optional[List[str]] = None, subscription_override_dict: Optional[Dict] = None, ) -> List["Subscription"]: """ @@ -86,6 +87,8 @@ class Subscription(SubscriptionDownload): Validated instance of the config subscription_path: File path to the subscription yaml file + subscription_matches: + Optional list, only output subscriptions that match one or more of these values subscription_override_dict: Optional dict containing overrides to every subscription @@ -99,16 +102,16 @@ class Subscription(SubscriptionDownload): If subscription file is misconfigured """ subscriptions: List["Subscription"] = [] - subscription_dict = load_yaml(file_path=subscription_path) + subscription_object = load_yaml(file_path=subscription_path) - has_file_preset = FILE_PRESET_APPLY_KEY in subscription_dict + has_file_preset = FILE_PRESET_APPLY_KEY in subscription_object # If a file preset is present... if has_file_preset: # Validate it (make sure it is a dict) file_preset = LiteralDictValidator( name=f"{subscription_path}.{FILE_PRESET_APPLY_KEY}", - value=subscription_dict[FILE_PRESET_APPLY_KEY], + value=subscription_object[FILE_PRESET_APPLY_KEY], ) # Deep copy the config and add this file preset to its preset list @@ -116,7 +119,9 @@ class Subscription(SubscriptionDownload): config.presets.dict[FILE_PRESET_APPLY_KEY] = file_preset.dict subscriptions_dict: Dict[str, Any] = { - key: obj for key, obj in subscription_dict.items() if key not in [FILE_PRESET_APPLY_KEY] + key: obj + for key, obj in subscription_object.items() + if key not in [FILE_PRESET_APPLY_KEY] } subscriptions_dicts = SubscriptionValidator( @@ -129,7 +134,16 @@ class Subscription(SubscriptionDownload): global_presets_to_apply=[FILE_PRESET_APPLY_KEY] if has_file_preset else [] ) - for subscription_key, subscription_object in subscriptions_dicts.items(): + if subscriptions_dicts and subscription_matches: + logger.info("Filtering subscriptions by name based on --match arguments") + subscriptions_dicts = { + subscription_name: subscription_object + for subscription_name, subscription_object in subscriptions_dicts.items() + if any(match in subscription_name for match in subscription_matches) + } + + for subscription_name, subscription_object in subscriptions_dicts.items(): + # Hard-override subscriptions here mergedeep.merge( subscription_object, @@ -140,7 +154,7 @@ class Subscription(SubscriptionDownload): subscriptions.append( cls.from_dict( config=config, - preset_name=subscription_key, + preset_name=subscription_name, preset_dict=subscription_object, ) ) From b9d05c5b8f8ffb0907185a77e7ac2520a3c28ece Mon Sep 17 00:00:00 2001 From: Jesse Bannon Date: Wed, 10 Jan 2024 09:31:37 -0800 Subject: [PATCH 4/7] [BUGFIX] keep_max_files=0 downloading 2 (#893) Fixes https://github.com/jmbannon/ytdl-sub/issues/892 Does not set max_downloads if keep_max_files = 0 --- src/ytdl_sub/subscriptions/subscription_ytdl_options.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/ytdl_sub/subscriptions/subscription_ytdl_options.py b/src/ytdl_sub/subscriptions/subscription_ytdl_options.py index 77ec1c6b..c407cd8a 100644 --- a/src/ytdl_sub/subscriptions/subscription_ytdl_options.py +++ b/src/ytdl_sub/subscriptions/subscription_ytdl_options.py @@ -95,10 +95,12 @@ class SubscriptionYTDLOptions: if self._preset.output_options.maintain_download_archive: ytdl_options["download_archive"] = self._enhanced_download_archive.working_file_path if self._preset.output_options.keep_max_files: - # yt-dlp has a weird bug with max_downloads=1, set to 2 for safe measure - ytdl_options["max_downloads"] = max( - int(self._overrides.apply_formatter(self._preset.output_options.keep_max_files)), 2 + keep_max_files = int( + self._overrides.apply_formatter(self._preset.output_options.keep_max_files) ) + if keep_max_files > 0: + # yt-dlp has a weird bug with max_downloads=1, set to 2 for safe measure + ytdl_options["max_downloads"] = max(keep_max_files, 2) return ytdl_options From 00e4cd8541a40dd6dd7b0e9a7386def656c6a6ec Mon Sep 17 00:00:00 2001 From: Jesse Bannon Date: Wed, 10 Jan 2024 09:33:03 -0800 Subject: [PATCH 5/7] [BUGFIX] Fix rare case when yt-dlp returns None and is downloaded (#894) --- src/ytdl_sub/downloaders/ytdlp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ytdl_sub/downloaders/ytdlp.py b/src/ytdl_sub/downloaders/ytdlp.py index 970acb1f..556ef54f 100644 --- a/src/ytdl_sub/downloaders/ytdlp.py +++ b/src/ytdl_sub/downloaders/ytdlp.py @@ -102,7 +102,7 @@ class YTDLP: ) if is_downloaded and is_thumbnail_downloaded: - return entry_dict + return entry_dict or {} # in-case yt-dlp returns None # Always add check_formats # See https://github.com/yt-dlp/yt-dlp/issues/502 From 2076de307463b9c659dab36c6e61b3531f9815c6 Mon Sep 17 00:00:00 2001 From: Jesse Bannon Date: Wed, 10 Jan 2024 18:16:12 -0800 Subject: [PATCH 6/7] [BUGFIX] Smarter subscription validation (#895) With the right functions, it was possible for subscription validation to raise a false-positive error. This should hopefully resolve that issue and give validation a small performance increase --- .../config/validators/variable_validation.py | 38 +++++++----- src/ytdl_sub/script/script.py | 53 ++++++++++++++++ src/ytdl_sub/utils/scriptable.py | 8 +-- .../validators/string_formatter_validators.py | 62 ++++++++++++++++--- tests/unit/config/test_preset.py | 28 +++++++-- 5 files changed, 157 insertions(+), 32 deletions(-) diff --git a/src/ytdl_sub/config/validators/variable_validation.py b/src/ytdl_sub/config/validators/variable_validation.py index cc554a70..f5de3ed5 100644 --- a/src/ytdl_sub/config/validators/variable_validation.py +++ b/src/ytdl_sub/config/validators/variable_validation.py @@ -13,9 +13,10 @@ from ytdl_sub.config.plugin.preset_plugins import PresetPlugins from ytdl_sub.config.preset_options import OutputOptions from ytdl_sub.config.validators.options import OptionsValidator from ytdl_sub.downloaders.url.validators import MultiUrlValidator -from ytdl_sub.entries.script.variable_definitions import VARIABLE_SCRIPTS from ytdl_sub.entries.variables.override_variables import SubscriptionVariables from ytdl_sub.script.script import Script +from ytdl_sub.utils.scriptable import BASE_SCRIPT +from ytdl_sub.validators.string_formatter_validators import to_variable_dependency_format_string from ytdl_sub.validators.string_formatter_validators import validate_formatters @@ -32,10 +33,10 @@ def _add_dummy_overrides(overrides: Overrides) -> Dict[str, str]: # Have the dummy override variable contain all variable deps that it uses in the string dummy_overrides: Dict[str, str] = {} for override_name in _override_variables(overrides): - dummy_overrides[override_name] = "" # pylint: disable=protected-access - for variable_dependency in overrides.script._variables[override_name].variables: - dummy_overrides[override_name] += f"{{ {variable_dependency.name } }}" + dummy_overrides[override_name] = to_variable_dependency_format_string( + script=overrides.script, parsed_format_string=overrides.script._variables[override_name] + ) # pylint: enable=protected-access return dummy_overrides @@ -72,8 +73,15 @@ def _override_variables(overrides: Overrides) -> Set[str]: } -def _entry_variables() -> Set[str]: - return set(list(VARIABLE_SCRIPTS.keys())) +_DUMMY_ENTRY_VARIABLES: Dict[str, str] = { + name: to_variable_dependency_format_string( + # pylint: disable=protected-access + script=BASE_SCRIPT, + parsed_format_string=BASE_SCRIPT._variables[name] + # pylint: enable=protected-access + ) + for name in BASE_SCRIPT.variable_names +} class VariableValidation: @@ -97,12 +105,12 @@ class VariableValidation: """ Do some gymnastics to initialize the Overrides script. """ - entry_variables = _entry_variables() override_variables = _override_variables(overrides) # Set resolved variables as all entry + override variables # at this point to generate every possible added/modified variable - self.resolved_variables = entry_variables | override_variables + self.resolved_variables = set(_DUMMY_ENTRY_VARIABLES.keys()) | override_variables + plugin_variables: Set[str] = set() for ( plugin_options, @@ -125,6 +133,7 @@ class VariableValidation: # Set unresolved as variables that are added but do not exist as # entry/override variables since they are created at run-time self.unresolved_variables |= added_variables | modified_variables + plugin_variables |= added_variables | modified_variables # Then update resolved variables to reflect that self.resolved_variables -= self.unresolved_variables @@ -137,10 +146,11 @@ class VariableValidation: ) # copy the script and mock entry variables - self.script = copy.deepcopy(overrides.script).add(_add_dummy_variables(entry_variables)) + self.script = copy.deepcopy(overrides.script) self.script.add( - variables=_add_dummy_overrides(overrides=overrides), - unresolvable=self.unresolved_variables, + variables=_add_dummy_overrides(overrides=overrides) + | _add_dummy_variables(variables=plugin_variables) + | _DUMMY_ENTRY_VARIABLES ) return self @@ -158,7 +168,6 @@ class VariableValidation: resolved_variables = added_variables | modified_variables - self.script.add(_add_dummy_variables(resolved_variables)) self.resolved_variables |= resolved_variables self.unresolved_variables -= resolved_variables @@ -177,13 +186,10 @@ class VariableValidation: ): self._add_variables(PluginOperation.MODIFY_ENTRY_METADATA, options=plugin_options) - self._update_script() for plugin_options in PluginMapping.order_options_by( self.plugins.zipped(), PluginOperation.MODIFY_ENTRY ): - added = self._add_variables(PluginOperation.MODIFY_ENTRY, options=plugin_options) - if added: - self._update_script() + self._add_variables(PluginOperation.MODIFY_ENTRY, options=plugin_options) # Validate that any formatter in the plugin options can resolve validate_formatters( diff --git a/src/ytdl_sub/script/script.py b/src/ytdl_sub/script/script.py index 0560984b..3f3b42ff 100644 --- a/src/ytdl_sub/script/script.py +++ b/src/ytdl_sub/script/script.py @@ -248,6 +248,52 @@ class Script: for variable_name, resolved in resolved_variables.items(): self._variables[variable_name] = SyntaxTree(ast=[resolved]) + def _recursive_get_unresolved_output_filter_variables( + self, current_var: SyntaxTree, subset_to_resolve: Set[str], unresolvable: Set[Variable] + ) -> Set[str]: + for var_dep in current_var.variables: + if var_dep in unresolvable: + raise ScriptVariableNotResolved( + f"Output filter variable contains the variable {var_dep} " + f"which is set as unresolvable" + ) + subset_to_resolve.add(var_dep.name) + subset_to_resolve |= self._recursive_get_unresolved_output_filter_variables( + current_var=self._variables[var_dep.name], + subset_to_resolve=subset_to_resolve, + unresolvable=unresolvable, + ) + + return subset_to_resolve + + def _get_unresolved_output_filter( + self, + unresolved: Dict[Variable, SyntaxTree], + output_filter: Set[str], + unresolvable: Set[Variable], + ) -> Dict[Variable, SyntaxTree]: + """ + When an output filter is applied, only a subset of variables that the filter + depends on need to be resolved. + """ + subset_to_resolve: Set[str] = set() + + for output_filter_variable in output_filter: + subset_to_resolve.add(output_filter_variable) + + if output_filter_variable not in self._variables: + raise ScriptVariableNotResolved( + "Tried to specify an output filter variable that does not exist" + ) + + subset_to_resolve |= self._recursive_get_unresolved_output_filter_variables( + current_var=self._variables[output_filter_variable], + subset_to_resolve=subset_to_resolve, + unresolvable=unresolvable, + ) + + return {var: syntax for var, syntax in unresolved.items() if var.name in subset_to_resolve} + def _resolve( self, pre_resolved: Optional[Dict[str, Resolvable]] = None, @@ -288,6 +334,13 @@ class Script: if Variable(name) not in unresolved_filter } + if output_filter: + unresolved = self._get_unresolved_output_filter( + unresolved=unresolved, + output_filter=output_filter, + unresolvable=unresolvable, + ) + while unresolved: unresolved_count: int = len(unresolved) diff --git a/src/ytdl_sub/utils/scriptable.py b/src/ytdl_sub/utils/scriptable.py index b1dc20e7..6aff1db3 100644 --- a/src/ytdl_sub/utils/scriptable.py +++ b/src/ytdl_sub/utils/scriptable.py @@ -14,10 +14,8 @@ from ytdl_sub.script.utils.exceptions import RuntimeException from ytdl_sub.utils.exceptions import StringFormattingException from ytdl_sub.utils.script import ScriptUtils -_BASE_SCRIPT: Script = Script( - ScriptUtils.add_sanitized_variables( - dict(copy.deepcopy(VARIABLE_SCRIPTS), **copy.deepcopy(CUSTOM_FUNCTION_SCRIPTS)) - ) +BASE_SCRIPT: Script = Script( + dict(ScriptUtils.add_sanitized_variables(VARIABLE_SCRIPTS), **CUSTOM_FUNCTION_SCRIPTS) ) @@ -37,7 +35,7 @@ class Scriptable(ABC): """ Initializes with base values """ - self._script = copy.deepcopy(_BASE_SCRIPT) + self._script = copy.deepcopy(BASE_SCRIPT) self._unresolvable = copy.deepcopy(UNRESOLVED_VARIABLES) @property diff --git a/src/ytdl_sub/validators/string_formatter_validators.py b/src/ytdl_sub/validators/string_formatter_validators.py index 7957169d..def6e6d8 100644 --- a/src/ytdl_sub/validators/string_formatter_validators.py +++ b/src/ytdl_sub/validators/string_formatter_validators.py @@ -6,8 +6,10 @@ from typing import final from ytdl_sub.entries.script.variable_definitions import VARIABLES from ytdl_sub.script.parser import parse from ytdl_sub.script.script import Script +from ytdl_sub.script.types.syntax_tree import SyntaxTree +from ytdl_sub.script.utils.exceptions import RuntimeException +from ytdl_sub.script.utils.exceptions import ScriptVariableNotResolved from ytdl_sub.script.utils.exceptions import UserException -from ytdl_sub.script.utils.exceptions import VariableDoesNotExist from ytdl_sub.utils.exceptions import StringFormattingVariableNotFoundException from ytdl_sub.validators.validators import DictValidator from ytdl_sub.validators.validators import ListValidator @@ -142,21 +144,67 @@ class OverridesDictFormatterValidator(DictFormatterValidator): _key_validator = OverridesStringFormatterValidator +def to_variable_dependency_format_string(script: Script, parsed_format_string: SyntaxTree) -> str: + """ + Create a dummy format string that contains all variable deps as a string. + """ + dummy_format_string = "" + for var in parsed_format_string.variables: + dummy_format_string += f"{{ {var.name} }}" + # pylint: disable=protected-access + for variable_dependency in script._variables[var.name].variables: + dummy_format_string += f"{{ {variable_dependency.name} }}" + # pylint: enable=protected-access + return dummy_format_string + + def _validate_formatter( mock_script: Script, unresolved_variables: Set[str], formatter_validator: Union[StringFormatterValidator, OverridesStringFormatterValidator], ) -> None: - try: - unresolvable = unresolved_variables - if isinstance(formatter_validator, OverridesStringFormatterValidator): - unresolvable = unresolved_variables.union({VARIABLES.entry_metadata.variable_name}) + is_static_formatter = False + unresolvable = unresolved_variables + if isinstance(formatter_validator, OverridesStringFormatterValidator): + is_static_formatter = True + unresolvable = unresolved_variables.union({VARIABLES.entry_metadata.variable_name}) + parsed = parse( + text=formatter_validator.format_string, + ) + variable_names = {var.name for var in parsed.variables} + custom_function_names = {f"%{func.name}" for func in parsed.custom_functions} + + if not variable_names.issubset(mock_script.variable_names): + raise StringFormattingVariableNotFoundException( + "contains the following variables that do not exist: " + f"{', '.join(sorted(variable_names - mock_script.variable_names))}" + ) + if not custom_function_names.issubset(mock_script.function_names): + raise StringFormattingVariableNotFoundException( + "contains the following custom functions that do not exist: " + f"{', '.join(sorted(custom_function_names - mock_script.function_names))}" + ) + if unresolved := variable_names.intersection(unresolvable): + raise StringFormattingVariableNotFoundException( + "contains the following variables that are unresolved when executing this " + f"formatter: {', '.join(sorted(unresolved))}" + ) + try: mock_script.resolve_once( - {"tmp_var": formatter_validator.format_string}, + { + "tmp_var": to_variable_dependency_format_string( + script=mock_script, parsed_format_string=parsed + ) + }, unresolvable=unresolvable, ) - except VariableDoesNotExist as exc: + except RuntimeException as exc: + if isinstance(exc, ScriptVariableNotResolved) and is_static_formatter: + raise StringFormattingVariableNotFoundException( + "static formatters must contain variables that have no dependency to " + "entry variables" + ) from exc raise StringFormattingVariableNotFoundException(exc) from exc diff --git a/tests/unit/config/test_preset.py b/tests/unit/config/test_preset.py index dff51a50..8afedd35 100644 --- a/tests/unit/config/test_preset.py +++ b/tests/unit/config/test_preset.py @@ -129,7 +129,7 @@ class TestPreset: ): with pytest.raises( StringFormattingVariableNotFoundException, - match="Variable dne_var does not exist.", + match="contains the following variables that do not exist: dne_var", ): _ = Preset( config=config_file, @@ -145,7 +145,7 @@ class TestPreset: ): with pytest.raises( StringFormattingVariableNotFoundException, - match="Variable dne_var does not exist", + match="contains the following variables that do not exist: dne_var", ): _ = Preset( config=config_file, @@ -161,7 +161,7 @@ class TestPreset: ): with pytest.raises( StringFormattingVariableNotFoundException, - match="Variable dne_var does not exist", + match="contains the following variables that do not exist: dne_var", ): _ = Preset( config=config_file, @@ -182,7 +182,7 @@ class TestPreset: ): with pytest.raises( StringFormattingVariableNotFoundException, - match="Variable dne_var does not exist", + match="contains the following variables that do not exist: dne_var", ): _ = Preset( config=config_file, @@ -198,6 +198,26 @@ class TestPreset: }, ) + def test_preset_error__dict_override_variable_not_static( + self, config_file, output_options, youtube_video + ): + with pytest.raises( + StringFormattingVariableNotFoundException, + match="static formatters must contain variables that " + "have no dependency to entry variables", + ): + _ = Preset( + config=config_file, + name="test", + value={ + "download": youtube_video, + "output_options": { + "output_directory": "{title}", + "file_name": "{uid}", + }, + }, + ) + def test_preset_with_multi_url__contains_empty_url(self, config_file, output_options): _ = Preset( config=config_file, From e3158583cabe7f0c41dcd3b635af71eee86fedfd Mon Sep 17 00:00:00 2001 From: Jesse Bannon Date: Thu, 11 Jan 2024 00:40:07 -0800 Subject: [PATCH 7/7] [FEATURE] Toggleable plugin field `enable` for all dict-based plugins (#897) For key/value-based plugins, add an `enable` field to make it easier for child presets to disable their functionality --- docs/source/config_reference/plugins.rst | 81 +++++++++++++++++++ src/ytdl_sub/config/plugin/plugin.py | 14 ++++ src/ytdl_sub/config/validators/options.py | 26 ++++++ src/ytdl_sub/plugins/audio_extract.py | 6 +- src/ytdl_sub/plugins/chapters.py | 5 +- src/ytdl_sub/plugins/date_range.py | 6 +- src/ytdl_sub/plugins/file_convert.py | 6 +- src/ytdl_sub/plugins/filter_exclude.py | 6 +- src/ytdl_sub/plugins/filter_include.py | 6 +- src/ytdl_sub/plugins/match_filters.py | 6 +- src/ytdl_sub/plugins/nfo_tags.py | 6 +- .../plugins/output_directory_nfo_tags.py | 2 +- src/ytdl_sub/plugins/regex.py | 6 +- src/ytdl_sub/plugins/subtitles.py | 5 +- src/ytdl_sub/plugins/throttle_protection.py | 5 +- src/ytdl_sub/plugins/video_tags.py | 6 +- .../subscriptions/subscription_download.py | 3 +- src/ytdl_sub/utils/script.py | 12 +++ .../validators/string_formatter_validators.py | 6 +- tests/conftest.py | 2 +- .../unit/plugins/test_throttle_protection.py | 57 +++++++++++++ tests/unit/utils/test_script_utils.py | 20 +++++ 22 files changed, 257 insertions(+), 35 deletions(-) diff --git a/docs/source/config_reference/plugins.rst b/docs/source/config_reference/plugins.rst index 69512e8f..d8535352 100644 --- a/docs/source/config_reference/plugins.rst +++ b/docs/source/config_reference/plugins.rst @@ -22,6 +22,15 @@ Extracts audio from a video file. opus, vorbis, wav, and best to grab the best possible format at runtime. +``enable`` + +:expected type: Optional[OverridesFormatter] +:description: + Can typically be left undefined to always default to enable. For preset convenience, + this field can be set using an override variable to easily toggle whether this plugin + is enabled or not via Boolean. + + ``quality`` :expected type: Float @@ -76,6 +85,15 @@ chapters and remove specific ones. Can also remove chapters using regex. Defaults to True. Embed chapters into the file. +``enable`` + +:expected type: Optional[OverridesFormatter] +:description: + Can typically be left undefined to always default to enable. For preset convenience, + this field can be set using an override variable to easily toggle whether this plugin + is enabled or not via Boolean. + + ``force_key_frames`` :expected type: Optional[Boolean] @@ -148,6 +166,15 @@ granularity possible. Only download videos before this datetime. +``enable`` + +:expected type: Optional[OverridesFormatter] +:description: + Can typically be left undefined to always default to enable. For preset convenience, + this field can be set using an override variable to easily toggle whether this plugin + is enabled or not via Boolean. + + ---------------------------------------------------------------------------------------------------- download @@ -259,6 +286,15 @@ Also supports custom ffmpeg conversions: with ``ffmpeg_post_process_args``. Defaults to ``yt-dlp``. +``enable`` + +:expected type: Optional[OverridesFormatter] +:description: + Can typically be left undefined to always default to enable. For preset convenience, + this field can be set using an override variable to easily toggle whether this plugin + is enabled or not via Boolean. + + ``ffmpeg_post_process_args`` :expected type: Optional[OverridesFormatter] @@ -395,6 +431,15 @@ with a ``.nfo`` extension. You can add any values into the NFO. episode: "{upload_month}{upload_day_padded}" kodi_safe: False +``enable`` + +:expected type: Optional[OverridesFormatter] +:description: + Can typically be left undefined to always default to enable. For preset convenience, + this field can be set using an override variable to easily toggle whether this plugin + is enabled or not via Boolean. + + ``kodi_safe`` :expected type: Optional[Boolean] @@ -484,6 +529,15 @@ Usage: # optional kodi_safe: False +``enable`` + +:expected type: Optional[OverridesFormatter] +:description: + Can typically be left undefined to always default to enable. For preset convenience, + this field can be set using an override variable to easily toggle whether this plugin + is enabled or not via Boolean. + + ``kodi_safe`` :expected type: Optional[Boolean] @@ -800,6 +854,15 @@ and using ``title_and_description`` can regex match/exclude from either ``title` - "{upload_month}" - "{upload_day}" +``enable`` + +:expected type: Optional[OverridesFormatter] +:description: + Can typically be left undefined to always default to enable. For preset convenience, + this field can be set using an override variable to easily toggle whether this plugin + is enabled or not via Boolean. + + ``skip_if_match_fails`` :expected type: Optional[Boolean] @@ -880,6 +943,15 @@ It will set the respective language to the correct subtitle file. webm files can only embed "vtt" subtitle types. +``enable`` + +:expected type: Optional[OverridesFormatter] +:description: + Can typically be left undefined to always default to enable. For preset convenience, + this field can be set using an override variable to easily toggle whether this plugin + is enabled or not via Boolean. + + ``languages`` :expected type: Optional[List[String]] @@ -931,6 +1003,15 @@ scripted. max: 36 subscription_download_probability: 1.0 +``enable`` + +:expected type: Optional[OverridesFormatter] +:description: + Can typically be left undefined to always default to enable. For preset convenience, + this field can be set using an override variable to easily toggle whether this plugin + is enabled or not via Boolean. + + ``max_downloads_per_subscription`` :expected type: Optional[Range] diff --git a/src/ytdl_sub/config/plugin/plugin.py b/src/ytdl_sub/config/plugin/plugin.py index 24d616f6..33e98a1b 100644 --- a/src/ytdl_sub/config/plugin/plugin.py +++ b/src/ytdl_sub/config/plugin/plugin.py @@ -1,5 +1,6 @@ from abc import ABC from abc import abstractmethod +from functools import cached_property from typing import Dict from typing import Generic from typing import List @@ -8,9 +9,11 @@ from typing import Tuple from typing import Type from ytdl_sub.config.overrides import Overrides +from ytdl_sub.config.validators.options import ToggleableOptionsDictValidator from ytdl_sub.config.validators.options import TOptionsValidator from ytdl_sub.entries.entry import Entry from ytdl_sub.utils.file_handler import FileMetadata +from ytdl_sub.utils.script import ScriptUtils from ytdl_sub.ytdl_additions.enhanced_download_archive import DownloadArchiver from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive @@ -40,6 +43,17 @@ class Plugin(BasePlugin[TOptionsValidator], Generic[TOptionsValidator], ABC): Class to define the new plugin functionality """ + @cached_property + def is_enabled(self) -> bool: + """ + Returns True if enabled, False if disabled. + """ + if isinstance(self.plugin_options, ToggleableOptionsDictValidator): + return ScriptUtils.bool_formatter_output( + self.overrides.apply_formatter(self.plugin_options.enable) + ) + return True + def ytdl_options_match_filters(self) -> Tuple[List[str], List[str]]: """ Returns diff --git a/src/ytdl_sub/config/validators/options.py b/src/ytdl_sub/config/validators/options.py index c35defa2..324b39e1 100644 --- a/src/ytdl_sub/config/validators/options.py +++ b/src/ytdl_sub/config/validators/options.py @@ -6,6 +6,7 @@ from typing import TypeVar from ytdl_sub.config.plugin.plugin_operation import PluginOperation from ytdl_sub.utils.exceptions import ValidationException from ytdl_sub.validators.strict_dict_validator import StrictDictValidator +from ytdl_sub.validators.string_formatter_validators import OverridesBooleanFormatterValidator from ytdl_sub.validators.validators import Validator # pylint: disable=no-self-use @@ -57,3 +58,28 @@ TOptionsValidator = TypeVar("TOptionsValidator", bound=OptionsValidator) class OptionsDictValidator(StrictDictValidator, OptionsValidator, ABC): pass + + +class ToggleableOptionsDictValidator(OptionsDictValidator): + _optional_keys = {"enable"} + + def __init__(self, name, value): + assert ( + "enable" in self._optional_keys + ), f"{self.__class__.__name__} does not have enable as an optional field" + super().__init__(name, value) + + self._enable = self._validate_key( + key="enable", validator=OverridesBooleanFormatterValidator, default="True" + ) + + @property + def enable(self) -> OverridesBooleanFormatterValidator: + """ + :expected type: Optional[OverridesFormatter] + :description: + Can typically be left undefined to always default to enable. For preset convenience, + this field can be set using an override variable to easily toggle whether this plugin + is enabled or not via Boolean. + """ + return self._enable diff --git a/src/ytdl_sub/plugins/audio_extract.py b/src/ytdl_sub/plugins/audio_extract.py index 644fa1c1..3c5368a4 100644 --- a/src/ytdl_sub/plugins/audio_extract.py +++ b/src/ytdl_sub/plugins/audio_extract.py @@ -6,7 +6,7 @@ from typing import Set from ytdl_sub.config.plugin.plugin import Plugin from ytdl_sub.config.plugin.plugin_operation import PluginOperation -from ytdl_sub.config.validators.options import OptionsDictValidator +from ytdl_sub.config.validators.options import ToggleableOptionsDictValidator from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder from ytdl_sub.entries.entry import Entry from ytdl_sub.entries.script.variable_definitions import VARIABLES @@ -21,7 +21,7 @@ from ytdl_sub.validators.validators import FloatValidator v: VariableDefinitions = VARIABLES -class AudioExtractOptions(OptionsDictValidator): +class AudioExtractOptions(ToggleableOptionsDictValidator): """ Extracts audio from a video file. @@ -35,7 +35,7 @@ class AudioExtractOptions(OptionsDictValidator): """ _required_keys = {"codec"} - _optional_keys = {"quality"} + _optional_keys = {"enable", "quality"} @classmethod def partial_validate(cls, name: str, value: Any) -> None: diff --git a/src/ytdl_sub/plugins/chapters.py b/src/ytdl_sub/plugins/chapters.py index fbd11c09..198f91b5 100644 --- a/src/ytdl_sub/plugins/chapters.py +++ b/src/ytdl_sub/plugins/chapters.py @@ -7,7 +7,7 @@ from typing import Set from ytdl_sub.config.plugin.plugin import Plugin from ytdl_sub.config.plugin.plugin_operation import PluginOperation -from ytdl_sub.config.validators.options import OptionsDictValidator +from ytdl_sub.config.validators.options import ToggleableOptionsDictValidator from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder from ytdl_sub.entries.entry import Entry from ytdl_sub.entries.entry import ytdl_sub_chapters_from_comments @@ -59,7 +59,7 @@ class SponsorBlockCategoryListValidator(ListValidator[SponsorBlockCategoriesVali _inner_list_type = SponsorBlockCategoriesValidator -class ChaptersOptions(OptionsDictValidator): +class ChaptersOptions(ToggleableOptionsDictValidator): """ Embeds chapters to video files if they are present. Additional options to add SponsorBlock chapters and remove specific ones. Can also remove chapters using regex. @@ -90,6 +90,7 @@ class ChaptersOptions(OptionsDictValidator): """ _optional_keys = { + "enable", "embed_chapters", "allow_chapters_from_comments", "sponsorblock_categories", diff --git a/src/ytdl_sub/plugins/date_range.py b/src/ytdl_sub/plugins/date_range.py index 3948cb1e..ff788bff 100644 --- a/src/ytdl_sub/plugins/date_range.py +++ b/src/ytdl_sub/plugins/date_range.py @@ -3,12 +3,12 @@ from typing import Optional from typing import Tuple from ytdl_sub.config.plugin.plugin import Plugin -from ytdl_sub.config.validators.options import OptionsDictValidator +from ytdl_sub.config.validators.options import ToggleableOptionsDictValidator from ytdl_sub.utils.datetime import to_date_str from ytdl_sub.validators.string_datetime import StringDatetimeValidator -class DateRangeOptions(OptionsDictValidator): +class DateRangeOptions(ToggleableOptionsDictValidator): """ Only download files uploaded within the specified date range. Dates must adhere to a yt-dlp datetime. From their docs: @@ -31,7 +31,7 @@ class DateRangeOptions(OptionsDictValidator): after: "today-2weeks" """ - _optional_keys = {"before", "after"} + _optional_keys = {"enable", "before", "after"} def __init__(self, name, value): super().__init__(name, value) diff --git a/src/ytdl_sub/plugins/file_convert.py b/src/ytdl_sub/plugins/file_convert.py index 8fccaedb..00182d69 100644 --- a/src/ytdl_sub/plugins/file_convert.py +++ b/src/ytdl_sub/plugins/file_convert.py @@ -7,7 +7,7 @@ from typing import Set from ytdl_sub.config.overrides import Overrides from ytdl_sub.config.plugin.plugin import Plugin from ytdl_sub.config.plugin.plugin_operation import PluginOperation -from ytdl_sub.config.validators.options import OptionsDictValidator +from ytdl_sub.config.validators.options import ToggleableOptionsDictValidator from ytdl_sub.entries.entry import Entry from ytdl_sub.entries.script.variable_definitions import VARIABLES from ytdl_sub.entries.script.variable_definitions import VariableDefinitions @@ -28,7 +28,7 @@ class FileConvertWithValidator(StringSelectValidator): _select_values = {"yt-dlp", "ffmpeg"} -class FileConvertOptions(OptionsDictValidator): +class FileConvertOptions(ToggleableOptionsDictValidator): """ Converts video files from one extension to another. @@ -56,7 +56,7 @@ class FileConvertOptions(OptionsDictValidator): """ _required_keys = {"convert_to"} - _optional_keys = {"convert_with", "ffmpeg_post_process_args"} + _optional_keys = {"enable", "convert_with", "ffmpeg_post_process_args"} @classmethod def partial_validate(cls, name: str, value: Any) -> None: diff --git a/src/ytdl_sub/plugins/filter_exclude.py b/src/ytdl_sub/plugins/filter_exclude.py index 72ad6c88..08f13939 100644 --- a/src/ytdl_sub/plugins/filter_exclude.py +++ b/src/ytdl_sub/plugins/filter_exclude.py @@ -1,4 +1,3 @@ -import json from typing import Dict from typing import Optional @@ -8,6 +7,7 @@ from ytdl_sub.config.validators.options import OptionsValidator from ytdl_sub.entries.entry import Entry from ytdl_sub.utils.exceptions import StringFormattingException from ytdl_sub.utils.logger import Logger +from ytdl_sub.utils.script import ScriptUtils from ytdl_sub.validators.string_formatter_validators import ListFormatterValidator from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive @@ -53,7 +53,9 @@ class FilterExcludePlugin(Plugin[FilterExcludeOptions]): return entry for formatter in self.plugin_options.list: - out = json.loads(self.overrides.apply_formatter(formatter=formatter, entry=entry)) + out = ScriptUtils.bool_formatter_output( + self.overrides.apply_formatter(formatter=formatter, entry=entry) + ) if bool(out): logger.info( "Filtering '%s' from the filter %s evaluating to True", diff --git a/src/ytdl_sub/plugins/filter_include.py b/src/ytdl_sub/plugins/filter_include.py index 5a54e8ec..5f7fcc4a 100644 --- a/src/ytdl_sub/plugins/filter_include.py +++ b/src/ytdl_sub/plugins/filter_include.py @@ -1,4 +1,3 @@ -import json from typing import Dict from typing import Optional @@ -8,6 +7,7 @@ from ytdl_sub.config.validators.options import OptionsValidator from ytdl_sub.entries.entry import Entry from ytdl_sub.utils.exceptions import StringFormattingException from ytdl_sub.utils.logger import Logger +from ytdl_sub.utils.script import ScriptUtils from ytdl_sub.validators.string_formatter_validators import ListFormatterValidator from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive @@ -61,7 +61,9 @@ class FilterIncludePlugin(Plugin[FilterIncludeOptions]): return entry for formatter in self.plugin_options.list: - out = json.loads(self.overrides.apply_formatter(formatter=formatter, entry=entry)) + out = ScriptUtils.bool_formatter_output( + self.overrides.apply_formatter(formatter=formatter, entry=entry) + ) if not bool(out): logger.info( "Filtering '%s' from the filter %s evaluating to False", diff --git a/src/ytdl_sub/plugins/match_filters.py b/src/ytdl_sub/plugins/match_filters.py index 90211e2c..9dda20ce 100644 --- a/src/ytdl_sub/plugins/match_filters.py +++ b/src/ytdl_sub/plugins/match_filters.py @@ -4,7 +4,7 @@ from typing import List from typing import Tuple from ytdl_sub.config.plugin.plugin import Plugin -from ytdl_sub.config.validators.options import OptionsDictValidator +from ytdl_sub.config.validators.options import ToggleableOptionsDictValidator from ytdl_sub.utils.logger import Logger from ytdl_sub.validators.validators import StringListValidator @@ -56,7 +56,7 @@ def combine_filters(filters: List[str], to_combine: List[str]) -> List[str]: return output_filters -class MatchFiltersOptions(OptionsDictValidator): +class MatchFiltersOptions(ToggleableOptionsDictValidator): """ Set ``--match-filters`` to pass into yt-dlp to filter entries from being downloaded. Uses the same syntax as yt-dlp. An entry will be downloaded if any one of the filters are met. @@ -74,7 +74,7 @@ class MatchFiltersOptions(OptionsDictValidator): # - "availability=?public" """ - _optional_keys = {"filters"} + _optional_keys = {"enable", "filters"} @classmethod def partial_validate(cls, name: str, value: Any) -> None: diff --git a/src/ytdl_sub/plugins/nfo_tags.py b/src/ytdl_sub/plugins/nfo_tags.py index 728259e1..86fe6568 100644 --- a/src/ytdl_sub/plugins/nfo_tags.py +++ b/src/ytdl_sub/plugins/nfo_tags.py @@ -8,7 +8,7 @@ from typing import List from typing import Optional from ytdl_sub.config.plugin.plugin import Plugin -from ytdl_sub.config.validators.options import OptionsDictValidator +from ytdl_sub.config.validators.options import ToggleableOptionsDictValidator from ytdl_sub.entries.entry import Entry from ytdl_sub.utils.file_handler import FileHandler from ytdl_sub.utils.file_handler import FileMetadata @@ -23,13 +23,13 @@ from ytdl_sub.validators.string_formatter_validators import StringFormatterValid from ytdl_sub.validators.validators import BoolValidator -class SharedNfoTagsOptions(OptionsDictValidator): +class SharedNfoTagsOptions(ToggleableOptionsDictValidator): """ Shared code between NFO tags and Ouptut Directory NFO Tags """ _required_keys = {"nfo_name", "nfo_root", "tags"} - _optional_keys = {"kodi_safe"} + _optional_keys = {"enable", "kodi_safe"} @classmethod def partial_validate(cls, name: str, value: Any) -> None: diff --git a/src/ytdl_sub/plugins/output_directory_nfo_tags.py b/src/ytdl_sub/plugins/output_directory_nfo_tags.py index f754ec44..aae44103 100644 --- a/src/ytdl_sub/plugins/output_directory_nfo_tags.py +++ b/src/ytdl_sub/plugins/output_directory_nfo_tags.py @@ -32,7 +32,7 @@ class OutputDirectoryNfoTagsOptions(SharedNfoTagsOptions): # Hack to make it so collection named seasons do not error # when adding output_directory_nfo info for plex _required_keys = set() - _optional_keys = {"kodi_safe", "nfo_name", "nfo_root", "tags"} + _optional_keys = {"enable", "kodi_safe", "nfo_name", "nfo_root", "tags"} @property def nfo_root(self) -> StringFormatterValidator: diff --git a/src/ytdl_sub/plugins/regex.py b/src/ytdl_sub/plugins/regex.py index 47b34262..0874b7f7 100644 --- a/src/ytdl_sub/plugins/regex.py +++ b/src/ytdl_sub/plugins/regex.py @@ -8,7 +8,7 @@ from typing import Set from ytdl_sub.config.overrides import Overrides from ytdl_sub.config.plugin.plugin import Plugin from ytdl_sub.config.plugin.plugin_operation import PluginOperation -from ytdl_sub.config.validators.options import OptionsDictValidator +from ytdl_sub.config.validators.options import ToggleableOptionsDictValidator from ytdl_sub.entries.entry import Entry from ytdl_sub.script.parser import parse from ytdl_sub.script.utils.exceptions import RuntimeException @@ -120,7 +120,7 @@ class FromSourceVariablesRegex(DictValidator): } -class RegexOptions(OptionsDictValidator): +class RegexOptions(ToggleableOptionsDictValidator): r""" .. attention:: @@ -218,7 +218,7 @@ class RegexOptions(OptionsDictValidator): """ _required_keys = {"from"} - _optional_keys = {"skip_if_match_fails"} + _optional_keys = {"enable", "skip_if_match_fails"} @classmethod def partial_validate(cls, name: str, value: Any) -> None: diff --git a/src/ytdl_sub/plugins/subtitles.py b/src/ytdl_sub/plugins/subtitles.py index 0e74d448..7be3d8f3 100644 --- a/src/ytdl_sub/plugins/subtitles.py +++ b/src/ytdl_sub/plugins/subtitles.py @@ -6,7 +6,7 @@ from typing import Set from ytdl_sub.config.plugin.plugin import Plugin from ytdl_sub.config.plugin.plugin_operation import PluginOperation -from ytdl_sub.config.validators.options import OptionsDictValidator +from ytdl_sub.config.validators.options import ToggleableOptionsDictValidator from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder from ytdl_sub.entries.entry import Entry from ytdl_sub.entries.script.variable_definitions import VARIABLES @@ -31,7 +31,7 @@ class SubtitlesTypeValidator(StringSelectValidator): _select_values = SUBTITLE_EXTENSIONS -class SubtitleOptions(OptionsDictValidator): +class SubtitleOptions(ToggleableOptionsDictValidator): """ Defines how to download and store subtitles. Using this plugin creates two new variables: ``lang`` and ``subtitles_ext``. ``lang`` is dynamic since you can download multiple subtitles. @@ -52,6 +52,7 @@ class SubtitleOptions(OptionsDictValidator): """ _optional_keys = { + "enable", "subtitles_name", "subtitles_type", "embed_subtitles", diff --git a/src/ytdl_sub/plugins/throttle_protection.py b/src/ytdl_sub/plugins/throttle_protection.py index 5295c4f2..9fb102db 100644 --- a/src/ytdl_sub/plugins/throttle_protection.py +++ b/src/ytdl_sub/plugins/throttle_protection.py @@ -6,7 +6,7 @@ from typing import Tuple from ytdl_sub.config.overrides import Overrides from ytdl_sub.config.plugin.plugin import Plugin -from ytdl_sub.config.validators.options import OptionsDictValidator +from ytdl_sub.config.validators.options import ToggleableOptionsDictValidator from ytdl_sub.entries.entry import Entry from ytdl_sub.utils.file_handler import FileMetadata from ytdl_sub.utils.logger import Logger @@ -59,7 +59,7 @@ class RandomizedRangeValidator(StrictDictValidator): return int(self.randomized_float()) -class ThrottleProtectionOptions(OptionsDictValidator): +class ThrottleProtectionOptions(ToggleableOptionsDictValidator): """ Provides options to make ytdl-sub look more 'human-like' to protect from throttling. For range-based values, a random number will be chosen within the range to avoid sleeps looking @@ -85,6 +85,7 @@ class ThrottleProtectionOptions(OptionsDictValidator): """ _optional_keys = { + "enable", "sleep_per_download_s", "sleep_per_subscription_s", "max_downloads_per_subscription", diff --git a/src/ytdl_sub/plugins/video_tags.py b/src/ytdl_sub/plugins/video_tags.py index 0d4614a8..b5fc3e77 100644 --- a/src/ytdl_sub/plugins/video_tags.py +++ b/src/ytdl_sub/plugins/video_tags.py @@ -3,7 +3,7 @@ from typing import Any from typing import Dict from ytdl_sub.config.plugin.plugin import Plugin -from ytdl_sub.config.validators.options import OptionsDictValidator +from ytdl_sub.config.validators.options import ToggleableOptionsDictValidator from ytdl_sub.entries.entry import Entry from ytdl_sub.utils.ffmpeg import add_ffmpeg_metadata_key_values from ytdl_sub.utils.file_handler import FileMetadata @@ -13,7 +13,7 @@ from ytdl_sub.validators.string_formatter_validators import DictFormatterValidat logger = Logger.get("video-tags") -class VideoTagsOptions(OptionsDictValidator): +class VideoTagsOptions(ToggleableOptionsDictValidator): """ Adds tags to every downloaded video file using ffmpeg ``-metadata key=value`` args. @@ -27,7 +27,7 @@ class VideoTagsOptions(OptionsDictValidator): description: "{description}" """ - _optional_keys = {"tags"} + _optional_keys = {"enable", "tags"} _allow_extra_keys = True @classmethod diff --git a/src/ytdl_sub/subscriptions/subscription_download.py b/src/ytdl_sub/subscriptions/subscription_download.py index 56e93954..06ce1bd2 100644 --- a/src/ytdl_sub/subscriptions/subscription_download.py +++ b/src/ytdl_sub/subscriptions/subscription_download.py @@ -190,7 +190,7 @@ class SubscriptionDownload(BaseSubscription, ABC): ------- List of plugins defined in the subscription, initialized and ready to use. """ - return [ + plugins = [ plugin_type( options=plugin_options, overrides=self.overrides, @@ -198,6 +198,7 @@ class SubscriptionDownload(BaseSubscription, ABC): ) for plugin_type, plugin_options in self.plugins.zipped() ] + return [plugin for plugin in plugins if plugin.is_enabled] @classmethod def _cleanup_entry_files(cls, entry: Entry): diff --git a/src/ytdl_sub/utils/script.py b/src/ytdl_sub/utils/script.py index d5d36b87..5653bc74 100644 --- a/src/ytdl_sub/utils/script.py +++ b/src/ytdl_sub/utils/script.py @@ -38,3 +38,15 @@ class ScriptUtils: out = f"{{%from_json('''{dumped_json}''')}}" return out + + @classmethod + def bool_formatter_output(cls, output: str) -> bool: + """ + Translate formatter output to a boolean + """ + if not output or output.lower() == "false": + return False + try: + return bool(json.loads(output)) + except Exception: # pylint: disable=broad-except + return True diff --git a/src/ytdl_sub/validators/string_formatter_validators.py b/src/ytdl_sub/validators/string_formatter_validators.py index def6e6d8..c561e47e 100644 --- a/src/ytdl_sub/validators/string_formatter_validators.py +++ b/src/ytdl_sub/validators/string_formatter_validators.py @@ -92,7 +92,7 @@ class OverridesStringFormatterValidator(StringFormatterValidator): # pylint: enable=line-too-long -class OverridesIntegerFormatterValidator(StringFormatterValidator): +class OverridesIntegerFormatterValidator(OverridesStringFormatterValidator): _expected_value_type_name = "integer" def post_process(self, resolved: str) -> str: @@ -106,6 +106,10 @@ class OverridesIntegerFormatterValidator(StringFormatterValidator): return resolved +class OverridesBooleanFormatterValidator(OverridesStringFormatterValidator): + _expected_value_type_name = "boolean" + + class ListFormatterValidator(ListValidator[StringFormatterValidator]): _inner_list_type = StringFormatterValidator diff --git a/tests/conftest.py b/tests/conftest.py index b7ec75ff..522ddb3b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -113,7 +113,7 @@ def assert_logs( for call_args in patched_debug.call_args_list: occurrences += int(expected_message in call_args.args[0]) - if expected_occurrences: + if expected_occurrences is not None: assert ( occurrences == expected_occurrences ), f"{expected_message} was expected {expected_occurrences} times, got {occurrences}" diff --git a/tests/unit/plugins/test_throttle_protection.py b/tests/unit/plugins/test_throttle_protection.py index 0d72e9fd..9dcdb89b 100644 --- a/tests/unit/plugins/test_throttle_protection.py +++ b/tests/unit/plugins/test_throttle_protection.py @@ -1,3 +1,4 @@ +import pytest from conftest import assert_logs from ytdl_sub.plugins.throttle_protection import logger as throttle_protection_logger @@ -57,3 +58,59 @@ class TestThrottleProtectionPlugin: expected_occurrences=1, ): _ = subscription.download(dry_run=False) + + @pytest.mark.parametrize( + "disable_value", + [ + "", + False, + "{tp_bool_string}", + "{tp_empty_string}", + ], + ) + def test_disabled( + self, + config, + subscription_name, + output_directory, + mock_download_collection_entries, + disable_value, + ): + preset_dict = { + "preset": [ + "Kodi Music Videos", + ], + "overrides": { + "url": "https://your.name.here", + "music_video_directory": output_directory, + "tp_bool_string": "{ %bool(False) }", + "tp_empty_string": "", + }, + "throttle_protection": { + "enable": disable_value, + "sleep_per_download_s": { + "min": 0.01, + "max": 0.01, + }, + "sleep_per_subscription_s": { + "min": 0.02, + "max": 0.02, + }, + }, + } + + subscription = Subscription.from_dict( + config=config, + preset_name=subscription_name, + preset_dict=preset_dict, + ) + + with mock_download_collection_entries( + is_youtube_channel=False, num_urls=1, is_extracted_audio=False + ), assert_logs( + logger=throttle_protection_logger, + expected_message="Sleeping between downloads for %0.2f seconds", + log_level="debug", + expected_occurrences=0, + ): + _ = subscription.download(dry_run=False) diff --git a/tests/unit/utils/test_script_utils.py b/tests/unit/utils/test_script_utils.py index 8b2b0e90..d3ff8921 100644 --- a/tests/unit/utils/test_script_utils.py +++ b/tests/unit/utils/test_script_utils.py @@ -1,5 +1,6 @@ import copy +import pytest from unit.script.conftest import single_variable_output from ytdl_sub.utils.script import ScriptUtils @@ -31,3 +32,22 @@ class TestScriptUtils: output = single_variable_output(ScriptUtils.to_script(json_dict)) assert output == expected_output + + @pytest.mark.parametrize( + "input_str, expected_output", + [ + ("", False), + ("true", True), + ("false", False), + ("[ ]", False), + ("{ }", False), + ("True", True), + ("False", False), + ("lol not False", True), + ("0", False), + ("-1", True), + ("1", True), + ], + ) + def test_bool_formatter_output(self, input_str: str, expected_output: bool): + assert ScriptUtils.bool_formatter_output(input_str) == expected_output