diff --git a/src/ytdl_sub/config/plugin/plugin_operation.py b/src/ytdl_sub/config/plugin/plugin_operation.py index 2c187f30..d3323aca 100644 --- a/src/ytdl_sub/config/plugin/plugin_operation.py +++ b/src/ytdl_sub/config/plugin/plugin_operation.py @@ -2,6 +2,7 @@ from enum import Enum class PluginOperation(Enum): + ANY = -2 DOWNLOADER = -1 MODIFY_ENTRY_METADATA = 0 MODIFY_ENTRY = 1 diff --git a/src/ytdl_sub/config/validators/options.py b/src/ytdl_sub/config/validators/options.py index 5c68bdc9..c35defa2 100644 --- a/src/ytdl_sub/config/validators/options.py +++ b/src/ytdl_sub/config/validators/options.py @@ -41,7 +41,10 @@ class OptionsValidator(Validator, ABC): return {} def added_variables( - self, resolved_variables: Set[str], unresolved_variables: Set[str] + self, + resolved_variables: Set[str], + unresolved_variables: Set[str], + plugin_op: PluginOperation, ) -> Dict[PluginOperation, Set[str]]: """ If the plugin adds source variables, list them here. diff --git a/src/ytdl_sub/config/validators/variable_validation.py b/src/ytdl_sub/config/validators/variable_validation.py index d80e7601..a0aa180d 100644 --- a/src/ytdl_sub/config/validators/variable_validation.py +++ b/src/ytdl_sub/config/validators/variable_validation.py @@ -28,7 +28,7 @@ def _add_dummy_variables(variables: Iterable[str]) -> Dict[str, str]: def _get_added_and_modified_variables( - plugins: PresetPlugins, downloader_options: MultiUrlValidator, resolved_variables: Set[str] + plugins: PresetPlugins, downloader_options: MultiUrlValidator ) -> Iterable[Tuple[OptionsValidator, Set[str], Set[str]]]: """ Iterates and returns the plugin options, added variables, modified variables @@ -41,8 +41,9 @@ def _get_added_and_modified_variables( modified_variables: Set[str] = set() for plugin_added_variables in plugin_options.added_variables( - resolved_variables=resolved_variables, + resolved_variables=set(), unresolved_variables=set(), + plugin_op=PluginOperation.ANY, ).values(): added_variables |= set(plugin_added_variables) @@ -93,7 +94,6 @@ class VariableValidation: ) in _get_added_and_modified_variables( plugins=self.plugins, downloader_options=self.downloader_options, - resolved_variables=self.resolved_variables, ): for added_variable in added_variables: @@ -129,12 +129,14 @@ class VariableValidation: added_variables = options.added_variables( resolved_variables=self.resolved_variables, unresolved_variables=self.unresolved_variables, + plugin_op=plugin_op, ).get(plugin_op, set()) modified_variables = options.modified_variables().get(plugin_op, set()) resolved_variables = added_variables | modified_variables self.script.add(_add_dummy_variables(resolved_variables)) + self.resolved_variables |= resolved_variables self.unresolved_variables -= resolved_variables return added_variables diff --git a/src/ytdl_sub/downloaders/url/validators.py b/src/ytdl_sub/downloaders/url/validators.py index 3fb454e8..4a22789b 100644 --- a/src/ytdl_sub/downloaders/url/validators.py +++ b/src/ytdl_sub/downloaders/url/validators.py @@ -6,6 +6,7 @@ from typing import Set from ytdl_sub.config.plugin.plugin_operation import PluginOperation from ytdl_sub.config.validators.options import OptionsValidator +from ytdl_sub.script.parser import parse from ytdl_sub.validators.strict_dict_validator import StrictDictValidator from ytdl_sub.validators.string_formatter_validators import DictFormatterValidator from ytdl_sub.validators.string_formatter_validators import OverridesStringFormatterValidator @@ -245,11 +246,25 @@ class MultiUrlValidator(OptionsValidator): return self._urls.list[0].variables def added_variables( - self, resolved_variables: Set[str], unresolved_variables: Set[str] + self, + resolved_variables: Set[str], + unresolved_variables: Set[str], + plugin_op: PluginOperation, ) -> Dict[PluginOperation, Set[str]]: """ Returns ------- List of variables added. The first collection url always contains all the variables. """ + if plugin_op != PluginOperation.ANY: + for url in self._urls.list: + for variable_name, definition in url.variables.dict_with_format_strings.items(): + used_variables = set(var.name for var in parse(definition).variables) + if unresolved := used_variables & unresolved_variables: + raise self._validation_exception( + f"variable {variable_name} cannot use the variables " + f"{', '.join(sorted(list(unresolved)))} because they depend on other" + " variables that are computed later in execution" + ) + return {PluginOperation.DOWNLOADER: set(self._urls.list[0].variables.keys)} diff --git a/src/ytdl_sub/plugins/chapters.py b/src/ytdl_sub/plugins/chapters.py index 0e185f66..bbb10ddd 100644 --- a/src/ytdl_sub/plugins/chapters.py +++ b/src/ytdl_sub/plugins/chapters.py @@ -197,7 +197,10 @@ class ChaptersOptions(OptionsDictValidator): return self._force_key_frames def added_variables( - self, resolved_variables: Set[str], unresolved_variables: Set[str] + self, + resolved_variables: Set[str], + unresolved_variables: Set[str], + plugin_op: PluginOperation, ) -> Dict[PluginOperation, Set[str]]: return {PluginOperation.MODIFY_ENTRY: {ytdl_sub_chapters_from_comments.variable_name}} diff --git a/src/ytdl_sub/plugins/regex.py b/src/ytdl_sub/plugins/regex.py index d9962e0c..d8f65c8a 100644 --- a/src/ytdl_sub/plugins/regex.py +++ b/src/ytdl_sub/plugins/regex.py @@ -203,6 +203,9 @@ class RegexOptions(OptionsDictValidator): key="skip_if_match_fails", validator=BoolValidator, default=True ).value + # Variables added by the regex plugin + self._added_variable_names: Set[str] = set() + @property def skip_if_match_fails(self) -> Optional[bool]: """ @@ -221,7 +224,7 @@ class RegexOptions(OptionsDictValidator): return self._from.variable_capture_dict @classmethod - def _can_evaluate_at_metadata_time( + def _can_resolve( cls, unresolved_variables: Set[str], input_variable_name: str, regex_options: VariableRegex ) -> bool: if input_variable_name in unresolved_variables: @@ -233,7 +236,10 @@ class RegexOptions(OptionsDictValidator): return True def added_variables( - self, resolved_variables: Set[str], unresolved_variables: Set[str] + self, + resolved_variables: Set[str], + unresolved_variables: Set[str], + plugin_op: PluginOperation, ) -> Dict[PluginOperation, Set[str]]: """ Returns @@ -245,28 +251,43 @@ class RegexOptions(OptionsDictValidator): PluginOperation.MODIFY_ENTRY: set(), } for input_variable_name, regex_options in self.source_variable_capture_dict.items(): + variables_to_add = set(regex_options.capture_group_names) - if input_variable_name not in resolved_variables: + if plugin_op != PluginOperation.ANY and input_variable_name not in ( + resolved_variables | unresolved_variables + ): raise self._validation_exception( - f"cannot regex capture '{input_variable_name}' because it is not a " - f"defined variable" + f"cannot regex capture '{input_variable_name}' because it is not a" + " defined variable." + ) + if ( + plugin_op.value >= PluginOperation.MODIFY_ENTRY.value + and input_variable_name in unresolved_variables + ): + raise self._validation_exception( + f"cannot regex capture '{input_variable_name}' because it is not " + f"computed until later in execution." ) - key = PluginOperation.MODIFY_ENTRY - if self._can_evaluate_at_metadata_time( + if plugin_op == PluginOperation.ANY: + added_source_vars[PluginOperation.MODIFY_ENTRY_METADATA] |= variables_to_add + self._added_variable_names |= variables_to_add + continue + + if not self._can_resolve( unresolved_variables=unresolved_variables, input_variable_name=input_variable_name, regex_options=regex_options, ): - key = PluginOperation.MODIFY_ENTRY_METADATA + continue for capture_group_name in regex_options.capture_group_names: - if capture_group_name in resolved_variables: + if capture_group_name in (resolved_variables - self._added_variable_names): raise self._validation_exception( f"cannot use '{capture_group_name}' as a capture group name because it is " - f"a defined variable" + f"an already defined variable." ) - added_source_vars[key] |= set(regex_options.capture_group_names) + added_source_vars[plugin_op] |= set(regex_options.capture_group_names) return added_source_vars diff --git a/src/ytdl_sub/plugins/split_by_chapters.py b/src/ytdl_sub/plugins/split_by_chapters.py index 481e8f5c..fc83a07c 100644 --- a/src/ytdl_sub/plugins/split_by_chapters.py +++ b/src/ytdl_sub/plugins/split_by_chapters.py @@ -82,7 +82,10 @@ class SplitByChaptersOptions(OptionsDictValidator): ).value def added_variables( - self, resolved_variables: Set[str], unresolved_variables: Set[str] + self, + resolved_variables: Set[str], + unresolved_variables: Set[str], + plugin_op: PluginOperation, ) -> Dict[PluginOperation, Set[str]]: return { PluginOperation.MODIFY_ENTRY: { diff --git a/src/ytdl_sub/plugins/subtitles.py b/src/ytdl_sub/plugins/subtitles.py index dd4d166d..445a00a9 100644 --- a/src/ytdl_sub/plugins/subtitles.py +++ b/src/ytdl_sub/plugins/subtitles.py @@ -120,7 +120,10 @@ class SubtitleOptions(OptionsDictValidator): return self._allow_auto_generated_subtitles def added_variables( - self, resolved_variables: Set[str], unresolved_variables: Set[str] + self, + resolved_variables: Set[str], + unresolved_variables: Set[str], + plugin_op: PluginOperation, ) -> Dict[PluginOperation, Set[str]]: """ Returns diff --git a/tests/e2e/plugins/test_split_by_chapters.py b/tests/e2e/plugins/test_split_by_chapters.py index 597a1fe4..cb02715e 100644 --- a/tests/e2e/plugins/test_split_by_chapters.py +++ b/tests/e2e/plugins/test_split_by_chapters.py @@ -34,7 +34,7 @@ def yt_album_as_chapters_with_regex_preset_dict(yt_album_as_chapters_preset_dict "from": { # Ensure regex can handle override variables that come from the # post-metadata stage - "override_chapter_title": { + "chapter_title": { "match": r"\d+\. (.+)", "capture_group_names": "captured_track_title", "capture_group_defaults": "{chapter_title}", @@ -56,7 +56,6 @@ def yt_album_as_chapters_with_regex_preset_dict(yt_album_as_chapters_preset_dict } }, "overrides": { - "override_chapter_title": "{chapter_title}", "track_title": "{captured_track_title}", "track_album": "{captured_track_album}", "track_artist": "{captured_track_artist}",