From 5ea1c83ef7a5b7085e8974536f5f1ac1c01ff201 Mon Sep 17 00:00:00 2001 From: Jesse Bannon Date: Wed, 13 Dec 2023 00:31:46 -0800 Subject: [PATCH] regex validation working --- src/ytdl_sub/config/validators/options.py | 4 +-- .../config/validators/variable_validation.py | 26 ++++++++++++------- src/ytdl_sub/downloaders/url/validators.py | 4 +-- src/ytdl_sub/plugins/regex.py | 11 ++++++-- src/ytdl_sub/plugins/split_by_chapters.py | 4 +-- src/ytdl_sub/plugins/subtitles.py | 4 +-- tests/e2e/plugins/test_regex.py | 8 +++--- 7 files changed, 36 insertions(+), 25 deletions(-) diff --git a/src/ytdl_sub/config/validators/options.py b/src/ytdl_sub/config/validators/options.py index cfba16dd..5c68bdc9 100644 --- a/src/ytdl_sub/config/validators/options.py +++ b/src/ytdl_sub/config/validators/options.py @@ -40,8 +40,8 @@ class OptionsValidator(Validator, ABC): """ return {} - def added_source_variables( - self, unresolved_variables: Set[str] + def added_variables( + self, resolved_variables: Set[str], unresolved_variables: Set[str] ) -> Dict[PluginOperation, Set[str]]: """ If the plugin adds source variables, list them here. diff --git a/src/ytdl_sub/config/validators/variable_validation.py b/src/ytdl_sub/config/validators/variable_validation.py index bb67527d..a9b367d4 100644 --- a/src/ytdl_sub/config/validators/variable_validation.py +++ b/src/ytdl_sub/config/validators/variable_validation.py @@ -17,14 +17,17 @@ from ytdl_sub.utils.script import ScriptUtils from ytdl_sub.validators.string_formatter_validators import validate_formatters -def _get_added_variables(plugins: PresetPlugins, downloader_options: MultiUrlValidator) -> Set[str]: +def _get_added_variables( + plugins: PresetPlugins, downloader_options: MultiUrlValidator, resolved_variables: Set[str] +) -> Set[str]: added_variables: Set[str] = set() options: List[OptionsValidator] = plugins.plugin_options options.append(downloader_options) for plugin_options in options: - for plugin_added_variables in plugin_options.added_source_variables( - unresolved_variables=set() + for plugin_added_variables in plugin_options.added_variables( + resolved_variables=resolved_variables, + unresolved_variables=set(), ).values(): added_variables |= set(plugin_added_variables) @@ -55,14 +58,17 @@ class VariableValidation: self.unresolved_variables: Set[str] = set() def initialize_overrides(self, overrides: Overrides) -> "VariableValidation": - override_variables = _override_variables(overrides) entry_variables = _entry_variables() + self.resolved_variables = entry_variables.union(_override_variables(overrides)) # Set unresolved as variables that are added but do not exist as entry/override variables self.unresolved_variables = ( - _get_added_variables(plugins=self.plugins, downloader_options=self.downloader_options) - - override_variables - - entry_variables + _get_added_variables( + plugins=self.plugins, + downloader_options=self.downloader_options, + resolved_variables=self.resolved_variables, + ) + - self.resolved_variables ) # Initialize overrides with unresolved variables to throw an error @@ -77,7 +83,6 @@ class VariableValidation: self.script = copy.deepcopy(overrides.script).add( ScriptUtils.add_dummy_variables(entry_variables) ) - self.resolved_variables = self.script.variable_names - self.unresolved_variables return self @@ -85,8 +90,9 @@ class VariableValidation: _ = self.script.resolve(unresolvable=self.unresolved_variables, update=True) def _add_variables(self, plugin_op: PluginOperation, options: OptionsValidator) -> Set[str]: - added_variables = options.added_source_variables( - unresolved_variables=self.unresolved_variables + added_variables = options.added_variables( + resolved_variables=self.resolved_variables, + unresolved_variables=self.unresolved_variables, ).get(plugin_op, set()) if added_variables: diff --git a/src/ytdl_sub/downloaders/url/validators.py b/src/ytdl_sub/downloaders/url/validators.py index 761a93bf..3fb454e8 100644 --- a/src/ytdl_sub/downloaders/url/validators.py +++ b/src/ytdl_sub/downloaders/url/validators.py @@ -244,8 +244,8 @@ class MultiUrlValidator(OptionsValidator): # keep for readthedocs documentation return self._urls.list[0].variables - def added_source_variables( - self, unresolved_variables: Set[str] + def added_variables( + self, resolved_variables: Set[str], unresolved_variables: Set[str] ) -> Dict[PluginOperation, Set[str]]: """ Returns diff --git a/src/ytdl_sub/plugins/regex.py b/src/ytdl_sub/plugins/regex.py index e4788bc7..cf21eab5 100644 --- a/src/ytdl_sub/plugins/regex.py +++ b/src/ytdl_sub/plugins/regex.py @@ -232,8 +232,8 @@ class RegexOptions(OptionsDictValidator): return False return True - def added_source_variables( - self, unresolved_variables: Set[str] + def added_variables( + self, resolved_variables: Set[str], unresolved_variables: Set[str] ) -> Dict[PluginOperation, Set[str]]: """ Returns @@ -245,6 +245,13 @@ class RegexOptions(OptionsDictValidator): PluginOperation.MODIFY_ENTRY: set(), } for input_variable_name, regex_options in self.source_variable_capture_dict.items(): + + if input_variable_name not in resolved_variables: + raise self._validation_exception( + f"cannot regex capture '{input_variable_name}' because it is not a " + f"defined variable" + ) + key = PluginOperation.MODIFY_ENTRY if self._can_evaluate_at_metadata_time( unresolved_variables=unresolved_variables, diff --git a/src/ytdl_sub/plugins/split_by_chapters.py b/src/ytdl_sub/plugins/split_by_chapters.py index 0ed74bce..6d4a5def 100644 --- a/src/ytdl_sub/plugins/split_by_chapters.py +++ b/src/ytdl_sub/plugins/split_by_chapters.py @@ -83,8 +83,8 @@ class SplitByChaptersOptions(OptionsDictValidator): key="when_no_chapters", validator=WhenNoChaptersValidator ).value - def added_source_variables( - self, unresolved_variables: Set[str] + def added_variables( + self, resolved_variables: Set[str], unresolved_variables: Set[str] ) -> Dict[PluginOperation, Set[str]]: return { PluginOperation.MODIFY_ENTRY: { diff --git a/src/ytdl_sub/plugins/subtitles.py b/src/ytdl_sub/plugins/subtitles.py index c4b27e69..0cabdd53 100644 --- a/src/ytdl_sub/plugins/subtitles.py +++ b/src/ytdl_sub/plugins/subtitles.py @@ -116,8 +116,8 @@ class SubtitleOptions(OptionsDictValidator): """ return self._allow_auto_generated_subtitles - def added_source_variables( - self, unresolved_variables: Set[str] + def added_variables( + self, resolved_variables: Set[str], unresolved_variables: Set[str] ) -> Dict[PluginOperation, Set[str]]: """ Returns diff --git a/tests/e2e/plugins/test_regex.py b/tests/e2e/plugins/test_regex.py index 2f128642..c61dad73 100644 --- a/tests/e2e/plugins/test_regex.py +++ b/tests/e2e/plugins/test_regex.py @@ -305,7 +305,7 @@ class TestRegex: def test_regex_fails_capture_group_is_entry_variable( self, regex_subscription_dict, default_config ): - regex_subscription_dict["regex"]["from"]["playlist_id"] = { + regex_subscription_dict["regex"]["from"]["playlist_uid"] = { "match": [".*http:\\/\\/(.+).com.*"], "capture_group_names": ["uid"], } @@ -325,7 +325,7 @@ class TestRegex: def test_regex_fails_capture_group_is_override_variable( self, regex_subscription_dict, default_config ): - regex_subscription_dict["regex"]["from"]["playlist_id"] = { + regex_subscription_dict["regex"]["from"]["playlist_uid"] = { "match": [".*http:\\/\\/(.+).com.*"], "capture_group_names": ["contains_regex_default"], } @@ -351,9 +351,7 @@ class TestRegex: ) with pytest.raises( ValidationException, - match=re.escape( - "cannot regex capture 'dne' because it is not a source or override variable" - ), + match=re.escape("cannot regex capture 'dne' because it is not a defined variable"), ): _ = Subscription.from_dict( config=default_config,