From 2dc4c9ef40893b3d8f20227372cea8248328ea7b Mon Sep 17 00:00:00 2001 From: Jesse Bannon Date: Wed, 13 Dec 2023 00:23:37 -0800 Subject: [PATCH] almost regex ready --- src/ytdl_sub/config/plugin/preset_plugins.py | 8 +- .../config/validators/variable_validation.py | 115 +++++++++++------- .../subscriptions/base_subscription.py | 2 +- tests/e2e/plugins/test_regex.py | 23 ++-- 4 files changed, 91 insertions(+), 57 deletions(-) diff --git a/src/ytdl_sub/config/plugin/preset_plugins.py b/src/ytdl_sub/config/plugin/preset_plugins.py index f9862060..2a86af6a 100644 --- a/src/ytdl_sub/config/plugin/preset_plugins.py +++ b/src/ytdl_sub/config/plugin/preset_plugins.py @@ -1,7 +1,11 @@ -from typing import List, Type, Tuple, Optional +from typing import List +from typing import Optional +from typing import Tuple +from typing import Type from ytdl_sub.config.plugin.plugin import Plugin -from ytdl_sub.config.validators.options import OptionsValidator, TOptionsValidator +from ytdl_sub.config.validators.options import OptionsValidator +from ytdl_sub.config.validators.options import TOptionsValidator class PresetPlugins: diff --git a/src/ytdl_sub/config/validators/variable_validation.py b/src/ytdl_sub/config/validators/variable_validation.py index c240e459..bb67527d 100644 --- a/src/ytdl_sub/config/validators/variable_validation.py +++ b/src/ytdl_sub/config/validators/variable_validation.py @@ -12,93 +12,116 @@ from ytdl_sub.config.validators.options import OptionsValidator from ytdl_sub.downloaders.url.validators import MultiUrlValidator from ytdl_sub.entries.script.variable_scripts import VARIABLE_SCRIPTS from ytdl_sub.script.script import Script +from ytdl_sub.utils.exceptions import ValidationException from ytdl_sub.utils.script import ScriptUtils from ytdl_sub.validators.string_formatter_validators import validate_formatters +def _get_added_variables(plugins: PresetPlugins, downloader_options: MultiUrlValidator) -> Set[str]: + added_variables: Set[str] = set() + options: List[OptionsValidator] = plugins.plugin_options + options.append(downloader_options) + + for plugin_options in options: + for plugin_added_variables in plugin_options.added_source_variables( + unresolved_variables=set() + ).values(): + added_variables |= set(plugin_added_variables) + + return added_variables + + +def _override_variables(overrides: Overrides) -> Set[str]: + return set(list(overrides.initial_variables(unresolved_variables={}).keys())) + + +def _entry_variables() -> Set[str]: + return set(list(VARIABLE_SCRIPTS.keys())) + + class VariableValidation: - @classmethod - def _get_added_variables( - cls, plugins: PresetPlugins, downloader_options: MultiUrlValidator - ) -> Set[str]: - added_variables: Set[str] = set() - options: List[OptionsValidator] = plugins.plugin_options - options.append(downloader_options) - - for plugin_options in options: - for plugin_added_variables in plugin_options.added_source_variables( - unresolved_variables=set() - ).values(): - added_variables |= set(plugin_added_variables) - - return added_variables - def __init__( self, downloader_options: MultiUrlValidator, output_options: OutputOptions, plugins: PresetPlugins, ): - self.script: Optional[Script] = None - self.resolved_variables: Set[str] = set() - self.downloader_options = downloader_options self.output_options = output_options self.plugins = plugins - self.unresolved_variables = VariableValidation._get_added_variables( - plugins=plugins, downloader_options=downloader_options - ) + + self.script: Optional[Script] = None + self.resolved_variables: Set[str] = set() + self.unresolved_variables: Set[str] = set() def initialize_overrides(self, overrides: Overrides) -> "VariableValidation": + override_variables = _override_variables(overrides) + entry_variables = _entry_variables() + + # Set unresolved as variables that are added but do not exist as entry/override variables + self.unresolved_variables = ( + _get_added_variables(plugins=self.plugins, downloader_options=self.downloader_options) + - override_variables + - entry_variables + ) + + # Initialize overrides with unresolved variables to throw an error overrides = overrides.initialize_script( unresolved_variables={ var_name: f"{{%throw('Plugin variable {var_name} has not been created yet')}}" for var_name in self.unresolved_variables } ) + + # copy the script and mock entry variables self.script = copy.deepcopy(overrides.script).add( - ScriptUtils.add_dummy_variables(list(VARIABLE_SCRIPTS.keys())) + ScriptUtils.add_dummy_variables(entry_variables) ) self.resolved_variables = self.script.variable_names - self.unresolved_variables return self + def _update_script(self) -> None: + _ = self.script.resolve(unresolvable=self.unresolved_variables, update=True) + + def _add_variables(self, plugin_op: PluginOperation, options: OptionsValidator) -> Set[str]: + added_variables = options.added_source_variables( + unresolved_variables=self.unresolved_variables + ).get(plugin_op, set()) + + if added_variables: + for added_variable in added_variables: + if added_variable in self.resolved_variables: + raise ValidationException( + f"Tried added the variable '{added_variable}', but it already " + f"exists as a defined variable." + ) + + self.script.add(ScriptUtils.add_dummy_variables(added_variables)) + self.unresolved_variables -= added_variables + + return added_variables + def ensure_proper_usage(self) -> None: """ Validate variables resolve as plugins are executed, and return a mock script which contains actualized added variables from the plugins """ + self._add_variables(PluginOperation.DOWNLOADER, options=self.downloader_options) - added_variables: Set[str] = self.downloader_options.added_source_variables( - self.unresolved_variables - ).get(PluginOperation.DOWNLOADER, set()) - self.script.add(ScriptUtils.add_dummy_variables(added_variables)) - self.unresolved_variables -= added_variables - + # Metadata variables to be added for plugin_options in PluginMapping.order_options_by( self.plugins.zipped(), PluginOperation.MODIFY_ENTRY_METADATA ): - added_variables = plugin_options.added_source_variables( - unresolved_variables=self.unresolved_variables - ).get(PluginOperation.MODIFY_ENTRY_METADATA, set()) + self._add_variables(PluginOperation.MODIFY_ENTRY_METADATA, options=plugin_options) - if added_variables: - self.script.add(ScriptUtils.add_dummy_variables(added_variables)) - self.unresolved_variables -= added_variables - - _ = self.script.resolve(unresolvable=self.unresolved_variables, update=True) + self._update_script() for plugin_options in PluginMapping.order_options_by( self.plugins.zipped(), PluginOperation.MODIFY_ENTRY ): - added_variables = plugin_options.added_source_variables( - unresolved_variables=self.unresolved_variables - ).get(PluginOperation.MODIFY_ENTRY, set()) - - if added_variables: - self.script.add(ScriptUtils.add_dummy_variables(added_variables)) - self.unresolved_variables -= added_variables - - _ = self.script.resolve(unresolvable=self.unresolved_variables, update=True) + added = self._add_variables(PluginOperation.MODIFY_ENTRY, options=plugin_options) + if added: + self._update_script() # Validate that any formatter in the plugin options can resolve validate_formatters( diff --git a/src/ytdl_sub/subscriptions/base_subscription.py b/src/ytdl_sub/subscriptions/base_subscription.py index fc382168..c7872e8a 100644 --- a/src/ytdl_sub/subscriptions/base_subscription.py +++ b/src/ytdl_sub/subscriptions/base_subscription.py @@ -4,8 +4,8 @@ from typing import Optional from ytdl_sub.config.config_validator import ConfigOptions from ytdl_sub.config.overrides import Overrides -from ytdl_sub.config.preset import Preset from ytdl_sub.config.plugin.preset_plugins import PresetPlugins +from ytdl_sub.config.preset import Preset from ytdl_sub.config.preset_options import OutputOptions from ytdl_sub.config.preset_options import YTDLOptions from ytdl_sub.downloaders.url.validators import MultiUrlValidator diff --git a/tests/e2e/plugins/test_regex.py b/tests/e2e/plugins/test_regex.py index 504f74bb..2f128642 100644 --- a/tests/e2e/plugins/test_regex.py +++ b/tests/e2e/plugins/test_regex.py @@ -302,32 +302,39 @@ class TestRegex: preset_dict=regex_subscription_dict, ) - def test_regex_fails_capture_group_is_source_variable( + def test_regex_fails_capture_group_is_entry_variable( self, regex_subscription_dict, default_config ): - regex_subscription_dict["regex"]["from"]["title"]["capture_group_names"][0] = "uid" + regex_subscription_dict["regex"]["from"]["playlist_id"] = { + "match": [".*http:\\/\\/(.+).com.*"], + "capture_group_names": ["uid"], + } + with pytest.raises( ValidationException, match=re.escape( - "'uid' cannot be used as a capture group name because it is a source variable" + "Tried added the variable 'uid', but it already exists as a defined variable" ), ): _ = Subscription.from_dict( config=default_config, - preset_name="test_regex_fails_capture_group_is_source_variable", + preset_name="test_regex_fails_capture_group_is_entry_variable", preset_dict=regex_subscription_dict, ) def test_regex_fails_capture_group_is_override_variable( self, regex_subscription_dict, default_config ): - regex_subscription_dict["regex"]["from"]["title"]["capture_group_names"][ - 0 - ] = "in_regex_default" + regex_subscription_dict["regex"]["from"]["playlist_id"] = { + "match": [".*http:\\/\\/(.+).com.*"], + "capture_group_names": ["contains_regex_default"], + } + with pytest.raises( ValidationException, match=re.escape( - "'in_regex_default' cannot be used as a capture group name because it is an override variable" + "Tried added the variable 'contains_regex_default', but it already exists " + "as a defined variable" ), ): _ = Subscription.from_dict(