danvics/ytdl-sub
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

almost regex ready

Browse source
This commit is contained in:
Jesse Bannon 2023-12-13 00:23:37 -08:00
parent cbfcd5154e
commit 2dc4c9ef40
4 changed files with 91 additions and 57 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
src/ytdl_sub/config/plugin/preset_plugins.py
View file

@ -1,7 +1,11 @@
from typing import List, Type, Tuple, Optional
from typing import List
from typing import Optional
from typing import Tuple
from typing import Type
from ytdl_sub.config.plugin.plugin import Plugin
from ytdl_sub.config.validators.options import OptionsValidator, TOptionsValidator
from ytdl_sub.config.validators.options import OptionsValidator
from ytdl_sub.config.validators.options import TOptionsValidator
class PresetPlugins:

115
src/ytdl_sub/config/validators/variable_validation.py
View file

@ -12,93 +12,116 @@ from ytdl_sub.config.validators.options import OptionsValidator
from ytdl_sub.downloaders.url.validators import MultiUrlValidator
from ytdl_sub.entries.script.variable_scripts import VARIABLE_SCRIPTS
from ytdl_sub.script.script import Script
from ytdl_sub.utils.exceptions import ValidationException
from ytdl_sub.utils.script import ScriptUtils
from ytdl_sub.validators.string_formatter_validators import validate_formatters
def _get_added_variables(plugins: PresetPlugins, downloader_options: MultiUrlValidator) -> Set[str]:
added_variables: Set[str] = set()
options: List[OptionsValidator] = plugins.plugin_options
options.append(downloader_options)
for plugin_options in options:
for plugin_added_variables in plugin_options.added_source_variables(
unresolved_variables=set()
).values():
added_variables |= set(plugin_added_variables)
return added_variables
def _override_variables(overrides: Overrides) -> Set[str]:
return set(list(overrides.initial_variables(unresolved_variables={}).keys()))
def _entry_variables() -> Set[str]:
return set(list(VARIABLE_SCRIPTS.keys()))
class VariableValidation:
@classmethod
def _get_added_variables(
cls, plugins: PresetPlugins, downloader_options: MultiUrlValidator
) -> Set[str]:
added_variables: Set[str] = set()
options: List[OptionsValidator] = plugins.plugin_options
options.append(downloader_options)
for plugin_options in options:
for plugin_added_variables in plugin_options.added_source_variables(
unresolved_variables=set()
).values():
added_variables |= set(plugin_added_variables)
return added_variables
def __init__(
self,
downloader_options: MultiUrlValidator,
output_options: OutputOptions,
plugins: PresetPlugins,
):
self.script: Optional[Script] = None
self.resolved_variables: Set[str] = set()
self.downloader_options = downloader_options
self.output_options = output_options
self.plugins = plugins
self.unresolved_variables = VariableValidation._get_added_variables(
plugins=plugins, downloader_options=downloader_options
)
self.script: Optional[Script] = None
self.resolved_variables: Set[str] = set()
self.unresolved_variables: Set[str] = set()
def initialize_overrides(self, overrides: Overrides) -> "VariableValidation":
override_variables = _override_variables(overrides)
entry_variables = _entry_variables()
# Set unresolved as variables that are added but do not exist as entry/override variables
self.unresolved_variables = (
_get_added_variables(plugins=self.plugins, downloader_options=self.downloader_options)
- override_variables
- entry_variables
)
# Initialize overrides with unresolved variables to throw an error
overrides = overrides.initialize_script(
unresolved_variables={
var_name: f"{{%throw('Plugin variable {var_name} has not been created yet')}}"
for var_name in self.unresolved_variables
}
)
# copy the script and mock entry variables
self.script = copy.deepcopy(overrides.script).add(
ScriptUtils.add_dummy_variables(list(VARIABLE_SCRIPTS.keys()))
ScriptUtils.add_dummy_variables(entry_variables)
)
self.resolved_variables = self.script.variable_names - self.unresolved_variables
return self
def _update_script(self) -> None:
_ = self.script.resolve(unresolvable=self.unresolved_variables, update=True)
def _add_variables(self, plugin_op: PluginOperation, options: OptionsValidator) -> Set[str]:
added_variables = options.added_source_variables(
unresolved_variables=self.unresolved_variables
).get(plugin_op, set())
if added_variables:
for added_variable in added_variables:
if added_variable in self.resolved_variables:
raise ValidationException(
f"Tried added the variable '{added_variable}', but it already "
f"exists as a defined variable."
)
self.script.add(ScriptUtils.add_dummy_variables(added_variables))
self.unresolved_variables -= added_variables
return added_variables
def ensure_proper_usage(self) -> None:
"""
Validate variables resolve as plugins are executed, and return
a mock script which contains actualized added variables from the plugins
"""
self._add_variables(PluginOperation.DOWNLOADER, options=self.downloader_options)
added_variables: Set[str] = self.downloader_options.added_source_variables(
self.unresolved_variables
).get(PluginOperation.DOWNLOADER, set())
self.script.add(ScriptUtils.add_dummy_variables(added_variables))
self.unresolved_variables -= added_variables
# Metadata variables to be added
for plugin_options in PluginMapping.order_options_by(
self.plugins.zipped(), PluginOperation.MODIFY_ENTRY_METADATA
):
added_variables = plugin_options.added_source_variables(
unresolved_variables=self.unresolved_variables
).get(PluginOperation.MODIFY_ENTRY_METADATA, set())
self._add_variables(PluginOperation.MODIFY_ENTRY_METADATA, options=plugin_options)
if added_variables:
self.script.add(ScriptUtils.add_dummy_variables(added_variables))
self.unresolved_variables -= added_variables
_ = self.script.resolve(unresolvable=self.unresolved_variables, update=True)
self._update_script()
for plugin_options in PluginMapping.order_options_by(
self.plugins.zipped(), PluginOperation.MODIFY_ENTRY
):
added_variables = plugin_options.added_source_variables(
unresolved_variables=self.unresolved_variables
).get(PluginOperation.MODIFY_ENTRY, set())
if added_variables:
self.script.add(ScriptUtils.add_dummy_variables(added_variables))
self.unresolved_variables -= added_variables
_ = self.script.resolve(unresolvable=self.unresolved_variables, update=True)
added = self._add_variables(PluginOperation.MODIFY_ENTRY, options=plugin_options)
if added:
self._update_script()
# Validate that any formatter in the plugin options can resolve
validate_formatters(

2
src/ytdl_sub/subscriptions/base_subscription.py
View file

@ -4,8 +4,8 @@ from typing import Optional
from ytdl_sub.config.config_validator import ConfigOptions
from ytdl_sub.config.overrides import Overrides
from ytdl_sub.config.preset import Preset
from ytdl_sub.config.plugin.preset_plugins import PresetPlugins
from ytdl_sub.config.preset import Preset
from ytdl_sub.config.preset_options import OutputOptions
from ytdl_sub.config.preset_options import YTDLOptions
from ytdl_sub.downloaders.url.validators import MultiUrlValidator

23
tests/e2e/plugins/test_regex.py
View file

@ -302,32 +302,39 @@ class TestRegex:
preset_dict=regex_subscription_dict,
)
def test_regex_fails_capture_group_is_source_variable(
def test_regex_fails_capture_group_is_entry_variable(
self, regex_subscription_dict, default_config
):
regex_subscription_dict["regex"]["from"]["title"]["capture_group_names"][0] = "uid"
regex_subscription_dict["regex"]["from"]["playlist_id"] = {
"match": [".*http:\\/\\/(.+).com.*"],
"capture_group_names": ["uid"],
}
with pytest.raises(
ValidationException,
match=re.escape(
"'uid' cannot be used as a capture group name because it is a source variable"
"Tried added the variable 'uid', but it already exists as a defined variable"
),
):
_ = Subscription.from_dict(
config=default_config,
preset_name="test_regex_fails_capture_group_is_source_variable",
preset_name="test_regex_fails_capture_group_is_entry_variable",
preset_dict=regex_subscription_dict,
)
def test_regex_fails_capture_group_is_override_variable(
self, regex_subscription_dict, default_config
):
regex_subscription_dict["regex"]["from"]["title"]["capture_group_names"][
0
] = "in_regex_default"
regex_subscription_dict["regex"]["from"]["playlist_id"] = {
"match": [".*http:\\/\\/(.+).com.*"],
"capture_group_names": ["contains_regex_default"],
}
with pytest.raises(
ValidationException,
match=re.escape(
"'in_regex_default' cannot be used as a capture group name because it is an override variable"
"Tried added the variable 'contains_regex_default', but it already exists "
"as a defined variable"
),
):
_ = Subscription.from_dict(