diff --git a/src/ytdl_sub/config/overrides.py b/src/ytdl_sub/config/overrides.py index 9bdeac33..ffb0e5ad 100644 --- a/src/ytdl_sub/config/overrides.py +++ b/src/ytdl_sub/config/overrides.py @@ -127,8 +127,8 @@ class Overrides(DictFormatterValidator, Scriptable): if function_overrides: script.add(function_overrides) - return str( - script.add({"tmp_var": formatter.format_string}).resolve(unresolvable=unresolvable)[ - "tmp_var" - ] + return ( + script.add({"tmp_var": formatter.format_string}) + .resolve(unresolvable=unresolvable) + .get_str("tmp_var") ) diff --git a/src/ytdl_sub/config/preset.py b/src/ytdl_sub/config/preset.py index 978e7526..f3195bbe 100644 --- a/src/ytdl_sub/config/preset.py +++ b/src/ytdl_sub/config/preset.py @@ -27,10 +27,12 @@ from ytdl_sub.entries.script.variable_scripts import VARIABLE_SCRIPTS from ytdl_sub.prebuilt_presets import PREBUILT_PRESET_NAMES from ytdl_sub.prebuilt_presets import PUBLISHED_PRESET_NAMES from ytdl_sub.script.script import Script +from ytdl_sub.script.script import ScriptBuilder from ytdl_sub.script.utils.exceptions import VariableDoesNotExist from ytdl_sub.utils.exceptions import StringFormattingVariableNotFoundException from ytdl_sub.utils.exceptions import ValidationException from ytdl_sub.utils.logger import Logger +from ytdl_sub.utils.scriptable import Scriptable from ytdl_sub.utils.yaml import dump_yaml from ytdl_sub.validators.strict_dict_validator import StrictDictValidator from ytdl_sub.validators.string_formatter_validators import DictFormatterValidator @@ -178,22 +180,41 @@ class Preset(_PresetShell): return added_variables @functools.cached_property - def _mock_script(self) -> Script: + def _cached_script_builder(self) -> ScriptBuilder: # Set the formatter variables to be the overrides - variable_dict = copy.deepcopy(self.overrides.dict_with_format_strings) - - source_variables = { - source_var: "dummy_string" - for source_var in self._source_variables - + self.downloader_options.added_source_variables() - } - variable_dict = dict(source_variables, **variable_dict) - variable_dict = dict(variable_dict, **self._added_variables) - - script = Script(variable_dict) - script.resolve(update=True) + script = ScriptBuilder( + Scriptable.add_sanitized_variables(self.overrides.dict_with_format_strings) + ) + script.add( + Scriptable.add_sanitized_variables( + {source_var: "dummy_string" for source_var in self._source_variables} + ) + ) return script + @property + def _script_builder(self) -> ScriptBuilder: + return copy.deepcopy(self._cached_script_builder) + + @functools.cached_property + def _script_builder_with_added_variables(self) -> ScriptBuilder: + return self._script_builder.add( + Scriptable.add_sanitized_variables( + {source_var: "dummy_string" for source_var in self._added_variables} + ) + ) + + @functools.cached_property + def _cached_script(self) -> Script: + """ + Contains actualized script which should hold all Override variables + """ + return self._script_builder_with_added_variables.partial_build(update=True) + + @property + def _script(self) -> Script: + return copy.deepcopy(self._cached_script) + def __validate_and_get_plugins(self) -> PresetPlugins: preset_plugins = PresetPlugins() @@ -209,30 +230,43 @@ class Preset(_PresetShell): return preset_plugins def __validate_added_variables(self): - self.downloader_options.validate_with_variables(script=copy.deepcopy(self._mock_script)) + script_builder = self._script_builder + self.downloader_options.validate_with_variables(script=copy.deepcopy(script_builder)) + script_builder.add( + Scriptable.add_sanitized_variables( + {name: "dummy_string" for name in self.downloader_options.added_source_variables()} + ) + ) for _, plugin_options in sorted( self.plugins.zipped(), key=lambda pl: pl[0].priority.modify_entry ): # Validate current plugin using source + added plugin variables - plugin_options.validate_with_variables(script=copy.deepcopy(self._mock_script)) + plugin_options.validate_with_variables(script=copy.deepcopy(script_builder)) + script_builder.add( + Scriptable.add_sanitized_variables( + { + name: "dummy_string" + for name in self.downloader_options.added_source_variables() + } + ) + ) def __validate_override_string_formatter_validator( self, formatter_validator: Union[StringFormatterValidator, OverridesStringFormatterValidator], - ): - script = copy.deepcopy(self._mock_script) - try: - script.add({"tmp_var": formatter_validator.format_string}) - except VariableDoesNotExist as exc: - raise StringFormattingVariableNotFoundException(exc) from exc - - unresolved: Optional[Set[str]] = ( - {VARIABLES.entry_metadata.variable_name} + ) -> None: + unresolvable = ( + set([VARIABLES.entry_metadata.variable_name] + list(self._added_variables.keys())) if isinstance(formatter_validator, OverridesStringFormatterValidator) else None ) - _ = script.resolve(unresolvable=unresolved)["tmp_var"] # TODO: error if not present + try: + self._script.add({"tmp_var": formatter_validator.format_string}).resolve( + unresolvable=unresolvable + ).get("tmp_var") + except VariableDoesNotExist as exc: + raise StringFormattingVariableNotFoundException(exc) from exc def __recursive_preset_validate( self, diff --git a/src/ytdl_sub/config/preset_options.py b/src/ytdl_sub/config/preset_options.py index 03e9c728..33404f00 100644 --- a/src/ytdl_sub/config/preset_options.py +++ b/src/ytdl_sub/config/preset_options.py @@ -6,6 +6,7 @@ from typing import TypeVar from ytdl_sub.config.defaults import DEFAULT_DOWNLOAD_ARCHIVE_NAME from ytdl_sub.script.script import Script +from ytdl_sub.script.script import ScriptBuilder from ytdl_sub.utils.exceptions import ValidationException from ytdl_sub.validators.file_path_validators import OverridesStringFormatterFilePathValidator from ytdl_sub.validators.file_path_validators import StringFormatterFileNameValidator @@ -53,7 +54,7 @@ class OptionsValidator(Validator, ABC): """ return [] - def validate_with_variables(self, script: Script) -> None: + def validate_with_variables(self, script: ScriptBuilder) -> None: """ Optional validation after init with the session's source and override variables. diff --git a/src/ytdl_sub/downloaders/url/validators.py b/src/ytdl_sub/downloaders/url/validators.py index 4ce88ca2..05fbac7c 100644 --- a/src/ytdl_sub/downloaders/url/validators.py +++ b/src/ytdl_sub/downloaders/url/validators.py @@ -6,6 +6,7 @@ from typing import Optional from ytdl_sub.config.preset_options import OptionsValidator from ytdl_sub.script.script import Script +from ytdl_sub.script.script import ScriptBuilder from ytdl_sub.validators.strict_dict_validator import StrictDictValidator from ytdl_sub.validators.string_formatter_validators import DictFormatterValidator from ytdl_sub.validators.string_formatter_validators import OverridesStringFormatterValidator @@ -252,20 +253,21 @@ class MultiUrlValidator(OptionsValidator): """ return list(self._urls.list[0].variables.keys) - def validate_with_variables(self, script: Script) -> None: + def validate_with_variables(self, script: ScriptBuilder) -> None: """ Ensures new variables added are not existing variables """ # Apply formatting to each new source variable, ensure it resolves for collection_url in self.urls.list: script.add(collection_url.variables.dict_with_format_strings) - script.resolve(update=True) + + resolved_script = script.partial_build(update=True) # Ensure at least URL is non-empty has_non_empty_url = False for url_validator in self.urls.list: - script.add({"tmp_var_url": url_validator.url.format_string}) - has_non_empty_url |= bool(str(script.resolve().get("tmp_var_url"))) + resolved_script.add({"tmp_var_url": url_validator.url.format_string}) + has_non_empty_url |= bool(str(resolved_script.resolve().get_native("tmp_var_url"))) if not has_non_empty_url: raise self._validation_exception("Must contain at least one url that is non-empty") diff --git a/src/ytdl_sub/plugins/regex.py b/src/ytdl_sub/plugins/regex.py index c640c05b..e8aea15f 100644 --- a/src/ytdl_sub/plugins/regex.py +++ b/src/ytdl_sub/plugins/regex.py @@ -12,6 +12,7 @@ from ytdl_sub.entries.entry import Entry from ytdl_sub.entries.script.variable_scripts import VARIABLE_SCRIPTS from ytdl_sub.entries.variables.kwargs import YTDL_SUB_REGEX_SOURCE_VARS from ytdl_sub.script.script import Script +from ytdl_sub.script.script import ScriptBuilder from ytdl_sub.utils.exceptions import RegexNoMatchException from ytdl_sub.utils.exceptions import StringFormattingVariableNotFoundException from ytdl_sub.utils.logger import Logger @@ -214,7 +215,7 @@ class RegexOptions(OptionsDictValidator): """ return self._skip_if_match_fails - def validate_with_variables(self, script: Script) -> None: + def validate_with_variables(self, script: ScriptBuilder) -> None: for key, regex_options in self.source_variable_capture_dict.items(): # Ensure each variable getting captured is a source variable if key not in script._variables: diff --git a/src/ytdl_sub/script/script.py b/src/ytdl_sub/script/script.py index 727f29d1..b8e63914 100644 --- a/src/ytdl_sub/script/script.py +++ b/src/ytdl_sub/script/script.py @@ -1,8 +1,10 @@ import copy +from collections import defaultdict from typing import Dict from typing import List from typing import Optional from typing import Set +from typing import Tuple from ytdl_sub.script.functions import Functions from ytdl_sub.script.parser import parse @@ -369,28 +371,45 @@ class ScriptBuilder: return self @property - def _missing_metadata(self) -> Dict[str, Set[str]]: - missing_metadata: Dict[str, Set[str]] = {} + def _missing_metadata(self) -> Tuple[Dict[str, Set[str]], Dict[str, Set[str]]]: + variables_missing_metadata: Dict[str, Set[str]] = defaultdict(set) + functions_missing_metadata: Dict[str, Set[str]] = defaultdict(set) defined_variables: Set[str] = set(self._variables.keys()) defined_functions: Set[str] = set(self._functions.keys()) - for name, variable in self._variables.items(): - missing_metadata[name] = {var.name for var in variable.variables}.difference( - defined_variables - ) - missing_metadata[name].update( - {fun.name for fun in variable.custom_functions}.difference(defined_functions) - ) - for name, function in self._functions.items(): - missing_metadata[name] = {var.name for var in function.variables}.difference( - defined_variables - ) - missing_metadata[name].update( - {fun.name for fun in function.custom_functions}.difference(defined_functions) - ) + while True: + variables_missing_metadata_snapshot = copy.deepcopy(variables_missing_metadata) + functions_missing_metadata_snapshot = copy.deepcopy(functions_missing_metadata) - return missing_metadata + for name, variable in self._variables.items(): + if diff := {var.name for var in variable.variables}.difference(defined_variables): + variables_missing_metadata[name].update(diff) + + if diff := {fun.name for fun in variable.custom_functions}.difference( + defined_functions + ): + variables_missing_metadata[name].update(diff) + + for name, function in self._functions.items(): + if diff := {var.name for var in function.variables}.difference(defined_variables): + functions_missing_metadata[name].update(diff) + + if diff := {fun.name for fun in function.custom_functions}.difference( + defined_functions + ): + functions_missing_metadata[name].update(diff) + + if ( + variables_missing_metadata == variables_missing_metadata_snapshot + and functions_missing_metadata == functions_missing_metadata_snapshot + ): + break + + defined_variables -= set(variables_missing_metadata.keys()) + defined_functions -= set(functions_missing_metadata.keys()) + + return variables_missing_metadata, functions_missing_metadata @classmethod def _build(cls, variables: Dict[str, SyntaxTree], functions: Dict[str, SyntaxTree]) -> Script: @@ -400,26 +419,29 @@ class ScriptBuilder: script._validate() return script - def partial_build(self) -> Script: - missing_metadata = self._missing_metadata + def partial_build(self, update: bool = False) -> Script: + missing_variables, missing_functions = self._missing_metadata maybe_resolvable_variables: Dict[str, SyntaxTree] = { name: variable for name, variable in self._variables.items() - if name not in missing_metadata + if name not in missing_variables } maybe_resolvable_functions: Dict[str, SyntaxTree] = { name: function for name, function in self._functions.items() - if name not in missing_metadata + if name not in missing_functions } - return self._build( + script = self._build( variables=maybe_resolvable_variables, functions=maybe_resolvable_functions ) + if update: + script.resolve(update=True) + return script def build(self) -> Script: - for name, missing_metadata in self._missing_metadata.items(): - if missing_metadata: + for missing_metadata in self._missing_metadata: + for name, missing in missing_metadata.items(): raise ScriptBuilderMissingDefinitions( f"{name} is missing the following definitions: {', '.join(missing_metadata)}" ) diff --git a/src/ytdl_sub/utils/scriptable.py b/src/ytdl_sub/utils/scriptable.py index c9056fd8..70b5b6aa 100644 --- a/src/ytdl_sub/utils/scriptable.py +++ b/src/ytdl_sub/utils/scriptable.py @@ -1,4 +1,5 @@ from abc import ABC +from typing import Dict from typing import Set from ytdl_sub.entries.script.variable_scripts import VARIABLE_SCRIPTS @@ -6,6 +7,13 @@ from ytdl_sub.script.script import Script class Scriptable(ABC): + @classmethod + def add_sanitized_variables(cls, variables: Dict[str, str]) -> Dict[str, str]: + sanitized_variables = { + f"{name}_sanitized": f"{{%sanitize({name})}}" for name in variables.keys() + } + return dict(variables, **sanitized_variables) + def __init__(self): self.script = Script(VARIABLE_SCRIPTS) self.unresolvable: Set[str] = set()