diff --git a/.gitignore b/.gitignore index e18f90d3..5e9f1aad 100644 --- a/.gitignore +++ b/.gitignore @@ -146,8 +146,11 @@ docker/testing/volumes .local/ .ytdl-sub-working-directory +.ytdl-sub-lock ffmpeg.exe ffprobe.exe -tools/docgen/out \ No newline at end of file +tools/docgen/out + +prof/ diff --git a/src/ytdl_sub/config/overrides.py b/src/ytdl_sub/config/overrides.py index c3b791b7..8f790022 100644 --- a/src/ytdl_sub/config/overrides.py +++ b/src/ytdl_sub/config/overrides.py @@ -4,15 +4,16 @@ from typing import Iterable from typing import Optional from typing import Set -import mergedeep - from ytdl_sub.entries.entry import Entry from ytdl_sub.entries.script.variable_definitions import VARIABLES from ytdl_sub.entries.variables.override_variables import REQUIRED_OVERRIDE_VARIABLE_NAMES from ytdl_sub.entries.variables.override_variables import OverrideHelpers from ytdl_sub.script.parser import parse from ytdl_sub.script.script import Script +from ytdl_sub.script.types.function import BuiltInFunction from ytdl_sub.script.types.resolvable import Resolvable +from ytdl_sub.script.types.resolvable import String +from ytdl_sub.script.types.syntax_tree import SyntaxTree from ytdl_sub.script.utils.exceptions import ScriptVariableNotResolved from ytdl_sub.utils.exceptions import InvalidVariableNameException from ytdl_sub.utils.exceptions import StringFormattingException @@ -134,29 +135,35 @@ class Overrides(UnstructuredDictFormatterValidator, Scriptable): ) def initial_variables( - self, unresolved_variables: Optional[Dict[str, str]] = None - ) -> Dict[str, str]: + self, unresolved_variables: Optional[Dict[str, SyntaxTree]] = None + ) -> Dict[str, SyntaxTree]: """ Returns ------- Variables and format strings for all Override variables + additional variables (Optional) """ - initial_variables: Dict[str, str] = {} - mergedeep.merge( - initial_variables, - self.dict_with_format_strings, - unresolved_variables if unresolved_variables else {}, - ) - return ScriptUtils.add_sanitized_variables(initial_variables) + initial_variables: Dict[str, SyntaxTree] = self.dict_with_parsed_format_strings + if unresolved_variables: + initial_variables |= unresolved_variables + return ScriptUtils.add_sanitized_parsed_variables(initial_variables) def initialize_script(self, unresolved_variables: Set[str]) -> "Overrides": """ Initialize the override script with any unresolved variables """ - self.script.add( + self.script.add_parsed( self.initial_variables( unresolved_variables={ - var_name: f"{{%throw('Plugin variable {var_name} has not been created yet')}}" + var_name: SyntaxTree( + ast=[ + BuiltInFunction( + name="throw", + args=[ + String(f"Plugin variable {var_name} has not been created yet") + ], + ) + ] + ) for var_name in unresolved_variables } ) diff --git a/src/ytdl_sub/config/plugin/preset_plugins.py b/src/ytdl_sub/config/plugin/preset_plugins.py index 7eda8a9d..61305a37 100644 --- a/src/ytdl_sub/config/plugin/preset_plugins.py +++ b/src/ytdl_sub/config/plugin/preset_plugins.py @@ -1,5 +1,7 @@ +from typing import Iterable from typing import List from typing import Optional +from typing import Set from typing import Tuple from typing import Type @@ -44,3 +46,34 @@ class PresetPlugins: if plugin_type in plugin_option_types: return self.plugin_options[plugin_option_types.index(plugin_type)] return None + + def get_added_and_modified_variables( + self, additional_options: List[OptionsValidator] + ) -> Iterable[Tuple[OptionsValidator, Set[str], Set[str]]]: + """ + Iterates and returns the plugin options, added variables, modified variables + """ + for plugin_options in self.plugin_options + additional_options: + added_variables: Set[str] = set() + modified_variables: Set[str] = set() + + for plugin_added_variables in plugin_options.added_variables( + unresolved_variables=set(), + ).values(): + added_variables |= set(plugin_added_variables) + + for plugin_modified_variables in plugin_options.modified_variables().values(): + modified_variables = plugin_modified_variables + + yield plugin_options, added_variables, modified_variables + + def get_all_variables(self, additional_options: List[OptionsValidator]) -> Set[str]: + """ + Returns set of all added and modified variables' names. + """ + all_variables: Set[str] = set() + for _, added, modified in self.get_added_and_modified_variables(additional_options): + all_variables.update(added) + all_variables.update(modified) + + return all_variables diff --git a/src/ytdl_sub/config/preset.py b/src/ytdl_sub/config/preset.py index 31ac34a8..d8380839 100644 --- a/src/ytdl_sub/config/preset.py +++ b/src/ytdl_sub/config/preset.py @@ -2,6 +2,7 @@ import copy from typing import Any from typing import Dict from typing import List +from typing import Set from mergedeep import mergedeep @@ -11,7 +12,6 @@ from ytdl_sub.config.plugin.plugin_mapping import PluginMapping from ytdl_sub.config.plugin.preset_plugins import PresetPlugins from ytdl_sub.config.preset_options import OutputOptions from ytdl_sub.config.preset_options import YTDLOptions -from ytdl_sub.config.validators.variable_validation import VariableValidation from ytdl_sub.downloaders.url.validators import MultiUrlValidator from ytdl_sub.prebuilt_presets import PREBUILT_PRESET_NAMES from ytdl_sub.prebuilt_presets import PUBLISHED_PRESET_NAMES @@ -172,6 +172,37 @@ class Preset(_PresetShell): mergedeep.merge({}, *reversed(presets_to_merge), strategy=mergedeep.Strategy.ADDITIVE) ) + def _initialize_overrides_script(self, overrides: Overrides) -> Overrides: + """ + Do some gymnastics to initialize the Overrides script. + """ + unresolved_variables: Set[str] = set() + + for ( + plugin_options, + added_variables, + modified_variables, + ) in self.plugins.get_added_and_modified_variables( + additional_options=[self.downloader_options, self.output_options] + ): + for added_variable in added_variables: + if not overrides.ensure_added_plugin_variable_valid(added_variable=added_variable): + # pylint: disable=protected-access + raise plugin_options._validation_exception( + f"Cannot use the variable name {added_variable} because it exists as a" + " built-in ytdl-sub variable name." + ) + # pylint: enable=protected-access + + # Set unresolved as variables that are added but do not exist as + # entry/override variables since they are created at run-time + unresolved_variables |= added_variables | modified_variables + + # Initialize overrides with unresolved variables + modified variables to throw an error. + # For modified variables, this is to prevent a resolve(update=True) to setting any + # dependencies until it has been explicitly added + return overrides.initialize_script(unresolved_variables=unresolved_variables) + def __init__(self, config: ConfigValidator, name: str, value: Any): super().__init__(name=name, value=value) @@ -192,16 +223,11 @@ class Preset(_PresetShell): ) self.plugins: PresetPlugins = self._validate_and_get_plugins() - self.overrides = self._validate_key(key="overrides", validator=Overrides, default={}) - + self.overrides = self._initialize_overrides_script( + overrides=self._validate_key(key="overrides", validator=Overrides, default={}) + ) self.overrides.ensure_variable_names_not_a_plugin(plugin_names=PRESET_KEYS) - VariableValidation( - downloader_options=self.downloader_options, - output_options=self.output_options, - plugins=self.plugins, - ).initialize_preset_overrides(overrides=self.overrides).ensure_proper_usage() - @property def name(self) -> str: """ diff --git a/src/ytdl_sub/config/validators/variable_validation.py b/src/ytdl_sub/config/validators/variable_validation.py index e2d5e85d..bb8ea705 100644 --- a/src/ytdl_sub/config/validators/variable_validation.py +++ b/src/ytdl_sub/config/validators/variable_validation.py @@ -1,10 +1,4 @@ -import copy from typing import Dict -from typing import Iterable -from typing import List -from typing import Optional -from typing import Set -from typing import Tuple from ytdl_sub.config.overrides import Overrides from ytdl_sub.config.plugin.plugin_mapping import PluginMapping @@ -13,155 +7,27 @@ from ytdl_sub.config.plugin.preset_plugins import PresetPlugins from ytdl_sub.config.preset_options import OutputOptions from ytdl_sub.config.validators.options import OptionsValidator from ytdl_sub.downloaders.url.validators import MultiUrlValidator -from ytdl_sub.entries.variables.override_variables import REQUIRED_OVERRIDE_VARIABLE_NAMES -from ytdl_sub.script.script import Script -from ytdl_sub.script.script import _is_function -from ytdl_sub.utils.scriptable import BASE_SCRIPT -from ytdl_sub.validators.string_formatter_validators import to_variable_dependency_format_string from ytdl_sub.validators.string_formatter_validators import validate_formatters -# Entry variables to mock during validation -_DUMMY_ENTRY_VARIABLES: Dict[str, str] = { - name: to_variable_dependency_format_string( - # pylint: disable=protected-access - script=BASE_SCRIPT, - parsed_format_string=BASE_SCRIPT._variables[name], - # pylint: enable=protected-access - ) - for name in BASE_SCRIPT.variable_names -} - - -def _add_dummy_variables(variables: Iterable[str]) -> Dict[str, str]: - dummy_variables: Dict[str, str] = {} - for var in variables: - dummy_variables[var] = "" - dummy_variables[f"{var}_sanitized"] = "" - - return dummy_variables - - -def _add_dummy_overrides(overrides: Overrides) -> Dict[str, str]: - # Have the dummy override variable contain all variable deps that it uses in the string - dummy_overrides: Dict[str, str] = {} - for override_name in _override_variables(overrides): - if _is_function(override_name): - continue - - # pylint: disable=protected-access - dummy_overrides[override_name] = to_variable_dependency_format_string( - script=overrides.script, parsed_format_string=overrides.script._variables[override_name] - ) - # pylint: enable=protected-access - return dummy_overrides - - -def _get_added_and_modified_variables( - plugins: PresetPlugins, downloader_options: MultiUrlValidator, output_options: OutputOptions -) -> Iterable[Tuple[OptionsValidator, Set[str], Set[str]]]: - """ - Iterates and returns the plugin options, added variables, modified variables - """ - options: List[OptionsValidator] = plugins.plugin_options - options.append(downloader_options) - options.append(output_options) - - for plugin_options in options: - added_variables: Set[str] = set() - modified_variables: Set[str] = set() - - for plugin_added_variables in plugin_options.added_variables( - unresolved_variables=set(), - ).values(): - added_variables |= set(plugin_added_variables) - - for plugin_modified_variables in plugin_options.modified_variables().values(): - modified_variables = plugin_modified_variables - - yield plugin_options, added_variables, modified_variables - - -def _override_variables(overrides: Overrides) -> Set[str]: - return set(list(overrides.initial_variables().keys())) - class VariableValidation: def __init__( self, + overrides: Overrides, downloader_options: MultiUrlValidator, output_options: OutputOptions, plugins: PresetPlugins, ): + self.overrides = overrides self.downloader_options = downloader_options self.output_options = output_options self.plugins = plugins - self.script: Optional[Script] = None - self.resolved_variables: Set[str] = set() - self.unresolved_variables: Set[str] = set() - - def initialize_preset_overrides(self, overrides: Overrides) -> "VariableValidation": - """ - Do some gymnastics to initialize the Overrides script. - """ - override_variables = set(list(overrides.initial_variables().keys())) - - # Set resolved variables as all entry + override variables - # at this point to generate every possible added/modified variable - self.resolved_variables = set(_DUMMY_ENTRY_VARIABLES.keys()) | override_variables - plugin_variables: Set[str] = set() - - for ( - plugin_options, - added_variables, - modified_variables, - ) in _get_added_and_modified_variables( - plugins=self.plugins, - downloader_options=self.downloader_options, - output_options=self.output_options, - ): - - for added_variable in added_variables: - if not overrides.ensure_added_plugin_variable_valid(added_variable=added_variable): - # pylint: disable=protected-access - raise plugin_options._validation_exception( - f"Cannot use the variable name {added_variable} because it exists as a" - " built-in ytdl-sub variable name." - ) - # pylint: enable=protected-access - - # Set unresolved as variables that are added but do not exist as - # entry/override variables since they are created at run-time - self.unresolved_variables |= added_variables | modified_variables - plugin_variables |= added_variables | modified_variables - - # Then update resolved variables to reflect that - self.resolved_variables -= self.unresolved_variables - - # Initialize overrides with unresolved variables + modified variables to throw an error. - # For modified variables, this is to prevent a resolve(update=True) to setting any - # dependencies until it has been explicitly added - overrides = overrides.initialize_script(unresolved_variables=self.unresolved_variables) - - # copy the script and mock entry variables - self.script = copy.deepcopy(overrides.script) - self.script.add( - variables=_add_dummy_overrides(overrides=overrides) - | _add_dummy_variables(variables=plugin_variables) - | _DUMMY_ENTRY_VARIABLES + self.script = self.overrides.script + self.unresolved_variables = self.plugins.get_all_variables( + additional_options=[self.output_options, self.downloader_options] ) - return self - - def _update_script(self) -> None: - _ = self.script.resolve(unresolvable=self.unresolved_variables, update=True) - - def _add_subscription_override_variables(self) -> None: - """ - Add dummy subscription variables for script validation - """ - self.resolved_variables |= REQUIRED_OVERRIDE_VARIABLE_NAMES - def _add_variables(self, plugin_op: PluginOperation, options: OptionsValidator) -> None: """ Add dummy variables for script validation @@ -171,19 +37,17 @@ class VariableValidation: ).get(plugin_op, set()) modified_variables = options.modified_variables().get(plugin_op, set()) - resolved_variables = added_variables | modified_variables + self.unresolved_variables -= added_variables | modified_variables - self.resolved_variables |= resolved_variables - self.unresolved_variables -= resolved_variables - - def ensure_proper_usage(self) -> None: + def ensure_proper_usage(self) -> Dict: """ Validate variables resolve as plugins are executed, and return a mock script which contains actualized added variables from the plugins """ + resolved_subscription: Dict = {} + self._add_variables(PluginOperation.DOWNLOADER, options=self.downloader_options) - self._add_subscription_override_variables() # Always add output options first self._add_variables(PluginOperation.MODIFY_ENTRY_METADATA, options=self.output_options) @@ -200,16 +64,28 @@ class VariableValidation: self._add_variables(PluginOperation.MODIFY_ENTRY, options=plugin_options) # Validate that any formatter in the plugin options can resolve - validate_formatters( + resolved_subscription |= validate_formatters( script=self.script, unresolved_variables=self.unresolved_variables, validator=plugin_options, ) - validate_formatters( + resolved_subscription |= validate_formatters( script=self.script, unresolved_variables=self.unresolved_variables, validator=self.output_options, ) + # TODO: make this a function + raw_download_output = validate_formatters( + script=self.script, + unresolved_variables=self.unresolved_variables, + validator=self.downloader_options.urls, + ) + resolved_subscription["download"] = [] + for url_output in raw_download_output["download"]: + if url_output["url"]: + resolved_subscription["download"].append(url_output) + assert not self.unresolved_variables + return resolved_subscription diff --git a/src/ytdl_sub/downloaders/url/validators.py b/src/ytdl_sub/downloaders/url/validators.py index 4504eccf..e518cdf8 100644 --- a/src/ytdl_sub/downloaders/url/validators.py +++ b/src/ytdl_sub/downloaders/url/validators.py @@ -21,7 +21,7 @@ class UrlThumbnailValidator(StrictDictValidator): def __init__(self, name, value): super().__init__(name, value) - self._name = self._validate_key(key="name", validator=StringFormatterValidator) + self._thumb_name = self._validate_key(key="name", validator=StringFormatterValidator) self._uid = self._validate_key(key="uid", validator=OverridesStringFormatterValidator) @property @@ -29,7 +29,7 @@ class UrlThumbnailValidator(StrictDictValidator): """ File name for the thumbnail """ - return self._name + return self._thumb_name @property def uid(self) -> OverridesStringFormatterValidator: diff --git a/src/ytdl_sub/script/parser.py b/src/ytdl_sub/script/parser.py index 9dd3f186..73dbb3f4 100644 --- a/src/ytdl_sub/script/parser.py +++ b/src/ytdl_sub/script/parser.py @@ -31,6 +31,8 @@ from ytdl_sub.script.utils.exceptions import InvalidSyntaxException from ytdl_sub.script.utils.exceptions import InvalidVariableName from ytdl_sub.script.utils.exceptions import UserException from ytdl_sub.script.utils.exceptions import VariableDoesNotExist +from ytdl_sub.script.utils.name_validation import is_function +from ytdl_sub.script.utils.name_validation import to_function_name from ytdl_sub.script.utils.name_validation import validate_variable_name # pylint: disable=invalid-name @@ -144,6 +146,9 @@ class _Parser: ): self._text = text self._name = name + if name and is_function(name): + self._name = to_function_name(name) + self._custom_function_names = custom_function_names self._variable_names = variable_names self._pos = 0 diff --git a/src/ytdl_sub/script/script.py b/src/ytdl_sub/script/script.py index d6e482fe..67eeeaf1 100644 --- a/src/ytdl_sub/script/script.py +++ b/src/ytdl_sub/script/script.py @@ -20,28 +20,13 @@ from ytdl_sub.script.utils.exceptions import IncompatibleFunctionArguments from ytdl_sub.script.utils.exceptions import InvalidCustomFunctionArguments from ytdl_sub.script.utils.exceptions import RuntimeException from ytdl_sub.script.utils.exceptions import ScriptVariableNotResolved +from ytdl_sub.script.utils.name_validation import is_function +from ytdl_sub.script.utils.name_validation import to_function_definition_name +from ytdl_sub.script.utils.name_validation import to_function_name from ytdl_sub.script.utils.name_validation import validate_variable_name from ytdl_sub.script.utils.type_checking import FunctionSpec -def _is_function(override_name: str): - return override_name.startswith("%") - - -def _function_name(function_key: str) -> str: - """ - Drop the % in %custom_function - """ - return function_key[1:] - - -def _to_function_definition_name(function_key: str) -> str: - """ - Add % in %custom_function - """ - return f"%{function_key}" - - class Script: """ Takes a dictionary of both @@ -241,23 +226,23 @@ class Script: def __init__(self, script: Dict[str, str]): function_names: Set[str] = { - _function_name(name) for name in script.keys() if _is_function(name) + to_function_name(name) for name in script.keys() if is_function(name) } variable_names: Set[str] = { - validate_variable_name(name) for name in script.keys() if not _is_function(name) + validate_variable_name(name) for name in script.keys() if not is_function(name) } self._functions: Dict[str, SyntaxTree] = { # custom_function_name must be passed to properly type custom function # arguments uniquely if they're nested (i.e. $0 to $custom_func___0) - _function_name(function_key): parse( + to_function_name(function_key): parse( text=function_value, - name=_function_name(function_key), + name=to_function_name(function_key), custom_function_names=function_names, variable_names=variable_names, ) for function_key, function_value in script.items() - if _is_function(function_key) + if is_function(function_key) } self._variables: Dict[str, SyntaxTree] = { @@ -268,7 +253,7 @@ class Script: variable_names=variable_names, ) for variable_key, variable_value in script.items() - if not _is_function(variable_key) + if not is_function(variable_key) } self._validate() @@ -485,12 +470,12 @@ class Script: added_variables_to_validate: Set[str] = set() functions_to_add = { - _function_name(name): definition + to_function_name(name): definition for name, definition in variables.items() - if _is_function(name) + if is_function(name) } variables_to_add = { - name: definition for name, definition in variables.items() if not _is_function(name) + name: definition for name, definition in variables.items() if not is_function(name) } custom_function_names = set(self._functions.keys()) | functions_to_add.keys() @@ -520,6 +505,46 @@ class Script: return self + def add_parsed(self, variables: Dict[str, SyntaxTree]) -> "Script": + """ + Adds already parsed, new variables to the script. + + Parameters + ---------- + variables + Mapping containing variable name to definition. + + Returns + ------- + Script + self + """ + added_variables_to_validate: Set[str] = set() + + functions_to_add = { + to_function_name(name): definition + for name, definition in variables.items() + if is_function(name) + } + variables_to_add = { + name: definition for name, definition in variables.items() if not is_function(name) + } + + for definitions in [functions_to_add, variables_to_add]: + for name, parsed in definitions.items(): + if parsed.maybe_resolvable is None: + added_variables_to_validate.add(name) + + if name in functions_to_add: + self._functions[name] = parsed + else: + self._variables[name] = parsed + + if added_variables_to_validate: + self._validate(added_variables=added_variables_to_validate) + + return self + def resolve_once( self, variable_definitions: Dict[str, str], @@ -560,6 +585,46 @@ class Script: for name in variable_definitions.keys(): self._variables.pop(name, None) + def resolve_once_parsed( + self, + variable_definitions: Dict[str, SyntaxTree], + resolved: Optional[Dict[str, Resolvable]] = None, + unresolvable: Optional[Set[str]] = None, + update: bool = False, + ) -> Dict[str, Resolvable]: + """ + Given a new set of variable definitions, resolve them using the Script, but do not + add them to the Script itself. + + Parameters + ---------- + variable_definitions + Variables to resolve, but not store in the Script + resolved + Optional. Pre-resolved variables that should be used instead of what is in the script. + unresolvable + Optional. Unresolvable variables that will be ignored in resolution, including all + variables with a dependency to them. + update + Whether to update the script's state with resolved variables. Defaults to False. + + Returns + ------- + Dict[str, Resolvable] + Dict containing the variable names to their resolved values. + """ + try: + self.add_parsed(variable_definitions) + return self._resolve( + pre_resolved=resolved, + unresolvable=unresolvable, + output_filter=set(list(variable_definitions.keys())), + update=update, + ).output + finally: + for name in variable_definitions.keys(): + self._variables.pop(name, None) + def get(self, variable_name: str) -> Resolvable: """ Parameters @@ -605,4 +670,4 @@ class Script: Set[str] Names of all functions within the Script. """ - return set(_to_function_definition_name(name) for name in self._functions.keys()) + return set(to_function_definition_name(name) for name in self._functions.keys()) diff --git a/src/ytdl_sub/script/utils/name_validation.py b/src/ytdl_sub/script/utils/name_validation.py index de187ec8..5b2597ce 100644 --- a/src/ytdl_sub/script/utils/name_validation.py +++ b/src/ytdl_sub/script/utils/name_validation.py @@ -58,3 +58,24 @@ def validate_custom_function_name(custom_function_name: str) -> None: f"Custom function name '%{custom_function_name}' is invalid:" " The name is used by a built-in function and cannot be overwritten." ) + + +def is_function(override_name: str): + """ + Whether the definition is a function or not. + """ + return override_name.startswith("%") + + +def to_function_name(function_key: str) -> str: + """ + Drop the % in %custom_function + """ + return function_key[1:] + + +def to_function_definition_name(function_key: str) -> str: + """ + Add % in %custom_function + """ + return f"%{function_key}" diff --git a/src/ytdl_sub/subscriptions/base_subscription.py b/src/ytdl_sub/subscriptions/base_subscription.py index 39a8a70a..656e98ed 100644 --- a/src/ytdl_sub/subscriptions/base_subscription.py +++ b/src/ytdl_sub/subscriptions/base_subscription.py @@ -8,12 +8,14 @@ from ytdl_sub.config.plugin.preset_plugins import PresetPlugins from ytdl_sub.config.preset import Preset from ytdl_sub.config.preset_options import OutputOptions from ytdl_sub.config.preset_options import YTDLOptions +from ytdl_sub.config.validators.variable_validation import VariableValidation from ytdl_sub.downloaders.url.validators import MultiUrlValidator from ytdl_sub.entries.variables.override_variables import SubscriptionVariables from ytdl_sub.utils.exceptions import SubscriptionPermissionError from ytdl_sub.utils.file_handler import FileHandler from ytdl_sub.utils.file_handler import FileHandlerTransactionLog from ytdl_sub.utils.logger import Logger +from ytdl_sub.utils.yaml import dump_yaml from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive logger = Logger.get("subscription") @@ -76,6 +78,14 @@ class BaseSubscription(ABC): } ) + # Validate after adding the subscription name + self._validated_dict = VariableValidation( + overrides=self.overrides, + downloader_options=self.downloader_options, + output_options=self.output_options, + plugins=self.plugins, + ).ensure_proper_usage() + self._enhanced_download_archive: Optional[EnhancedDownloadArchive] = ( _initialize_download_archive( output_options=self.output_options, @@ -88,9 +98,9 @@ class BaseSubscription(ABC): # Add post-archive variables self.overrides.add( { - SubscriptionVariables.subscription_has_download_archive(): f"""{{ - %bool({self.download_archive.num_entries > 0}) - }}""", + SubscriptionVariables.subscription_has_download_archive(): ( + f"{{%bool({self.download_archive.num_entries > 0})}}" + ), } ) @@ -245,3 +255,11 @@ class BaseSubscription(ABC): Subscription in yaml format """ return self._preset_options.yaml + + def resolved_yaml(self) -> str: + """ + Returns + ------- + Human-readable, condensed YAML definition of the subscription. + """ + return dump_yaml(self._validated_dict) diff --git a/src/ytdl_sub/utils/script.py b/src/ytdl_sub/utils/script.py index e3a17b2f..027a0bd8 100644 --- a/src/ytdl_sub/utils/script.py +++ b/src/ytdl_sub/utils/script.py @@ -4,7 +4,6 @@ from typing import Any from typing import Dict from ytdl_sub.script.parser import parse -from ytdl_sub.script.script import _is_function from ytdl_sub.script.types.array import UnresolvedArray from ytdl_sub.script.types.function import BuiltInFunction from ytdl_sub.script.types.function import Function @@ -15,8 +14,10 @@ from ytdl_sub.script.types.resolvable import Float from ytdl_sub.script.types.resolvable import Integer from ytdl_sub.script.types.resolvable import Lambda from ytdl_sub.script.types.resolvable import String +from ytdl_sub.script.types.syntax_tree import SyntaxTree from ytdl_sub.script.types.variable import Variable from ytdl_sub.script.utils.exceptions import UNREACHABLE +from ytdl_sub.script.utils.name_validation import is_function # pylint: disable=too-many-return-statements @@ -30,10 +31,26 @@ class ScriptUtils: sanitized_variables = { f"{name}_sanitized": f"{{%sanitize({name})}}" for name in variables.keys() - if not _is_function(name) + if not is_function(name) } return dict(variables, **sanitized_variables) + @classmethod + def add_sanitized_parsed_variables( + cls, variables: Dict[str, SyntaxTree] + ) -> Dict[str, SyntaxTree]: + """ + Helper to add sanitized variables to a Script + """ + sanitized_variables = { + f"{name}_sanitized": SyntaxTree( + ast=[BuiltInFunction(name="sanitize", args=[Variable(name)])] + ) + for name in variables.keys() + if not is_function(name) + } + return variables | sanitized_variables + @classmethod def to_script(cls, value: Any, sort_keys: bool = True) -> str: """ diff --git a/src/ytdl_sub/validators/string_formatter_validators.py b/src/ytdl_sub/validators/string_formatter_validators.py index 567f0f8d..19a59344 100644 --- a/src/ytdl_sub/validators/string_formatter_validators.py +++ b/src/ytdl_sub/validators/string_formatter_validators.py @@ -11,6 +11,7 @@ from ytdl_sub.script.types.syntax_tree import SyntaxTree from ytdl_sub.script.utils.exceptions import RuntimeException from ytdl_sub.script.utils.exceptions import ScriptVariableNotResolved from ytdl_sub.script.utils.exceptions import UserException +from ytdl_sub.script.utils.exceptions import UserThrownRuntimeError from ytdl_sub.utils.exceptions import StringFormattingVariableNotFoundException from ytdl_sub.utils.script import ScriptUtils from ytdl_sub.validators.validators import DictValidator @@ -19,6 +20,8 @@ from ytdl_sub.validators.validators import LiteralDictValidator from ytdl_sub.validators.validators import StringValidator from ytdl_sub.validators.validators import Validator +# pylint: disable=protected-access + class StringFormatterValidator(StringValidator): """ @@ -51,7 +54,10 @@ class StringFormatterValidator(StringValidator): def __init__(self, name, value: str): super().__init__(name=name, value=value) try: - _ = parse(str(value)) + self._parsed = parse( + text=str(value), + name=self.leaf_name, + ) except UserException as exc: raise self._validation_exception(exc) from exc @@ -65,6 +71,16 @@ class StringFormatterValidator(StringValidator): """ return self._value + @property + @final + def parsed(self) -> SyntaxTree: + """ + Returns + ------- + The parsed format string. + """ + return self._parsed + def post_process(self, resolved: str) -> str: """ Returns @@ -167,9 +183,14 @@ class DictFormatterValidator(LiteralDictValidator): @property def dict_with_format_strings(self) -> Dict[str, str]: - """Returns dict with the format strings themselves""" + """Returns dict with the format strings themselves.""" return {key: string_formatter.format_string for key, string_formatter in self.dict.items()} + @property + def dict_with_parsed_format_strings(self) -> Dict[str, SyntaxTree]: + """Returns dict with the parsed format strings.""" + return {key: string_formatter.parsed for key, string_formatter in self.dict.items()} + class OverridesDictFormatterValidator(DictFormatterValidator): """ @@ -199,10 +220,8 @@ def to_variable_dependency_format_string(script: Script, parsed_format_string: S dummy_format_string = "" for var in parsed_format_string.variables: dummy_format_string += f"{{ {var.name} }}" - # pylint: disable=protected-access for variable_dependency in script._variables[var.name].variables: dummy_format_string += f"{{ {variable_dependency.name} }}" - # pylint: enable=protected-access return dummy_format_string @@ -210,16 +229,15 @@ def _validate_formatter( mock_script: Script, unresolved_variables: Set[str], formatter_validator: Union[StringFormatterValidator, OverridesStringFormatterValidator], -) -> None: - is_static_formatter = False - unresolvable = unresolved_variables - if isinstance(formatter_validator, OverridesStringFormatterValidator): - is_static_formatter = True - unresolvable = unresolved_variables.union({VARIABLES.entry_metadata.variable_name}) +) -> str: + parsed = formatter_validator.parsed + if resolved := parsed.maybe_resolvable: + return resolved.native + + is_static_formatter = isinstance(formatter_validator, OverridesStringFormatterValidator) + if is_static_formatter: + unresolved_variables = unresolved_variables.union({VARIABLES.entry_metadata.variable_name}) - parsed = parse( - text=formatter_validator.format_string, - ) variable_names = {var.name for var in parsed.variables} custom_function_names = {f"%{func.name}" for func in parsed.custom_functions} @@ -233,21 +251,20 @@ def _validate_formatter( "contains the following custom functions that do not exist: " f"{', '.join(sorted(custom_function_names - mock_script.function_names))}" ) - if unresolved := variable_names.intersection(unresolvable): + if unresolved := variable_names.intersection(unresolved_variables): raise StringFormattingVariableNotFoundException( "contains the following variables that are unresolved when executing this " f"formatter: {', '.join(sorted(unresolved))}" ) try: - mock_script.resolve_once( - { - "tmp_var": to_variable_dependency_format_string( - script=mock_script, parsed_format_string=parsed - ) - }, - unresolvable=unresolvable, - update=True, - ) + if is_static_formatter: + return mock_script.resolve_once_parsed( + {"tmp_var": formatter_validator.parsed}, + unresolvable=unresolved_variables, + update=True, + )["tmp_var"].native + + return formatter_validator.format_string except RuntimeException as exc: if isinstance(exc, ScriptVariableNotResolved) and is_static_formatter: raise StringFormattingVariableNotFoundException( @@ -255,45 +272,60 @@ def _validate_formatter( "entry variables" ) from exc raise StringFormattingVariableNotFoundException(exc) from exc + except UserThrownRuntimeError as exc: + # Errors are expected for non-static formatters due to missing entry + # data. Raise otherwise. + if not is_static_formatter: + return formatter_validator.format_string + raise exc def validate_formatters( script: Script, unresolved_variables: Set[str], validator: Validator, -) -> None: +) -> Dict: """ Ensure all OverridesStringFormatterValidator's only contain variables from the overrides and resolve. """ + resolved_dict: Dict = {} + if isinstance(validator, DictValidator): - # pylint: disable=protected-access + resolved_dict[validator.leaf_name] = {} # Usage of protected variables in other validators is fine. The reason to keep # them protected is for readability when using them in subscriptions. for validator_value in validator._validator_dict.values(): - validate_formatters( + resolved_dict[validator.leaf_name] |= validate_formatters( script=script, unresolved_variables=unresolved_variables, validator=validator_value, ) - # pylint: enable=protected-access elif isinstance(validator, ListValidator): + resolved_dict[validator.leaf_name] = [] for list_value in validator.list: - validate_formatters( + list_output = validate_formatters( script=script, unresolved_variables=unresolved_variables, validator=list_value, ) + assert len(list_output) == 1 + resolved_dict[validator.leaf_name].append(list(list_output.values())[0]) elif isinstance(validator, (StringFormatterValidator, OverridesStringFormatterValidator)): - _validate_formatter( + resolved_dict[validator.leaf_name] = _validate_formatter( mock_script=script, unresolved_variables=unresolved_variables, formatter_validator=validator, ) elif isinstance(validator, (DictFormatterValidator, OverridesDictFormatterValidator)): + resolved_dict[validator.leaf_name] = {} for validator_value in validator.dict.values(): - _validate_formatter( + resolved_dict[validator.leaf_name] |= _validate_formatter( mock_script=script, unresolved_variables=unresolved_variables, formatter_validator=validator_value, ) + else: + resolved_dict[validator.leaf_name] = validator._value + + return resolved_dict diff --git a/src/ytdl_sub/validators/validators.py b/src/ytdl_sub/validators/validators.py index fbb13c95..6c1356ae 100644 --- a/src/ytdl_sub/validators/validators.py +++ b/src/ytdl_sub/validators/validators.py @@ -95,21 +95,11 @@ class Validator(ABC): @final @property - def _root_name(self) -> str: + def leaf_name(self) -> str: """ Returns ------- - "first" from the first.element.of.the.name - """ - return self._name.split(".")[0] - - @final - @property - def _leaf_name(self) -> str: - """ - Returns - ------- - "first" from the first.element.of.the.name + "name" from the first.element.of.the.name """ return self._name.split(".")[-1] @@ -274,7 +264,7 @@ class DictValidator(Validator): value=self._dict.get(key, default), ) - self.__validator_dict[validator_name] = validator_instance + self.__validator_dict[key] = validator_instance return validator_instance @final diff --git a/tests/unit/config/test_preset.py b/tests/unit/config/test_preset.py index 8e9b4c89..a42439be 100644 --- a/tests/unit/config/test_preset.py +++ b/tests/unit/config/test_preset.py @@ -6,7 +6,6 @@ import pytest from ytdl_sub.config.preset import Preset from ytdl_sub.plugins.nfo_tags import NfoTagsOptions -from ytdl_sub.utils.exceptions import StringFormattingVariableNotFoundException from ytdl_sub.utils.exceptions import ValidationException @@ -157,100 +156,6 @@ class TestPreset: }, ) - def test_preset_error__source_variable_does_not_exist( - self, config_file, output_options, youtube_video - ): - with pytest.raises( - StringFormattingVariableNotFoundException, - match="contains the following variables that do not exist: dne_var", - ): - _ = Preset( - config=config_file, - name="test", - value={ - "download": youtube_video, - "output_options": {"output_directory": "dir", "file_name": "{dne_var}"}, - }, - ) - - def test_preset_error__override_variable_does_not_exist( - self, config_file, output_options, youtube_video - ): - with pytest.raises( - StringFormattingVariableNotFoundException, - match="contains the following variables that do not exist: dne_var", - ): - _ = Preset( - config=config_file, - name="test", - value={ - "download": youtube_video, - "output_options": {"output_directory": "{dne_var}", "file_name": "file"}, - }, - ) - - def test_preset_error__dict_source_variable_does_not_exist( - self, config_file, output_options, youtube_video - ): - with pytest.raises( - StringFormattingVariableNotFoundException, - match="contains the following variables that do not exist: dne_var", - ): - _ = Preset( - config=config_file, - name="test", - value={ - "download": youtube_video, - "output_options": {"output_directory": "dir", "file_name": "file"}, - "nfo_tags": { - "nfo_name": "the nfo name", - "nfo_root": "the root", - "tags": {"tag_a": "{dne_var}"}, - }, - }, - ) - - def test_preset_error__dict_override_variable_does_not_exist( - self, config_file, output_options, youtube_video - ): - with pytest.raises( - StringFormattingVariableNotFoundException, - match="contains the following variables that do not exist: dne_var", - ): - _ = Preset( - config=config_file, - name="test", - value={ - "download": youtube_video, - "output_options": output_options, - "output_directory_nfo_tags": { - "nfo_name": "the nfo name", - "nfo_root": "the root", - "tags": {"tag_a": "{dne_var}"}, - }, - }, - ) - - def test_preset_error__dict_override_variable_not_static( - self, config_file, output_options, youtube_video - ): - with pytest.raises( - StringFormattingVariableNotFoundException, - match="static formatters must contain variables that " - "have no dependency to entry variables", - ): - _ = Preset( - config=config_file, - name="test", - value={ - "download": youtube_video, - "output_options": { - "output_directory": "{title}", - "file_name": "{uid}", - }, - }, - ) - def test_preset_with_multi_url__contains_empty_url(self, config_file, output_options): _ = Preset( config=config_file, @@ -412,50 +317,3 @@ class TestPreset: "overrides": {name: "ack"}, }, ) - - def test_preset_error_added_url_variable_cannot_resolve(self, config_file, output_options): - with pytest.raises( - ValidationException, - match=re.escape( - "variable the_bad_one cannot use the variables subtitles_ext because it " - "depends on other variables that are computed later in execution" - ), - ): - _ = Preset( - config=config_file, - name="test", - value={ - "download": { - "url": "youtube.com/watch?v=123abc", - "variables": {"the_bad_one": "{subtitles_ext}"}, - }, - "subtitles": { - "embed_subtitles": True, - }, - "output_options": {"output_directory": "dir", "file_name": "acjk"}, - }, - ) - - def test_preset_error_override_name_conflicts_with_plugin(self, config_file, output_options): - with pytest.raises( - ValidationException, - match=re.escape( - "Override variable with name throttle_protection cannot be used since it is the " - "name of a plugin. Perhaps you meant to define it as a plugin? If so, indent it " - "left to make it at the same level as overrides." - ), - ): - _ = Preset( - config=config_file, - name="test", - value={ - "download": { - "url": "youtube.com/watch?v=123abc", - }, - "subtitles": { - "embed_subtitles": True, - }, - "output_options": {"output_directory": "dir", "file_name": "acjk"}, - "overrides": {"throttle_protection": "nope"}, - }, - ) diff --git a/tests/unit/config/test_subscription.py b/tests/unit/config/test_subscription.py index 423ff8b0..7e14fe05 100644 --- a/tests/unit/config/test_subscription.py +++ b/tests/unit/config/test_subscription.py @@ -6,6 +6,7 @@ from typing import Dict from unittest.mock import patch import pytest +import yaml from ytdl_sub.config.config_file import ConfigFile from ytdl_sub.plugins.nfo_tags import NfoTagsOptions @@ -540,9 +541,67 @@ def test_music_video_subscriptions(default_config: ConfigFile, music_video_subsc assert gnr.get("url2").native == "https://www.youtube.com/watch?v=OldpIhHPsbs" -def test_default_docker_config_and_subscriptions(docker_default_subscription_path: Path): +def test_default_docker_config_and_subscriptions( + docker_default_subscription_path: Path, output_directory: str +): default_config = ConfigFile.from_file_path("docker/root/defaults/config.yaml") default_subs = Subscription.from_file_path( config=default_config, subscription_path=docker_default_subscription_path ) assert len(default_subs) == 1 + + resolved_yaml_as_json = yaml.safe_load(default_subs[0].resolved_yaml()) + + # Since this creates random values, ignore it for this test + assert "throttle_protection" in resolved_yaml_as_json + del resolved_yaml_as_json["throttle_protection"] + + assert resolved_yaml_as_json == { + "chapters": { + "allow_chapters_from_comments": False, + "embed_chapters": True, + "enable": "True", + "force_key_frames": False, + }, + "date_range": {"breaks": "True", "enable": "True", "type": "upload_date"}, + "download": [ + { + "download_reverse": "True", + "include_sibling_metadata": False, + "playlist_thumbnails": [ + {"name": "{avatar_uncropped_thumbnail_file_name}", "uid": "avatar_uncropped"}, + {"name": "{banner_uncropped_thumbnail_file_name}", "uid": "banner_uncropped"}, + ], + "source_thumbnails": [ + {"name": "{avatar_uncropped_thumbnail_file_name}", "uid": "avatar_uncropped"}, + {"name": "{banner_uncropped_thumbnail_file_name}", "uid": "banner_uncropped"}, + ], + "url": "https://www.youtube.com/@novapbs", + "variables": {}, + "webpage_url": "{modified_webpage_url}", + "ytdl_options": {}, + } + ], + "file_convert": {"convert_to": "mp4", "convert_with": "yt-dlp", "enable": "True"}, + "format": "(bv*[ext=mp4][vcodec~='^((he|a)vc|h26[45])']+ba[ext=m4a]) / (bv[ext=mp4]*+ba[ext=m4a]/b)", + "output_options": { + "download_archive_name": ".ytdl-sub-NOVA PBS-download-archive.json", + "file_name": "{episode_file_path}.{ext}", + "info_json_name": "{episode_file_path}.{info_json_ext}", + "keep_files_date_eval": "{episode_date_standardized}", + "maintain_download_archive": True, + "output_directory": f"{output_directory}/NOVA PBS", + "preserve_mtime": False, + "thumbnail_name": "{thumbnail_file_name}", + }, + "video_tags": { + "contentRating": "{episode_content_rating}", + "date": "{episode_date_standardized}", + "episode_id": "{episode_number}", + "genre": "{tv_show_genre}", + "show": "{tv_show_name}", + "synopsis": "{episode_plot}", + "title": "{episode_title}", + "year": "{episode_year}", + }, + } diff --git a/tests/unit/config/test_subscription_validation.py b/tests/unit/config/test_subscription_validation.py new file mode 100644 index 00000000..3a6fd13d --- /dev/null +++ b/tests/unit/config/test_subscription_validation.py @@ -0,0 +1,172 @@ +import re + +import pytest + +from ytdl_sub.config.preset import Preset +from ytdl_sub.subscriptions.subscription import Subscription +from ytdl_sub.utils.exceptions import StringFormattingVariableNotFoundException +from ytdl_sub.utils.exceptions import ValidationException + + +class TestSubscriptionValidation: + def test_preset_error__source_variable_does_not_exist( + self, config_file, output_options, youtube_video + ): + with pytest.raises( + StringFormattingVariableNotFoundException, + match="contains the following variables that do not exist: dne_var", + ): + _ = Subscription.from_preset( + preset=Preset( + config=config_file, + name="test", + value={ + "download": youtube_video, + "output_options": {"output_directory": "dir", "file_name": "{dne_var}"}, + }, + ), + config=config_file, + ) + + def test_preset_error__override_variable_does_not_exist( + self, config_file, output_options, youtube_video + ): + with pytest.raises( + StringFormattingVariableNotFoundException, + match="contains the following variables that do not exist: dne_var", + ): + _ = Subscription.from_preset( + preset=Preset( + config=config_file, + name="test", + value={ + "download": youtube_video, + "output_options": {"output_directory": "{dne_var}", "file_name": "file"}, + }, + ), + config=config_file, + ) + + def test_preset_error__dict_source_variable_does_not_exist( + self, config_file, output_options, youtube_video + ): + with pytest.raises( + StringFormattingVariableNotFoundException, + match="contains the following variables that do not exist: dne_var", + ): + _ = Subscription.from_preset( + preset=Preset( + config=config_file, + name="test", + value={ + "download": youtube_video, + "output_options": {"output_directory": "dir", "file_name": "file"}, + "nfo_tags": { + "nfo_name": "the nfo name", + "nfo_root": "the root", + "tags": {"tag_a": "{dne_var}"}, + }, + }, + ), + config=config_file, + ) + + def test_preset_error__dict_override_variable_does_not_exist( + self, config_file, output_options, youtube_video + ): + with pytest.raises( + StringFormattingVariableNotFoundException, + match="contains the following variables that do not exist: dne_var", + ): + _ = Subscription.from_preset( + preset=Preset( + config=config_file, + name="test", + value={ + "download": youtube_video, + "output_options": output_options, + "output_directory_nfo_tags": { + "nfo_name": "the nfo name", + "nfo_root": "the root", + "tags": {"tag_a": "{dne_var}"}, + }, + }, + ), + config=config_file, + ) + + def test_preset_error__dict_override_variable_not_static( + self, config_file, output_options, youtube_video + ): + with pytest.raises( + StringFormattingVariableNotFoundException, + match="static formatters must contain variables that " + "have no dependency to entry variables", + ): + _ = Subscription.from_preset( + preset=Preset( + config=config_file, + name="test", + value={ + "download": youtube_video, + "output_options": { + "output_directory": "{title}", + "file_name": "{uid}", + }, + }, + ), + config=config_file, + ) + + def test_preset_error_added_url_variable_cannot_resolve(self, config_file, output_options): + with pytest.raises( + ValidationException, + match=re.escape( + "variable the_bad_one cannot use the variables subtitles_ext because it " + "depends on other variables that are computed later in execution" + ), + ): + _ = Subscription.from_preset( + preset=Preset( + config=config_file, + name="test", + value={ + "download": { + "url": "youtube.com/watch?v=123abc", + "variables": {"the_bad_one": "{subtitles_ext}"}, + }, + "subtitles": { + "embed_subtitles": True, + }, + "output_options": {"output_directory": "dir", "file_name": "acjk"}, + }, + ), + config=config_file, + ) + + def test_preset_error_override_name_conflicts_with_plugin(self, config_file, output_options): + with pytest.raises( + ValidationException, + match=re.escape( + "Override variable with name throttle_protection cannot be used since it is the " + "name of a plugin. Perhaps you meant to define it as a plugin? If so, indent it " + "left to make it at the same level as overrides." + ), + ): + _ = Subscription.from_preset( + preset=Preset( + config=config_file, + name="test", + value={ + "download": { + "url": "youtube.com/watch?v=123abc", + }, + "subtitles": { + "embed_subtitles": True, + }, + "output_options": {"output_directory": "dir", "file_name": "acjk"}, + "overrides": {"throttle_protection": "nope"}, + }, + ), + config=config_file, + ) diff --git a/tools/docgen/plugins.py b/tools/docgen/plugins.py index a7f7bed7..d2be80d3 100644 --- a/tools/docgen/plugins.py +++ b/tools/docgen/plugins.py @@ -37,10 +37,12 @@ def should_filter_property(property_name: str) -> bool: "dict", "keys", "dict_with_format_strings", + "dict_with_parsed_format_strings", "subscription_name", "list", "script", "unresolvable", + "leaf_name", )