From 7d6666a8f26eb007e7e2347cffdc7758a0658162 Mon Sep 17 00:00:00 2001 From: Jesse Bannon Date: Sat, 30 Mar 2024 15:16:48 -0700 Subject: [PATCH] massive --- src/ytdl_sub/config/overrides.py | 11 +--- src/ytdl_sub/config/preset.py | 4 +- .../config/validators/variable_validation.py | 56 ++++++++++--------- src/ytdl_sub/downloaders/url/downloader.py | 21 +++++-- src/ytdl_sub/entries/script/variable_types.py | 8 +++ .../entries/variables/override_variables.py | 56 +++++++++++++++---- src/ytdl_sub/script/script.py | 4 +- .../subscriptions/subscription_download.py | 12 ++++ .../subscriptions/subscription_validators.py | 16 +++--- src/ytdl_sub/utils/scriptable.py | 5 +- tests/e2e/youtube/test_playlist.py | 18 +++--- 11 files changed, 139 insertions(+), 72 deletions(-) diff --git a/src/ytdl_sub/config/overrides.py b/src/ytdl_sub/config/overrides.py index 3372e936..9ad6ca56 100644 --- a/src/ytdl_sub/config/overrides.py +++ b/src/ytdl_sub/config/overrides.py @@ -127,17 +127,10 @@ class Overrides(DictFormatterValidator, Scriptable): ) return ScriptUtils.add_sanitized_variables(initial_variables) - def initialize_script( - self, subscription_name: str, unresolved_variables: Set[str] - ) -> "Overrides": + def initialize_script(self, unresolved_variables: Set[str]) -> "Overrides": """ - Initialize the override script with override variables + any unresolved variables + Initialize the override script with any unresolved variables """ - self.script.add( - ScriptUtils.add_sanitized_variables( - {SubscriptionVariables.subscription_name(): subscription_name} - ) - ) self.script.add( self.initial_variables( unresolved_variables={ diff --git a/src/ytdl_sub/config/preset.py b/src/ytdl_sub/config/preset.py index 4fec587d..914ebc77 100644 --- a/src/ytdl_sub/config/preset.py +++ b/src/ytdl_sub/config/preset.py @@ -198,9 +198,7 @@ class Preset(_PresetShell): downloader_options=self.downloader_options, output_options=self.output_options, plugins=self.plugins, - ).initialize_overrides( - subscription_name=self.name, overrides=self.overrides - ).ensure_proper_usage() + ).initialize_preset_overrides(overrides=self.overrides).ensure_proper_usage() @property def name(self) -> str: diff --git a/src/ytdl_sub/config/validators/variable_validation.py b/src/ytdl_sub/config/validators/variable_validation.py index bdeb9fac..b838c714 100644 --- a/src/ytdl_sub/config/validators/variable_validation.py +++ b/src/ytdl_sub/config/validators/variable_validation.py @@ -13,13 +13,25 @@ from ytdl_sub.config.plugin.preset_plugins import PresetPlugins from ytdl_sub.config.preset_options import OutputOptions from ytdl_sub.config.validators.options import OptionsValidator from ytdl_sub.downloaders.url.validators import MultiUrlValidator -from ytdl_sub.entries.variables.override_variables import SubscriptionVariables +from ytdl_sub.entries.variables.override_variables import REQUIRED_OVERRIDE_VARIABLE_DEFINITIONS +from ytdl_sub.entries.variables.override_variables import REQUIRED_OVERRIDE_VARIABLE_NAMES from ytdl_sub.script.script import Script from ytdl_sub.script.script import _is_function from ytdl_sub.utils.scriptable import BASE_SCRIPT from ytdl_sub.validators.string_formatter_validators import to_variable_dependency_format_string from ytdl_sub.validators.string_formatter_validators import validate_formatters +# Entry variables to mock during validation +_DUMMY_ENTRY_VARIABLES: Dict[str, str] = { + name: to_variable_dependency_format_string( + # pylint: disable=protected-access + script=BASE_SCRIPT, + parsed_format_string=BASE_SCRIPT._variables[name] + # pylint: enable=protected-access + ) + for name in BASE_SCRIPT.variable_names +} + def _add_dummy_variables(variables: Iterable[str]) -> Dict[str, str]: dummy_variables: Dict[str, str] = {} @@ -72,20 +84,7 @@ def _get_added_and_modified_variables( def _override_variables(overrides: Overrides) -> Set[str]: - return set(list(overrides.initial_variables().keys())) | { - SubscriptionVariables.subscription_name() - } - - -_DUMMY_ENTRY_VARIABLES: Dict[str, str] = { - name: to_variable_dependency_format_string( - # pylint: disable=protected-access - script=BASE_SCRIPT, - parsed_format_string=BASE_SCRIPT._variables[name] - # pylint: enable=protected-access - ) - for name in BASE_SCRIPT.variable_names -} + return set(list(overrides.initial_variables().keys())) class VariableValidation: @@ -103,13 +102,13 @@ class VariableValidation: self.resolved_variables: Set[str] = set() self.unresolved_variables: Set[str] = set() - def initialize_overrides( - self, subscription_name: str, overrides: Overrides - ) -> "VariableValidation": + def initialize_preset_overrides(self, overrides: Overrides) -> "VariableValidation": """ Do some gymnastics to initialize the Overrides script. """ - override_variables = _override_variables(overrides) + override_variables = ( + set(list(overrides.initial_variables().keys())) | REQUIRED_OVERRIDE_VARIABLE_NAMES + ) # Set resolved variables as all entry + override variables # at this point to generate every possible added/modified variable @@ -145,9 +144,7 @@ class VariableValidation: # Initialize overrides with unresolved variables + modified variables to throw an error. # For modified variables, this is to prevent a resolve(update=True) to setting any # dependencies until it has been explicitly added - overrides = overrides.initialize_script( - subscription_name=subscription_name, unresolved_variables=self.unresolved_variables - ) + overrides = overrides.initialize_script(unresolved_variables=self.unresolved_variables) # copy the script and mock entry variables self.script = copy.deepcopy(overrides.script) @@ -162,7 +159,16 @@ class VariableValidation: def _update_script(self) -> None: _ = self.script.resolve(unresolvable=self.unresolved_variables, update=True) - def _add_variables(self, plugin_op: PluginOperation, options: OptionsValidator) -> Set[str]: + def _add_subscription_override_variables(self) -> None: + """ + Add dummy subscription variables for script validation + """ + self.resolved_variables |= REQUIRED_OVERRIDE_VARIABLE_NAMES + + def _add_variables(self, plugin_op: PluginOperation, options: OptionsValidator) -> None: + """ + Add dummy variables for script validation + """ added_variables = options.added_variables( resolved_variables=self.resolved_variables, unresolved_variables=self.unresolved_variables, @@ -175,14 +181,14 @@ class VariableValidation: self.resolved_variables |= resolved_variables self.unresolved_variables -= resolved_variables - return added_variables - def ensure_proper_usage(self) -> None: """ Validate variables resolve as plugins are executed, and return a mock script which contains actualized added variables from the plugins """ + self._add_variables(PluginOperation.DOWNLOADER, options=self.downloader_options) + self._add_subscription_override_variables() # Metadata variables to be added for plugin_options in PluginMapping.order_options_by( diff --git a/src/ytdl_sub/downloaders/url/downloader.py b/src/ytdl_sub/downloaders/url/downloader.py index 7c7f4e93..c38aedfa 100644 --- a/src/ytdl_sub/downloaders/url/downloader.py +++ b/src/ytdl_sub/downloaders/url/downloader.py @@ -134,7 +134,9 @@ class UrlDownloaderThumbnailPlugin(SourcePluginExtension): try_convert_download_thumbnail(entry=entry) self._download_url_thumbnails( - collection_url=self.plugin_options.urls.list[entry.get(v.ytdl_sub_input_url_index, int)], + collection_url=self.plugin_options.urls.list[ + entry.get(v.ytdl_sub_input_url_index, int) + ], entry=entry, ) return entry @@ -224,7 +226,12 @@ class MultiUrlDownloader(SourcePlugin[MultiUrlValidator]): return ( self._download_ytdl_options_builder.clone() .add(self.ytdl_option_defaults(), before=True) - .add(self.plugin_options.urls.list[url_idx].ytdl_options.dict if url_idx is not None else None, before=True) + .add( + self.plugin_options.urls.list[url_idx].ytdl_options.dict + if url_idx is not None + else None, + before=True, + ) .to_dict() ) @@ -463,10 +470,12 @@ class MultiUrlDownloader(SourcePlugin[MultiUrlValidator]): continue for entry in self._download_metadata(url=url, validator=url_validator): - entry.initialize_script(self.overrides).add({ - v.ytdl_sub_input_url: url, - v.ytdl_sub_input_url_index: idx, - }) + entry.initialize_script(self.overrides).add( + { + v.ytdl_sub_input_url: url, + v.ytdl_sub_input_url_index: idx, + } + ) yield entry diff --git a/src/ytdl_sub/entries/script/variable_types.py b/src/ytdl_sub/entries/script/variable_types.py index 7cca58cd..214d8a7c 100644 --- a/src/ytdl_sub/entries/script/variable_types.py +++ b/src/ytdl_sub/entries/script/variable_types.py @@ -9,6 +9,7 @@ from typing import TypeVar from ytdl_sub.script.types.array import Array from ytdl_sub.script.types.map import Map +from ytdl_sub.script.types.resolvable import Boolean from ytdl_sub.script.types.resolvable import Integer from ytdl_sub.script.types.resolvable import String @@ -63,6 +64,13 @@ class Variable(ABC): """ +@dataclass(frozen=True) +class BooleanVariable(Variable): + @classmethod + def human_readable_type(cls) -> str: + return Boolean.__name__ + + @dataclass(frozen=True) class StringVariable(Variable): @classmethod diff --git a/src/ytdl_sub/entries/variables/override_variables.py b/src/ytdl_sub/entries/variables/override_variables.py index 007aaf2b..8eee70f1 100644 --- a/src/ytdl_sub/entries/variables/override_variables.py +++ b/src/ytdl_sub/entries/variables/override_variables.py @@ -1,6 +1,14 @@ +from typing import Dict +from typing import Set + from ytdl_sub.entries.script.function_scripts import CUSTOM_FUNCTION_SCRIPTS from ytdl_sub.entries.script.variable_definitions import VARIABLE_SCRIPTS +from ytdl_sub.entries.script.variable_types import BooleanVariable +from ytdl_sub.entries.script.variable_types import MapVariable +from ytdl_sub.entries.script.variable_types import StringVariable +from ytdl_sub.entries.script.variable_types import Variable from ytdl_sub.script.functions import Functions +from ytdl_sub.script.types.syntax_tree import SyntaxTree from ytdl_sub.script.utils.name_validation import is_valid_name # TODO: use this @@ -9,15 +17,15 @@ SUBSCRIPTION_ARRAY = "subscription_array" class SubscriptionVariables: @staticmethod - def subscription_name() -> str: + def subscription_name() -> StringVariable: """ Name of the subscription. For subscriptions types that use a prefix (``~``, ``+``), the prefix and all whitespace afterwards is stripped from the subscription name. """ - return "subscription_name" + return StringVariable(variable_name="subscription_name", definition="{ %string('') }") @staticmethod - def subscription_value() -> str: + def subscription_value() -> StringVariable: """ For subscriptions in the form of @@ -27,10 +35,10 @@ class SubscriptionVariables: ``subscription_value`` gets set to ``https://...``. """ - return "subscription_value" + return StringVariable(variable_name="subscription_value", definition="{ %string('') }") @staticmethod - def subscription_indent_i(index: int) -> str: + def subscription_indent_i(index: int) -> StringVariable: """ For subscriptions in the form of @@ -43,10 +51,12 @@ class SubscriptionVariables: ``subscription_indent_1`` and ``subscription_indent_2`` get set to ``Indent Value 1`` and ``Indent Value 2``. """ - return f"subscription_indent_{index + 1}" + return StringVariable( + variable_name=f"subscription_indent_{index + 1}", definition="{ %string('') }" + ) @staticmethod - def subscription_value_i(index: int) -> str: + def subscription_value_i(index: int) -> StringVariable: """ For subscriptions in the form of @@ -60,10 +70,12 @@ class SubscriptionVariables: and ``https://url2.com/...``. Note that ``subscription_value_1`` also gets set to ``subscription_value``. """ - return f"subscription_value_{index + 1}" + return StringVariable( + variable_name=f"subscription_value_{index + 1}", definition="{ %string('') }" + ) @staticmethod - def subscription_map() -> str: + def subscription_map() -> MapVariable: """ For subscriptions in the form of @@ -89,7 +101,17 @@ class SubscriptionVariables: ] } """ - return "subscription_map" + return MapVariable(variable_name="subscription_map", definition="{ {} }") + + @staticmethod + def subscription_has_download_archive() -> BooleanVariable: + """ + Returns True if the subscription has any entries recorded in a download archive. False + otherwise. + """ + return BooleanVariable( + variable_name="subscription_has_download_archive", definition="{ %bool(True) }" + ) class OverrideHelpers: @@ -124,3 +146,17 @@ class OverrideHelpers: return is_valid_name(name=name[1:]) return is_valid_name(name=name) + + +REQUIRED_OVERRIDE_VARIABLES: Set[Variable] = { + SubscriptionVariables.subscription_name(), + SubscriptionVariables.subscription_has_download_archive(), +} + +REQUIRED_OVERRIDE_VARIABLE_DEFINITIONS: Dict[str, str] = { + var.variable_name: var.definition for var in REQUIRED_OVERRIDE_VARIABLES +} + +REQUIRED_OVERRIDE_VARIABLE_NAMES: Set[str] = { + var.variable_name for var in REQUIRED_OVERRIDE_VARIABLES +} diff --git a/src/ytdl_sub/script/script.py b/src/ytdl_sub/script/script.py index 602b1a0e..f285eae6 100644 --- a/src/ytdl_sub/script/script.py +++ b/src/ytdl_sub/script/script.py @@ -454,7 +454,9 @@ class Script: added_variables_to_validate: Set[str] = set() functions_to_add = { - _function_name(name): definition for name, definition in variables.items() if _is_function(name) + _function_name(name): definition + for name, definition in variables.items() + if _is_function(name) } variables_to_add = { name: definition for name, definition in variables.items() if not _is_function(name) diff --git a/src/ytdl_sub/subscriptions/subscription_download.py b/src/ytdl_sub/subscriptions/subscription_download.py index 06ce1bd2..076096b7 100644 --- a/src/ytdl_sub/subscriptions/subscription_download.py +++ b/src/ytdl_sub/subscriptions/subscription_download.py @@ -17,6 +17,7 @@ from ytdl_sub.downloaders.source_plugin import SourcePlugin from ytdl_sub.downloaders.url.downloader import MultiUrlDownloader from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder from ytdl_sub.entries.entry import Entry +from ytdl_sub.entries.variables.override_variables import SubscriptionVariables from ytdl_sub.subscriptions.base_subscription import BaseSubscription from ytdl_sub.subscriptions.subscription_ytdl_options import SubscriptionYTDLOptions from ytdl_sub.utils.datetime import to_date_range @@ -325,6 +326,16 @@ class SubscriptionDownload(BaseSubscription, ABC): return self._enhanced_download_archive.get_file_handler_transaction_log() + def _initialize_subscription_overrides(self): + required_overrides = { + SubscriptionVariables.subscription_name(): self.name, + SubscriptionVariables.subscription_has_download_archive(): f"""{{ + %bool({self._enhanced_download_archive.num_entries > 0}) + }}""", + } + assert SubscriptionVariables + self.overrides.add(required_overrides) + def download(self, dry_run: bool = False) -> FileHandlerTransactionLog: """ Performs the subscription download @@ -338,6 +349,7 @@ class SubscriptionDownload(BaseSubscription, ABC): self._exception = None self._enhanced_download_archive.reinitialize(dry_run=dry_run) + self._initialize_subscription_overrides() plugins = self._initialize_plugins() subscription_ytdl_options = SubscriptionYTDLOptions( diff --git a/src/ytdl_sub/subscriptions/subscription_validators.py b/src/ytdl_sub/subscriptions/subscription_validators.py index c247779a..6ce95cc6 100644 --- a/src/ytdl_sub/subscriptions/subscription_validators.py +++ b/src/ytdl_sub/subscriptions/subscription_validators.py @@ -32,7 +32,7 @@ class SubscriptionOutput(Validator, ABC): indent overrides to merge with the preset dict's overrides """ return { - SubscriptionVariables.subscription_indent_i(i): self._indent_overrides[i] + SubscriptionVariables.subscription_indent_i(i).variable_name: self._indent_overrides[i] for i in range(len(self._indent_overrides)) } @@ -143,7 +143,9 @@ class SubscriptionValueValidator(SubscriptionLeafValidator, StringValidator): presets=presets, indent_overrides=indent_overrides, ) - self._overrides_to_add[SubscriptionVariables.subscription_value()] = self.value + self._overrides_to_add[ + SubscriptionVariables.subscription_value().variable_name + ] = self.value class SubscriptionListValuesValidator(SubscriptionLeafValidator, StringListValidator): @@ -169,11 +171,11 @@ class SubscriptionListValuesValidator(SubscriptionLeafValidator, StringListValid # Write the first list value into subscription_value as well if idx == 0: self._overrides_to_add[ - SubscriptionVariables.subscription_value() + SubscriptionVariables.subscription_value().variable_name ] = list_value.value self._overrides_to_add[ - SubscriptionVariables.subscription_value_i(index=idx) + SubscriptionVariables.subscription_value_i(index=idx).variable_name ] = list_value.value @@ -217,9 +219,9 @@ class SubscriptionMapValidator(SubscriptionLeafValidator, LiteralDictValidator): presets=presets, indent_overrides=indent_overrides, ) - self._overrides_to_add[SubscriptionVariables.subscription_map()] = ScriptUtils.to_script( - self.dict - ) + self._overrides_to_add[ + SubscriptionVariables.subscription_map().variable_name + ] = ScriptUtils.to_script(self.dict) class SubscriptionValidator(SubscriptionOutput): diff --git a/src/ytdl_sub/utils/scriptable.py b/src/ytdl_sub/utils/scriptable.py index 6aff1db3..38780024 100644 --- a/src/ytdl_sub/utils/scriptable.py +++ b/src/ytdl_sub/utils/scriptable.py @@ -9,13 +9,16 @@ from ytdl_sub.entries.script.function_scripts import CUSTOM_FUNCTION_SCRIPTS from ytdl_sub.entries.script.variable_definitions import UNRESOLVED_VARIABLES from ytdl_sub.entries.script.variable_definitions import VARIABLE_SCRIPTS from ytdl_sub.entries.script.variable_types import Variable +from ytdl_sub.entries.variables.override_variables import REQUIRED_OVERRIDE_VARIABLE_DEFINITIONS from ytdl_sub.script.script import Script from ytdl_sub.script.utils.exceptions import RuntimeException from ytdl_sub.utils.exceptions import StringFormattingException from ytdl_sub.utils.script import ScriptUtils BASE_SCRIPT: Script = Script( - dict(ScriptUtils.add_sanitized_variables(VARIABLE_SCRIPTS), **CUSTOM_FUNCTION_SCRIPTS) + ScriptUtils.add_sanitized_variables(VARIABLE_SCRIPTS) + | ScriptUtils.add_sanitized_variables(REQUIRED_OVERRIDE_VARIABLE_DEFINITIONS) + | CUSTOM_FUNCTION_SCRIPTS ) diff --git a/tests/e2e/youtube/test_playlist.py b/tests/e2e/youtube/test_playlist.py index 7f237526..1223bef6 100644 --- a/tests/e2e/youtube/test_playlist.py +++ b/tests/e2e/youtube/test_playlist.py @@ -56,7 +56,7 @@ def playlist_bilateral_dict(output_directory): return { "preset": [ "Jellyfin TV Show by Date", - ], + ], "format": "worst[ext=mp4]", "overrides": { "url": "https://www.youtube.com/playlist?list=PLd4Q7G88JqoekF0b30NYQcOTnTiIe9Ali", @@ -238,14 +238,12 @@ class TestPlaylist: assert subscriptions[0].transaction_log.is_empty def test_tv_show_downloads_bilateral( - self, - playlist_bilateral_dict: Dict, - output_directory: str, - default_config: ConfigFile, + self, + playlist_bilateral_dict: Dict, + output_directory: str, + default_config: ConfigFile, ): - playlist_bilateral_dict['filter_include'] = [ - "{ %contains(title, 'Feb.1') }" - ] + playlist_bilateral_dict["filter_include"] = ["{ %contains(title, 'Feb.1') }"] playlist_subscription = Subscription.from_dict( config=default_config, preset_name="bilateral_test", @@ -261,7 +259,7 @@ class TestPlaylist: # Now that one vid is downloaded, attempt to download all and see if bilateral # logic kicks in - del playlist_bilateral_dict['filter_include'] + del playlist_bilateral_dict["filter_include"] playlist_subscription = Subscription.from_dict( config=default_config, preset_name="bilateral_test", @@ -272,4 +270,4 @@ class TestPlaylist: output_directory=output_directory, transaction_log=transaction_log, transaction_log_summary_file_name="youtube/test_playlist_bilateral_p2.txt", - ) \ No newline at end of file + )