From 6ad5c47068630bac7cbdd5e616aa42bf86d497d6 Mon Sep 17 00:00:00 2001 From: Jesse Bannon Date: Thu, 7 Dec 2023 12:24:37 -0800 Subject: [PATCH] really need partial script --- src/ytdl_sub/config/overrides.py | 134 ++++++++++++++++++ src/ytdl_sub/config/plugin.py | 2 +- src/ytdl_sub/config/preset.py | 94 +++++++----- src/ytdl_sub/config/preset_options.py | 119 +--------------- .../info_json/info_json_downloader.py | 3 +- src/ytdl_sub/downloaders/source_plugin.py | 2 +- src/ytdl_sub/downloaders/url/downloader.py | 13 +- src/ytdl_sub/downloaders/url/validators.py | 28 +--- src/ytdl_sub/entries/entry.py | 15 +- src/ytdl_sub/entries/entry_parent.py | 12 +- .../entries/script/variable_scripts.py | 4 +- src/ytdl_sub/plugins/internal/view.py | 2 +- .../plugins/output_directory_nfo_tags.py | 2 +- src/ytdl_sub/plugins/regex.py | 22 +-- src/ytdl_sub/plugins/throttle_protection.py | 2 +- .../subscriptions/base_subscription.py | 2 +- .../subscriptions/subscription_validators.py | 2 +- src/ytdl_sub/utils/datetime.py | 2 +- src/ytdl_sub/utils/scriptable.py | 14 ++ .../validators/string_formatter_validators.py | 58 -------- .../enhanced_download_archive.py | 1 + tests/unit/config/test_preset.py | 8 +- tests/unit/entries/conftest.py | 2 +- 23 files changed, 271 insertions(+), 272 deletions(-) create mode 100644 src/ytdl_sub/config/overrides.py create mode 100644 src/ytdl_sub/utils/scriptable.py diff --git a/src/ytdl_sub/config/overrides.py b/src/ytdl_sub/config/overrides.py new file mode 100644 index 00000000..9bdeac33 --- /dev/null +++ b/src/ytdl_sub/config/overrides.py @@ -0,0 +1,134 @@ +import copy +from typing import Any +from typing import Dict +from typing import Optional +from typing import Set + +from yt_dlp.utils import sanitize_filename + +from ytdl_sub.entries.entry import Entry +from ytdl_sub.entries.script.variable_definitions import VARIABLES +from ytdl_sub.entries.variables.override_variables import SUBSCRIPTION_NAME +from ytdl_sub.script.parser import parse +from ytdl_sub.utils.scriptable import Scriptable +from ytdl_sub.validators.string_formatter_validators import DictFormatterValidator +from ytdl_sub.validators.string_formatter_validators import StringFormatterValidator + + +class Overrides(DictFormatterValidator, Scriptable): + """ + Optional. This section allows you to define variables that can be used in any string formatter. + For example, if you want your file and thumbnail files to match without copy-pasting a large + format string, you can define something like: + + .. code-block:: yaml + + presets: + my_example_preset: + overrides: + output_directory: "/path/to/media" + custom_file_name: "{upload_year}.{upload_month_padded}.{upload_day_padded}.{title_sanitized}" + + # Then use the override variables in the output options + output_options: + output_directory: "{output_directory}" + file_name: "{custom_file_name}.{ext}" + thumbnail_name: "{custom_file_name}.{thumbnail_ext}" + + Override variables can contain explicit values and other variables, including both override + and source variables. + + In addition, any override variable defined will automatically create a ``sanitized`` variable + for use. In the example above, ``output_directory_sanitized`` will exist and perform + sanitization on the value when used. + """ + + @classmethod + def partial_validate(cls, name: str, value: Any) -> None: + dict_formatter = DictFormatterValidator(name=name, value=value) + _ = [parse(format_string) for format_string in dict_formatter.dict_with_format_strings] + + # pylint: enable=line-too-long + + def _add_override_variable(self, key_name: str, format_string: str, sanitize: bool = False): + if sanitize: + key_name = f"{key_name}_sanitized" + format_string = sanitize_filename(format_string) + + self._value[key_name] = StringFormatterValidator( + name="__should_never_fail__", + value=format_string, + ) + + def __init__(self, name, value): + DictFormatterValidator.__init__(self, name, value) + Scriptable.__init__(self) + + # Add sanitized overrides + for key in self._keys: + self._add_override_variable( + key_name=key, + format_string=self._value[key].format_string, + sanitize=True, + ) + + if SUBSCRIPTION_NAME not in self._value: + for sanitized in [True, False]: + self._add_override_variable( + key_name=SUBSCRIPTION_NAME, + format_string=self.subscription_name, + sanitize=sanitized, + ) + + self.unresolvable.add(VARIABLES.entry_metadata.variable_name) + + def initialize_script(self, unresolved_variables: Dict[str, str]) -> None: + self.script.add(dict(self.dict_with_format_strings, **unresolved_variables)) + self.unresolvable.update(set(unresolved_variables.keys())) + + self.update_script() + + @property + def subscription_name(self) -> str: + """ + Returns + ------- + Name of the subscription + """ + return self._root_name + + def apply_formatter( + self, + formatter: StringFormatterValidator, + entry: Optional[Entry] = None, + function_overrides: Dict[str, str] = None, + ) -> str: + """ + Parameters + ---------- + formatter + Formatter to apply + entry + Optional. Entry to add source variables to the formatter + function_overrides + Optional. Explicit values to override the overrides themselves and source variables + + Returns + ------- + The format_string after .format has been called + """ + if entry: + script = copy.deepcopy(entry.script) + unresolvable = entry.unresolvable + else: + script = copy.deepcopy(self.script) + unresolvable = self.unresolvable + + if function_overrides: + script.add(function_overrides) + + return str( + script.add({"tmp_var": formatter.format_string}).resolve(unresolvable=unresolvable)[ + "tmp_var" + ] + ) diff --git a/src/ytdl_sub/config/plugin.py b/src/ytdl_sub/config/plugin.py index 9ddc083c..45468333 100644 --- a/src/ytdl_sub/config/plugin.py +++ b/src/ytdl_sub/config/plugin.py @@ -7,7 +7,7 @@ from typing import Optional from typing import Tuple from typing import Type -from ytdl_sub.config.preset_options import Overrides +from ytdl_sub.config.overrides import Overrides from ytdl_sub.config.preset_options import TOptionsValidator from ytdl_sub.entries.entry import Entry from ytdl_sub.utils.file_handler import FileMetadata diff --git a/src/ytdl_sub/config/preset.py b/src/ytdl_sub/config/preset.py index e1b4c6c3..978e7526 100644 --- a/src/ytdl_sub/config/preset.py +++ b/src/ytdl_sub/config/preset.py @@ -1,9 +1,11 @@ import copy +import functools from typing import Any from typing import Dict from typing import Iterable from typing import List from typing import Optional +from typing import Set from typing import Tuple from typing import Type from typing import Union @@ -11,18 +13,22 @@ from typing import Union from mergedeep import mergedeep from ytdl_sub.config.config_validator import ConfigValidator +from ytdl_sub.config.overrides import Overrides from ytdl_sub.config.plugin import Plugin from ytdl_sub.config.plugin_mapping import PluginMapping from ytdl_sub.config.preset_options import OptionsValidator from ytdl_sub.config.preset_options import OutputOptions -from ytdl_sub.config.preset_options import Overrides from ytdl_sub.config.preset_options import TOptionsValidator from ytdl_sub.config.preset_options import YTDLOptions from ytdl_sub.downloaders.url.validators import MultiUrlValidator from ytdl_sub.entries.entry import Entry +from ytdl_sub.entries.script.variable_definitions import VARIABLES from ytdl_sub.entries.script.variable_scripts import VARIABLE_SCRIPTS from ytdl_sub.prebuilt_presets import PREBUILT_PRESET_NAMES from ytdl_sub.prebuilt_presets import PUBLISHED_PRESET_NAMES +from ytdl_sub.script.script import Script +from ytdl_sub.script.utils.exceptions import VariableDoesNotExist +from ytdl_sub.utils.exceptions import StringFormattingVariableNotFoundException from ytdl_sub.utils.exceptions import ValidationException from ytdl_sub.utils.logger import Logger from ytdl_sub.utils.yaml import dump_yaml @@ -159,6 +165,35 @@ class Preset(_PresetShell): def _source_variables(self) -> List[str]: return list(VARIABLE_SCRIPTS.keys()) + @property + def _added_variables(self) -> Dict[str, str]: + added_variables: Dict[str, str] = { + var_name: "dummy_string" + for var_name in self.downloader_options.added_source_variables() + } + + for plugin_options in self.plugins.plugin_options: + for source_var in plugin_options.added_source_variables(): + added_variables[source_var] = "dummy_string" + return added_variables + + @functools.cached_property + def _mock_script(self) -> Script: + # Set the formatter variables to be the overrides + variable_dict = copy.deepcopy(self.overrides.dict_with_format_strings) + + source_variables = { + source_var: "dummy_string" + for source_var in self._source_variables + + self.downloader_options.added_source_variables() + } + variable_dict = dict(source_variables, **variable_dict) + variable_dict = dict(variable_dict, **self._added_variables) + + script = Script(variable_dict) + script.resolve(update=True) + return script + def __validate_and_get_plugins(self) -> PresetPlugins: preset_plugins = PresetPlugins() @@ -174,54 +209,30 @@ class Preset(_PresetShell): return preset_plugins def __validate_added_variables(self): - source_variables = copy.deepcopy(self._source_variables) - - # Validate added download option variables here since plugins could subsequently use them - self.downloader_options.validate_with_variables( - source_variables=source_variables, - override_variables=self.overrides.dict_with_format_strings, - ) - source_variables.extend(self.downloader_options.added_source_variables()) + self.downloader_options.validate_with_variables(script=copy.deepcopy(self._mock_script)) for _, plugin_options in sorted( self.plugins.zipped(), key=lambda pl: pl[0].priority.modify_entry ): # Validate current plugin using source + added plugin variables - plugin_options.validate_with_variables( - source_variables=source_variables, - override_variables=self.overrides.dict_with_format_strings, - ) - - # Extend existing source variables with ones created from this plugin - source_variables.extend(plugin_options.added_source_variables()) + plugin_options.validate_with_variables(script=copy.deepcopy(self._mock_script)) def __validate_override_string_formatter_validator( self, formatter_validator: Union[StringFormatterValidator, OverridesStringFormatterValidator], ): - # Set the formatter variables to be the overrides - variable_dict = copy.deepcopy(self.overrides.dict_with_format_strings) + script = copy.deepcopy(self._mock_script) + try: + script.add({"tmp_var": formatter_validator.format_string}) + except VariableDoesNotExist as exc: + raise StringFormattingVariableNotFoundException(exc) from exc - source_variables = { - source_var: "dummy_string" - for source_var in self._source_variables - + self.downloader_options.added_source_variables() - } - variable_dict = dict(source_variables, **variable_dict) - - # For all plugins, add in any extra added source variables - # TODO: Check in order variables are added - for plugin_options in self.plugins.plugin_options: - added_plugin_variables = { - source_var: "dummy_string" for source_var in plugin_options.added_source_variables() - } - # sanity check plugin variables do not override source variables - expected_len = len(variable_dict) + len(added_plugin_variables) - variable_dict = dict(variable_dict, **added_plugin_variables) - - assert len(variable_dict) == expected_len, "plugin variables overwrote source variables" - - _ = formatter_validator.apply_formatter(variable_dict=variable_dict) + unresolved: Optional[Set[str]] = ( + {VARIABLES.entry_metadata.variable_name} + if isinstance(formatter_validator, OverridesStringFormatterValidator) + else None + ) + _ = script.resolve(unresolvable=unresolved)["tmp_var"] # TODO: error if not present def __recursive_preset_validate( self, @@ -333,6 +344,13 @@ class Preset(_PresetShell): # values from multiple validators self.__recursive_preset_validate() + self.overrides.initialize_script( + unresolved_variables={ + var_name: f"{{%throw('Plugin variable {var_name} has not been created yet')}}" + for var_name in self._added_variables + } + ) + @property def name(self) -> str: """ diff --git a/src/ytdl_sub/config/preset_options.py b/src/ytdl_sub/config/preset_options.py index 188eb6a3..03e9c728 100644 --- a/src/ytdl_sub/config/preset_options.py +++ b/src/ytdl_sub/config/preset_options.py @@ -1,26 +1,16 @@ -import copy -import json from abc import ABC from typing import Any -from typing import Dict from typing import List from typing import Optional from typing import TypeVar -import mergedeep -from yt_dlp.utils import sanitize_filename - from ytdl_sub.config.defaults import DEFAULT_DOWNLOAD_ARCHIVE_NAME -from ytdl_sub.entries.entry import Entry -from ytdl_sub.entries.script.variable_definitions import VARIABLES -from ytdl_sub.entries.script.variable_scripts import VARIABLE_SCRIPTS -from ytdl_sub.entries.variables.override_variables import SUBSCRIPTION_NAME +from ytdl_sub.script.script import Script from ytdl_sub.utils.exceptions import ValidationException from ytdl_sub.validators.file_path_validators import OverridesStringFormatterFilePathValidator from ytdl_sub.validators.file_path_validators import StringFormatterFileNameValidator from ytdl_sub.validators.strict_dict_validator import StrictDictValidator from ytdl_sub.validators.string_datetime import StringDatetimeValidator -from ytdl_sub.validators.string_formatter_validators import DictFormatterValidator from ytdl_sub.validators.string_formatter_validators import OverridesIntegerFormatterValidator from ytdl_sub.validators.string_formatter_validators import OverridesStringFormatterValidator from ytdl_sub.validators.string_formatter_validators import StringFormatterValidator @@ -63,18 +53,14 @@ class OptionsValidator(Validator, ABC): """ return [] - def validate_with_variables( - self, source_variables: List[str], override_variables: Dict[str, str] - ) -> None: + def validate_with_variables(self, script: Script) -> None: """ Optional validation after init with the session's source and override variables. Parameters ---------- - source_variables - Available source variables when running the plugin - override_variables - Available override variables when running the plugin + script + Script containing all current variables """ return None @@ -129,103 +115,6 @@ class YTDLOptions(LiteralDictValidator): # Disable for proper docstring formatting # pylint: disable=line-too-long -class Overrides(DictFormatterValidator): - """ - Optional. This section allows you to define variables that can be used in any string formatter. - For example, if you want your file and thumbnail files to match without copy-pasting a large - format string, you can define something like: - - .. code-block:: yaml - - presets: - my_example_preset: - overrides: - output_directory: "/path/to/media" - custom_file_name: "{upload_year}.{upload_month_padded}.{upload_day_padded}.{title_sanitized}" - - # Then use the override variables in the output options - output_options: - output_directory: "{output_directory}" - file_name: "{custom_file_name}.{ext}" - thumbnail_name: "{custom_file_name}.{thumbnail_ext}" - - Override variables can contain explicit values and other variables, including both override - and source variables. - - In addition, any override variable defined will automatically create a ``sanitized`` variable - for use. In the example above, ``output_directory_sanitized`` will exist and perform - sanitization on the value when used. - """ - - # pylint: enable=line-too-long - - def _add_override_variable(self, key_name: str, format_string: str, sanitize: bool = False): - if sanitize: - key_name = f"{key_name}_sanitized" - format_string = sanitize_filename(format_string) - - self._value[key_name] = StringFormatterValidator( - name="__should_never_fail__", - value=format_string, - ) - - def __init__(self, name, value): - super().__init__(name, value) - - # Add sanitized overrides - for key in self._keys: - self._add_override_variable( - key_name=key, - format_string=self._value[key].format_string, - sanitize=True, - ) - - if SUBSCRIPTION_NAME not in self._value: - for sanitized in [True, False]: - self._add_override_variable( - key_name=SUBSCRIPTION_NAME, - format_string=self.subscription_name, - sanitize=sanitized, - ) - - @property - def subscription_name(self) -> str: - """ - Returns - ------- - Name of the subscription - """ - return self._root_name - - def apply_formatter( - self, - formatter: StringFormatterValidator, - entry: Optional[Entry] = None, - function_overrides: Dict[str, str] = None, - ) -> str: - """ - Parameters - ---------- - formatter - Formatter to apply - entry - Optional. Entry to add source variables to the formatter - function_overrides - Optional. Explicit values to override the overrides themselves and source variables - - Returns - ------- - The format_string after .format has been called - """ - variable_dict = copy.deepcopy(VARIABLE_SCRIPTS) - mergedeep.merge(variable_dict, self.dict_with_format_strings) - - if entry: - mergedeep.merge(variable_dict, {VARIABLES.entry_metadata.variable_name: json.dumps(entry._kwargs)}) - if function_overrides: - mergedeep.merge(variable_dict, function_overrides) - - return formatter.apply_formatter(variable_dict) class OutputOptions(StrictDictValidator): diff --git a/src/ytdl_sub/downloaders/info_json/info_json_downloader.py b/src/ytdl_sub/downloaders/info_json/info_json_downloader.py index a7fac294..bbdfe1d9 100644 --- a/src/ytdl_sub/downloaders/info_json/info_json_downloader.py +++ b/src/ytdl_sub/downloaders/info_json/info_json_downloader.py @@ -6,8 +6,8 @@ from typing import Iterable from typing import List from typing import Optional +from ytdl_sub.config.overrides import Overrides from ytdl_sub.config.preset_options import OptionsDictValidator -from ytdl_sub.config.preset_options import Overrides from ytdl_sub.downloaders.source_plugin import SourcePlugin from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder from ytdl_sub.entries.entry import Entry @@ -82,6 +82,7 @@ class InfoJsonDownloader(SourcePlugin[InfoJsonDownloaderOptions]): return Entry( entry_dict=entry_dict, working_directory=self.working_directory, + override_variables=self.overrides.dict_with_format_strings, ) raise ValidationException( diff --git a/src/ytdl_sub/downloaders/source_plugin.py b/src/ytdl_sub/downloaders/source_plugin.py index b11b1940..f9ebb510 100644 --- a/src/ytdl_sub/downloaders/source_plugin.py +++ b/src/ytdl_sub/downloaders/source_plugin.py @@ -8,9 +8,9 @@ from typing import Optional from typing import Type from typing import final +from ytdl_sub.config.overrides import Overrides from ytdl_sub.config.plugin import BasePlugin from ytdl_sub.config.plugin import Plugin -from ytdl_sub.config.preset_options import Overrides from ytdl_sub.config.preset_options import TOptionsValidator from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder from ytdl_sub.entries.entry import Entry diff --git a/src/ytdl_sub/downloaders/url/downloader.py b/src/ytdl_sub/downloaders/url/downloader.py index 4fb8f273..6b03cf3d 100644 --- a/src/ytdl_sub/downloaders/url/downloader.py +++ b/src/ytdl_sub/downloaders/url/downloader.py @@ -11,8 +11,8 @@ from typing import Tuple from yt_dlp.utils import RejectedVideoReached +from ytdl_sub.config.overrides import Overrides from ytdl_sub.config.plugin import PluginPriority -from ytdl_sub.config.preset_options import Overrides from ytdl_sub.downloaders.source_plugin import SourcePlugin from ytdl_sub.downloaders.source_plugin import SourcePluginExtension from ytdl_sub.downloaders.url.validators import MultiUrlValidator @@ -356,7 +356,11 @@ class MultiUrlDownloader(SourcePlugin[MultiUrlValidator]): else entry.is_thumbnail_downloaded_via_ytdlp, url=entry.webpage_url, ) - return Entry(download_entry_dict, working_directory=self.working_directory) + return Entry( + download_entry_dict, + working_directory=self.working_directory, + override_variables=self.overrides.dict_with_format_strings, + ) def _iterate_child_entries( self, url_validator: UrlValidator, entries: List[Entry] @@ -413,7 +417,10 @@ class MultiUrlDownloader(SourcePlugin[MultiUrlValidator]): working_directory=self.working_directory, ) orphans = EntryParent.from_entry_dicts_with_no_parents( - parents=parents, entry_dicts=entry_dicts, working_directory=self.working_directory + parents=parents, + entry_dicts=entry_dicts, + working_directory=self.working_directory, + override_variables=self.overrides.dict_with_format_strings, ) return parents, orphans diff --git a/src/ytdl_sub/downloaders/url/validators.py b/src/ytdl_sub/downloaders/url/validators.py index cfca4dad..4ce88ca2 100644 --- a/src/ytdl_sub/downloaders/url/validators.py +++ b/src/ytdl_sub/downloaders/url/validators.py @@ -5,6 +5,7 @@ from typing import List from typing import Optional from ytdl_sub.config.preset_options import OptionsValidator +from ytdl_sub.script.script import Script from ytdl_sub.validators.strict_dict_validator import StrictDictValidator from ytdl_sub.validators.string_formatter_validators import DictFormatterValidator from ytdl_sub.validators.string_formatter_validators import OverridesStringFormatterValidator @@ -251,37 +252,20 @@ class MultiUrlValidator(OptionsValidator): """ return list(self._urls.list[0].variables.keys) - def validate_with_variables( - self, source_variables: List[str], override_variables: Dict[str, str] - ) -> None: + def validate_with_variables(self, script: Script) -> None: """ Ensures new variables added are not existing variables """ - for source_var_name in self.added_source_variables(): - if source_var_name in source_variables: - raise self._validation_exception( - f"'{source_var_name}' cannot be used as a variable name because it " - f"is an existing source variable" - ) - - base_variables = dict( - override_variables, **{source_var: "dummy_string" for source_var in source_variables} - ) - # Apply formatting to each new source variable, ensure it resolves for collection_url in self.urls.list: - for ( - source_var_name, - source_var_formatter_str, - ) in collection_url.variables.dict_with_format_strings.items(): - _ = StringFormatterValidator( - name=f"{self._name}.{source_var_name}", value=source_var_formatter_str - ).apply_formatter(base_variables) + script.add(collection_url.variables.dict_with_format_strings) + script.resolve(update=True) # Ensure at least URL is non-empty has_non_empty_url = False for url_validator in self.urls.list: - has_non_empty_url |= bool(url_validator.url.apply_formatter(base_variables)) + script.add({"tmp_var_url": url_validator.url.format_string}) + has_non_empty_url |= bool(str(script.resolve().get("tmp_var_url"))) if not has_non_empty_url: raise self._validation_exception("Must contain at least one url that is non-empty") diff --git a/src/ytdl_sub/entries/entry.py b/src/ytdl_sub/entries/entry.py index cd7e5aad..751e719d 100644 --- a/src/ytdl_sub/entries/entry.py +++ b/src/ytdl_sub/entries/entry.py @@ -2,19 +2,32 @@ import copy import json import os from pathlib import Path +from typing import Dict from typing import Optional from typing import final from ytdl_sub.entries.base_entry import BaseEntry +from ytdl_sub.entries.script.variable_definitions import VARIABLES +from ytdl_sub.utils.scriptable import Scriptable from ytdl_sub.validators.audo_codec_validator import AUDIO_CODEC_EXTS from ytdl_sub.validators.audo_codec_validator import VIDEO_CODEC_EXTS -class Entry(BaseEntry): +class Entry(BaseEntry, Scriptable): """ Entry object to represent a single media object returned from yt-dlp. """ + def __init__( + self, entry_dict: Dict, working_directory: str, override_variables: Dict[str, str] + ): + BaseEntry.__init__(self, entry_dict=entry_dict, working_directory=working_directory) + Scriptable.__init__(self) + + self.script.add({VARIABLES.entry_metadata.variable_name: json.dumps(self._kwargs)}) + self.script.add(override_variables) + self.script.resolve(update=True) + @property def ext(self) -> str: """ diff --git a/src/ytdl_sub/entries/entry_parent.py b/src/ytdl_sub/entries/entry_parent.py index 24e184af..252c1030 100644 --- a/src/ytdl_sub/entries/entry_parent.py +++ b/src/ytdl_sub/entries/entry_parent.py @@ -279,7 +279,11 @@ class EntryParent(BaseEntry): @classmethod def from_entry_dicts_with_no_parents( - cls, parents: List["EntryParent"], entry_dicts: List[Dict], working_directory: str + cls, + parents: List["EntryParent"], + entry_dicts: List[Dict], + working_directory: str, + override_variables: Dict[str, str], ) -> List[Entry]: """ Reads all entries that do not have any parents @@ -289,7 +293,11 @@ class EntryParent(BaseEntry): return any(entry_dict in parent for parent in parents) return [ - Entry(entry_dict=entry_dict, working_directory=working_directory) + Entry( + entry_dict=entry_dict, + working_directory=working_directory, + override_variables=override_variables, + ) for entry_dict in entry_dicts if cls.is_entry(entry_dict) and not _in_any_parents(entry_dict) ] diff --git a/src/ytdl_sub/entries/script/variable_scripts.py b/src/ytdl_sub/entries/script/variable_scripts.py index f273606e..14990456 100644 --- a/src/ytdl_sub/entries/script/variable_scripts.py +++ b/src/ytdl_sub/entries/script/variable_scripts.py @@ -98,9 +98,7 @@ def source_get_int(key: MetadataVariable, default: Optional[Variable | int] = No ############################################################################################### # Scripts -ENTRY_EMPTY_METADATA: Dict[Variable, str] = { - v.entry_metadata: "{ {} }" -} +ENTRY_EMPTY_METADATA: Dict[Variable, str] = {v.entry_metadata: "{ {} }"} ENTRY_HARDCODED_VARIABLES: Dict[Variable, str] = { v.info_json_ext: "info.json", diff --git a/src/ytdl_sub/plugins/internal/view.py b/src/ytdl_sub/plugins/internal/view.py index fa8d5471..940452eb 100644 --- a/src/ytdl_sub/plugins/internal/view.py +++ b/src/ytdl_sub/plugins/internal/view.py @@ -1,10 +1,10 @@ import copy from typing import Optional +from ytdl_sub.config.overrides import Overrides from ytdl_sub.config.plugin import Plugin from ytdl_sub.config.plugin import PluginPriority from ytdl_sub.config.preset_options import OptionsDictValidator -from ytdl_sub.config.preset_options import Overrides from ytdl_sub.entries.entry import Entry from ytdl_sub.utils.file_handler import FileMetadata from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive diff --git a/src/ytdl_sub/plugins/output_directory_nfo_tags.py b/src/ytdl_sub/plugins/output_directory_nfo_tags.py index 1244c870..72f1828d 100644 --- a/src/ytdl_sub/plugins/output_directory_nfo_tags.py +++ b/src/ytdl_sub/plugins/output_directory_nfo_tags.py @@ -1,4 +1,4 @@ -from ytdl_sub.config.preset_options import Overrides +from ytdl_sub.config.overrides import Overrides from ytdl_sub.entries.entry import Entry from ytdl_sub.plugins.nfo_tags import NfoTagsValidator from ytdl_sub.plugins.nfo_tags import SharedNfoTagsOptions diff --git a/src/ytdl_sub/plugins/regex.py b/src/ytdl_sub/plugins/regex.py index d2228190..c640c05b 100644 --- a/src/ytdl_sub/plugins/regex.py +++ b/src/ytdl_sub/plugins/regex.py @@ -9,7 +9,9 @@ from ytdl_sub.config.plugin import Plugin from ytdl_sub.config.plugin import PluginPriority from ytdl_sub.config.preset_options import OptionsDictValidator from ytdl_sub.entries.entry import Entry +from ytdl_sub.entries.script.variable_scripts import VARIABLE_SCRIPTS from ytdl_sub.entries.variables.kwargs import YTDL_SUB_REGEX_SOURCE_VARS +from ytdl_sub.script.script import Script from ytdl_sub.utils.exceptions import RegexNoMatchException from ytdl_sub.utils.exceptions import StringFormattingVariableNotFoundException from ytdl_sub.utils.logger import Logger @@ -212,34 +214,22 @@ class RegexOptions(OptionsDictValidator): """ return self._skip_if_match_fails - def validate_with_variables( - self, source_variables: List[str], override_variables: Dict[str, str] - ) -> None: - """ - Ensures each source variable capture group is valid - - Parameters - ---------- - source_variables - Available source variables when running the plugin - override_variables - Available override variables when running the plugin - """ + def validate_with_variables(self, script: Script) -> None: for key, regex_options in self.source_variable_capture_dict.items(): # Ensure each variable getting captured is a source variable - if key not in source_variables and key not in override_variables: + if key not in script._variables: raise self._validation_exception( f"cannot regex capture '{key}' because it is not a source or override variable" ) # Ensure the capture group names are not existing source/override variables for capture_group_name in regex_options.capture_group_names: - if capture_group_name in source_variables: + if capture_group_name in VARIABLE_SCRIPTS: raise self._validation_exception( f"'{capture_group_name}' cannot be used as a capture group name because it " f"is a source variable" ) - if capture_group_name in override_variables: + if capture_group_name in script._variables: raise self._validation_exception( f"'{capture_group_name}' cannot be used as a capture group name because it " f"is an override variable" diff --git a/src/ytdl_sub/plugins/throttle_protection.py b/src/ytdl_sub/plugins/throttle_protection.py index 25b5791b..40727ca7 100644 --- a/src/ytdl_sub/plugins/throttle_protection.py +++ b/src/ytdl_sub/plugins/throttle_protection.py @@ -4,9 +4,9 @@ from typing import List from typing import Optional from typing import Tuple +from ytdl_sub.config.overrides import Overrides from ytdl_sub.config.plugin import Plugin from ytdl_sub.config.preset_options import OptionsDictValidator -from ytdl_sub.config.preset_options import Overrides from ytdl_sub.entries.entry import Entry from ytdl_sub.utils.file_handler import FileMetadata from ytdl_sub.utils.logger import Logger diff --git a/src/ytdl_sub/subscriptions/base_subscription.py b/src/ytdl_sub/subscriptions/base_subscription.py index 0c61f7e1..08bc344c 100644 --- a/src/ytdl_sub/subscriptions/base_subscription.py +++ b/src/ytdl_sub/subscriptions/base_subscription.py @@ -3,10 +3,10 @@ from pathlib import Path from typing import Optional from ytdl_sub.config.config_validator import ConfigOptions +from ytdl_sub.config.overrides import Overrides from ytdl_sub.config.preset import Preset from ytdl_sub.config.preset import PresetPlugins from ytdl_sub.config.preset_options import OutputOptions -from ytdl_sub.config.preset_options import Overrides from ytdl_sub.config.preset_options import YTDLOptions from ytdl_sub.downloaders.url.validators import MultiUrlValidator from ytdl_sub.utils.file_handler import FileHandlerTransactionLog diff --git a/src/ytdl_sub/subscriptions/subscription_validators.py b/src/ytdl_sub/subscriptions/subscription_validators.py index 678a9758..dbcd5516 100644 --- a/src/ytdl_sub/subscriptions/subscription_validators.py +++ b/src/ytdl_sub/subscriptions/subscription_validators.py @@ -8,7 +8,7 @@ from typing import Optional from typing import final from ytdl_sub.config.config_file import ConfigFile -from ytdl_sub.config.preset_options import Overrides +from ytdl_sub.config.overrides import Overrides from ytdl_sub.entries.variables.override_variables import SUBSCRIPTION_NAME from ytdl_sub.entries.variables.override_variables import SUBSCRIPTION_VALUE from ytdl_sub.entries.variables.override_variables import OverrideVariables diff --git a/src/ytdl_sub/utils/datetime.py b/src/ytdl_sub/utils/datetime.py index 07fcaacf..cce045d7 100644 --- a/src/ytdl_sub/utils/datetime.py +++ b/src/ytdl_sub/utils/datetime.py @@ -3,7 +3,7 @@ from typing import Optional from yt_dlp import DateRange from yt_dlp.utils import datetime_from_str -from ytdl_sub.config.preset_options import Overrides +from ytdl_sub.config.overrides import Overrides from ytdl_sub.validators.string_datetime import StringDatetimeValidator diff --git a/src/ytdl_sub/utils/scriptable.py b/src/ytdl_sub/utils/scriptable.py new file mode 100644 index 00000000..c9056fd8 --- /dev/null +++ b/src/ytdl_sub/utils/scriptable.py @@ -0,0 +1,14 @@ +from abc import ABC +from typing import Set + +from ytdl_sub.entries.script.variable_scripts import VARIABLE_SCRIPTS +from ytdl_sub.script.script import Script + + +class Scriptable(ABC): + def __init__(self): + self.script = Script(VARIABLE_SCRIPTS) + self.unresolvable: Set[str] = set() + + def update_script(self) -> None: + self.script.resolve(unresolvable=self.unresolvable, update=True) diff --git a/src/ytdl_sub/validators/string_formatter_validators.py b/src/ytdl_sub/validators/string_formatter_validators.py index e0f114f1..e8f4548e 100644 --- a/src/ytdl_sub/validators/string_formatter_validators.py +++ b/src/ytdl_sub/validators/string_formatter_validators.py @@ -93,33 +93,6 @@ class StringFormatterValidator(StringValidator): """ return self._value - def _variable_dict(self, variable_dict: Dict[str, str]) -> Dict[str, str]: - sanitized_variables = { - f"{var_name}_sanitized": f"{{%sanitize({var_name})}}" - for var_name in variable_dict.keys() - } - return dict(variable_dict, **sanitized_variables) - - def apply_formatter(self, variable_dict: Dict[str, str]) -> str: - """ - Calls `format` on the format string using the variable_dict as input kwargs - - Parameters - ---------- - variable_dict - kwargs to pass to the format string - - Returns - ------- - Format string formatted - """ - out = ( - Script(self._variable_dict(variable_dict)) - .add({"tmp_var": self.format_string}) - .resolve()["tmp_var"] - ) - return str(out) - # pylint: disable=line-too-long class OverridesStringFormatterValidator(StringFormatterValidator): @@ -133,37 +106,6 @@ class OverridesStringFormatterValidator(StringFormatterValidator): :class:`nfo_output_directory ` """ - _variable_not_found_error_msg_formatter = ( - "Override variable '{variable_name}' does not exist. For this field, ensure your override " - "variable does not contain any source variables - it is a requirement that this be a " - "static string. Available override variables: {available_fields}" - ) - - def apply_formatter(self, variable_dict: Dict[str, str]) -> str: - """ - Calls `format` on the format string using the variable_dict as input kwargs - - Parameters - ---------- - variable_dict - kwargs to pass to the format string - - Returns - ------- - Format string formatted - """ - output = ( - Script(self._variable_dict(variable_dict)) - .add({"tmp_var": self.format_string}) - .resolve(unresolvable={VARIABLES.entry_metadata.variable_name}) - ) - if "tmp_var" not in output: - raise self._validation_exception( - "Has a dependency on entry variables when it is not allowed", - exception_class=StringFormattingVariableNotFoundException, - ) - return str(output["tmp_var"]) - # pylint: enable=line-too-long diff --git a/src/ytdl_sub/ytdl_additions/enhanced_download_archive.py b/src/ytdl_sub/ytdl_additions/enhanced_download_archive.py index 584f5541..e614176f 100644 --- a/src/ytdl_sub/ytdl_additions/enhanced_download_archive.py +++ b/src/ytdl_sub/ytdl_additions/enhanced_download_archive.py @@ -652,6 +652,7 @@ class EnhancedDownloadArchive: parent_entry = Entry( entry_dict=entry.kwargs(SPLIT_BY_CHAPTERS_PARENT_ENTRY), working_directory=entry.working_directory(), + override_variables={}, ) self.mapping.add_entry(parent_entry, entry_file_path=output_file_name) elif entry: diff --git a/tests/unit/config/test_preset.py b/tests/unit/config/test_preset.py index 158b2434..9e6cff71 100644 --- a/tests/unit/config/test_preset.py +++ b/tests/unit/config/test_preset.py @@ -129,7 +129,7 @@ class TestPreset: ): with pytest.raises( StringFormattingVariableNotFoundException, - match="Format variable 'dne_var' does not exist", + match="Variable dne_var does not exist.", ): _ = Preset( config=config_file, @@ -145,7 +145,7 @@ class TestPreset: ): with pytest.raises( StringFormattingVariableNotFoundException, - match="Override variable 'dne_var' does not exist", + match="Variable dne_var does not exist", ): _ = Preset( config=config_file, @@ -161,7 +161,7 @@ class TestPreset: ): with pytest.raises( StringFormattingVariableNotFoundException, - match="Format variable 'dne_var' does not exist", + match="Variable dne_var does not exist", ): _ = Preset( config=config_file, @@ -182,7 +182,7 @@ class TestPreset: ): with pytest.raises( StringFormattingVariableNotFoundException, - match="Format variable 'dne_var' does not exist", + match="Variable dne_var does not exist", ): _ = Preset( config=config_file, diff --git a/tests/unit/entries/conftest.py b/tests/unit/entries/conftest.py index 6fdf8e62..ff888a76 100644 --- a/tests/unit/entries/conftest.py +++ b/tests/unit/entries/conftest.py @@ -174,7 +174,7 @@ def mock_entry_kwargs( @pytest.fixture def mock_entry(mock_entry_kwargs): - return Entry(entry_dict=mock_entry_kwargs, working_directory=".") + return Entry(entry_dict=mock_entry_kwargs, working_directory=".", override_variables={}) @pytest.fixture