From cbfcd5154eea26590c1ecfdc0e982367ebd2c722 Mon Sep 17 00:00:00 2001 From: Jesse Bannon Date: Tue, 12 Dec 2023 23:23:24 -0800 Subject: [PATCH] huge refactor, maybe revert? --- src/ytdl_sub/config/overrides.py | 1 - src/ytdl_sub/config/plugin/__init__.py | 0 src/ytdl_sub/config/{ => plugin}/plugin.py | 2 +- .../config/{ => plugin}/plugin_mapping.py | 10 +- .../config/{ => plugin}/plugin_operation.py | 0 src/ytdl_sub/config/plugin/preset_plugins.py | 42 +++++ src/ytdl_sub/config/preset.py | 158 +----------------- src/ytdl_sub/config/preset_options.py | 55 ------ src/ytdl_sub/config/validators/__init__.py | 0 src/ytdl_sub/config/validators/options.py | 56 +++++++ .../config/validators/variable_validation.py | 116 +++++++++++++ .../info_json/info_json_downloader.py | 2 +- src/ytdl_sub/downloaders/source_plugin.py | 6 +- src/ytdl_sub/downloaders/url/validators.py | 4 +- src/ytdl_sub/plugins/audio_extract.py | 10 +- src/ytdl_sub/plugins/chapters.py | 4 +- src/ytdl_sub/plugins/date_range.py | 4 +- src/ytdl_sub/plugins/embed_thumbnail.py | 4 +- src/ytdl_sub/plugins/file_convert.py | 10 +- src/ytdl_sub/plugins/format.py | 4 +- src/ytdl_sub/plugins/internal/view.py | 4 +- src/ytdl_sub/plugins/match_filters.py | 4 +- src/ytdl_sub/plugins/music_tags.py | 4 +- src/ytdl_sub/plugins/nfo_tags.py | 4 +- src/ytdl_sub/plugins/regex.py | 6 +- src/ytdl_sub/plugins/split_by_chapters.py | 6 +- src/ytdl_sub/plugins/subtitles.py | 6 +- src/ytdl_sub/plugins/throttle_protection.py | 4 +- src/ytdl_sub/plugins/video_tags.py | 4 +- src/ytdl_sub/script/script.py | 4 + .../subscriptions/base_subscription.py | 2 +- .../subscriptions/subscription_download.py | 8 +- .../subscription_ytdl_options.py | 2 +- tests/unit/config/test_config_file.py | 2 +- 34 files changed, 282 insertions(+), 266 deletions(-) create mode 100644 src/ytdl_sub/config/plugin/__init__.py rename src/ytdl_sub/config/{ => plugin}/plugin.py (98%) rename src/ytdl_sub/config/{ => plugin}/plugin_mapping.py (96%) rename src/ytdl_sub/config/{ => plugin}/plugin_operation.py (100%) create mode 100644 src/ytdl_sub/config/plugin/preset_plugins.py create mode 100644 src/ytdl_sub/config/validators/__init__.py create mode 100644 src/ytdl_sub/config/validators/options.py create mode 100644 src/ytdl_sub/config/validators/variable_validation.py diff --git a/src/ytdl_sub/config/overrides.py b/src/ytdl_sub/config/overrides.py index 4a1d285a..e5d9974d 100644 --- a/src/ytdl_sub/config/overrides.py +++ b/src/ytdl_sub/config/overrides.py @@ -5,7 +5,6 @@ from typing import Optional from typing import Set import mergedeep -from yt_dlp.utils import sanitize_filename from ytdl_sub.entries.entry import Entry from ytdl_sub.entries.script.variable_definitions import VARIABLES diff --git a/src/ytdl_sub/config/plugin/__init__.py b/src/ytdl_sub/config/plugin/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/ytdl_sub/config/plugin.py b/src/ytdl_sub/config/plugin/plugin.py similarity index 98% rename from src/ytdl_sub/config/plugin.py rename to src/ytdl_sub/config/plugin/plugin.py index 1e57c6a8..24d616f6 100644 --- a/src/ytdl_sub/config/plugin.py +++ b/src/ytdl_sub/config/plugin/plugin.py @@ -8,7 +8,7 @@ from typing import Tuple from typing import Type from ytdl_sub.config.overrides import Overrides -from ytdl_sub.config.preset_options import TOptionsValidator +from ytdl_sub.config.validators.options import TOptionsValidator from ytdl_sub.entries.entry import Entry from ytdl_sub.utils.file_handler import FileMetadata from ytdl_sub.ytdl_additions.enhanced_download_archive import DownloadArchiver diff --git a/src/ytdl_sub/config/plugin_mapping.py b/src/ytdl_sub/config/plugin/plugin_mapping.py similarity index 96% rename from src/ytdl_sub/config/plugin_mapping.py rename to src/ytdl_sub/config/plugin/plugin_mapping.py index 62bceb7a..ea0073e0 100644 --- a/src/ytdl_sub/config/plugin_mapping.py +++ b/src/ytdl_sub/config/plugin/plugin_mapping.py @@ -1,15 +1,13 @@ -import copy from typing import Dict -from typing import Iterable from typing import List from typing import Optional from typing import Tuple from typing import Type -from ytdl_sub.config.plugin import Plugin -from ytdl_sub.config.plugin import SplitPlugin -from ytdl_sub.config.plugin_operation import PluginOperation -from ytdl_sub.config.preset_options import OptionsValidator +from ytdl_sub.config.plugin.plugin import Plugin +from ytdl_sub.config.plugin.plugin import SplitPlugin +from ytdl_sub.config.plugin.plugin_operation import PluginOperation +from ytdl_sub.config.validators.options import OptionsValidator from ytdl_sub.plugins.audio_extract import AudioExtractPlugin from ytdl_sub.plugins.chapters import ChaptersPlugin from ytdl_sub.plugins.date_range import DateRangePlugin diff --git a/src/ytdl_sub/config/plugin_operation.py b/src/ytdl_sub/config/plugin/plugin_operation.py similarity index 100% rename from src/ytdl_sub/config/plugin_operation.py rename to src/ytdl_sub/config/plugin/plugin_operation.py diff --git a/src/ytdl_sub/config/plugin/preset_plugins.py b/src/ytdl_sub/config/plugin/preset_plugins.py new file mode 100644 index 00000000..f9862060 --- /dev/null +++ b/src/ytdl_sub/config/plugin/preset_plugins.py @@ -0,0 +1,42 @@ +from typing import List, Type, Tuple, Optional + +from ytdl_sub.config.plugin.plugin import Plugin +from ytdl_sub.config.validators.options import OptionsValidator, TOptionsValidator + + +class PresetPlugins: + def __init__(self): + self.plugin_types: List[Type[Plugin]] = [] + self.plugin_options: List[OptionsValidator] = [] + + def add(self, plugin_type: Type[Plugin], plugin_options: OptionsValidator) -> "PresetPlugins": + """ + Add a pair of plugin type and options to the list + """ + self.plugin_types.append(plugin_type) + self.plugin_options.append(plugin_options) + return self + + def zipped(self) -> List[Tuple[Type[Plugin], OptionsValidator]]: + """ + Returns + ------- + Plugin and PluginOptions zipped + """ + return list(zip(self.plugin_types, self.plugin_options)) + + def get(self, plugin_type: Type[TOptionsValidator]) -> Optional[TOptionsValidator]: + """ + Parameters + ---------- + plugin_type + Fetch the plugin options for this type + + Returns + ------- + Options of this plugin if they exit. Otherwise, return None. + """ + plugin_option_types = [type(plugin_options) for plugin_options in self.plugin_options] + if plugin_type in plugin_option_types: + return self.plugin_options[plugin_option_types.index(plugin_type)] + return None diff --git a/src/ytdl_sub/config/preset.py b/src/ytdl_sub/config/preset.py index e7e23713..2e5d9fb6 100644 --- a/src/ytdl_sub/config/preset.py +++ b/src/ytdl_sub/config/preset.py @@ -1,48 +1,25 @@ import copy -import functools from typing import Any from typing import Dict -from typing import Iterable from typing import List -from typing import Optional -from typing import Set -from typing import Tuple -from typing import Type -from typing import Union from mergedeep import mergedeep from ytdl_sub.config.config_validator import ConfigValidator from ytdl_sub.config.overrides import Overrides -from ytdl_sub.config.plugin import Plugin -from ytdl_sub.config.plugin_mapping import PluginMapping -from ytdl_sub.config.plugin_operation import PluginOperation -from ytdl_sub.config.preset_options import OptionsValidator +from ytdl_sub.config.plugin.plugin_mapping import PluginMapping +from ytdl_sub.config.plugin.preset_plugins import PresetPlugins from ytdl_sub.config.preset_options import OutputOptions -from ytdl_sub.config.preset_options import TOptionsValidator from ytdl_sub.config.preset_options import YTDLOptions +from ytdl_sub.config.validators.variable_validation import VariableValidation from ytdl_sub.downloaders.url.validators import MultiUrlValidator -from ytdl_sub.entries.script.variable_definitions import VARIABLES -from ytdl_sub.entries.script.variable_scripts import VARIABLE_SCRIPTS from ytdl_sub.prebuilt_presets import PREBUILT_PRESET_NAMES from ytdl_sub.prebuilt_presets import PUBLISHED_PRESET_NAMES -from ytdl_sub.script.script import Script -from ytdl_sub.script.utils.exceptions import VariableDoesNotExist -from ytdl_sub.utils.exceptions import StringFormattingVariableNotFoundException from ytdl_sub.utils.exceptions import ValidationException from ytdl_sub.utils.logger import Logger -from ytdl_sub.utils.script import ScriptUtils from ytdl_sub.utils.yaml import dump_yaml from ytdl_sub.validators.strict_dict_validator import StrictDictValidator -from ytdl_sub.validators.string_formatter_validators import DictFormatterValidator -from ytdl_sub.validators.string_formatter_validators import OverridesDictFormatterValidator -from ytdl_sub.validators.string_formatter_validators import OverridesStringFormatterValidator -from ytdl_sub.validators.string_formatter_validators import StringFormatterValidator -from ytdl_sub.validators.string_formatter_validators import validate_formatters -from ytdl_sub.validators.validators import DictValidator -from ytdl_sub.validators.validators import ListValidator from ytdl_sub.validators.validators import StringListValidator -from ytdl_sub.validators.validators import Validator from ytdl_sub.validators.validators import validation_exception PRESET_KEYS = { @@ -70,44 +47,6 @@ def _parent_preset_error_message( ) -class PresetPlugins: - def __init__(self): - self.plugin_types: List[Type[Plugin]] = [] - self.plugin_options: List[OptionsValidator] = [] - - def add(self, plugin_type: Type[Plugin], plugin_options: OptionsValidator) -> "PresetPlugins": - """ - Add a pair of plugin type and options to the list - """ - self.plugin_types.append(plugin_type) - self.plugin_options.append(plugin_options) - return self - - def zipped(self) -> List[Tuple[Type[Plugin], OptionsValidator]]: - """ - Returns - ------- - Plugin and PluginOptions zipped - """ - return list(zip(self.plugin_types, self.plugin_options)) - - def get(self, plugin_type: Type[TOptionsValidator]) -> Optional[TOptionsValidator]: - """ - Parameters - ---------- - plugin_type - Fetch the plugin options for this type - - Returns - ------- - Options of this plugin if they exit. Otherwise, return None. - """ - plugin_option_types = [type(plugin_options) for plugin_options in self.plugin_options] - if plugin_type in plugin_option_types: - return self.plugin_options[plugin_option_types.index(plugin_type)] - return None - - class _PresetShell(StrictDictValidator): # Have all present keys optional since parent presets could not have all the # required keys. They will get validated in the init after the mergedeep of dicts @@ -163,24 +102,6 @@ class Preset(_PresetShell): validator=PluginMapping.get(plugin_name).plugin_options_type, ) - @property - def _source_variables(self) -> List[str]: - return list(VARIABLE_SCRIPTS.keys()) - - @property - def _added_variables(self) -> Set[str]: - added_variables: Set[str] = set() - options: List[OptionsValidator] = self.plugins.plugin_options - options.append(self.downloader_options) - - for plugin_options in options: - for plugin_added_variables in plugin_options.added_source_variables( - unresolved_variables=set() - ).values(): - added_variables |= set(plugin_added_variables) - - return added_variables - def _validate_and_get_plugins(self) -> PresetPlugins: preset_plugins = PresetPlugins() @@ -195,64 +116,6 @@ class Preset(_PresetShell): return preset_plugins - def _validate_variable_usage(self) -> None: - """ - Validate variables resolve as plugins are executed, and return - a mock script which contains actualized added variables from the plugins - """ - script = copy.deepcopy(self.overrides.script).add( - ScriptUtils.add_dummy_variables(self._source_variables) - ) - unresolved_variables = self._added_variables - - added_variables: Set[str] = self.downloader_options.added_source_variables( - unresolved_variables - ).get(PluginOperation.DOWNLOADER, set()) - script.add(ScriptUtils.add_dummy_variables(added_variables)) - unresolved_variables -= added_variables - - for plugin_options in PluginMapping.order_options_by( - self.plugins.zipped(), PluginOperation.MODIFY_ENTRY_METADATA - ): - added_variables = plugin_options.added_source_variables( - unresolved_variables=unresolved_variables - ).get(PluginOperation.MODIFY_ENTRY_METADATA, set()) - - if added_variables: - script.add(ScriptUtils.add_dummy_variables(added_variables)) - unresolved_variables -= added_variables - - _ = script.resolve(unresolvable=unresolved_variables, update=True) - for plugin_options in PluginMapping.order_options_by( - self.plugins.zipped(), PluginOperation.MODIFY_ENTRY - ): - added_variables = plugin_options.added_source_variables( - unresolved_variables=unresolved_variables - ).get(PluginOperation.MODIFY_ENTRY, set()) - - if added_variables: - script.add(ScriptUtils.add_dummy_variables(added_variables)) - unresolved_variables -= added_variables - - _ = script.resolve(unresolvable=unresolved_variables, update=True) - - # Validate that any formatter in the plugin options can resolve - validate_formatters( - script=script, - unresolved_variables=unresolved_variables, - validator=plugin_options, - ) - - validate_formatters( - script=script, - unresolved_variables=unresolved_variables, - validator=self.output_options, - ) - - assert not unresolved_variables - - return script - def _get_presets_to_merge( self, parent_presets: str | List[str], seen_presets: List[str], config: ConfigValidator ) -> List[Dict]: @@ -329,16 +192,13 @@ class Preset(_PresetShell): ) self.plugins: PresetPlugins = self._validate_and_get_plugins() - self.overrides = self._validate_key( - key="overrides", validator=Overrides, default={} - ).initialize_script( - unresolved_variables={ - var_name: f"{{%throw('Plugin variable {var_name} has not been created yet')}}" - for var_name in self._added_variables - } - ) + self.overrides = self._validate_key(key="overrides", validator=Overrides, default={}) - self._validate_variable_usage() + VariableValidation( + downloader_options=self.downloader_options, + output_options=self.output_options, + plugins=self.plugins, + ).initialize_overrides(overrides=self.overrides).ensure_proper_usage() @property def name(self) -> str: diff --git a/src/ytdl_sub/config/preset_options.py b/src/ytdl_sub/config/preset_options.py index 6eb88054..d6b2bc2e 100644 --- a/src/ytdl_sub/config/preset_options.py +++ b/src/ytdl_sub/config/preset_options.py @@ -1,13 +1,7 @@ -from abc import ABC from typing import Any -from typing import Dict from typing import Optional -from typing import Set -from typing import TypeVar from ytdl_sub.config.defaults import DEFAULT_DOWNLOAD_ARCHIVE_NAME -from ytdl_sub.config.plugin_operation import PluginOperation -from ytdl_sub.utils.exceptions import ValidationException from ytdl_sub.validators.file_path_validators import OverridesStringFormatterFilePathValidator from ytdl_sub.validators.file_path_validators import StringFormatterFileNameValidator from ytdl_sub.validators.strict_dict_validator import StrictDictValidator @@ -17,55 +11,6 @@ from ytdl_sub.validators.string_formatter_validators import OverridesStringForma from ytdl_sub.validators.string_formatter_validators import StringFormatterValidator from ytdl_sub.validators.validators import BoolValidator from ytdl_sub.validators.validators import LiteralDictValidator -from ytdl_sub.validators.validators import Validator - - -# pylint: disable=no-self-use -# pylint: disable=unused-argument -class OptionsValidator(Validator, ABC): - """ - Abstract class that validates options for preset sections (plugins, downloaders) - """ - - def validation_exception( - self, - error_message: str | Exception, - ) -> ValidationException: - """ - Parameters - ---------- - error_message - Error message to include in the validation exception - - Returns - ------- - Validation exception that points to the location in the config. To be used to throw good - validation exceptions at runtime from code outside this class. - """ - return self._validation_exception(error_message=error_message) - - def added_source_variables( - self, unresolved_variables: Set[str] - ) -> Dict[PluginOperation, Set[str]]: - """ - If the plugin adds source variables, list them here. - - Returns - ------- - List of added source variables this plugin creates - """ - return {} - - -TOptionsValidator = TypeVar("TOptionsValidator", bound=OptionsValidator) - - -class OptionsDictValidator(StrictDictValidator, OptionsValidator, ABC): - pass - - -# pylint: enable=no-self-use -# pylint: enable=unused-argument class YTDLOptions(LiteralDictValidator): diff --git a/src/ytdl_sub/config/validators/__init__.py b/src/ytdl_sub/config/validators/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/ytdl_sub/config/validators/options.py b/src/ytdl_sub/config/validators/options.py new file mode 100644 index 00000000..cfba16dd --- /dev/null +++ b/src/ytdl_sub/config/validators/options.py @@ -0,0 +1,56 @@ +from abc import ABC +from typing import Dict +from typing import Set +from typing import TypeVar + +from ytdl_sub.config.plugin.plugin_operation import PluginOperation +from ytdl_sub.utils.exceptions import ValidationException +from ytdl_sub.validators.strict_dict_validator import StrictDictValidator +from ytdl_sub.validators.validators import Validator + +# pylint: disable=no-self-use +# pylint: disable=unused-argument + + +class OptionsValidator(Validator, ABC): + """ + Abstract class that validates options for preset sections (plugins, downloaders) + """ + + def validation_exception( + self, + error_message: str | Exception, + ) -> ValidationException: + """ + Parameters + ---------- + error_message + Error message to include in the validation exception + + Returns + ------- + Validation exception that points to the location in the config. To be used to throw good + validation exceptions at runtime from code outside this class. + """ + return self._validation_exception(error_message=error_message) + + def modified_variables(self) -> Dict[PluginOperation, Set[str]]: + """ + If the plugin modifies existing variables, define them here + """ + return {} + + def added_source_variables( + self, unresolved_variables: Set[str] + ) -> Dict[PluginOperation, Set[str]]: + """ + If the plugin adds source variables, list them here. + """ + return {} + + +TOptionsValidator = TypeVar("TOptionsValidator", bound=OptionsValidator) + + +class OptionsDictValidator(StrictDictValidator, OptionsValidator, ABC): + pass diff --git a/src/ytdl_sub/config/validators/variable_validation.py b/src/ytdl_sub/config/validators/variable_validation.py new file mode 100644 index 00000000..c240e459 --- /dev/null +++ b/src/ytdl_sub/config/validators/variable_validation.py @@ -0,0 +1,116 @@ +import copy +from typing import List +from typing import Optional +from typing import Set + +from ytdl_sub.config.overrides import Overrides +from ytdl_sub.config.plugin.plugin_mapping import PluginMapping +from ytdl_sub.config.plugin.plugin_operation import PluginOperation +from ytdl_sub.config.plugin.preset_plugins import PresetPlugins +from ytdl_sub.config.preset_options import OutputOptions +from ytdl_sub.config.validators.options import OptionsValidator +from ytdl_sub.downloaders.url.validators import MultiUrlValidator +from ytdl_sub.entries.script.variable_scripts import VARIABLE_SCRIPTS +from ytdl_sub.script.script import Script +from ytdl_sub.utils.script import ScriptUtils +from ytdl_sub.validators.string_formatter_validators import validate_formatters + + +class VariableValidation: + @classmethod + def _get_added_variables( + cls, plugins: PresetPlugins, downloader_options: MultiUrlValidator + ) -> Set[str]: + added_variables: Set[str] = set() + options: List[OptionsValidator] = plugins.plugin_options + options.append(downloader_options) + + for plugin_options in options: + for plugin_added_variables in plugin_options.added_source_variables( + unresolved_variables=set() + ).values(): + added_variables |= set(plugin_added_variables) + + return added_variables + + def __init__( + self, + downloader_options: MultiUrlValidator, + output_options: OutputOptions, + plugins: PresetPlugins, + ): + self.script: Optional[Script] = None + self.resolved_variables: Set[str] = set() + + self.downloader_options = downloader_options + self.output_options = output_options + self.plugins = plugins + self.unresolved_variables = VariableValidation._get_added_variables( + plugins=plugins, downloader_options=downloader_options + ) + + def initialize_overrides(self, overrides: Overrides) -> "VariableValidation": + overrides = overrides.initialize_script( + unresolved_variables={ + var_name: f"{{%throw('Plugin variable {var_name} has not been created yet')}}" + for var_name in self.unresolved_variables + } + ) + self.script = copy.deepcopy(overrides.script).add( + ScriptUtils.add_dummy_variables(list(VARIABLE_SCRIPTS.keys())) + ) + self.resolved_variables = self.script.variable_names - self.unresolved_variables + + return self + + def ensure_proper_usage(self) -> None: + """ + Validate variables resolve as plugins are executed, and return + a mock script which contains actualized added variables from the plugins + """ + + added_variables: Set[str] = self.downloader_options.added_source_variables( + self.unresolved_variables + ).get(PluginOperation.DOWNLOADER, set()) + self.script.add(ScriptUtils.add_dummy_variables(added_variables)) + self.unresolved_variables -= added_variables + + for plugin_options in PluginMapping.order_options_by( + self.plugins.zipped(), PluginOperation.MODIFY_ENTRY_METADATA + ): + added_variables = plugin_options.added_source_variables( + unresolved_variables=self.unresolved_variables + ).get(PluginOperation.MODIFY_ENTRY_METADATA, set()) + + if added_variables: + self.script.add(ScriptUtils.add_dummy_variables(added_variables)) + self.unresolved_variables -= added_variables + + _ = self.script.resolve(unresolvable=self.unresolved_variables, update=True) + for plugin_options in PluginMapping.order_options_by( + self.plugins.zipped(), PluginOperation.MODIFY_ENTRY + ): + added_variables = plugin_options.added_source_variables( + unresolved_variables=self.unresolved_variables + ).get(PluginOperation.MODIFY_ENTRY, set()) + + if added_variables: + self.script.add(ScriptUtils.add_dummy_variables(added_variables)) + self.unresolved_variables -= added_variables + + _ = self.script.resolve(unresolvable=self.unresolved_variables, update=True) + + # Validate that any formatter in the plugin options can resolve + validate_formatters( + script=self.script, + unresolved_variables=self.unresolved_variables, + validator=plugin_options, + ) + + validate_formatters( + script=self.script, + unresolved_variables=self.unresolved_variables, + validator=self.output_options, + ) + + assert not self.unresolved_variables diff --git a/src/ytdl_sub/downloaders/info_json/info_json_downloader.py b/src/ytdl_sub/downloaders/info_json/info_json_downloader.py index 9f3ea5dc..a339eaed 100644 --- a/src/ytdl_sub/downloaders/info_json/info_json_downloader.py +++ b/src/ytdl_sub/downloaders/info_json/info_json_downloader.py @@ -7,7 +7,7 @@ from typing import List from typing import Optional from ytdl_sub.config.overrides import Overrides -from ytdl_sub.config.preset_options import OptionsDictValidator +from ytdl_sub.config.validators.options import OptionsDictValidator from ytdl_sub.downloaders.source_plugin import SourcePlugin from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder from ytdl_sub.entries.entry import YTDL_SUB_ENTRY_VARIABLES_KWARG_KEY diff --git a/src/ytdl_sub/downloaders/source_plugin.py b/src/ytdl_sub/downloaders/source_plugin.py index f9ebb510..167f67fd 100644 --- a/src/ytdl_sub/downloaders/source_plugin.py +++ b/src/ytdl_sub/downloaders/source_plugin.py @@ -9,9 +9,9 @@ from typing import Type from typing import final from ytdl_sub.config.overrides import Overrides -from ytdl_sub.config.plugin import BasePlugin -from ytdl_sub.config.plugin import Plugin -from ytdl_sub.config.preset_options import TOptionsValidator +from ytdl_sub.config.plugin.plugin import BasePlugin +from ytdl_sub.config.plugin.plugin import Plugin +from ytdl_sub.config.validators.options import TOptionsValidator from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder from ytdl_sub.entries.entry import Entry from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive diff --git a/src/ytdl_sub/downloaders/url/validators.py b/src/ytdl_sub/downloaders/url/validators.py index e3727440..761a93bf 100644 --- a/src/ytdl_sub/downloaders/url/validators.py +++ b/src/ytdl_sub/downloaders/url/validators.py @@ -4,8 +4,8 @@ from typing import Dict from typing import Optional from typing import Set -from ytdl_sub.config.plugin_operation import PluginOperation -from ytdl_sub.config.preset_options import OptionsValidator +from ytdl_sub.config.plugin.plugin_operation import PluginOperation +from ytdl_sub.config.validators.options import OptionsValidator from ytdl_sub.validators.strict_dict_validator import StrictDictValidator from ytdl_sub.validators.string_formatter_validators import DictFormatterValidator from ytdl_sub.validators.string_formatter_validators import OverridesStringFormatterValidator diff --git a/src/ytdl_sub/plugins/audio_extract.py b/src/ytdl_sub/plugins/audio_extract.py index 8f80ed18..97ebdb3d 100644 --- a/src/ytdl_sub/plugins/audio_extract.py +++ b/src/ytdl_sub/plugins/audio_extract.py @@ -4,9 +4,9 @@ from typing import Dict from typing import Optional from typing import Set -from ytdl_sub.config.plugin import Plugin -from ytdl_sub.config.plugin_operation import PluginOperation -from ytdl_sub.config.preset_options import OptionsDictValidator +from ytdl_sub.config.plugin.plugin import Plugin +from ytdl_sub.config.plugin.plugin_operation import PluginOperation +from ytdl_sub.config.validators.options import OptionsDictValidator from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder from ytdl_sub.entries.entry import Entry from ytdl_sub.entries.script.variable_definitions import VARIABLES as v @@ -68,9 +68,7 @@ class AudioExtractOptions(OptionsDictValidator): return self._quality.value return None - def added_source_variables( - self, unresolved_variables: Set[str] - ) -> Dict[PluginOperation, Set[str]]: + def modified_variables(self) -> Dict[PluginOperation, Set[str]]: """ Possibly changes ``ext``, so do not resolve until this has run """ diff --git a/src/ytdl_sub/plugins/chapters.py b/src/ytdl_sub/plugins/chapters.py index 18330764..7e353028 100644 --- a/src/ytdl_sub/plugins/chapters.py +++ b/src/ytdl_sub/plugins/chapters.py @@ -5,8 +5,8 @@ from typing import List from typing import Optional from typing import Set -from ytdl_sub.config.plugin import Plugin -from ytdl_sub.config.preset_options import OptionsDictValidator +from ytdl_sub.config.plugin.plugin import Plugin +from ytdl_sub.config.validators.options import OptionsDictValidator from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder from ytdl_sub.entries.entry import Entry from ytdl_sub.entries.script.variable_definitions import VARIABLES as v diff --git a/src/ytdl_sub/plugins/date_range.py b/src/ytdl_sub/plugins/date_range.py index 1d93a483..092d1054 100644 --- a/src/ytdl_sub/plugins/date_range.py +++ b/src/ytdl_sub/plugins/date_range.py @@ -2,8 +2,8 @@ from typing import List from typing import Optional from typing import Tuple -from ytdl_sub.config.plugin import Plugin -from ytdl_sub.config.preset_options import OptionsDictValidator +from ytdl_sub.config.plugin.plugin import Plugin +from ytdl_sub.config.validators.options import OptionsDictValidator from ytdl_sub.utils.datetime import to_date_str from ytdl_sub.validators.string_datetime import StringDatetimeValidator diff --git a/src/ytdl_sub/plugins/embed_thumbnail.py b/src/ytdl_sub/plugins/embed_thumbnail.py index 5946d550..64703870 100644 --- a/src/ytdl_sub/plugins/embed_thumbnail.py +++ b/src/ytdl_sub/plugins/embed_thumbnail.py @@ -3,8 +3,8 @@ from typing import Optional import mediafile -from ytdl_sub.config.plugin import Plugin -from ytdl_sub.config.preset_options import OptionsValidator +from ytdl_sub.config.plugin.plugin import Plugin +from ytdl_sub.config.validators.options import OptionsValidator from ytdl_sub.entries.entry import Entry from ytdl_sub.utils.ffmpeg import FFMPEG from ytdl_sub.utils.file_handler import FileHandler diff --git a/src/ytdl_sub/plugins/file_convert.py b/src/ytdl_sub/plugins/file_convert.py index 2f53f471..cfba3eb0 100644 --- a/src/ytdl_sub/plugins/file_convert.py +++ b/src/ytdl_sub/plugins/file_convert.py @@ -5,9 +5,9 @@ from typing import Optional from typing import Set from ytdl_sub.config.overrides import Overrides -from ytdl_sub.config.plugin import Plugin -from ytdl_sub.config.plugin_operation import PluginOperation -from ytdl_sub.config.preset_options import OptionsDictValidator +from ytdl_sub.config.plugin.plugin import Plugin +from ytdl_sub.config.plugin.plugin_operation import PluginOperation +from ytdl_sub.config.validators.options import OptionsDictValidator from ytdl_sub.entries.entry import Entry from ytdl_sub.entries.script.variable_definitions import VARIABLES as v from ytdl_sub.utils.exceptions import FileNotDownloadedException @@ -118,9 +118,7 @@ class FileConvertOptions(OptionsDictValidator): """ return self._ffmpeg_post_process_args - def added_source_variables( - self, unresolved_variables: Set[str] - ) -> Dict[PluginOperation, Set[str]]: + def modified_variables(self) -> Dict[PluginOperation, Set[str]]: return {PluginOperation.MODIFY_ENTRY: {v.ext.variable_name}} diff --git a/src/ytdl_sub/plugins/format.py b/src/ytdl_sub/plugins/format.py index 7f1ccc06..3b5c334f 100644 --- a/src/ytdl_sub/plugins/format.py +++ b/src/ytdl_sub/plugins/format.py @@ -1,8 +1,8 @@ from typing import Dict from typing import Optional -from ytdl_sub.config.plugin import Plugin -from ytdl_sub.config.preset_options import OptionsValidator +from ytdl_sub.config.plugin.plugin import Plugin +from ytdl_sub.config.validators.options import OptionsValidator from ytdl_sub.validators.validators import StringValidator diff --git a/src/ytdl_sub/plugins/internal/view.py b/src/ytdl_sub/plugins/internal/view.py index 1aecd189..39685ad3 100644 --- a/src/ytdl_sub/plugins/internal/view.py +++ b/src/ytdl_sub/plugins/internal/view.py @@ -2,8 +2,8 @@ import copy from typing import Optional from ytdl_sub.config.overrides import Overrides -from ytdl_sub.config.plugin import Plugin -from ytdl_sub.config.preset_options import OptionsDictValidator +from ytdl_sub.config.plugin.plugin import Plugin +from ytdl_sub.config.validators.options import OptionsDictValidator from ytdl_sub.entries.entry import Entry from ytdl_sub.utils.file_handler import FileMetadata from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive diff --git a/src/ytdl_sub/plugins/match_filters.py b/src/ytdl_sub/plugins/match_filters.py index ef4d6d4a..4e1ad498 100644 --- a/src/ytdl_sub/plugins/match_filters.py +++ b/src/ytdl_sub/plugins/match_filters.py @@ -3,8 +3,8 @@ from typing import Any from typing import List from typing import Tuple -from ytdl_sub.config.plugin import Plugin -from ytdl_sub.config.preset_options import OptionsDictValidator +from ytdl_sub.config.plugin.plugin import Plugin +from ytdl_sub.config.validators.options import OptionsDictValidator from ytdl_sub.utils.logger import Logger from ytdl_sub.validators.validators import StringListValidator diff --git a/src/ytdl_sub/plugins/music_tags.py b/src/ytdl_sub/plugins/music_tags.py index f0de6158..4d6bafc9 100644 --- a/src/ytdl_sub/plugins/music_tags.py +++ b/src/ytdl_sub/plugins/music_tags.py @@ -6,8 +6,8 @@ from typing import List import mediafile -from ytdl_sub.config.plugin import Plugin -from ytdl_sub.config.preset_options import OptionsDictValidator +from ytdl_sub.config.plugin.plugin import Plugin +from ytdl_sub.config.validators.options import OptionsDictValidator from ytdl_sub.entries.entry import Entry from ytdl_sub.entries.script.variable_definitions import VARIABLES as v from ytdl_sub.utils.file_handler import FileMetadata diff --git a/src/ytdl_sub/plugins/nfo_tags.py b/src/ytdl_sub/plugins/nfo_tags.py index 2ff09d55..92e2b968 100644 --- a/src/ytdl_sub/plugins/nfo_tags.py +++ b/src/ytdl_sub/plugins/nfo_tags.py @@ -7,8 +7,8 @@ from typing import Dict from typing import List from typing import Optional -from ytdl_sub.config.plugin import Plugin -from ytdl_sub.config.preset_options import OptionsDictValidator +from ytdl_sub.config.plugin.plugin import Plugin +from ytdl_sub.config.validators.options import OptionsDictValidator from ytdl_sub.entries.entry import Entry from ytdl_sub.utils.file_handler import FileHandler from ytdl_sub.utils.file_handler import FileMetadata diff --git a/src/ytdl_sub/plugins/regex.py b/src/ytdl_sub/plugins/regex.py index e53a9873..e4788bc7 100644 --- a/src/ytdl_sub/plugins/regex.py +++ b/src/ytdl_sub/plugins/regex.py @@ -6,9 +6,9 @@ from typing import Optional from typing import Set from ytdl_sub.config.overrides import Overrides -from ytdl_sub.config.plugin import Plugin -from ytdl_sub.config.plugin_operation import PluginOperation -from ytdl_sub.config.preset_options import OptionsDictValidator +from ytdl_sub.config.plugin.plugin import Plugin +from ytdl_sub.config.plugin.plugin_operation import PluginOperation +from ytdl_sub.config.validators.options import OptionsDictValidator from ytdl_sub.entries.entry import Entry from ytdl_sub.script.parser import parse from ytdl_sub.script.utils.exceptions import ScriptVariableNotResolved diff --git a/src/ytdl_sub/plugins/split_by_chapters.py b/src/ytdl_sub/plugins/split_by_chapters.py index 6c24c5df..0ed74bce 100644 --- a/src/ytdl_sub/plugins/split_by_chapters.py +++ b/src/ytdl_sub/plugins/split_by_chapters.py @@ -9,9 +9,9 @@ from typing import Tuple from yt_dlp.utils import sanitize_filename -from ytdl_sub.config.plugin import SplitPlugin -from ytdl_sub.config.plugin_operation import PluginOperation -from ytdl_sub.config.preset_options import OptionsDictValidator +from ytdl_sub.config.plugin.plugin import SplitPlugin +from ytdl_sub.config.plugin.plugin_operation import PluginOperation +from ytdl_sub.config.validators.options import OptionsDictValidator from ytdl_sub.entries.entry import Entry from ytdl_sub.entries.variables.kwargs import CHAPTERS from ytdl_sub.entries.variables.kwargs import SPLIT_BY_CHAPTERS_PARENT_ENTRY diff --git a/src/ytdl_sub/plugins/subtitles.py b/src/ytdl_sub/plugins/subtitles.py index 356cf398..c4b27e69 100644 --- a/src/ytdl_sub/plugins/subtitles.py +++ b/src/ytdl_sub/plugins/subtitles.py @@ -4,9 +4,9 @@ from typing import List from typing import Optional from typing import Set -from ytdl_sub.config.plugin import Plugin -from ytdl_sub.config.plugin_operation import PluginOperation -from ytdl_sub.config.preset_options import OptionsDictValidator +from ytdl_sub.config.plugin.plugin import Plugin +from ytdl_sub.config.plugin.plugin_operation import PluginOperation +from ytdl_sub.config.validators.options import OptionsDictValidator from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder from ytdl_sub.entries.entry import Entry from ytdl_sub.entries.script.variable_definitions import VARIABLES as v diff --git a/src/ytdl_sub/plugins/throttle_protection.py b/src/ytdl_sub/plugins/throttle_protection.py index 40727ca7..5bab172c 100644 --- a/src/ytdl_sub/plugins/throttle_protection.py +++ b/src/ytdl_sub/plugins/throttle_protection.py @@ -5,8 +5,8 @@ from typing import Optional from typing import Tuple from ytdl_sub.config.overrides import Overrides -from ytdl_sub.config.plugin import Plugin -from ytdl_sub.config.preset_options import OptionsDictValidator +from ytdl_sub.config.plugin.plugin import Plugin +from ytdl_sub.config.validators.options import OptionsDictValidator from ytdl_sub.entries.entry import Entry from ytdl_sub.utils.file_handler import FileMetadata from ytdl_sub.utils.logger import Logger diff --git a/src/ytdl_sub/plugins/video_tags.py b/src/ytdl_sub/plugins/video_tags.py index e27af4d8..9ce3d8fd 100644 --- a/src/ytdl_sub/plugins/video_tags.py +++ b/src/ytdl_sub/plugins/video_tags.py @@ -2,8 +2,8 @@ import copy from typing import Any from typing import Dict -from ytdl_sub.config.plugin import Plugin -from ytdl_sub.config.preset_options import OptionsDictValidator +from ytdl_sub.config.plugin.plugin import Plugin +from ytdl_sub.config.validators.options import OptionsDictValidator from ytdl_sub.entries.entry import Entry from ytdl_sub.utils.ffmpeg import add_ffmpeg_metadata_key_values from ytdl_sub.utils.file_handler import FileMetadata diff --git a/src/ytdl_sub/script/script.py b/src/ytdl_sub/script/script.py index ad1a93c7..f86e1e46 100644 --- a/src/ytdl_sub/script/script.py +++ b/src/ytdl_sub/script/script.py @@ -402,3 +402,7 @@ class Script: return resolvable raise RuntimeException(f"Tried to get unresolved variable {variable_name}") + + @property + def variable_names(self) -> Set[str]: + return set(list(self._variables.keys())) diff --git a/src/ytdl_sub/subscriptions/base_subscription.py b/src/ytdl_sub/subscriptions/base_subscription.py index 08bc344c..fc382168 100644 --- a/src/ytdl_sub/subscriptions/base_subscription.py +++ b/src/ytdl_sub/subscriptions/base_subscription.py @@ -5,7 +5,7 @@ from typing import Optional from ytdl_sub.config.config_validator import ConfigOptions from ytdl_sub.config.overrides import Overrides from ytdl_sub.config.preset import Preset -from ytdl_sub.config.preset import PresetPlugins +from ytdl_sub.config.plugin.preset_plugins import PresetPlugins from ytdl_sub.config.preset_options import OutputOptions from ytdl_sub.config.preset_options import YTDLOptions from ytdl_sub.downloaders.url.validators import MultiUrlValidator diff --git a/src/ytdl_sub/subscriptions/subscription_download.py b/src/ytdl_sub/subscriptions/subscription_download.py index 6686df55..7bbd09ca 100644 --- a/src/ytdl_sub/subscriptions/subscription_download.py +++ b/src/ytdl_sub/subscriptions/subscription_download.py @@ -7,10 +7,10 @@ from pathlib import Path from typing import List from typing import Optional -from ytdl_sub.config.plugin import Plugin -from ytdl_sub.config.plugin import SplitPlugin -from ytdl_sub.config.plugin_mapping import PluginMapping -from ytdl_sub.config.plugin_operation import PluginOperation +from ytdl_sub.config.plugin.plugin import Plugin +from ytdl_sub.config.plugin.plugin import SplitPlugin +from ytdl_sub.config.plugin.plugin_mapping import PluginMapping +from ytdl_sub.config.plugin.plugin_operation import PluginOperation from ytdl_sub.downloaders.info_json.info_json_downloader import InfoJsonDownloader from ytdl_sub.downloaders.info_json.info_json_downloader import InfoJsonDownloaderOptions from ytdl_sub.downloaders.source_plugin import SourcePlugin diff --git a/src/ytdl_sub/subscriptions/subscription_ytdl_options.py b/src/ytdl_sub/subscriptions/subscription_ytdl_options.py index 6a7a7cea..14916c2e 100644 --- a/src/ytdl_sub/subscriptions/subscription_ytdl_options.py +++ b/src/ytdl_sub/subscriptions/subscription_ytdl_options.py @@ -7,7 +7,7 @@ from typing import TypeVar from yt_dlp import match_filter_func -from ytdl_sub.config.plugin import Plugin +from ytdl_sub.config.plugin.plugin import Plugin from ytdl_sub.config.preset import Preset from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder from ytdl_sub.plugins.audio_extract import AudioExtractPlugin diff --git a/tests/unit/config/test_config_file.py b/tests/unit/config/test_config_file.py index 265cf840..56b97498 100644 --- a/tests/unit/config/test_config_file.py +++ b/tests/unit/config/test_config_file.py @@ -5,7 +5,7 @@ from typing import Optional import pytest from ytdl_sub.config.config_file import ConfigFile -from ytdl_sub.config.plugin_mapping import PluginMapping +from ytdl_sub.config.plugin.plugin_mapping import PluginMapping from ytdl_sub.config.preset import PRESET_KEYS from ytdl_sub.utils.exceptions import ValidationException