From 6d84e524670d93db3ba94f3c70741f788b4f7a1f Mon Sep 17 00:00:00 2001 From: Jesse Bannon Date: Thu, 1 Sep 2022 10:43:07 -0700 Subject: [PATCH] [FEATURE] Add multiple preset inheritance (#204) --- docs/config.rst | 15 ++++ src/ytdl_sub/config/preset.py | 74 +++++++++++------ tests/unit/config/test_preset.py | 138 +++++++++++++++++++++++++------ 3 files changed, 176 insertions(+), 51 deletions(-) diff --git a/docs/config.rst b/docs/config.rst index 27af4263..ee549655 100644 --- a/docs/config.rst +++ b/docs/config.rst @@ -157,6 +157,8 @@ Presets support inheritance by defining a parent preset: .. code-block:: yaml presets: + custom_preset: + ... parent_preset: ... child_preset: @@ -166,6 +168,19 @@ In the example above, ``child_preset`` inherits all fields defined in ``parent_p It is advantageous to use parent presets where possible to reduce duplicate yaml definitions. +Presets also support inheritance from multiple presets: + +.. code-block:: yaml + + child_preset: + preset: + - "custom_preset" + - "parent_preset" + +In this example, ``child_preset`` will inherit all fields from ``custom_preset`` +and ``parent_preset`` in that order. The bottom-most preset has the highest +priority. + ------------------------------------------------------------------------------- diff --git a/src/ytdl_sub/config/preset.py b/src/ytdl_sub/config/preset.py index 3de9d7ea..20df6ef3 100644 --- a/src/ytdl_sub/config/preset.py +++ b/src/ytdl_sub/config/preset.py @@ -6,6 +6,7 @@ from typing import List from typing import Optional from typing import Tuple from typing import Type +from typing import TypeVar from typing import Union from mergedeep import mergedeep @@ -27,6 +28,7 @@ from ytdl_sub.validators.string_formatter_validators import OverridesDictFormatt from ytdl_sub.validators.string_formatter_validators import OverridesStringFormatterValidator from ytdl_sub.validators.string_formatter_validators import StringFormatterValidator from ytdl_sub.validators.validators import DictValidator +from ytdl_sub.validators.validators import StringListValidator from ytdl_sub.validators.validators import StringValidator from ytdl_sub.validators.validators import Validator @@ -41,6 +43,8 @@ PRESET_KEYS = { class PresetPlugins: + _TPluginOptions = TypeVar("_TPluginOptions", bound=PluginOptions) + def __init__(self): self.plugin_types: List[Type[Plugin]] = [] self.plugin_options: List[PluginOptions] = [] @@ -61,6 +65,22 @@ class PresetPlugins: """ return zip(self.plugin_types, self.plugin_options) + def get(self, plugin_type: Type[_TPluginOptions]) -> Optional[_TPluginOptions]: + """ + Parameters + ---------- + plugin_type + Fetch the plugin options for this type + + Returns + ------- + Options of this plugin if they exit. Otherwise, return None. + """ + plugin_option_types = [type(plugin_options) for plugin_options in self.plugin_options] + if plugin_type in plugin_option_types: + return self.plugin_options[plugin_option_types.index(plugin_type)] + return None + class DownloadStrategyValidator(StrictDictValidator): """ @@ -248,36 +268,40 @@ class Preset(StrictDictValidator): self.__validate_override_string_formatter_validator(validator_value) def __merge_parent_preset_dicts_if_present(self, config: ConfigFile): - parent_presets = set() parent_preset_validator = self._validate_key_if_present( - key="preset", validator=StringValidator + key="preset", validator=StringListValidator ) - parent_preset = parent_preset_validator.value if parent_preset_validator else None - while parent_preset: - # Make sure the parent preset actually exists - if parent_preset not in config.presets.keys: - raise self._validation_exception( - f"preset '{parent_preset}' does not exist in the provided config. " - f"Available presets: {', '.join(config.presets.keys)}" + if parent_preset_validator is None: + return + + for parent_preset in [preset.value for preset in parent_preset_validator.list]: + parent_presets = set() + + while parent_preset: + # Make sure the parent preset actually exists + if parent_preset not in config.presets.keys: + raise self._validation_exception( + f"preset '{parent_preset}' does not exist in the provided config. " + f"Available presets: {', '.join(config.presets.keys)}" + ) + + # Make sure we do not hit an infinite loop + if parent_preset in parent_presets: + raise self._validation_exception( + f"preset loop detected with the preset '{parent_preset}'" + ) + + parent_preset_dict = copy.deepcopy(config.presets.dict[parent_preset]) + + parent_presets.add(parent_preset) + parent_preset = parent_preset_dict.get("preset") + + # Override the parent preset with the contents of this preset + self._value = mergedeep.merge( + parent_preset_dict, self._value, strategy=mergedeep.Strategy.REPLACE ) - # Make sure we do not hit an infinite loop - if parent_preset in parent_presets: - raise self._validation_exception( - f"preset loop detected with the preset '{parent_preset}'" - ) - - parent_preset_dict = copy.deepcopy(config.presets.dict[parent_preset]) - - parent_presets.add(parent_preset) - parent_preset = parent_preset_dict.get("preset") - - # Override the parent preset with the contents of this preset - self._value = mergedeep.merge( - parent_preset_dict, self._value, strategy=mergedeep.Strategy.REPLACE - ) - def __init__(self, config: ConfigFile, name: str, value: Any): super().__init__(name=name, value=value) diff --git a/tests/unit/config/test_preset.py b/tests/unit/config/test_preset.py index fa571631..8cf6f353 100644 --- a/tests/unit/config/test_preset.py +++ b/tests/unit/config/test_preset.py @@ -4,13 +4,39 @@ import pytest from ytdl_sub.config.config_file import ConfigFile from ytdl_sub.config.preset import Preset +from ytdl_sub.plugins.nfo_tags import NfoTagsOptions from ytdl_sub.utils.exceptions import StringFormattingVariableNotFoundException +from ytdl_sub.utils.exceptions import ValidationException @pytest.fixture def config_file() -> ConfigFile: return ConfigFile( - name="config", value={"configuration": {"working_directory": "."}, "presets": {}} + name="config", + value={ + "configuration": {"working_directory": "."}, + "presets": { + "parent_preset_0": {"nfo_tags": {"tags": {"key-1": "preset_0"}}}, + "parent_preset_1": { + "preset": "parent_preset_0", + "nfo_tags": { + "nfo_name": "{uid}.nfo", + "nfo_root": "root", + "tags": {"key-2": "preset_1"}, + }, + }, + "parent_preset_2": { + "nfo_tags": { + "nfo_name": "{uid}.nfo", + "nfo_root": "root", + "tags": {"key-2": "preset_2", "key-3": "preset_2"}, + } + }, + "preset_self_loop": {"preset": "preset_self_loop"}, + "preset_loop_0": {"preset": "preset_loop_1"}, + "preset_loop_1": {"preset": "preset_loop_0"}, + }, + }, ) @@ -22,6 +48,14 @@ def output_options() -> Dict: } +@pytest.fixture +def youtube_video() -> Dict: + return { + "download_strategy": "video", + "video_url": "youtube.com/watch?v=123abc", + } + + class TestPreset: @pytest.mark.parametrize( "source, download_strategy", @@ -48,21 +82,79 @@ class TestPreset: value={source: download_strategy, "output_options": output_options}, ) - def test_preset_with_override_variable(self, config_file, output_options): + def test_preset_with_override_variable(self, config_file, output_options, youtube_video): _ = Preset( config=config_file, name="test", value={ - "youtube": { - "download_strategy": "video", - "video_url": "youtube.com/watch?v=123abc", - }, + "youtube": youtube_video, "output_options": {"output_directory": "dir", "file_name": "{dne_var}"}, "overrides": {"dne_var": "not dne"}, }, ) - def test_preset_error__source_variable_does_not_exist(self, config_file, output_options): + def test_preset_parent(self, config_file, output_options, youtube_video): + preset = Preset( + config=config_file, + name="test", + value={ + "preset": "parent_preset_1", + "youtube": youtube_video, + "output_options": output_options, + "nfo_tags": {"tags": {"key-2": "this-preset"}}, + }, + ) + + nfo_options: NfoTagsOptions = preset.plugins.get(NfoTagsOptions) + tags_string_dict = { + key: formatter.format_string for key, formatter in nfo_options.tags.string_tags.items() + } + + assert tags_string_dict == {"key-1": "preset_0", "key-2": "this-preset"} + + def test_preset_multiple_parents(self, config_file, output_options, youtube_video): + preset = Preset( + config=config_file, + name="test", + value={ + "preset": ["parent_preset_1", "parent_preset_2"], + "youtube": youtube_video, + "output_options": output_options, + "nfo_tags": {"tags": {"key-3": "this-preset"}}, + }, + ) + + nfo_options: NfoTagsOptions = preset.plugins.get(NfoTagsOptions) + tags_string_dict = { + key: formatter.format_string for key, formatter in nfo_options.tags.string_tags.items() + } + + assert tags_string_dict == { + "key-1": "preset_0", + "key-2": "preset_1", + "key-3": "this-preset", + } + + @pytest.mark.parametrize( + "parent_preset", ["preset_self_loop", "preset_loop_0", "preset_loop_1"] + ) + def test_preset_error__parent_loop( + self, config_file, output_options, youtube_video, parent_preset + ): + with pytest.raises(ValidationException, match="preset loop detected"): + _ = Preset( + config=config_file, + name="test", + value={ + "preset": parent_preset, + "youtube": youtube_video, + "output_options": output_options, + }, + ) + + def test_preset_error__source_variable_does_not_exist( + self, config_file, output_options, youtube_video + ): with pytest.raises( StringFormattingVariableNotFoundException, match="Format variable 'dne_var' does not exist", @@ -71,15 +163,14 @@ class TestPreset: config=config_file, name="test", value={ - "youtube": { - "download_strategy": "video", - "video_url": "youtube.com/watch?v=123abc", - }, + "youtube": youtube_video, "output_options": {"output_directory": "dir", "file_name": "{dne_var}"}, }, ) - def test_preset_error__override_variable_does_not_exist(self, config_file, output_options): + def test_preset_error__override_variable_does_not_exist( + self, config_file, output_options, youtube_video + ): with pytest.raises( StringFormattingVariableNotFoundException, match="Override variable 'dne_var' does not exist", @@ -88,15 +179,14 @@ class TestPreset: config=config_file, name="test", value={ - "youtube": { - "download_strategy": "video", - "video_url": "youtube.com/watch?v=123abc", - }, + "youtube": youtube_video, "output_options": {"output_directory": "{dne_var}", "file_name": "file"}, }, ) - def test_preset_error__dict_source_variable_does_not_exist(self, config_file, output_options): + def test_preset_error__dict_source_variable_does_not_exist( + self, config_file, output_options, youtube_video + ): with pytest.raises( StringFormattingVariableNotFoundException, match="Format variable 'dne_var' does not exist", @@ -105,10 +195,7 @@ class TestPreset: config=config_file, name="test", value={ - "youtube": { - "download_strategy": "video", - "video_url": "youtube.com/watch?v=123abc", - }, + "youtube": youtube_video, "output_options": {"output_directory": "dir", "file_name": "file"}, "nfo_tags": { "nfo_name": "the nfo name", @@ -118,7 +205,9 @@ class TestPreset: }, ) - def test_preset_error__dict_override_variable_does_not_exist(self, config_file, output_options): + def test_preset_error__dict_override_variable_does_not_exist( + self, config_file, output_options, youtube_video + ): with pytest.raises( StringFormattingVariableNotFoundException, match="Override variable 'dne_var' does not exist", @@ -127,10 +216,7 @@ class TestPreset: config=config_file, name="test", value={ - "youtube": { - "download_strategy": "video", - "video_url": "youtube.com/watch?v=123abc", - }, + "youtube": youtube_video, "output_options": {"output_directory": "dir", "file_name": "file"}, "output_directory_nfo_tags": { "nfo_name": "the nfo name",