diff --git a/src/ytdl_sub/config/preset.py b/src/ytdl_sub/config/preset.py index fdc622ef..68a5d281 100644 --- a/src/ytdl_sub/config/preset.py +++ b/src/ytdl_sub/config/preset.py @@ -1,3 +1,4 @@ +import copy from typing import Any from typing import Dict from typing import List @@ -5,6 +6,10 @@ from typing import Optional from typing import Tuple from typing import Type +import yaml +from mergedeep import mergedeep + +from ytdl_sub.config.config_file import ConfigFile from ytdl_sub.config.preset_class_mappings import DownloadStrategyMapping from ytdl_sub.config.preset_class_mappings import PluginMapping from ytdl_sub.config.preset_options import OutputOptions @@ -21,8 +26,9 @@ from ytdl_sub.validators.validators import DictValidator from ytdl_sub.validators.validators import StringValidator from ytdl_sub.validators.validators import Validator -PRESET_REQUIRED_KEYS = {"output_options"} -PRESET_OPTIONAL_KEYS = { +PRESET_KEYS = { + "preset", + "output_options", "ytdl_options", "overrides", *DownloadStrategyMapping.sources(), @@ -69,8 +75,10 @@ class DownloadStrategyValidator(StrictDictValidator): class Preset(StrictDictValidator): - _required_keys = PRESET_REQUIRED_KEYS - _optional_keys = PRESET_OPTIONAL_KEYS + # Have all present keys optional since parent presets could not have all the + # required keys. They will get validated in the init after the mergedeep of dicts + # and ensure required keys are present. + _optional_keys = PRESET_KEYS def __validate_and_get_downloader(self, downloader_source: str) -> Type[Downloader]: return self._validate_key(key=downloader_source, validator=DownloadStrategyValidator).get( @@ -176,9 +184,43 @@ class Preset(StrictDictValidator): if isinstance(validator, OverridesStringFormatterValidator): self.__validate_override_string_formatter_validator(validator) - def __init__(self, name: str, value: Any): + def __merge_parent_preset_dicts_if_present(self, config: ConfigFile): + parent_presets = set() + parent_preset_validator = self._validate_key_if_present( + key="preset", validator=StringValidator + ) + parent_preset = parent_preset_validator.value if parent_preset_validator else None + + while parent_preset: + # Make sure the parent preset actually exists + if parent_preset not in config.presets.keys: + raise self._validation_exception( + f"preset '{parent_preset}' does not exist in the provided config. " + f"Available presets: {', '.join(config.presets.keys)}" + ) + + # Make sure we do not hit an infinite loop + if parent_preset in parent_presets: + raise self._validation_exception( + f"preset loop detected with the preset '{parent_preset}'" + ) + + parent_preset_dict = copy.deepcopy(config.presets.dict[parent_preset]) + + parent_presets.add(parent_preset) + parent_preset = parent_preset_dict.get("preset") + + # Override the parent preset with the contents of this preset + self._value = mergedeep.merge( + parent_preset_dict, self._value, strategy=mergedeep.Strategy.REPLACE + ) + + def __init__(self, config: ConfigFile, name: str, value: Any): super().__init__(name=name, value=value) + # Perform the merge of parent presets before validating any keys + self.__merge_parent_preset_dicts_if_present(config=config) + self.downloader, self.downloader_options = self.__validate_and_get_downloader_and_options() self.output_options = self._validate_key( @@ -196,3 +238,60 @@ class Preset(StrictDictValidator): # After all options are initialized, perform a recursive post-validate that requires # values from multiple validators self.__recursive_preset_validate() + + @property + def name(self) -> str: + """ + Returns + ------- + Name of the preset + """ + return self._name + + @classmethod + def from_dict(cls, config: ConfigFile, preset_name: str, preset_dict: Dict) -> "Preset": + """ + Parameters + ---------- + config: + Validated instance of the config + preset_name: + Name of the preset + preset_dict: + The preset config in dict format + + Returns + ------- + The Subscription validator + """ + return cls(config=config, name=preset_name, value=preset_dict) + + @classmethod + def from_file_path(cls, config: ConfigFile, subscription_path: str) -> List["Preset"]: + """ + Parameters + ---------- + config: + Validated instance of the config + subscription_path: + File path to the subscription yaml file + + Returns + ------- + List of presets, for each one in the subscription yaml + """ + # TODO: Create separate yaml file loader class + with open(subscription_path, "r", encoding="utf-8") as file: + subscription_dict = yaml.safe_load(file) + + subscriptions: List["Preset"] = [] + for subscription_key, subscription_object in subscription_dict.items(): + subscriptions.append( + Preset.from_dict( + config=config, + preset_name=subscription_key, + preset_dict=subscription_object, + ) + ) + + return subscriptions diff --git a/src/ytdl_sub/config/subscription.py b/src/ytdl_sub/config/subscription.py deleted file mode 100644 index 6d046a00..00000000 --- a/src/ytdl_sub/config/subscription.py +++ /dev/null @@ -1,131 +0,0 @@ -import copy -from typing import Any -from typing import Dict -from typing import List - -import yaml -from mergedeep import mergedeep - -from ytdl_sub.config.config_file import ConfigFile -from ytdl_sub.config.preset import PRESET_OPTIONAL_KEYS -from ytdl_sub.config.preset import PRESET_REQUIRED_KEYS -from ytdl_sub.config.preset import Preset -from ytdl_sub.config.preset_options import Overrides -from ytdl_sub.subscriptions.subscription import Subscription -from ytdl_sub.validators.strict_dict_validator import StrictDictValidator -from ytdl_sub.validators.validators import StringValidator - - -class SubscriptionValidator(StrictDictValidator): - """ - A Subscription is a preset but overrides it with specific values - """ - - _required_keys = {"preset"} - _optional_keys = PRESET_REQUIRED_KEYS.union(PRESET_OPTIONAL_KEYS) - - def __init__(self, config: ConfigFile, name: str, value: Any): - super().__init__(name, value) - self.config = config - - # Ensure the overrides defined here are valid - _ = self._validate_key( - key="overrides", - validator=Overrides, - default={}, - ) - - preset_name = self._validate_key( - key="preset", - validator=StringValidator, - ).value - - if preset_name not in self.config.presets.keys: - raise self._validation_exception( - f"preset '{preset_name}' does not exist in the provided config. " - f"Available presets: {', '.join(self.config.presets.keys)}" - ) - - # A little hacky, we will override the preset with the contents of this subscription, - # then validate it - preset_dict = copy.deepcopy(self.config.presets.dict[preset_name]) - preset_dict = mergedeep.merge(preset_dict, self._dict, strategy=mergedeep.Strategy.REPLACE) - del preset_dict["preset"] - - self.preset = Preset( - name=f"{self._name}.{preset_name}", - value=preset_dict, - ) - - def to_subscription(self) -> Subscription: - """ - Returns - ------- - The subscription after the config and preset have been validated - """ - return Subscription( - name=self._name, - config_options=self.config.config_options, - preset_options=self.preset, - ) - - @property - def name(self) -> str: - """ - Returns - ------- - Name of the subscription - """ - return self._name - - @classmethod - def from_dict( - cls, config: ConfigFile, subscription_name: str, subscription_dict: Dict - ) -> "SubscriptionValidator": - """ - Parameters - ---------- - config: - Validated instance of the config - subscription_name: - Name of the subscription - subscription_dict: - The subscription config in dict format - - Returns - ------- - The Subscription validator - """ - return SubscriptionValidator(config=config, name=subscription_name, value=subscription_dict) - - @classmethod - def from_file_path( - cls, config: ConfigFile, subscription_path: str - ) -> List["SubscriptionValidator"]: - """ - Parameters - ---------- - config: - Validated instance of the config - subscription_path: - File path to the subscription yaml file - - Returns - ------- - List of subscription validators, for each one in the subscription yaml - """ - # TODO: Create separate yaml file loader class - with open(subscription_path, "r", encoding="utf-8") as file: - subscription_dict = yaml.safe_load(file) - - subscriptions: List["SubscriptionValidator"] = [] - for subscription_key, subscription_object in subscription_dict.items(): - subscriptions.append( - SubscriptionValidator.from_dict( - config=config, - subscription_name=subscription_key, - subscription_dict=subscription_object, - ) - ) - - return subscriptions diff --git a/src/ytdl_sub/main.py b/src/ytdl_sub/main.py index 7226f480..c68e546e 100644 --- a/src/ytdl_sub/main.py +++ b/src/ytdl_sub/main.py @@ -5,7 +5,8 @@ from typing import List from ytdl_sub.cli.download_args_parser import DownloadArgsParser from ytdl_sub.cli.main_args_parser import parser from ytdl_sub.config.config_file import ConfigFile -from ytdl_sub.config.subscription import SubscriptionValidator +from ytdl_sub.config.preset import Preset +from ytdl_sub.subscriptions.subscription import Subscription from ytdl_sub.utils.exceptions import ValidationException from ytdl_sub.utils.logger import Logger @@ -19,17 +20,17 @@ def _download_subscriptions_from_yaml_files(config: ConfigFile, args: argparse.N :param config: Configuration file :param args: Arguments from argparse """ - subscription_paths: List[str] = args.subscription_paths - subscriptions: List[SubscriptionValidator] = [] + preset_paths: List[str] = args.subscription_paths + presets: List[Preset] = [] - for subscription_path in subscription_paths: - subscriptions += SubscriptionValidator.from_file_path( - config=config, subscription_path=subscription_path - ) + for preset_path in preset_paths: + presets += Preset.from_file_path(config=config, subscription_path=preset_path) + + for preset in presets: + subscription = Subscription.from_preset(preset=preset, config=config) - for subscription in subscriptions: logger.info("Beginning subscription download for %s", subscription.name) - subscription.to_subscription().download() + subscription.download() def _download_subscription_from_cli(config: ConfigFile, extra_args: List[str]) -> None: @@ -43,11 +44,18 @@ def _download_subscription_from_cli(config: ConfigFile, extra_args: List[str]) - subscription_args_dict = dl_args_parser.to_subscription_dict() subscription_name = f"cli-dl-{dl_args_parser.get_args_hash()}" - SubscriptionValidator.from_dict( + subscription_preset = Preset.from_dict( config=config, - subscription_name=subscription_name, - subscription_dict=subscription_args_dict, - ).to_subscription().download() + preset_name=subscription_name, + preset_dict=subscription_args_dict, + ) + + subscription = Subscription.from_preset( + preset=subscription_preset, + config=config, + ) + + subscription.download() def main(): diff --git a/src/ytdl_sub/subscriptions/subscription.py b/src/ytdl_sub/subscriptions/subscription.py index 11333731..299526fe 100644 --- a/src/ytdl_sub/subscriptions/subscription.py +++ b/src/ytdl_sub/subscriptions/subscription.py @@ -7,6 +7,7 @@ from typing import List from typing import Tuple from typing import Type +from ytdl_sub.config.config_file import ConfigFile from ytdl_sub.config.config_file import ConfigOptions from ytdl_sub.config.preset import Preset from ytdl_sub.config.preset_options import OutputOptions @@ -254,3 +255,25 @@ class Subscription: downloader.post_download( overrides=self.overrides, output_directory=self.output_directory ) + + @classmethod + def from_preset(cls, preset: Preset, config: ConfigFile) -> "Subscription": + """ + Creates a subscription from a preset + + Parameters + ---------- + preset + Preset to make the subscription out of + config + The config file that should contain this preset + + Returns + ------- + Initialized subscription + """ + return cls( + name=preset.name, + preset_options=preset, + config_options=config.config_options, + ) diff --git a/tests/e2e/youtube/test_channel_as_kodi_tv_show.py b/tests/e2e/youtube/test_channel_as_kodi_tv_show.py index 03c1e22e..a1fc5fda 100644 --- a/tests/e2e/youtube/test_channel_as_kodi_tv_show.py +++ b/tests/e2e/youtube/test_channel_as_kodi_tv_show.py @@ -6,7 +6,8 @@ import pytest from e2e.expected_download import ExpectedDownload from ytdl_sub.config.config_file import ConfigFile -from ytdl_sub.config.subscription import SubscriptionValidator +from ytdl_sub.config.preset import Preset +from ytdl_sub.subscriptions.subscription import Subscription @pytest.fixture() @@ -52,11 +53,16 @@ def subscription_dict(output_directory, subscription_name): @pytest.fixture def full_channel_subscription(config, subscription_name, subscription_dict): - return SubscriptionValidator.from_dict( + full_channel_preset = Preset.from_dict( config=config, - subscription_name=subscription_name, - subscription_dict=subscription_dict, - ).to_subscription() + preset_name=subscription_name, + preset_dict=subscription_dict, + ) + + return Subscription.from_preset( + preset=full_channel_preset, + config=config, + ) @pytest.fixture @@ -144,11 +150,16 @@ def recent_channel_subscription_dict(subscription_dict): @pytest.fixture def recent_channel_subscription(config, subscription_name, recent_channel_subscription_dict): - return SubscriptionValidator.from_dict( + recent_channel_preset = Preset.from_dict( config=config, - subscription_name=subscription_name, - subscription_dict=recent_channel_subscription_dict, - ).to_subscription() + preset_name=subscription_name, + preset_dict=recent_channel_subscription_dict, + ) + + return Subscription.from_preset( + preset=recent_channel_preset, + config=config, + ) @pytest.fixture @@ -191,11 +202,16 @@ def rolling_recent_channel_subscription_dict(recent_channel_subscription_dict): def rolling_recent_channel_subscription( config, subscription_name, rolling_recent_channel_subscription_dict ): - return SubscriptionValidator.from_dict( + rolling_recent_channel_preset = Preset.from_dict( config=config, - subscription_name=subscription_name, - subscription_dict=rolling_recent_channel_subscription_dict, - ).to_subscription() + preset_name=subscription_name, + preset_dict=rolling_recent_channel_subscription_dict, + ) + + return Subscription.from_preset( + preset=rolling_recent_channel_preset, + config=config, + ) @pytest.fixture