danvics/ytdl-sub
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

parent preset support

Browse source
This commit is contained in:
jbannon 2022-05-11 06:28:59 +00:00
parent f67a4dc7dc
commit ca7b8d08c9
5 changed files with 177 additions and 162 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

109
src/ytdl_sub/config/preset.py
View file

@ -1,3 +1,4 @@
import copy
from typing import Any
from typing import Dict
from typing import List
@ -5,6 +6,10 @@ from typing import Optional
from typing import Tuple
from typing import Type
import yaml
from mergedeep import mergedeep
from ytdl_sub.config.config_file import ConfigFile
from ytdl_sub.config.preset_class_mappings import DownloadStrategyMapping
from ytdl_sub.config.preset_class_mappings import PluginMapping
from ytdl_sub.config.preset_options import OutputOptions
@ -21,8 +26,9 @@ from ytdl_sub.validators.validators import DictValidator
from ytdl_sub.validators.validators import StringValidator
from ytdl_sub.validators.validators import Validator
PRESET_REQUIRED_KEYS = {"output_options"}
PRESET_OPTIONAL_KEYS = {
PRESET_KEYS = {
"preset",
"output_options",
"ytdl_options",
"overrides",
*DownloadStrategyMapping.sources(),
@ -69,8 +75,10 @@ class DownloadStrategyValidator(StrictDictValidator):
class Preset(StrictDictValidator):
_required_keys = PRESET_REQUIRED_KEYS
_optional_keys = PRESET_OPTIONAL_KEYS
# Have all present keys optional since parent presets could not have all the
# required keys. They will get validated in the init after the mergedeep of dicts
# and ensure required keys are present.
_optional_keys = PRESET_KEYS
def __validate_and_get_downloader(self, downloader_source: str) -> Type[Downloader]:
return self._validate_key(key=downloader_source, validator=DownloadStrategyValidator).get(
@ -176,9 +184,43 @@ class Preset(StrictDictValidator):
if isinstance(validator, OverridesStringFormatterValidator):
self.__validate_override_string_formatter_validator(validator)
def __init__(self, name: str, value: Any):
def __merge_parent_preset_dicts_if_present(self, config: ConfigFile):
parent_presets = set()
parent_preset_validator = self._validate_key_if_present(
key="preset", validator=StringValidator
)
parent_preset = parent_preset_validator.value if parent_preset_validator else None
while parent_preset:
# Make sure the parent preset actually exists
if parent_preset not in config.presets.keys:
raise self._validation_exception(
f"preset '{parent_preset}' does not exist in the provided config. "
f"Available presets: {', '.join(config.presets.keys)}"
)
# Make sure we do not hit an infinite loop
if parent_preset in parent_presets:
raise self._validation_exception(
f"preset loop detected with the preset '{parent_preset}'"
)
parent_preset_dict = copy.deepcopy(config.presets.dict[parent_preset])
parent_presets.add(parent_preset)
parent_preset = parent_preset_dict.get("preset")
# Override the parent preset with the contents of this preset
self._value = mergedeep.merge(
parent_preset_dict, self._value, strategy=mergedeep.Strategy.REPLACE
)
def __init__(self, config: ConfigFile, name: str, value: Any):
super().__init__(name=name, value=value)
# Perform the merge of parent presets before validating any keys
self.__merge_parent_preset_dicts_if_present(config=config)
self.downloader, self.downloader_options = self.__validate_and_get_downloader_and_options()
self.output_options = self._validate_key(
@ -196,3 +238,60 @@ class Preset(StrictDictValidator):
# After all options are initialized, perform a recursive post-validate that requires
# values from multiple validators
self.__recursive_preset_validate()
@property
def name(self) -> str:
"""
Returns
-------
Name of the preset
"""
return self._name
@classmethod
def from_dict(cls, config: ConfigFile, preset_name: str, preset_dict: Dict) -> "Preset":
"""
Parameters
----------
config:
Validated instance of the config
preset_name:
Name of the preset
preset_dict:
The preset config in dict format
Returns
-------
The Subscription validator
"""
return cls(config=config, name=preset_name, value=preset_dict)
@classmethod
def from_file_path(cls, config: ConfigFile, subscription_path: str) -> List["Preset"]:
"""
Parameters
----------
config:
Validated instance of the config
subscription_path:
File path to the subscription yaml file
Returns
-------
List of presets, for each one in the subscription yaml
"""
# TODO: Create separate yaml file loader class
with open(subscription_path, "r", encoding="utf-8") as file:
subscription_dict = yaml.safe_load(file)
subscriptions: List["Preset"] = []
for subscription_key, subscription_object in subscription_dict.items():
subscriptions.append(
Preset.from_dict(
config=config,
preset_name=subscription_key,
preset_dict=subscription_object,
)
)
return subscriptions

131
src/ytdl_sub/config/subscription.py
View file

@ -1,131 +0,0 @@
import copy
from typing import Any
from typing import Dict
from typing import List
import yaml
from mergedeep import mergedeep
from ytdl_sub.config.config_file import ConfigFile
from ytdl_sub.config.preset import PRESET_OPTIONAL_KEYS
from ytdl_sub.config.preset import PRESET_REQUIRED_KEYS
from ytdl_sub.config.preset import Preset
from ytdl_sub.config.preset_options import Overrides
from ytdl_sub.subscriptions.subscription import Subscription
from ytdl_sub.validators.strict_dict_validator import StrictDictValidator
from ytdl_sub.validators.validators import StringValidator
class SubscriptionValidator(StrictDictValidator):
"""
A Subscription is a preset but overrides it with specific values
"""
_required_keys = {"preset"}
_optional_keys = PRESET_REQUIRED_KEYS.union(PRESET_OPTIONAL_KEYS)
def __init__(self, config: ConfigFile, name: str, value: Any):
super().__init__(name, value)
self.config = config
# Ensure the overrides defined here are valid
_ = self._validate_key(
key="overrides",
validator=Overrides,
default={},
)
preset_name = self._validate_key(
key="preset",
validator=StringValidator,
).value
if preset_name not in self.config.presets.keys:
raise self._validation_exception(
f"preset '{preset_name}' does not exist in the provided config. "
f"Available presets: {', '.join(self.config.presets.keys)}"
)
# A little hacky, we will override the preset with the contents of this subscription,
# then validate it
preset_dict = copy.deepcopy(self.config.presets.dict[preset_name])
preset_dict = mergedeep.merge(preset_dict, self._dict, strategy=mergedeep.Strategy.REPLACE)
del preset_dict["preset"]
self.preset = Preset(
name=f"{self._name}.{preset_name}",
value=preset_dict,
)
def to_subscription(self) -> Subscription:
"""
Returns
-------
The subscription after the config and preset have been validated
"""
return Subscription(
name=self._name,
config_options=self.config.config_options,
preset_options=self.preset,
)
@property
def name(self) -> str:
"""
Returns
-------
Name of the subscription
"""
return self._name
@classmethod
def from_dict(
cls, config: ConfigFile, subscription_name: str, subscription_dict: Dict
) -> "SubscriptionValidator":
"""
Parameters
----------
config:
Validated instance of the config
subscription_name:
Name of the subscription
subscription_dict:
The subscription config in dict format
Returns
-------
The Subscription validator
"""
return SubscriptionValidator(config=config, name=subscription_name, value=subscription_dict)
@classmethod
def from_file_path(
cls, config: ConfigFile, subscription_path: str
) -> List["SubscriptionValidator"]:
"""
Parameters
----------
config:
Validated instance of the config
subscription_path:
File path to the subscription yaml file
Returns
-------
List of subscription validators, for each one in the subscription yaml
"""
# TODO: Create separate yaml file loader class
with open(subscription_path, "r", encoding="utf-8") as file:
subscription_dict = yaml.safe_load(file)
subscriptions: List["SubscriptionValidator"] = []
for subscription_key, subscription_object in subscription_dict.items():
subscriptions.append(
SubscriptionValidator.from_dict(
config=config,
subscription_name=subscription_key,
subscription_dict=subscription_object,
)
)
return subscriptions

34
src/ytdl_sub/main.py
View file

@ -5,7 +5,8 @@ from typing import List
from ytdl_sub.cli.download_args_parser import DownloadArgsParser
from ytdl_sub.cli.main_args_parser import parser
from ytdl_sub.config.config_file import ConfigFile
from ytdl_sub.config.subscription import SubscriptionValidator
from ytdl_sub.config.preset import Preset
from ytdl_sub.subscriptions.subscription import Subscription
from ytdl_sub.utils.exceptions import ValidationException
from ytdl_sub.utils.logger import Logger
@ -19,17 +20,17 @@ def _download_subscriptions_from_yaml_files(config: ConfigFile, args: argparse.N
:param config: Configuration file
:param args: Arguments from argparse
"""
subscription_paths: List[str] = args.subscription_paths
subscriptions: List[SubscriptionValidator] = []
preset_paths: List[str] = args.subscription_paths
presets: List[Preset] = []
for subscription_path in subscription_paths:
subscriptions += SubscriptionValidator.from_file_path(
config=config, subscription_path=subscription_path
)
for preset_path in preset_paths:
presets += Preset.from_file_path(config=config, subscription_path=preset_path)
for preset in presets:
subscription = Subscription.from_preset(preset=preset, config=config)
for subscription in subscriptions:
logger.info("Beginning subscription download for %s", subscription.name)
subscription.to_subscription().download()
subscription.download()
def _download_subscription_from_cli(config: ConfigFile, extra_args: List[str]) -> None:
@ -43,11 +44,18 @@ def _download_subscription_from_cli(config: ConfigFile, extra_args: List[str]) -
subscription_args_dict = dl_args_parser.to_subscription_dict()
subscription_name = f"cli-dl-{dl_args_parser.get_args_hash()}"
SubscriptionValidator.from_dict(
subscription_preset = Preset.from_dict(
config=config,
subscription_name=subscription_name,
subscription_dict=subscription_args_dict,
).to_subscription().download()
preset_name=subscription_name,
preset_dict=subscription_args_dict,
)
subscription = Subscription.from_preset(
preset=subscription_preset,
config=config,
)
subscription.download()
def main():

23
src/ytdl_sub/subscriptions/subscription.py
View file

@ -7,6 +7,7 @@ from typing import List
from typing import Tuple
from typing import Type
from ytdl_sub.config.config_file import ConfigFile
from ytdl_sub.config.config_file import ConfigOptions
from ytdl_sub.config.preset import Preset
from ytdl_sub.config.preset_options import OutputOptions
@ -254,3 +255,25 @@ class Subscription:
downloader.post_download(
overrides=self.overrides, output_directory=self.output_directory
)
@classmethod
def from_preset(cls, preset: Preset, config: ConfigFile) -> "Subscription":
"""
Creates a subscription from a preset
Parameters
----------
preset
Preset to make the subscription out of
config
The config file that should contain this preset
Returns
-------
Initialized subscription
"""
return cls(
name=preset.name,
preset_options=preset,
config_options=config.config_options,
)

42
tests/e2e/youtube/test_channel_as_kodi_tv_show.py
View file

@ -6,7 +6,8 @@ import pytest
from e2e.expected_download import ExpectedDownload
from ytdl_sub.config.config_file import ConfigFile
from ytdl_sub.config.subscription import SubscriptionValidator
from ytdl_sub.config.preset import Preset
from ytdl_sub.subscriptions.subscription import Subscription
@pytest.fixture()
@ -52,11 +53,16 @@ def subscription_dict(output_directory, subscription_name):
@pytest.fixture
def full_channel_subscription(config, subscription_name, subscription_dict):
return SubscriptionValidator.from_dict(
full_channel_preset = Preset.from_dict(
config=config,
subscription_name=subscription_name,
subscription_dict=subscription_dict,
).to_subscription()
preset_name=subscription_name,
preset_dict=subscription_dict,
)
return Subscription.from_preset(
preset=full_channel_preset,
config=config,
)
@pytest.fixture
@ -144,11 +150,16 @@ def recent_channel_subscription_dict(subscription_dict):
@pytest.fixture
def recent_channel_subscription(config, subscription_name, recent_channel_subscription_dict):
return SubscriptionValidator.from_dict(
recent_channel_preset = Preset.from_dict(
config=config,
subscription_name=subscription_name,
subscription_dict=recent_channel_subscription_dict,
).to_subscription()
preset_name=subscription_name,
preset_dict=recent_channel_subscription_dict,
)
return Subscription.from_preset(
preset=recent_channel_preset,
config=config,
)
@pytest.fixture
@ -191,11 +202,16 @@ def rolling_recent_channel_subscription_dict(recent_channel_subscription_dict):
def rolling_recent_channel_subscription(
config, subscription_name, rolling_recent_channel_subscription_dict
):
return SubscriptionValidator.from_dict(
rolling_recent_channel_preset = Preset.from_dict(
config=config,
subscription_name=subscription_name,
subscription_dict=rolling_recent_channel_subscription_dict,
).to_subscription()
preset_name=subscription_name,
preset_dict=rolling_recent_channel_subscription_dict,
)
return Subscription.from_preset(
preset=rolling_recent_channel_preset,
config=config,
)
@pytest.fixture