From 87338853bc2d872575405f49b1250213c2c740e0 Mon Sep 17 00:00:00 2001 From: jbannon Date: Fri, 22 Apr 2022 17:33:33 +0000 Subject: [PATCH] more yolo refactors --- ytdl_subscribe/config/config_file.py | 42 +++++++++ .../config/config_file_validator.py | 31 ------- .../config/config_options_validator.py | 17 ---- .../config/output_options_validator.py | 44 --------- ytdl_subscribe/config/overrides_validator.py | 36 -------- .../config/{preset_validator.py => preset.py} | 91 +++++++++++++++++-- ...scription_validator.py => subscription.py} | 18 ++-- .../config/ytdl_options_validator.py | 5 - ytdl_subscribe/main.py | 12 +-- ytdl_subscribe/plugins/convert_thumbnail.py | 4 +- ytdl_subscribe/plugins/plugin.py | 4 +- ytdl_subscribe/subscriptions/subscription.py | 16 ++-- 12 files changed, 151 insertions(+), 169 deletions(-) create mode 100644 ytdl_subscribe/config/config_file.py delete mode 100644 ytdl_subscribe/config/config_file_validator.py delete mode 100644 ytdl_subscribe/config/config_options_validator.py delete mode 100644 ytdl_subscribe/config/output_options_validator.py delete mode 100644 ytdl_subscribe/config/overrides_validator.py rename ytdl_subscribe/config/{preset_validator.py => preset.py} (60%) rename ytdl_subscribe/config/{subscription_validator.py => subscription.py} (87%) delete mode 100644 ytdl_subscribe/config/ytdl_options_validator.py diff --git a/ytdl_subscribe/config/config_file.py b/ytdl_subscribe/config/config_file.py new file mode 100644 index 00000000..4a84c3d2 --- /dev/null +++ b/ytdl_subscribe/config/config_file.py @@ -0,0 +1,42 @@ +from typing import Any + +import yaml + +from ytdl_subscribe.validators.strict_dict_validator import StrictDictValidator +from ytdl_subscribe.validators.validators import LiteralDictValidator +from ytdl_subscribe.validators.validators import StringValidator + + +class ConfigOptions(StrictDictValidator): + """Validation for global config options""" + + _required_keys = {"working_directory"} + + def __init__(self, name: str, value: Any): + super().__init__(name, value) + + self.working_directory = self._validate_key( + key="working_directory", validator=StringValidator + ) + + +class ConfigFile(StrictDictValidator): + _required_keys = {"configuration", "presets"} + + def __init__(self, name: str, value: Any): + super().__init__(name, value) + self.config_options = self._validate_key("configuration", ConfigOptions) + + # Make sure presets is a dictionary. Will be validated in `PresetValidator` + self.presets = self._validate_key("presets", LiteralDictValidator) + + @classmethod + def from_dict(cls, config_dict) -> "ConfigFile": + return ConfigFile(name="", value=config_dict) + + @classmethod + def from_file_path(cls, config_path) -> "ConfigFile": + # TODO: Create separate yaml file loader class + with open(config_path, "r", encoding="utf-8") as file: + config_dict = yaml.safe_load(file) + return ConfigFile.from_dict(config_dict) diff --git a/ytdl_subscribe/config/config_file_validator.py b/ytdl_subscribe/config/config_file_validator.py deleted file mode 100644 index 82009314..00000000 --- a/ytdl_subscribe/config/config_file_validator.py +++ /dev/null @@ -1,31 +0,0 @@ -from typing import Any - -import yaml - -from ytdl_subscribe.config.config_options_validator import ConfigOptionsValidator -from ytdl_subscribe.validators.strict_dict_validator import StrictDictValidator -from ytdl_subscribe.validators.validators import LiteralDictValidator - - -class ConfigPresetsValidator(LiteralDictValidator): - """Shallow validator checking for the presets dict in the config""" - - -class ConfigFileValidator(StrictDictValidator): - _required_keys = {"configuration", "presets"} - - def __init__(self, name: str, value: Any): - super().__init__(name, value) - self.config_options = self._validate_key("configuration", ConfigOptionsValidator) - self.presets = self._validate_key("presets", ConfigPresetsValidator) - - @classmethod - def from_dict(cls, config_dict) -> "ConfigFileValidator": - return ConfigFileValidator(name="", value=config_dict) - - @classmethod - def from_file_path(cls, config_path) -> "ConfigFileValidator": - # TODO: Create separate yaml file loader class - with open(config_path, "r", encoding="utf-8") as file: - config_dict = yaml.safe_load(file) - return ConfigFileValidator.from_dict(config_dict) diff --git a/ytdl_subscribe/config/config_options_validator.py b/ytdl_subscribe/config/config_options_validator.py deleted file mode 100644 index caad9905..00000000 --- a/ytdl_subscribe/config/config_options_validator.py +++ /dev/null @@ -1,17 +0,0 @@ -from typing import Any - -from ytdl_subscribe.validators.strict_dict_validator import StrictDictValidator -from ytdl_subscribe.validators.validators import StringValidator - - -class ConfigOptionsValidator(StrictDictValidator): - """Validation for the config options""" - - _required_keys = {"working_directory"} - - def __init__(self, name: str, value: Any): - super().__init__(name, value) - - self.working_directory = self._validate_key( - key="working_directory", validator=StringValidator - ) diff --git a/ytdl_subscribe/config/output_options_validator.py b/ytdl_subscribe/config/output_options_validator.py deleted file mode 100644 index a31bc973..00000000 --- a/ytdl_subscribe/config/output_options_validator.py +++ /dev/null @@ -1,44 +0,0 @@ -from ytdl_subscribe.validators.strict_dict_validator import StrictDictValidator -from ytdl_subscribe.validators.string_formatter_validators import OverridesStringFormatterValidator -from ytdl_subscribe.validators.string_formatter_validators import StringFormatterValidator -from ytdl_subscribe.validators.validators import BoolValidator - - -class OutputOptionsValidator(StrictDictValidator): - """Where to output the final files and thumbnails""" - - _required_keys = {"output_directory", "file_name"} - _optional_keys = { - "thumbnail_name", - "maintain_download_archive", - "maintain_stale_file_deletion", - } - - def __init__(self, name, value): - super().__init__(name, value) - - # Output directory should resolve without any entry variables. - # This is to check the directory for any download-archives before any downloads begin - self.output_directory: OverridesStringFormatterValidator = self._validate_key( - key="output_directory", validator=OverridesStringFormatterValidator - ) - - # file name and thumbnails however can use entry variables - self.file_name: StringFormatterValidator = self._validate_key( - key="file_name", validator=StringFormatterValidator - ) - self.thumbnail_name = self._validate_key_if_present( - key="thumbnail_name", validator=StringFormatterValidator - ) - - self.maintain_download_archive = self._validate_key_if_present( - key="maintain_download_archive", validator=BoolValidator, default=False - ) - self.maintain_stale_file_deletion = self._validate_key_if_present( - key="maintain_stale_file_deletion", validator=BoolValidator, default=False - ) - - if self.maintain_stale_file_deletion.value and not self.maintain_download_archive.value: - raise self._validation_exception( - "maintain_stale_file_deletion requires maintain_download_archive set to True" - ) diff --git a/ytdl_subscribe/config/overrides_validator.py b/ytdl_subscribe/config/overrides_validator.py deleted file mode 100644 index 5bf96e86..00000000 --- a/ytdl_subscribe/config/overrides_validator.py +++ /dev/null @@ -1,36 +0,0 @@ -from typing import Optional - -from yt_dlp.utils import sanitize_filename - -from ytdl_subscribe.entries.entry import Entry -from ytdl_subscribe.validators.string_formatter_validators import DictFormatterValidator -from ytdl_subscribe.validators.string_formatter_validators import StringFormatterValidator - - -class OverridesValidator(DictFormatterValidator): - """Ensures `overrides` is a dict""" - - def __init__(self, name, value): - super().__init__(name, value) - for key in self._keys: - sanitized_key_name = f"sanitized_{key}" - # First, sanitize the format string - self._value[sanitized_key_name] = sanitize_filename(self._value[key].format_string) - - # Then, convert it into a StringFormatterValidator - self._value[sanitized_key_name] = StringFormatterValidator( - name="__should_never_fail__", - value=self._value[sanitized_key_name], - ) - - def apply_formatter( - self, formatter: StringFormatterValidator, entry: Optional[Entry] = None - ) -> str: - """ - Returns the format_string after .format has been called on it using entry (if provided) and - override values - """ - variable_dict = self.dict_with_format_strings - if entry: - variable_dict = dict(entry.to_dict(), **variable_dict) - return formatter.apply_formatter(variable_dict) diff --git a/ytdl_subscribe/config/preset_validator.py b/ytdl_subscribe/config/preset.py similarity index 60% rename from ytdl_subscribe/config/preset_validator.py rename to ytdl_subscribe/config/preset.py index bef626ab..3fe67e94 100644 --- a/ytdl_subscribe/config/preset_validator.py +++ b/ytdl_subscribe/config/preset.py @@ -4,18 +4,22 @@ from typing import List from typing import Optional from typing import Type +from yt_dlp.utils import sanitize_filename + from ytdl_subscribe.config.download_strategy_validators import DownloadStrategyValidator from ytdl_subscribe.config.download_strategy_validators import SoundcloudDownloadStrategyValidator from ytdl_subscribe.config.download_strategy_validators import YoutubeDownloadStrategyValidator -from ytdl_subscribe.config.output_options_validator import OutputOptionsValidator -from ytdl_subscribe.config.overrides_validator import OverridesValidator -from ytdl_subscribe.config.ytdl_options_validator import YTDLOptionsValidator from ytdl_subscribe.downloaders.downloader import DownloaderValidator +from ytdl_subscribe.entries.entry import Entry from ytdl_subscribe.utils.exceptions import StringFormattingVariableNotFoundException from ytdl_subscribe.utils.exceptions import ValidationException from ytdl_subscribe.validators.strict_dict_validator import StrictDictValidator +from ytdl_subscribe.validators.string_formatter_validators import DictFormatterValidator from ytdl_subscribe.validators.string_formatter_validators import OverridesStringFormatterValidator +from ytdl_subscribe.validators.string_formatter_validators import StringFormatterValidator +from ytdl_subscribe.validators.validators import BoolValidator from ytdl_subscribe.validators.validators import DictValidator +from ytdl_subscribe.validators.validators import LiteralDictValidator from ytdl_subscribe.validators.validators import Validator PRESET_SOURCE_VALIDATOR_MAPPING: Dict[str, Type[DownloadStrategyValidator]] = { @@ -32,6 +36,79 @@ PRESET_OPTIONAL_KEYS = { } +class YTDLOptions(LiteralDictValidator): + """Ensures `ytdl_options` is a dict""" + + +class Overrides(DictFormatterValidator): + """Ensures `overrides` is a dict""" + + def __init__(self, name, value): + super().__init__(name, value) + for key in self._keys: + sanitized_key_name = f"sanitized_{key}" + # First, sanitize the format string + self._value[sanitized_key_name] = sanitize_filename(self._value[key].format_string) + + # Then, convert it into a StringFormatterValidator + self._value[sanitized_key_name] = StringFormatterValidator( + name="__should_never_fail__", + value=self._value[sanitized_key_name], + ) + + def apply_formatter( + self, formatter: StringFormatterValidator, entry: Optional[Entry] = None + ) -> str: + """ + Returns the format_string after .format has been called on it using entry (if provided) and + override values + """ + variable_dict = self.dict_with_format_strings + if entry: + variable_dict = dict(entry.to_dict(), **variable_dict) + return formatter.apply_formatter(variable_dict) + + +class OutputOptions(StrictDictValidator): + """Where to output the final files and thumbnails""" + + _required_keys = {"output_directory", "file_name"} + _optional_keys = { + "thumbnail_name", + "maintain_download_archive", + "maintain_stale_file_deletion", + } + + def __init__(self, name, value): + super().__init__(name, value) + + # Output directory should resolve without any entry variables. + # This is to check the directory for any download-archives before any downloads begin + self.output_directory: OverridesStringFormatterValidator = self._validate_key( + key="output_directory", validator=OverridesStringFormatterValidator + ) + + # file name and thumbnails however can use entry variables + self.file_name: StringFormatterValidator = self._validate_key( + key="file_name", validator=StringFormatterValidator + ) + self.thumbnail_name = self._validate_key_if_present( + key="thumbnail_name", validator=StringFormatterValidator + ) + + self.maintain_download_archive = self._validate_key_if_present( + key="maintain_download_archive", validator=BoolValidator, default=False + ) + self.maintain_stale_file_deletion = self._validate_key_if_present( + key="maintain_stale_file_deletion", validator=BoolValidator, default=False + ) + + if self.maintain_stale_file_deletion.value and not self.maintain_download_archive.value: + raise self._validation_exception( + "maintain_stale_file_deletion requires maintain_download_archive set to True" + ) + + class PresetValidator(StrictDictValidator): _required_keys = PRESET_REQUIRED_KEYS _optional_keys = PRESET_OPTIONAL_KEYS @@ -113,18 +190,16 @@ class PresetValidator(StrictDictValidator): self.output_options = self._validate_key( key="output_options", - validator=OutputOptionsValidator, + validator=OutputOptions, ) # TODO: REPLACE METADATA OPTIONS WITH PLUGINS self.ytdl_options = self._validate_key( - key="ytdl_options", validator=YTDLOptionsValidator, default={} + key="ytdl_options", validator=YTDLOptions, default={} ) - self.overrides = self._validate_key( - key="overrides", validator=OverridesValidator, default={} - ) + self.overrides = self._validate_key(key="overrides", validator=Overrides, default={}) # After all options are initialized, perform a recursive post-validate that requires # values from multiple validators diff --git a/ytdl_subscribe/config/subscription_validator.py b/ytdl_subscribe/config/subscription.py similarity index 87% rename from ytdl_subscribe/config/subscription_validator.py rename to ytdl_subscribe/config/subscription.py index 99bffde3..9811dfe1 100644 --- a/ytdl_subscribe/config/subscription_validator.py +++ b/ytdl_subscribe/config/subscription.py @@ -8,11 +8,11 @@ from typing import Type import yaml from mergedeep import mergedeep -from ytdl_subscribe.config.config_file_validator import ConfigFileValidator -from ytdl_subscribe.config.overrides_validator import OverridesValidator -from ytdl_subscribe.config.preset_validator import PRESET_OPTIONAL_KEYS -from ytdl_subscribe.config.preset_validator import PRESET_REQUIRED_KEYS -from ytdl_subscribe.config.preset_validator import PresetValidator +from ytdl_subscribe.config.config_file import ConfigFile +from ytdl_subscribe.config.preset import PRESET_OPTIONAL_KEYS +from ytdl_subscribe.config.preset import PRESET_REQUIRED_KEYS +from ytdl_subscribe.config.preset import Overrides +from ytdl_subscribe.config.preset import PresetValidator from ytdl_subscribe.downloaders.soundcloud_downloader import ( SoundcloudAlbumsAndSinglesSourceValidator, ) @@ -36,14 +36,14 @@ class SubscriptionValidator(StrictDictValidator): _required_keys = {"preset"} _optional_keys = PRESET_REQUIRED_KEYS.union(PRESET_OPTIONAL_KEYS) - def __init__(self, config: ConfigFileValidator, name: str, value: Any): + def __init__(self, config: ConfigFile, name: str, value: Any): super().__init__(name, value) self.config = config # Ensure the overrides defined here are valid _ = self._validate_key( key="overrides", - validator=OverridesValidator, + validator=Overrides, default={}, ) @@ -90,13 +90,13 @@ class SubscriptionValidator(StrictDictValidator): @classmethod def from_dict( - cls, config: ConfigFileValidator, subscription_name, subscription_dict: Dict + cls, config: ConfigFile, subscription_name, subscription_dict: Dict ) -> "SubscriptionValidator": return SubscriptionValidator(config=config, name=subscription_name, value=subscription_dict) @classmethod def from_file_path( - cls, config: ConfigFileValidator, subscription_path: str + cls, config: ConfigFile, subscription_path: str ) -> List["SubscriptionValidator"]: # TODO: Create separate yaml file loader class with open(subscription_path, "r", encoding="utf-8") as file: diff --git a/ytdl_subscribe/config/ytdl_options_validator.py b/ytdl_subscribe/config/ytdl_options_validator.py deleted file mode 100644 index 07191e0e..00000000 --- a/ytdl_subscribe/config/ytdl_options_validator.py +++ /dev/null @@ -1,5 +0,0 @@ -from ytdl_subscribe.validators.validators import LiteralDictValidator - - -class YTDLOptionsValidator(LiteralDictValidator): - """Ensures `ytdl_options` is a dict""" diff --git a/ytdl_subscribe/main.py b/ytdl_subscribe/main.py index 55e2a1ea..6f616c90 100644 --- a/ytdl_subscribe/main.py +++ b/ytdl_subscribe/main.py @@ -5,14 +5,12 @@ from typing import List from ytdl_subscribe.cli.download_args_parser import DownloadArgsParser from ytdl_subscribe.cli.main_args_parser import parser -from ytdl_subscribe.config.config_file_validator import ConfigFileValidator -from ytdl_subscribe.config.subscription_validator import SubscriptionValidator +from ytdl_subscribe.config.config_file import ConfigFile +from ytdl_subscribe.config.subscription import SubscriptionValidator from ytdl_subscribe.utils.exceptions import ValidationException -def _download_subscriptions_from_yaml_files( - config: ConfigFileValidator, args: argparse.Namespace -) -> None: +def _download_subscriptions_from_yaml_files(config: ConfigFile, args: argparse.Namespace) -> None: """ Downloads all subscriptions from one or many subscription yaml files. @@ -31,7 +29,7 @@ def _download_subscriptions_from_yaml_files( subscription.to_subscription().download() -def _download_subscription_from_cli(config: ConfigFileValidator, extra_args: List[str]) -> None: +def _download_subscription_from_cli(config: ConfigFile, extra_args: List[str]) -> None: """ Downloads a one-off subscription using the CLI @@ -53,7 +51,7 @@ def _main(): """Entrypoint for ytdl-subscribe""" args, extra_args = parser.parse_known_args() - config: ConfigFileValidator = ConfigFileValidator.from_file_path(args.config) + config: ConfigFile = ConfigFile.from_file_path(args.config) if args.subparser == "sub": _download_subscriptions_from_yaml_files(config=config, args=args) print("Subscription download complete!") diff --git a/ytdl_subscribe/plugins/convert_thumbnail.py b/ytdl_subscribe/plugins/convert_thumbnail.py index 4f2db5fc..79b13771 100644 --- a/ytdl_subscribe/plugins/convert_thumbnail.py +++ b/ytdl_subscribe/plugins/convert_thumbnail.py @@ -19,7 +19,7 @@ class ConvertThumbnailValidator(PluginValidator): def __init__(self, name, value): super().__init__(name, value) - self.to = self._validate_key(key="to", validator=ThumbnailTypes) + self.convert_to = self._validate_key(key="to", validator=ThumbnailTypes) class ConvertThumbnail(Plugin[ConvertThumbnailValidator]): @@ -34,7 +34,7 @@ class ConvertThumbnail(Plugin[ConvertThumbnailValidator]): image = Image.open(entry.download_thumbnail_path).convert("RGB") # Pillow likes the formal 'jpeg' name and not 'jpg' - thumbnail_format = self.plugin_options.to.value + thumbnail_format = self.plugin_options.convert_to.value if thumbnail_format == "jpg": thumbnail_format = "jpeg" diff --git a/ytdl_subscribe/plugins/plugin.py b/ytdl_subscribe/plugins/plugin.py index ec196042..b74f3c62 100644 --- a/ytdl_subscribe/plugins/plugin.py +++ b/ytdl_subscribe/plugins/plugin.py @@ -3,7 +3,7 @@ from typing import Generic from typing import TypeVar from typing import final -from ytdl_subscribe.config.overrides_validator import OverridesValidator +from ytdl_subscribe.config.preset import Overrides from ytdl_subscribe.entries.entry import Entry from ytdl_subscribe.validators.strict_dict_validator import StrictDictValidator from ytdl_subscribe.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive @@ -30,7 +30,7 @@ class Plugin(Generic[PluginValidatorT], ABC): self, plugin_options: PluginValidatorT, output_directory: str, - overrides: OverridesValidator, + overrides: Overrides, enhanced_download_archive: EnhancedDownloadArchive, ): self.plugin_options = plugin_options diff --git a/ytdl_subscribe/subscriptions/subscription.py b/ytdl_subscribe/subscriptions/subscription.py index f1ee74b7..25dae28f 100644 --- a/ytdl_subscribe/subscriptions/subscription.py +++ b/ytdl_subscribe/subscriptions/subscription.py @@ -11,10 +11,10 @@ from typing import Optional from typing import Type from typing import TypeVar -from ytdl_subscribe.config.config_options_validator import ConfigOptionsValidator -from ytdl_subscribe.config.output_options_validator import OutputOptionsValidator -from ytdl_subscribe.config.overrides_validator import OverridesValidator -from ytdl_subscribe.config.preset_validator import PresetValidator +from ytdl_subscribe.config.config_file import ConfigOptions +from ytdl_subscribe.config.preset import OutputOptions +from ytdl_subscribe.config.preset import Overrides +from ytdl_subscribe.config.preset import PresetValidator from ytdl_subscribe.downloaders.downloader import Downloader from ytdl_subscribe.downloaders.downloader import DownloaderValidator from ytdl_subscribe.entries.entry import Entry @@ -43,7 +43,7 @@ class Subscription(Generic[SourceT, EntryT], ABC): def __init__( self, name: str, - config_options: ConfigOptionsValidator, + config_options: ConfigOptions, preset_options: PresetValidator, ): """ @@ -51,7 +51,7 @@ class Subscription(Generic[SourceT, EntryT], ABC): ---------- name: str Name of the subscription - config_options: ConfigOptionsValidator + config_options: ConfigOptions preset_options: PresetValidator """ self.name = name @@ -77,12 +77,12 @@ class Subscription(Generic[SourceT, EntryT], ABC): return self.__preset_options.subscription_source @property - def output_options(self) -> OutputOptionsValidator: + def output_options(self) -> OutputOptions: """Returns the output options defined for this subscription""" return self.__preset_options.output_options @property - def overrides(self) -> OverridesValidator: + def overrides(self) -> Overrides: """Returns the overrides defined for this subscription""" return self.__preset_options.overrides