From 44ec463ab3e50cc6caf9b56052a82b7df9763b31 Mon Sep 17 00:00:00 2001 From: Jesse Bannon Date: Wed, 26 Oct 2022 00:00:36 -0700 Subject: [PATCH] [BACKEND] More strict validation for config files (#301) --- examples/tv_show_config.yaml | 15 +- src/ytdl_sub/cli/download_args_parser.py | 2 +- src/ytdl_sub/config/config_file.py | 90 +--------- src/ytdl_sub/config/config_validator.py | 92 +++++++++++ src/ytdl_sub/config/preset.py | 119 +++++++++++++- src/ytdl_sub/config/preset_options.py | 13 ++ .../generic/collection_validator.py | 19 +++ .../soundcloud/albums_and_singles.py | 10 ++ src/ytdl_sub/downloaders/youtube/channel.py | 12 ++ src/ytdl_sub/downloaders/youtube/playlist.py | 12 ++ src/ytdl_sub/downloaders/youtube/video.py | 10 ++ src/ytdl_sub/plugins/audio_extract.py | 13 +- src/ytdl_sub/plugins/file_convert.py | 10 ++ src/ytdl_sub/plugins/music_tags.py | 10 ++ src/ytdl_sub/plugins/nfo_tags.py | 13 ++ src/ytdl_sub/plugins/regex.py | 10 ++ src/ytdl_sub/plugins/split_by_chapters.py | 7 + src/ytdl_sub/plugins/video_tags.py | 10 ++ src/ytdl_sub/prebuilt_presets/__init__.py | 6 + .../music_videos/jellyfin-music-videos.yaml | 6 - ...sic-videos.yaml => kodi_music_videos.yaml} | 2 +- .../tv_show/tv_show_collection.yaml | 5 + .../subscriptions/base_subscription.py | 2 +- src/ytdl_sub/validators/validators.py | 47 +++++- tests/unit/cli/test_download_args_parser.py | 2 +- tests/unit/config/test_config_file.py | 155 ++++++++++++++++++ tests/unit/config/test_preset.py | 4 +- 27 files changed, 579 insertions(+), 117 deletions(-) create mode 100644 src/ytdl_sub/config/config_validator.py delete mode 100644 src/ytdl_sub/prebuilt_presets/music_videos/jellyfin-music-videos.yaml rename src/ytdl_sub/prebuilt_presets/music_videos/{kodi-music-videos.yaml => kodi_music_videos.yaml} (95%) create mode 100644 tests/unit/config/test_config_file.py diff --git a/examples/tv_show_config.yaml b/examples/tv_show_config.yaml index c21f6230..e2763980 100644 --- a/examples/tv_show_config.yaml +++ b/examples/tv_show_config.yaml @@ -86,14 +86,15 @@ presets: # Preset to download subtitles (either by file or embedded) add_subtitles: - # Embed subtitles into the video - embed_subtitles: True - # And/or download them as a file. Uncomment to download as file: - # subtitles_name: "{episode_file_path}.{lang}.{subtitles_ext}" - # subtitles_type: "srt" + subtitles: + # Embed subtitles into the video + embed_subtitles: True + # And/or download them as a file. Uncomment to download as file: + # subtitles_name: "{episode_file_path}.{lang}.{subtitles_ext}" + # subtitles_type: "srt" - languages: "en" # supports list of multiple languages - allow_auto_generated_subtitles: True # allow auto subtitles + languages: "en" # supports list of multiple languages + allow_auto_generated_subtitles: True # allow auto subtitles #################################################################################################### diff --git a/src/ytdl_sub/cli/download_args_parser.py b/src/ytdl_sub/cli/download_args_parser.py index 95d086ef..938bed5e 100644 --- a/src/ytdl_sub/cli/download_args_parser.py +++ b/src/ytdl_sub/cli/download_args_parser.py @@ -8,7 +8,7 @@ from typing import Tuple from mergedeep import mergedeep from ytdl_sub.cli.main_args_parser import MainArgs -from ytdl_sub.config.config_file import ConfigOptions +from ytdl_sub.config.config_validator import ConfigOptions from ytdl_sub.utils.exceptions import InvalidDlArguments diff --git a/src/ytdl_sub/config/config_file.py b/src/ytdl_sub/config/config_file.py index d8f48920..197a82ce 100644 --- a/src/ytdl_sub/config/config_file.py +++ b/src/ytdl_sub/config/config_file.py @@ -1,99 +1,19 @@ import os from typing import Any -from typing import Dict -from typing import Optional -import mergedeep - -from ytdl_sub.prebuilt_presets import PREBUILT_PRESETS +from ytdl_sub.config.config_validator import ConfigValidator +from ytdl_sub.config.preset import Preset from ytdl_sub.utils.yaml import load_yaml -from ytdl_sub.validators.strict_dict_validator import StrictDictValidator -from ytdl_sub.validators.validators import LiteralDictValidator -from ytdl_sub.validators.validators import StringValidator -class ConfigOptions(StrictDictValidator): - _required_keys = {"working_directory"} - _optional_keys = {"umask", "dl_aliases"} - - def __init__(self, name: str, value: Any): - super().__init__(name, value) - - self._working_directory = self._validate_key( - key="working_directory", validator=StringValidator - ) - self._umask = self._validate_key_if_present( - key="umask", validator=StringValidator, default="022" - ) - self._dl_aliases = self._validate_key_if_present( - key="dl_aliases", validator=LiteralDictValidator - ) - - @property - def working_directory(self) -> str: - """ - The directory to temporarily store downloaded files before moving them into their final - directory. - """ - return self._working_directory.value - - @property - def umask(self) -> Optional[str]: - """ - Umask (octal format) to apply to every created file. Defaults to "022". - """ - return self._umask.value - - @property - def dl_aliases(self) -> Optional[Dict[str, str]]: - """ - Alias definitions to shorten ``ytdl-sub dl`` arguments. For example, - - .. code-block:: yaml - - configuration: - dl_aliases: - mv: "--preset yt_music_video" - v: "--youtube.video_url" - - Simplifies - - .. code-block:: bash - - ytdl-sub dl --preset "yt_music_video" --youtube.video_url "youtube.com/watch?v=a1b2c3" - - to - - .. code-block:: bash - - ytdl-sub dl --mv --v "youtube.com/watch?v=a1b2c3" - """ - if self._dl_aliases: - return self._dl_aliases.dict - return {} - - -class ConfigFile(StrictDictValidator): +class ConfigFile(ConfigValidator): _required_keys = {"configuration", "presets"} def __init__(self, name: str, value: Any): super().__init__(name, value) - self.config_options = self._validate_key("configuration", ConfigOptions) - prebuilt_presets = PREBUILT_PRESETS - - # Make sure presets is a dictionary. Will be validated in `PresetValidator` - self.presets = self._validate_key("presets", LiteralDictValidator) - - # Ensure custom presets do not collide with prebuilt presets - for preset_name in self.presets.keys: - if preset_name in prebuilt_presets: - raise self._validation_exception( - f"preset name '{preset_name}' conflicts with a prebuilt preset" - ) - - # Merge prebuilt presets into the config so custom presets can use them - mergedeep.merge(self.presets._value, prebuilt_presets) + for preset_name, preset_dict in self.presets.dict.items(): + Preset.preset_partial_validate(config=self, name=preset_name, value=preset_dict) def initialize(self): """ diff --git a/src/ytdl_sub/config/config_validator.py b/src/ytdl_sub/config/config_validator.py new file mode 100644 index 00000000..31a15b04 --- /dev/null +++ b/src/ytdl_sub/config/config_validator.py @@ -0,0 +1,92 @@ +from typing import Any +from typing import Dict +from typing import Optional + +from mergedeep import mergedeep + +from ytdl_sub.prebuilt_presets import PREBUILT_PRESETS +from ytdl_sub.validators.strict_dict_validator import StrictDictValidator +from ytdl_sub.validators.validators import LiteralDictValidator +from ytdl_sub.validators.validators import StringValidator + + +class ConfigOptions(StrictDictValidator): + _required_keys = {"working_directory"} + _optional_keys = {"umask", "dl_aliases"} + + def __init__(self, name: str, value: Any): + super().__init__(name, value) + + self._working_directory = self._validate_key( + key="working_directory", validator=StringValidator + ) + self._umask = self._validate_key_if_present( + key="umask", validator=StringValidator, default="022" + ) + self._dl_aliases = self._validate_key_if_present( + key="dl_aliases", validator=LiteralDictValidator + ) + + @property + def working_directory(self) -> str: + """ + The directory to temporarily store downloaded files before moving them into their final + directory. + """ + return self._working_directory.value + + @property + def umask(self) -> Optional[str]: + """ + Umask (octal format) to apply to every created file. Defaults to "022". + """ + return self._umask.value + + @property + def dl_aliases(self) -> Optional[Dict[str, str]]: + """ + Alias definitions to shorten ``ytdl-sub dl`` arguments. For example, + + .. code-block:: yaml + + configuration: + dl_aliases: + mv: "--preset yt_music_video" + v: "--youtube.video_url" + + Simplifies + + .. code-block:: bash + + ytdl-sub dl --preset "yt_music_video" --youtube.video_url "youtube.com/watch?v=a1b2c3" + + to + + .. code-block:: bash + + ytdl-sub dl --mv --v "youtube.com/watch?v=a1b2c3" + """ + if self._dl_aliases: + return self._dl_aliases.dict + return {} + + +class ConfigValidator(StrictDictValidator): + _required_keys = {"configuration", "presets"} + + def __init__(self, name: str, value: Any): + super().__init__(name, value) + self.config_options = self._validate_key("configuration", ConfigOptions) + + # Make sure presets is a dictionary. Will be validated in `PresetValidator` + self.presets = self._validate_key("presets", LiteralDictValidator) + + # Ensure custom presets do not collide with prebuilt presets + for preset_name in self.presets.keys: + if preset_name in PREBUILT_PRESETS: + raise self._validation_exception( + f"preset name '{preset_name}' conflicts with a prebuilt preset" + ) + + # Merge prebuilt presets into the config so custom presets can use them + mergedeep.merge(self.presets._value, PREBUILT_PRESETS) diff --git a/src/ytdl_sub/config/preset.py b/src/ytdl_sub/config/preset.py index 077443ee..84974d35 100644 --- a/src/ytdl_sub/config/preset.py +++ b/src/ytdl_sub/config/preset.py @@ -11,7 +11,7 @@ from typing import Union from mergedeep import mergedeep -from ytdl_sub.config.config_file import ConfigFile +from ytdl_sub.config.config_validator import ConfigValidator from ytdl_sub.config.preset_class_mappings import DownloadStrategyMapping from ytdl_sub.config.preset_class_mappings import PluginMapping from ytdl_sub.config.preset_options import OutputOptions @@ -22,6 +22,9 @@ from ytdl_sub.downloaders.downloader import DownloaderValidator from ytdl_sub.entries.entry import Entry from ytdl_sub.plugins.plugin import Plugin from ytdl_sub.plugins.plugin import PluginOptions +from ytdl_sub.prebuilt_presets import PREBUILT_PRESET_NAMES +from ytdl_sub.prebuilt_presets import PUBLISHED_PRESET_NAMES +from ytdl_sub.utils.exceptions import ValidationException from ytdl_sub.utils.yaml import dump_yaml from ytdl_sub.validators.strict_dict_validator import StrictDictValidator from ytdl_sub.validators.string_formatter_validators import DictFormatterValidator @@ -33,6 +36,7 @@ from ytdl_sub.validators.validators import ListValidator from ytdl_sub.validators.validators import StringListValidator from ytdl_sub.validators.validators import StringValidator from ytdl_sub.validators.validators import Validator +from ytdl_sub.validators.validators import validation_exception PRESET_KEYS = { "preset", @@ -44,6 +48,19 @@ PRESET_KEYS = { } +def _parent_preset_error_message( + current_preset_name: str, parent_preset_name: str, presets: List[str] +) -> ValidationException: + user_defined_presets = set(presets) - PREBUILT_PRESET_NAMES - {current_preset_name} + + return validation_exception( + name=current_preset_name, + error_message=f"preset '{parent_preset_name}' does not exist in the provided config.\n" + f"Available prebuilt presets: {', '.join(sorted(PUBLISHED_PRESET_NAMES))}\n" + f"Your presets: {', '.join(sorted(user_defined_presets))}", + ) + + class PresetPlugins: _TPluginOptions = TypeVar("_TPluginOptions", bound=PluginOptions) @@ -113,6 +130,11 @@ class DownloadStrategyValidator(StrictDictValidator): Returns ------- The downloader class + + Raises + ------ + ValidationException + If the download strategy is invalid """ try: return DownloadStrategyMapping.get( @@ -122,12 +144,90 @@ class DownloadStrategyValidator(StrictDictValidator): raise self._validation_exception(error_message=value_exc) -class Preset(StrictDictValidator): +class _PresetShell(StrictDictValidator): # Have all present keys optional since parent presets could not have all the # required keys. They will get validated in the init after the mergedeep of dicts # and ensure required keys are present. _optional_keys = PRESET_KEYS + +class Preset(_PresetShell): + @classmethod + def _validate_download_strategy(cls, name: str, value: Dict) -> None: + sources: List[str] = [] + for source_name in DownloadStrategyMapping.sources(): + if source_name in value: + sources.append(source_name) + + if len(sources) > 1: + raise validation_exception( + name=name, + error_message=f"Contains the sources {', '.join(sources)} but can only have one", + ) + + # If no sources, nothing more to validate + if not sources: + return + + source_name = sources[0] + source_dict = copy.deepcopy(value[source_name]) + + downloader = DownloadStrategyValidator(name=f"{name}.{source_name}", value=source_dict).get( + downloader_source=source_name + ) + del source_dict["download_strategy"] + + downloader.downloader_options_type.partial_validate( + name=f"{name}.{source_name}", value=source_dict + ) + + @classmethod + def preset_partial_validate(cls, config: ConfigValidator, name: str, value: Any) -> None: + """ + Partially validates a preset. Used to ensure every preset in a ConfigFile looks sane. + Cannot fully validate each preset using the Preset init because required fields could + be missing, which become filled in a child preset. + + Parameters + ---------- + config + Config that this preset belongs to + name + Preset name + value + Preset value + + Raises + ------ + ValidationException + If validation fails + """ + # Ensure value is a dict + _ = _PresetShell(name=name, value=value) + assert isinstance(value, dict) + + cls._validate_download_strategy(name, value) + cls._partial_validate_key(name, value, "output_options", OutputOptions) + cls._partial_validate_key(name, value, "ytdl_options", YTDLOptions) + cls._partial_validate_key(name, value, "overrides", Overrides) + + for plugin_name in PluginMapping.plugins(): + cls._partial_validate_key( + name, + value, + key=plugin_name, + validator=PluginMapping.get(plugin_name).plugin_options_type, + ) + + parent_presets = StringListValidator(name=f"{name}.preset", value=value.get("preset", [])) + for parent_preset_name in parent_presets.list: + if parent_preset_name.value not in config.presets.keys: + raise _parent_preset_error_message( + current_preset_name=name, + parent_preset_name=parent_preset_name.value, + presets=config.presets.keys, + ) + @property def _source_variables(self) -> List[str]: return Entry.source_variables() @@ -278,7 +378,7 @@ class Preset(StrictDictValidator): self.__validate_override_string_formatter_validator(validator_value) def _get_presets_to_merge( - self, parent_presets: str | List[str], seen_presets: List[str], config: ConfigFile + self, parent_presets: str | List[str], seen_presets: List[str], config: ConfigValidator ) -> List[Dict]: presets_to_merge: List[Dict] = [] @@ -294,9 +394,10 @@ class Preset(StrictDictValidator): # Make sure the parent preset actually exists if parent_preset not in config.presets.keys: - raise self._validation_exception( - f"preset '{parent_preset}' does not exist in the provided config. " - f"Available presets: {', '.join(config.presets.keys)}" + raise _parent_preset_error_message( + current_preset_name=self._name, + parent_preset_name=parent_preset, + presets=config.presets.keys, ) parent_preset_dict = copy.deepcopy(config.presets.dict[parent_preset]) @@ -313,7 +414,7 @@ class Preset(StrictDictValidator): return presets_to_merge - def __merge_parent_preset_dicts_if_present(self, config: ConfigFile): + def __merge_parent_preset_dicts_if_present(self, config: ConfigValidator): parent_preset_validator = self._validate_key_if_present( key="preset", validator=StringListValidator ) @@ -332,7 +433,7 @@ class Preset(StrictDictValidator): mergedeep.merge({}, *reversed(presets_to_merge), strategy=mergedeep.Strategy.ADDITIVE) ) - def __init__(self, config: ConfigFile, name: str, value: Any): + def __init__(self, config: ConfigValidator, name: str, value: Any): super().__init__(name=name, value=value) # Perform the merge of parent presets before validating any keys @@ -367,7 +468,7 @@ class Preset(StrictDictValidator): return self._name @classmethod - def from_dict(cls, config: ConfigFile, preset_name: str, preset_dict: Dict) -> "Preset": + def from_dict(cls, config: ConfigValidator, preset_name: str, preset_dict: Dict) -> "Preset": """ Parameters ---------- diff --git a/src/ytdl_sub/config/preset_options.py b/src/ytdl_sub/config/preset_options.py index 4ed67789..47cf2d8c 100644 --- a/src/ytdl_sub/config/preset_options.py +++ b/src/ytdl_sub/config/preset_options.py @@ -1,4 +1,5 @@ from abc import ABC +from typing import Any from typing import Dict from typing import List from typing import Optional @@ -182,6 +183,18 @@ class OutputOptions(StrictDictValidator): "keep_files_after", } + @classmethod + def partial_validate(cls, name: str, value: Any) -> None: + """ + Partially validate output options + """ + if isinstance(value, dict): + value["output_directory"] = value.get("output_directory", "placeholder") + value["file_name"] = value.get("file_name", "placeholder") + # Set this to True by default in partial validate to avoid failing from keep_files + value["maintain_download_archive"] = value.get("maintain_download_archive", True) + _ = cls(name, value) + def __init__(self, name, value): super().__init__(name, value) diff --git a/src/ytdl_sub/downloaders/generic/collection_validator.py b/src/ytdl_sub/downloaders/generic/collection_validator.py index 5af9ebaf..ba10467f 100644 --- a/src/ytdl_sub/downloaders/generic/collection_validator.py +++ b/src/ytdl_sub/downloaders/generic/collection_validator.py @@ -1,3 +1,4 @@ +from typing import Any from typing import Dict from typing import List from typing import Optional @@ -42,6 +43,15 @@ class CollectionUrlValidator(StrictDictValidator): _required_keys = {"url"} _optional_keys = {"variables", "source_thumbnails", "playlist_thumbnails"} + @classmethod + def partial_validate(cls, name: str, value: Any) -> None: + """ + Partially validate a YouTube collection url + """ + if isinstance(value, dict): + value["url"] = value.get("url", "placeholder") + _ = cls(name, value) + def __init__(self, name, value): super().__init__(name, value) @@ -149,6 +159,15 @@ class CollectionValidator(StrictDictValidator, AddsVariablesMixin): _required_keys = {"urls"} + @classmethod + def partial_validate(cls, name: str, value: Any) -> None: + """ + Partially validate a generic collection + """ + if isinstance(value, dict): + value["urls"] = value.get("urls", [{"url": "placeholder"}]) + _ = cls(name, value) + def __init__(self, name, value): super().__init__(name, value) self._urls = self._validate_key(key="urls", validator=CollectionUrlListValidator) diff --git a/src/ytdl_sub/downloaders/soundcloud/albums_and_singles.py b/src/ytdl_sub/downloaders/soundcloud/albums_and_singles.py index b6529f7e..70a89e97 100644 --- a/src/ytdl_sub/downloaders/soundcloud/albums_and_singles.py +++ b/src/ytdl_sub/downloaders/soundcloud/albums_and_singles.py @@ -1,3 +1,4 @@ +from typing import Any from typing import Dict from typing import Generator @@ -32,6 +33,15 @@ class SoundcloudAlbumsAndSinglesDownloadOptions(DownloaderValidator): _required_keys = {"url"} _optional_keys = {"skip_premiere_tracks"} + @classmethod + def partial_validate(cls, name: str, value: Any) -> None: + """ + Partially validate a Soundcloud source + """ + if isinstance(value, dict): + value["url"] = value.get("url", "https://soundcloud.com/jessebannon") + _ = cls(name, value) + def __init__(self, name, value): super().__init__(name, value) self._url = self._validate_key( diff --git a/src/ytdl_sub/downloaders/youtube/channel.py b/src/ytdl_sub/downloaders/youtube/channel.py index 166ed6e4..ca019a64 100644 --- a/src/ytdl_sub/downloaders/youtube/channel.py +++ b/src/ytdl_sub/downloaders/youtube/channel.py @@ -1,3 +1,4 @@ +from typing import Any from typing import Dict from typing import List from typing import Optional @@ -34,6 +35,17 @@ class YoutubeChannelDownloaderOptions(DownloaderValidator): "channel_banner_path", } + @classmethod + def partial_validate(cls, name: str, value: Any) -> None: + """ + Partially validate a YouTube channel source + """ + if isinstance(value, dict): + value["channel_url"] = value.get( + "channel_url", "https://www.youtube.com/c/ProjectZombie603" + ) + _ = cls(name, value) + def __init__(self, name, value): super().__init__(name, value) self._channel_url = self._validate_key( diff --git a/src/ytdl_sub/downloaders/youtube/playlist.py b/src/ytdl_sub/downloaders/youtube/playlist.py index 7a3c7e91..d8c42835 100644 --- a/src/ytdl_sub/downloaders/youtube/playlist.py +++ b/src/ytdl_sub/downloaders/youtube/playlist.py @@ -1,3 +1,4 @@ +from typing import Any from typing import Dict from typing import List from typing import Optional @@ -31,6 +32,17 @@ class YoutubePlaylistDownloaderOptions(DownloaderValidator): _required_keys = {"playlist_url"} _optional_keys = {"playlist_thumbnail_name"} + @classmethod + def partial_validate(cls, name: str, value: Any) -> None: + """ + Partially validate a YouTube playlist source + """ + if isinstance(value, dict): + value["playlist_url"] = value.get( + "playlist_url", "https://www.youtube.com/playlist?list=UCsvn_Po0SmunchJYtttWpOxMg" + ) + _ = cls(name, value) + def __init__(self, name, value): super().__init__(name, value) self._playlist_url = self._validate_key( diff --git a/src/ytdl_sub/downloaders/youtube/video.py b/src/ytdl_sub/downloaders/youtube/video.py index c9b9f951..0c30c11e 100644 --- a/src/ytdl_sub/downloaders/youtube/video.py +++ b/src/ytdl_sub/downloaders/youtube/video.py @@ -1,3 +1,4 @@ +from typing import Any from typing import Dict from ytdl_sub.downloaders.downloader import Downloader @@ -32,6 +33,15 @@ class YoutubeVideoDownloaderOptions(DownloaderValidator): _required_keys = {"video_url"} + @classmethod + def partial_validate(cls, name: str, value: Any) -> None: + """ + Partially validate a YouTube video source + """ + if isinstance(value, dict): + value["video_url"] = value.get("video_url", "youtube.com/watch?v=VMAPTo7RVDo") + _ = cls(name, value) + def __init__(self, name, value): super().__init__(name, value) self._video_url = self._validate_key("video_url", YoutubeVideoUrlValidator).video_url diff --git a/src/ytdl_sub/plugins/audio_extract.py b/src/ytdl_sub/plugins/audio_extract.py index 7703a786..0840a37d 100644 --- a/src/ytdl_sub/plugins/audio_extract.py +++ b/src/ytdl_sub/plugins/audio_extract.py @@ -1,4 +1,5 @@ import os.path +from typing import Any from typing import Dict from typing import Optional @@ -27,7 +28,17 @@ class AudioExtractOptions(PluginOptions): quality: 128 """ - _optional_keys = {"codec", "quality"} + _required_keys = {"codec"} + _optional_keys = {"quality"} + + @classmethod + def partial_validate(cls, name: str, value: Any) -> None: + """ + Partially validate audio extract options + """ + if isinstance(value, dict): + value["codec"] = value.get("codec", "mp3") + _ = cls(name, value) def __init__(self, name, value): super().__init__(name, value) diff --git a/src/ytdl_sub/plugins/file_convert.py b/src/ytdl_sub/plugins/file_convert.py index bdc1b07d..f1428dd5 100644 --- a/src/ytdl_sub/plugins/file_convert.py +++ b/src/ytdl_sub/plugins/file_convert.py @@ -1,4 +1,5 @@ import os +from typing import Any from typing import Dict from typing import Optional @@ -27,6 +28,15 @@ class FileConvertOptions(PluginOptions): _required_keys = {"convert_to"} + @classmethod + def partial_validate(cls, name: str, value: Any) -> None: + """ + Partially validate file_convert + """ + if isinstance(value, dict): + value["convert_to"] = value.get("convert_to", "mp3") + _ = cls(name, value) + def __init__(self, name, value): super().__init__(name, value) self._convert_to = self._validate_key(key="convert_to", validator=FileTypeValidator).value diff --git a/src/ytdl_sub/plugins/music_tags.py b/src/ytdl_sub/plugins/music_tags.py index 8b4b3480..0f85c55b 100644 --- a/src/ytdl_sub/plugins/music_tags.py +++ b/src/ytdl_sub/plugins/music_tags.py @@ -1,3 +1,4 @@ +from typing import Any from typing import Dict import mediafile @@ -34,6 +35,15 @@ class MusicTagsOptions(PluginOptions): _required_keys = {"tags"} + @classmethod + def partial_validate(cls, name: str, value: Any) -> None: + """ + Partially validate music tags + """ + if isinstance(value, dict): + value["tags"] = value.get("tags", {}) + _ = cls(name, value) + def __init__(self, name, value): super().__init__(name, value) diff --git a/src/ytdl_sub/plugins/nfo_tags.py b/src/ytdl_sub/plugins/nfo_tags.py index f44c0546..ffd8c225 100644 --- a/src/ytdl_sub/plugins/nfo_tags.py +++ b/src/ytdl_sub/plugins/nfo_tags.py @@ -2,6 +2,7 @@ import os from abc import ABC from collections import defaultdict from pathlib import Path +from typing import Any from typing import Dict from typing import List from typing import Optional @@ -28,6 +29,18 @@ class SharedNfoTagsOptions(PluginOptions): _required_keys = {"nfo_name", "nfo_root", "tags"} _optional_keys = {"kodi_safe"} + @classmethod + def partial_validate(cls, name: str, value: Any) -> None: + """ + Partially validate NFO tag options + """ + if isinstance(value, dict): + value["nfo_name"] = value.get("nfo_name", "placeholder") + value["nfo_root"] = value.get("nfo_root", "placeholder") + value["tags"] = value.get("tags", {}) + + _ = cls(name=name, value=value) + def __init__(self, name, value): super().__init__(name, value) diff --git a/src/ytdl_sub/plugins/regex.py b/src/ytdl_sub/plugins/regex.py index b818e9da..9f20b560 100644 --- a/src/ytdl_sub/plugins/regex.py +++ b/src/ytdl_sub/plugins/regex.py @@ -1,3 +1,4 @@ +from typing import Any from typing import Dict from typing import List from typing import Optional @@ -149,6 +150,15 @@ class RegexOptions(PluginOptions): _required_keys = {"from"} _optional_keys = {"skip_if_match_fails"} + @classmethod + def partial_validate(cls, name: str, value: Any) -> None: + """ + Partially validate regex + """ + if isinstance(value, dict): + value["from"] = value.get("from", {}) + _ = cls(name, value) + def __init__(self, name, value): super().__init__(name, value) self._from = self._validate_key(key="from", validator=FromSourceVariablesRegex) diff --git a/src/ytdl_sub/plugins/split_by_chapters.py b/src/ytdl_sub/plugins/split_by_chapters.py index 1dfbceb8..c7fa4b50 100644 --- a/src/ytdl_sub/plugins/split_by_chapters.py +++ b/src/ytdl_sub/plugins/split_by_chapters.py @@ -1,6 +1,7 @@ import copy import os.path from pathlib import Path +from typing import Any from typing import List from typing import Optional from typing import Tuple @@ -67,6 +68,12 @@ class SplitByChaptersOptions(PluginOptions): _required_keys = {"when_no_chapters"} + @classmethod + def partial_validate(cls, name: str, value: Any) -> None: + if isinstance(value, dict): + value["when_no_chapters"] = value.get("when_no_chapters", "pass") + _ = cls(name, value) + def __init__(self, name, value): super().__init__(name, value) self._when_no_chapters = self._validate_key( diff --git a/src/ytdl_sub/plugins/video_tags.py b/src/ytdl_sub/plugins/video_tags.py index 61d89a78..e1fb85a4 100644 --- a/src/ytdl_sub/plugins/video_tags.py +++ b/src/ytdl_sub/plugins/video_tags.py @@ -1,3 +1,4 @@ +from typing import Any from typing import Dict from ytdl_sub.entries.entry import Entry @@ -27,6 +28,15 @@ class VideoTagsOptions(PluginOptions): _required_keys = {"tags"} + @classmethod + def partial_validate(cls, name: str, value: Any) -> None: + """ + Partially validate video tags + """ + if isinstance(value, dict): + value["tags"] = value.get("tags", {}) + _ = cls(name, value) + def __init__(self, name, value): super().__init__(name, value) self._tags = self._validate_key(key="tags", validator=DictFormatterValidator) diff --git a/src/ytdl_sub/prebuilt_presets/__init__.py b/src/ytdl_sub/prebuilt_presets/__init__.py index 96701605..43ae2433 100644 --- a/src/ytdl_sub/prebuilt_presets/__init__.py +++ b/src/ytdl_sub/prebuilt_presets/__init__.py @@ -1,6 +1,7 @@ import pathlib from typing import Any from typing import Dict +from typing import Set import mergedeep @@ -21,3 +22,8 @@ def _merge_presets() -> Dict[str, Any]: PREBUILT_PRESETS: Dict[str, Any] = _merge_presets() + +PREBUILT_PRESET_NAMES: Set[str] = set(PREBUILT_PRESETS.keys()) +PUBLISHED_PRESET_NAMES: Set[str] = { + name for name in PREBUILT_PRESET_NAMES if not name.startswith("_") +} diff --git a/src/ytdl_sub/prebuilt_presets/music_videos/jellyfin-music-videos.yaml b/src/ytdl_sub/prebuilt_presets/music_videos/jellyfin-music-videos.yaml deleted file mode 100644 index 2db3c8cc..00000000 --- a/src/ytdl_sub/prebuilt_presets/music_videos/jellyfin-music-videos.yaml +++ /dev/null @@ -1,6 +0,0 @@ -presets: - - kodi-music-videos: - preset: - - "jellyfin-music-videos" - - "kodi-safe" diff --git a/src/ytdl_sub/prebuilt_presets/music_videos/kodi-music-videos.yaml b/src/ytdl_sub/prebuilt_presets/music_videos/kodi_music_videos.yaml similarity index 95% rename from src/ytdl_sub/prebuilt_presets/music_videos/kodi-music-videos.yaml rename to src/ytdl_sub/prebuilt_presets/music_videos/kodi_music_videos.yaml index 84623470..86999852 100644 --- a/src/ytdl_sub/prebuilt_presets/music_videos/kodi-music-videos.yaml +++ b/src/ytdl_sub/prebuilt_presets/music_videos/kodi_music_videos.yaml @@ -3,7 +3,7 @@ presets: kodi_music_video: generic: download_strategy: "source" - music_video_url: "{music_video_url}" + url: "{music_video_url}" output_options: output_directory: "{music_video_directory}" diff --git a/src/ytdl_sub/prebuilt_presets/tv_show/tv_show_collection.yaml b/src/ytdl_sub/prebuilt_presets/tv_show/tv_show_collection.yaml index fbaf2c4f..f05fe036 100644 --- a/src/ytdl_sub/prebuilt_presets/tv_show/tv_show_collection.yaml +++ b/src/ytdl_sub/prebuilt_presets/tv_show/tv_show_collection.yaml @@ -18,6 +18,7 @@ presets: collection_season_1: generic: + download_strategy: "collection" urls: - url: "{collection_season_1_url}" variables: @@ -42,6 +43,7 @@ presets: collection_season_2: generic: + download_strategy: "collection" urls: - url: "{collection_season_2_url}" variables: @@ -60,6 +62,7 @@ presets: collection_season_3: generic: + download_strategy: "collection" urls: - url: "{collection_season_3_url}" variables: @@ -78,6 +81,7 @@ presets: collection_season_4: generic: + download_strategy: "collection" urls: - url: "{collection_season_4_url}" variables: @@ -96,6 +100,7 @@ presets: collection_season_5: generic: + download_strategy: "collection" urls: - url: "{collection_season_5_url}" variables: diff --git a/src/ytdl_sub/subscriptions/base_subscription.py b/src/ytdl_sub/subscriptions/base_subscription.py index 3b1405c3..d3a2cf06 100644 --- a/src/ytdl_sub/subscriptions/base_subscription.py +++ b/src/ytdl_sub/subscriptions/base_subscription.py @@ -2,7 +2,7 @@ from abc import ABC from pathlib import Path from typing import Type -from ytdl_sub.config.config_file import ConfigOptions +from ytdl_sub.config.config_validator import ConfigOptions from ytdl_sub.config.preset import Preset from ytdl_sub.config.preset import PresetPlugins from ytdl_sub.config.preset_options import OutputOptions diff --git a/src/ytdl_sub/validators/validators.py b/src/ytdl_sub/validators/validators.py index e2235587..eb8a49b3 100644 --- a/src/ytdl_sub/validators/validators.py +++ b/src/ytdl_sub/validators/validators.py @@ -16,6 +16,29 @@ ValidationExceptionT = TypeVar("ValidationExceptionT", bound=ValidationException ValidatorT = TypeVar("ValidatorT", bound="Validator") +def validation_exception( + name: str, + error_message: str | Exception, + exception_class: Type[ValidationExceptionT] = ValidationException, +) -> ValidationExceptionT: + """ + Parameters + ---------- + name + Name of the validator + error_message + Error message to include in the ValidationException + exception_class + Class of the exception + + Returns + ------- + Validation exception with a consistent prefix. + """ + prefix = f"Validation error in {name}: " + return exception_class(f"{prefix}{error_message}") + + class Validator(ABC): """ Used to validate the value of a python object. This is the 'base' class that will first @@ -29,6 +52,18 @@ class Validator(ABC): # When raising an error, call the type this value instead of its python name _expected_value_type_name: Optional[str] = None + @classmethod + def partial_validate(cls, name: str, value: Any) -> None: + """ + Parameters + ---------- + name + Name of the validator + value + Value of the validator + """ + _ = cls(name=name, value=value) + def __init__(self, name: str, value: Any): self._name = name self._value = copy.deepcopy(value) # Always deep copy to avoid editing references @@ -56,8 +91,7 @@ class Validator(ABC): ------- Validation exception with a consistent prefix. """ - prefix = f"Validation error in {self._name}: " - return exception_class(f"{prefix}{error_message}") + return validation_exception(self._name, error_message, exception_class) class ValueValidator(Validator, ABC, Generic[ValueT]): @@ -231,6 +265,15 @@ class DictValidator(Validator): return self._validate_key(key=key, validator=validator, default=default) + @final + @classmethod + def _partial_validate_key( + cls, name: str, value: Any, key: str, validator: Type[ValidatorT] + ) -> None: + value_dict = DictValidator(name=name, value=value) + if key in value_dict._dict: + validator.partial_validate(name=f"{name}.{key}", value=value_dict._dict[key]) + class LiteralDictValidator(DictValidator): """DictValidator with exposed dict and keys method""" diff --git a/tests/unit/cli/test_download_args_parser.py b/tests/unit/cli/test_download_args_parser.py index 1388e7dc..d528c6c1 100644 --- a/tests/unit/cli/test_download_args_parser.py +++ b/tests/unit/cli/test_download_args_parser.py @@ -9,7 +9,7 @@ import pytest from ytdl_sub.cli.download_args_parser import DownloadArgsParser from ytdl_sub.cli.main_args_parser import MainArgs from ytdl_sub.cli.main_args_parser import parser -from ytdl_sub.config.config_file import ConfigOptions +from ytdl_sub.config.config_validator import ConfigOptions from ytdl_sub.utils.exceptions import InvalidDlArguments diff --git a/tests/unit/config/test_config_file.py b/tests/unit/config/test_config_file.py new file mode 100644 index 00000000..e83ac7f9 --- /dev/null +++ b/tests/unit/config/test_config_file.py @@ -0,0 +1,155 @@ +import re +from typing import Dict +from typing import Optional + +import pytest + +from ytdl_sub.config.config_file import ConfigFile +from ytdl_sub.config.preset import PRESET_KEYS +from ytdl_sub.config.preset_class_mappings import DownloadStrategyMapping +from ytdl_sub.config.preset_class_mappings import PluginMapping +from ytdl_sub.utils.exceptions import ValidationException + + +class TestConfigFilePartiallyValidatesPresets: + @classmethod + def _partial_validate( + cls, preset_dict: Dict, expected_error_message: Optional[str] = None + ) -> None: + def _config_create() -> None: + _ = ConfigFile( + name="test_partial_validate", + value={ + "configuration": {"working_directory": "."}, + "presets": {"partial_preset": preset_dict}, + }, + ) + + if expected_error_message: + with pytest.raises(ValidationException, match=re.escape(expected_error_message)): + _config_create() + else: + _config_create() + + @pytest.mark.parametrize( + "preset_dict", + [ + {"nfo_tags": {"tags": {"key-1": "preset_0"}}}, + {"output_directory_nfo_tags": {"nfo_root": "test"}}, + {"output_options": {"file_name": "test"}}, + {"output_options": {"keep_files_after": "today"}}, + {"ytdl_options": {"format": "best"}}, + {"overrides": {"a": "b"}}, + ], + ) + def test_success(self, preset_dict: Dict): + self._partial_validate(preset_dict) + + @pytest.mark.parametrize("key", ["output_options", "overrides", "ytdl_options"]) + def test_success__empty_preset_keys(self, key): + self._partial_validate({key: {}}) + + @pytest.mark.parametrize("plugin", PluginMapping.plugins()) + def test_success__empty_plugins(self, plugin: str): + self._partial_validate({plugin: {}}) + + @pytest.mark.parametrize("source", DownloadStrategyMapping.sources()) + def test_success__empty_sources(self, source: str): + for download_strategy in DownloadStrategyMapping.source_download_strategies(source): + self._partial_validate({source: {"download_strategy": download_strategy}}) + + def test_error__bad_preset_section(self): + self._partial_validate( + preset_dict={"does_not_exist": "lol"}, + expected_error_message="Validation error in partial_preset: " + "'partial_preset' contains the field 'does_not_exist' which is not allowed. " + f"Allowed fields: {', '.join(sorted(PRESET_KEYS))}", + ) + + def test_error__multiple_sources(self): + self._partial_validate( + preset_dict={"youtube": {}, "generic": {}}, + expected_error_message="Validation error in partial_preset: " + "Contains the sources generic, youtube but can only have one", + ) + + def test_error__no_download_strategy(self): + self._partial_validate( + preset_dict={"generic": {}}, + expected_error_message="Validation error in partial_preset.generic: " + "missing the required field 'download_strategy'", + ) + + def test_error__bad_download_strategy(self): + self._partial_validate( + preset_dict={"generic": {"download_strategy": "fail"}}, + expected_error_message="Validation error in partial_preset.generic: " + "Tried to use download strategy 'fail' with source 'generic', " + "which does not exist. Available download strategies: collection, source", + ) + + def test_error__bad_download_strategy_args(self): + self._partial_validate( + preset_dict={"generic": {"download_strategy": "collection", "bad_key": "nope"}}, + expected_error_message="Validation error in partial_preset.generic: " + "'partial_preset.generic' contains the field 'bad_key' which is not allowed. " + "Allowed fields: urls", + ) + + @pytest.mark.parametrize( + "preset_dict", + [ + {"nfo_tags": {"tags": {"key-1": {"attributes": {"test": "2"}}}}}, + {"generic": {"urls": [{"variables_to_set": {"name": "value"}}]}}, + ], + ) + def test_partial_validate__incomplete_list_item(self, preset_dict): + with pytest.raises(ValidationException): + _ = ConfigFile( + name="test_partial_validate", + value={ + "configuration": {"working_directory": "."}, + "presets": {"partial_preset": preset_dict}, + }, + ) + + @pytest.mark.parametrize( + "preset_dict", + [ + {"overrides": "not a dict"}, + {"overrides": {"nested": {"dict": "value"}}}, + {"overrides": ["list"]}, + ], + ) + def test_error__bad_overrides(self, preset_dict): + self._partial_validate( + preset_dict=preset_dict, + expected_error_message="Validation error in partial_preset.overrides", + ) + + @pytest.mark.parametrize( + "preset_dict", + [ + {"ytdl_options": "not a dict"}, + {"ytdl_options": ["list"]}, + ], + ) + def test_error__bad_ytdl_options(self, preset_dict): + self._partial_validate( + preset_dict=preset_dict, + expected_error_message="Validation error in partial_preset.ytdl_options", + ) + + @pytest.mark.parametrize( + "preset_dict", + [ + {"preset": "DNE"}, + {"preset": ["DNE"]}, + ], + ) + def test_error__non_existent_preset(self, preset_dict): + self._partial_validate( + preset_dict=preset_dict, + expected_error_message="Validation error in partial_preset: " + "preset 'DNE' does not exist in the provided config.", + ) diff --git a/tests/unit/config/test_preset.py b/tests/unit/config/test_preset.py index c2d2efb9..d924c82c 100644 --- a/tests/unit/config/test_preset.py +++ b/tests/unit/config/test_preset.py @@ -1,5 +1,3 @@ -from typing import Dict - import pytest from ytdl_sub.config.preset import Preset @@ -102,7 +100,7 @@ class TestPreset: "youtube": youtube_video, "output_options": dict( output_options, - **{"maintain_download_archive": True, "keep_files_after": "today-{ttl}"} + **{"maintain_download_archive": True, "keep_files_after": "today-{ttl}"}, ), "overrides": {"ttl": "2months"}, },