danvics/ytdl-sub
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

[BACKEND] More strict validation for config files (#301)

Browse source
This commit is contained in:
Jesse Bannon 2022-10-26 00:00:36 -07:00 committed by GitHub
parent 3e4a995f6e
commit 44ec463ab3
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
27 changed files with 579 additions and 117 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

15
examples/tv_show_config.yaml
View file

@ -86,14 +86,15 @@ presets:
# Preset to download subtitles (either by file or embedded)
add_subtitles:
# Embed subtitles into the video
embed_subtitles: True
# And/or download them as a file. Uncomment to download as file:
# subtitles_name: "{episode_file_path}.{lang}.{subtitles_ext}"
# subtitles_type: "srt"
subtitles:
# Embed subtitles into the video
embed_subtitles: True
# And/or download them as a file. Uncomment to download as file:
# subtitles_name: "{episode_file_path}.{lang}.{subtitles_ext}"
# subtitles_type: "srt"
languages: "en" # supports list of multiple languages
allow_auto_generated_subtitles: True # allow auto subtitles
languages: "en" # supports list of multiple languages
allow_auto_generated_subtitles: True # allow auto subtitles
####################################################################################################

2
src/ytdl_sub/cli/download_args_parser.py
View file

@ -8,7 +8,7 @@ from typing import Tuple
from mergedeep import mergedeep
from ytdl_sub.cli.main_args_parser import MainArgs
from ytdl_sub.config.config_file import ConfigOptions
from ytdl_sub.config.config_validator import ConfigOptions
from ytdl_sub.utils.exceptions import InvalidDlArguments

90
src/ytdl_sub/config/config_file.py
View file

@ -1,99 +1,19 @@
import os
from typing import Any
from typing import Dict
from typing import Optional
import mergedeep
from ytdl_sub.prebuilt_presets import PREBUILT_PRESETS
from ytdl_sub.config.config_validator import ConfigValidator
from ytdl_sub.config.preset import Preset
from ytdl_sub.utils.yaml import load_yaml
from ytdl_sub.validators.strict_dict_validator import StrictDictValidator
from ytdl_sub.validators.validators import LiteralDictValidator
from ytdl_sub.validators.validators import StringValidator
class ConfigOptions(StrictDictValidator):
_required_keys = {"working_directory"}
_optional_keys = {"umask", "dl_aliases"}
def __init__(self, name: str, value: Any):
super().__init__(name, value)
self._working_directory = self._validate_key(
key="working_directory", validator=StringValidator
)
self._umask = self._validate_key_if_present(
key="umask", validator=StringValidator, default="022"
)
self._dl_aliases = self._validate_key_if_present(
key="dl_aliases", validator=LiteralDictValidator
)
@property
def working_directory(self) -> str:
"""
The directory to temporarily store downloaded files before moving them into their final
directory.
"""
return self._working_directory.value
@property
def umask(self) -> Optional[str]:
"""
Umask (octal format) to apply to every created file. Defaults to "022".
"""
return self._umask.value
@property
def dl_aliases(self) -> Optional[Dict[str, str]]:
"""
Alias definitions to shorten ``ytdl-sub dl`` arguments. For example,
.. code-block:: yaml
configuration:
dl_aliases:
mv: "--preset yt_music_video"
v: "--youtube.video_url"
Simplifies
.. code-block:: bash
ytdl-sub dl --preset "yt_music_video" --youtube.video_url "youtube.com/watch?v=a1b2c3"
to
.. code-block:: bash
ytdl-sub dl --mv --v "youtube.com/watch?v=a1b2c3"
"""
if self._dl_aliases:
return self._dl_aliases.dict
return {}
class ConfigFile(StrictDictValidator):
class ConfigFile(ConfigValidator):
_required_keys = {"configuration", "presets"}
def __init__(self, name: str, value: Any):
super().__init__(name, value)
self.config_options = self._validate_key("configuration", ConfigOptions)
prebuilt_presets = PREBUILT_PRESETS
# Make sure presets is a dictionary. Will be validated in `PresetValidator`
self.presets = self._validate_key("presets", LiteralDictValidator)
# Ensure custom presets do not collide with prebuilt presets
for preset_name in self.presets.keys:
if preset_name in prebuilt_presets:
raise self._validation_exception(
f"preset name '{preset_name}' conflicts with a prebuilt preset"
)
# Merge prebuilt presets into the config so custom presets can use them
mergedeep.merge(self.presets._value, prebuilt_presets)
for preset_name, preset_dict in self.presets.dict.items():
Preset.preset_partial_validate(config=self, name=preset_name, value=preset_dict)
def initialize(self):
"""

92
src/ytdl_sub/config/config_validator.py Normal file
View file

@ -0,0 +1,92 @@
from typing import Any
from typing import Dict
from typing import Optional
from mergedeep import mergedeep
from ytdl_sub.prebuilt_presets import PREBUILT_PRESETS
from ytdl_sub.validators.strict_dict_validator import StrictDictValidator
from ytdl_sub.validators.validators import LiteralDictValidator
from ytdl_sub.validators.validators import StringValidator
class ConfigOptions(StrictDictValidator):
_required_keys = {"working_directory"}
_optional_keys = {"umask", "dl_aliases"}
def __init__(self, name: str, value: Any):
super().__init__(name, value)
self._working_directory = self._validate_key(
key="working_directory", validator=StringValidator
)
self._umask = self._validate_key_if_present(
key="umask", validator=StringValidator, default="022"
)
self._dl_aliases = self._validate_key_if_present(
key="dl_aliases", validator=LiteralDictValidator
)
@property
def working_directory(self) -> str:
"""
The directory to temporarily store downloaded files before moving them into their final
directory.
"""
return self._working_directory.value
@property
def umask(self) -> Optional[str]:
"""
Umask (octal format) to apply to every created file. Defaults to "022".
"""
return self._umask.value
@property
def dl_aliases(self) -> Optional[Dict[str, str]]:
"""
Alias definitions to shorten ``ytdl-sub dl`` arguments. For example,
.. code-block:: yaml
configuration:
dl_aliases:
mv: "--preset yt_music_video"
v: "--youtube.video_url"
Simplifies
.. code-block:: bash
ytdl-sub dl --preset "yt_music_video" --youtube.video_url "youtube.com/watch?v=a1b2c3"
to
.. code-block:: bash
ytdl-sub dl --mv --v "youtube.com/watch?v=a1b2c3"
"""
if self._dl_aliases:
return self._dl_aliases.dict
return {}
class ConfigValidator(StrictDictValidator):
_required_keys = {"configuration", "presets"}
def __init__(self, name: str, value: Any):
super().__init__(name, value)
self.config_options = self._validate_key("configuration", ConfigOptions)
# Make sure presets is a dictionary. Will be validated in `PresetValidator`
self.presets = self._validate_key("presets", LiteralDictValidator)
# Ensure custom presets do not collide with prebuilt presets
for preset_name in self.presets.keys:
if preset_name in PREBUILT_PRESETS:
raise self._validation_exception(
f"preset name '{preset_name}' conflicts with a prebuilt preset"
)
# Merge prebuilt presets into the config so custom presets can use them
mergedeep.merge(self.presets._value, PREBUILT_PRESETS)

119
src/ytdl_sub/config/preset.py
View file

@ -11,7 +11,7 @@ from typing import Union
from mergedeep import mergedeep
from ytdl_sub.config.config_file import ConfigFile
from ytdl_sub.config.config_validator import ConfigValidator
from ytdl_sub.config.preset_class_mappings import DownloadStrategyMapping
from ytdl_sub.config.preset_class_mappings import PluginMapping
from ytdl_sub.config.preset_options import OutputOptions
@ -22,6 +22,9 @@ from ytdl_sub.downloaders.downloader import DownloaderValidator
from ytdl_sub.entries.entry import Entry
from ytdl_sub.plugins.plugin import Plugin
from ytdl_sub.plugins.plugin import PluginOptions
from ytdl_sub.prebuilt_presets import PREBUILT_PRESET_NAMES
from ytdl_sub.prebuilt_presets import PUBLISHED_PRESET_NAMES
from ytdl_sub.utils.exceptions import ValidationException
from ytdl_sub.utils.yaml import dump_yaml
from ytdl_sub.validators.strict_dict_validator import StrictDictValidator
from ytdl_sub.validators.string_formatter_validators import DictFormatterValidator
@ -33,6 +36,7 @@ from ytdl_sub.validators.validators import ListValidator
from ytdl_sub.validators.validators import StringListValidator
from ytdl_sub.validators.validators import StringValidator
from ytdl_sub.validators.validators import Validator
from ytdl_sub.validators.validators import validation_exception
PRESET_KEYS = {
"preset",
@ -44,6 +48,19 @@ PRESET_KEYS = {
}
def _parent_preset_error_message(
current_preset_name: str, parent_preset_name: str, presets: List[str]
) -> ValidationException:
user_defined_presets = set(presets) - PREBUILT_PRESET_NAMES - {current_preset_name}
return validation_exception(
name=current_preset_name,
error_message=f"preset '{parent_preset_name}' does not exist in the provided config.\n"
f"Available prebuilt presets: {', '.join(sorted(PUBLISHED_PRESET_NAMES))}\n"
f"Your presets: {', '.join(sorted(user_defined_presets))}",
)
class PresetPlugins:
_TPluginOptions = TypeVar("_TPluginOptions", bound=PluginOptions)
@ -113,6 +130,11 @@ class DownloadStrategyValidator(StrictDictValidator):
Returns
-------
The downloader class
Raises
------
ValidationException
If the download strategy is invalid
"""
try:
return DownloadStrategyMapping.get(
@ -122,12 +144,90 @@ class DownloadStrategyValidator(StrictDictValidator):
raise self._validation_exception(error_message=value_exc)
class Preset(StrictDictValidator):
class _PresetShell(StrictDictValidator):
# Have all present keys optional since parent presets could not have all the
# required keys. They will get validated in the init after the mergedeep of dicts
# and ensure required keys are present.
_optional_keys = PRESET_KEYS
class Preset(_PresetShell):
@classmethod
def _validate_download_strategy(cls, name: str, value: Dict) -> None:
sources: List[str] = []
for source_name in DownloadStrategyMapping.sources():
if source_name in value:
sources.append(source_name)
if len(sources) > 1:
raise validation_exception(
name=name,
error_message=f"Contains the sources {', '.join(sources)} but can only have one",
)
# If no sources, nothing more to validate
if not sources:
return
source_name = sources[0]
source_dict = copy.deepcopy(value[source_name])
downloader = DownloadStrategyValidator(name=f"{name}.{source_name}", value=source_dict).get(
downloader_source=source_name
)
del source_dict["download_strategy"]
downloader.downloader_options_type.partial_validate(
name=f"{name}.{source_name}", value=source_dict
)
@classmethod
def preset_partial_validate(cls, config: ConfigValidator, name: str, value: Any) -> None:
"""
Partially validates a preset. Used to ensure every preset in a ConfigFile looks sane.
Cannot fully validate each preset using the Preset init because required fields could
be missing, which become filled in a child preset.
Parameters
----------
config
Config that this preset belongs to
name
Preset name
value
Preset value
Raises
------
ValidationException
If validation fails
"""
# Ensure value is a dict
_ = _PresetShell(name=name, value=value)
assert isinstance(value, dict)
cls._validate_download_strategy(name, value)
cls._partial_validate_key(name, value, "output_options", OutputOptions)
cls._partial_validate_key(name, value, "ytdl_options", YTDLOptions)
cls._partial_validate_key(name, value, "overrides", Overrides)
for plugin_name in PluginMapping.plugins():
cls._partial_validate_key(
name,
value,
key=plugin_name,
validator=PluginMapping.get(plugin_name).plugin_options_type,
)
parent_presets = StringListValidator(name=f"{name}.preset", value=value.get("preset", []))
for parent_preset_name in parent_presets.list:
if parent_preset_name.value not in config.presets.keys:
raise _parent_preset_error_message(
current_preset_name=name,
parent_preset_name=parent_preset_name.value,
presets=config.presets.keys,
)
@property
def _source_variables(self) -> List[str]:
return Entry.source_variables()
@ -278,7 +378,7 @@ class Preset(StrictDictValidator):
self.__validate_override_string_formatter_validator(validator_value)
def _get_presets_to_merge(
self, parent_presets: str | List[str], seen_presets: List[str], config: ConfigFile
self, parent_presets: str | List[str], seen_presets: List[str], config: ConfigValidator
) -> List[Dict]:
presets_to_merge: List[Dict] = []
@ -294,9 +394,10 @@ class Preset(StrictDictValidator):
# Make sure the parent preset actually exists
if parent_preset not in config.presets.keys:
raise self._validation_exception(
f"preset '{parent_preset}' does not exist in the provided config. "
f"Available presets: {', '.join(config.presets.keys)}"
raise _parent_preset_error_message(
current_preset_name=self._name,
parent_preset_name=parent_preset,
presets=config.presets.keys,
)
parent_preset_dict = copy.deepcopy(config.presets.dict[parent_preset])
@ -313,7 +414,7 @@ class Preset(StrictDictValidator):
return presets_to_merge
def __merge_parent_preset_dicts_if_present(self, config: ConfigFile):
def __merge_parent_preset_dicts_if_present(self, config: ConfigValidator):
parent_preset_validator = self._validate_key_if_present(
key="preset", validator=StringListValidator
)
@ -332,7 +433,7 @@ class Preset(StrictDictValidator):
mergedeep.merge({}, *reversed(presets_to_merge), strategy=mergedeep.Strategy.ADDITIVE)
)
def __init__(self, config: ConfigFile, name: str, value: Any):
def __init__(self, config: ConfigValidator, name: str, value: Any):
super().__init__(name=name, value=value)
# Perform the merge of parent presets before validating any keys
@ -367,7 +468,7 @@ class Preset(StrictDictValidator):
return self._name
@classmethod
def from_dict(cls, config: ConfigFile, preset_name: str, preset_dict: Dict) -> "Preset":
def from_dict(cls, config: ConfigValidator, preset_name: str, preset_dict: Dict) -> "Preset":
"""
Parameters
----------

13
src/ytdl_sub/config/preset_options.py
View file

@ -1,4 +1,5 @@
from abc import ABC
from typing import Any
from typing import Dict
from typing import List
from typing import Optional
@ -182,6 +183,18 @@ class OutputOptions(StrictDictValidator):
"keep_files_after",
}
@classmethod
def partial_validate(cls, name: str, value: Any) -> None:
"""
Partially validate output options
"""
if isinstance(value, dict):
value["output_directory"] = value.get("output_directory", "placeholder")
value["file_name"] = value.get("file_name", "placeholder")
# Set this to True by default in partial validate to avoid failing from keep_files
value["maintain_download_archive"] = value.get("maintain_download_archive", True)
_ = cls(name, value)
def __init__(self, name, value):
super().__init__(name, value)

19
src/ytdl_sub/downloaders/generic/collection_validator.py
View file

@ -1,3 +1,4 @@
from typing import Any
from typing import Dict
from typing import List
from typing import Optional
@ -42,6 +43,15 @@ class CollectionUrlValidator(StrictDictValidator):
_required_keys = {"url"}
_optional_keys = {"variables", "source_thumbnails", "playlist_thumbnails"}
@classmethod
def partial_validate(cls, name: str, value: Any) -> None:
"""
Partially validate a YouTube collection url
"""
if isinstance(value, dict):
value["url"] = value.get("url", "placeholder")
_ = cls(name, value)
def __init__(self, name, value):
super().__init__(name, value)
@ -149,6 +159,15 @@ class CollectionValidator(StrictDictValidator, AddsVariablesMixin):
_required_keys = {"urls"}
@classmethod
def partial_validate(cls, name: str, value: Any) -> None:
"""
Partially validate a generic collection
"""
if isinstance(value, dict):
value["urls"] = value.get("urls", [{"url": "placeholder"}])
_ = cls(name, value)
def __init__(self, name, value):
super().__init__(name, value)
self._urls = self._validate_key(key="urls", validator=CollectionUrlListValidator)

10
src/ytdl_sub/downloaders/soundcloud/albums_and_singles.py
View file

@ -1,3 +1,4 @@
from typing import Any
from typing import Dict
from typing import Generator
@ -32,6 +33,15 @@ class SoundcloudAlbumsAndSinglesDownloadOptions(DownloaderValidator):
_required_keys = {"url"}
_optional_keys = {"skip_premiere_tracks"}
@classmethod
def partial_validate(cls, name: str, value: Any) -> None:
"""
Partially validate a Soundcloud source
"""
if isinstance(value, dict):
value["url"] = value.get("url", "https://soundcloud.com/jessebannon")
_ = cls(name, value)
def __init__(self, name, value):
super().__init__(name, value)
self._url = self._validate_key(

12
src/ytdl_sub/downloaders/youtube/channel.py
View file

@ -1,3 +1,4 @@
from typing import Any
from typing import Dict
from typing import List
from typing import Optional
@ -34,6 +35,17 @@ class YoutubeChannelDownloaderOptions(DownloaderValidator):
"channel_banner_path",
}
@classmethod
def partial_validate(cls, name: str, value: Any) -> None:
"""
Partially validate a YouTube channel source
"""
if isinstance(value, dict):
value["channel_url"] = value.get(
"channel_url", "https://www.youtube.com/c/ProjectZombie603"
)
_ = cls(name, value)
def __init__(self, name, value):
super().__init__(name, value)
self._channel_url = self._validate_key(

12
src/ytdl_sub/downloaders/youtube/playlist.py
View file

@ -1,3 +1,4 @@
from typing import Any
from typing import Dict
from typing import List
from typing import Optional
@ -31,6 +32,17 @@ class YoutubePlaylistDownloaderOptions(DownloaderValidator):
_required_keys = {"playlist_url"}
_optional_keys = {"playlist_thumbnail_name"}
@classmethod
def partial_validate(cls, name: str, value: Any) -> None:
"""
Partially validate a YouTube playlist source
"""
if isinstance(value, dict):
value["playlist_url"] = value.get(
"playlist_url", "https://www.youtube.com/playlist?list=UCsvn_Po0SmunchJYtttWpOxMg"
)
_ = cls(name, value)
def __init__(self, name, value):
super().__init__(name, value)
self._playlist_url = self._validate_key(

10
src/ytdl_sub/downloaders/youtube/video.py
View file

@ -1,3 +1,4 @@
from typing import Any
from typing import Dict
from ytdl_sub.downloaders.downloader import Downloader
@ -32,6 +33,15 @@ class YoutubeVideoDownloaderOptions(DownloaderValidator):
_required_keys = {"video_url"}
@classmethod
def partial_validate(cls, name: str, value: Any) -> None:
"""
Partially validate a YouTube video source
"""
if isinstance(value, dict):
value["video_url"] = value.get("video_url", "youtube.com/watch?v=VMAPTo7RVDo")
_ = cls(name, value)
def __init__(self, name, value):
super().__init__(name, value)
self._video_url = self._validate_key("video_url", YoutubeVideoUrlValidator).video_url

13
src/ytdl_sub/plugins/audio_extract.py
View file

@ -1,4 +1,5 @@
import os.path
from typing import Any
from typing import Dict
from typing import Optional
@ -27,7 +28,17 @@ class AudioExtractOptions(PluginOptions):
quality: 128
"""
_optional_keys = {"codec", "quality"}
_required_keys = {"codec"}
_optional_keys = {"quality"}
@classmethod
def partial_validate(cls, name: str, value: Any) -> None:
"""
Partially validate audio extract options
"""
if isinstance(value, dict):
value["codec"] = value.get("codec", "mp3")
_ = cls(name, value)
def __init__(self, name, value):
super().__init__(name, value)

10
src/ytdl_sub/plugins/file_convert.py
View file

@ -1,4 +1,5 @@
import os
from typing import Any
from typing import Dict
from typing import Optional
@ -27,6 +28,15 @@ class FileConvertOptions(PluginOptions):
_required_keys = {"convert_to"}
@classmethod
def partial_validate(cls, name: str, value: Any) -> None:
"""
Partially validate file_convert
"""
if isinstance(value, dict):
value["convert_to"] = value.get("convert_to", "mp3")
_ = cls(name, value)
def __init__(self, name, value):
super().__init__(name, value)
self._convert_to = self._validate_key(key="convert_to", validator=FileTypeValidator).value

10
src/ytdl_sub/plugins/music_tags.py
View file

@ -1,3 +1,4 @@
from typing import Any
from typing import Dict
import mediafile
@ -34,6 +35,15 @@ class MusicTagsOptions(PluginOptions):
_required_keys = {"tags"}
@classmethod
def partial_validate(cls, name: str, value: Any) -> None:
"""
Partially validate music tags
"""
if isinstance(value, dict):
value["tags"] = value.get("tags", {})
_ = cls(name, value)
def __init__(self, name, value):
super().__init__(name, value)

13
src/ytdl_sub/plugins/nfo_tags.py
View file

@ -2,6 +2,7 @@ import os
from abc import ABC
from collections import defaultdict
from pathlib import Path
from typing import Any
from typing import Dict
from typing import List
from typing import Optional
@ -28,6 +29,18 @@ class SharedNfoTagsOptions(PluginOptions):
_required_keys = {"nfo_name", "nfo_root", "tags"}
_optional_keys = {"kodi_safe"}
@classmethod
def partial_validate(cls, name: str, value: Any) -> None:
"""
Partially validate NFO tag options
"""
if isinstance(value, dict):
value["nfo_name"] = value.get("nfo_name", "placeholder")
value["nfo_root"] = value.get("nfo_root", "placeholder")
value["tags"] = value.get("tags", {})
_ = cls(name=name, value=value)
def __init__(self, name, value):
super().__init__(name, value)

10
src/ytdl_sub/plugins/regex.py
View file

@ -1,3 +1,4 @@
from typing import Any
from typing import Dict
from typing import List
from typing import Optional
@ -149,6 +150,15 @@ class RegexOptions(PluginOptions):
_required_keys = {"from"}
_optional_keys = {"skip_if_match_fails"}
@classmethod
def partial_validate(cls, name: str, value: Any) -> None:
"""
Partially validate regex
"""
if isinstance(value, dict):
value["from"] = value.get("from", {})
_ = cls(name, value)
def __init__(self, name, value):
super().__init__(name, value)
self._from = self._validate_key(key="from", validator=FromSourceVariablesRegex)

7
src/ytdl_sub/plugins/split_by_chapters.py
View file

@ -1,6 +1,7 @@
import copy
import os.path
from pathlib import Path
from typing import Any
from typing import List
from typing import Optional
from typing import Tuple
@ -67,6 +68,12 @@ class SplitByChaptersOptions(PluginOptions):
_required_keys = {"when_no_chapters"}
@classmethod
def partial_validate(cls, name: str, value: Any) -> None:
if isinstance(value, dict):
value["when_no_chapters"] = value.get("when_no_chapters", "pass")
_ = cls(name, value)
def __init__(self, name, value):
super().__init__(name, value)
self._when_no_chapters = self._validate_key(

10
src/ytdl_sub/plugins/video_tags.py
View file

@ -1,3 +1,4 @@
from typing import Any
from typing import Dict
from ytdl_sub.entries.entry import Entry
@ -27,6 +28,15 @@ class VideoTagsOptions(PluginOptions):
_required_keys = {"tags"}
@classmethod
def partial_validate(cls, name: str, value: Any) -> None:
"""
Partially validate video tags
"""
if isinstance(value, dict):
value["tags"] = value.get("tags", {})
_ = cls(name, value)
def __init__(self, name, value):
super().__init__(name, value)
self._tags = self._validate_key(key="tags", validator=DictFormatterValidator)

6
src/ytdl_sub/prebuilt_presets/__init__.py
View file

@ -1,6 +1,7 @@
import pathlib
from typing import Any
from typing import Dict
from typing import Set
import mergedeep
@ -21,3 +22,8 @@ def _merge_presets() -> Dict[str, Any]:
PREBUILT_PRESETS: Dict[str, Any] = _merge_presets()
PREBUILT_PRESET_NAMES: Set[str] = set(PREBUILT_PRESETS.keys())
PUBLISHED_PRESET_NAMES: Set[str] = {
name for name in PREBUILT_PRESET_NAMES if not name.startswith("_")
}

6
src/ytdl_sub/prebuilt_presets/music_videos/jellyfin-music-videos.yaml
View file

@ -1,6 +0,0 @@
presets:
kodi-music-videos:
preset:
- "jellyfin-music-videos"
- "kodi-safe"

2
src/ytdl_sub/prebuilt_presets/music_videos/kodi-music-videos.yaml → src/ytdl_sub/prebuilt_presets/music_videos/kodi_music_videos.yaml
View file

@ -3,7 +3,7 @@ presets:
kodi_music_video:
generic:
download_strategy: "source"
music_video_url: "{music_video_url}"
url: "{music_video_url}"
output_options:
output_directory: "{music_video_directory}"

5
src/ytdl_sub/prebuilt_presets/tv_show/tv_show_collection.yaml
View file

@ -18,6 +18,7 @@ presets:
collection_season_1:
generic:
download_strategy: "collection"
urls:
- url: "{collection_season_1_url}"
variables:
@ -42,6 +43,7 @@ presets:
collection_season_2:
generic:
download_strategy: "collection"
urls:
- url: "{collection_season_2_url}"
variables:
@ -60,6 +62,7 @@ presets:
collection_season_3:
generic:
download_strategy: "collection"
urls:
- url: "{collection_season_3_url}"
variables:
@ -78,6 +81,7 @@ presets:
collection_season_4:
generic:
download_strategy: "collection"
urls:
- url: "{collection_season_4_url}"
variables:
@ -96,6 +100,7 @@ presets:
collection_season_5:
generic:
download_strategy: "collection"
urls:
- url: "{collection_season_5_url}"
variables:

2
src/ytdl_sub/subscriptions/base_subscription.py
View file

@ -2,7 +2,7 @@ from abc import ABC
from pathlib import Path
from typing import Type
from ytdl_sub.config.config_file import ConfigOptions
from ytdl_sub.config.config_validator import ConfigOptions
from ytdl_sub.config.preset import Preset
from ytdl_sub.config.preset import PresetPlugins
from ytdl_sub.config.preset_options import OutputOptions

47
src/ytdl_sub/validators/validators.py
View file

@ -16,6 +16,29 @@ ValidationExceptionT = TypeVar("ValidationExceptionT", bound=ValidationException
ValidatorT = TypeVar("ValidatorT", bound="Validator")
def validation_exception(
name: str,
error_message: str | Exception,
exception_class: Type[ValidationExceptionT] = ValidationException,
) -> ValidationExceptionT:
"""
Parameters
----------
name
Name of the validator
error_message
Error message to include in the ValidationException
exception_class
Class of the exception
Returns
-------
Validation exception with a consistent prefix.
"""
prefix = f"Validation error in {name}: "
return exception_class(f"{prefix}{error_message}")
class Validator(ABC):
"""
Used to validate the value of a python object. This is the 'base' class that will first
@ -29,6 +52,18 @@ class Validator(ABC):
# When raising an error, call the type this value instead of its python name
_expected_value_type_name: Optional[str] = None
@classmethod
def partial_validate(cls, name: str, value: Any) -> None:
"""
Parameters
----------
name
Name of the validator
value
Value of the validator
"""
_ = cls(name=name, value=value)
def __init__(self, name: str, value: Any):
self._name = name
self._value = copy.deepcopy(value) # Always deep copy to avoid editing references
@ -56,8 +91,7 @@ class Validator(ABC):
-------
Validation exception with a consistent prefix.
"""
prefix = f"Validation error in {self._name}: "
return exception_class(f"{prefix}{error_message}")
return validation_exception(self._name, error_message, exception_class)
class ValueValidator(Validator, ABC, Generic[ValueT]):
@ -231,6 +265,15 @@ class DictValidator(Validator):
return self._validate_key(key=key, validator=validator, default=default)
@final
@classmethod
def _partial_validate_key(
cls, name: str, value: Any, key: str, validator: Type[ValidatorT]
) -> None:
value_dict = DictValidator(name=name, value=value)
if key in value_dict._dict:
validator.partial_validate(name=f"{name}.{key}", value=value_dict._dict[key])
class LiteralDictValidator(DictValidator):
"""DictValidator with exposed dict and keys method"""

2
tests/unit/cli/test_download_args_parser.py
View file

@ -9,7 +9,7 @@ import pytest
from ytdl_sub.cli.download_args_parser import DownloadArgsParser
from ytdl_sub.cli.main_args_parser import MainArgs
from ytdl_sub.cli.main_args_parser import parser
from ytdl_sub.config.config_file import ConfigOptions
from ytdl_sub.config.config_validator import ConfigOptions
from ytdl_sub.utils.exceptions import InvalidDlArguments

155
tests/unit/config/test_config_file.py Normal file
View file

@ -0,0 +1,155 @@
import re
from typing import Dict
from typing import Optional
import pytest
from ytdl_sub.config.config_file import ConfigFile
from ytdl_sub.config.preset import PRESET_KEYS
from ytdl_sub.config.preset_class_mappings import DownloadStrategyMapping
from ytdl_sub.config.preset_class_mappings import PluginMapping
from ytdl_sub.utils.exceptions import ValidationException
class TestConfigFilePartiallyValidatesPresets:
@classmethod
def _partial_validate(
cls, preset_dict: Dict, expected_error_message: Optional[str] = None
) -> None:
def _config_create() -> None:
_ = ConfigFile(
name="test_partial_validate",
value={
"configuration": {"working_directory": "."},
"presets": {"partial_preset": preset_dict},
},
)
if expected_error_message:
with pytest.raises(ValidationException, match=re.escape(expected_error_message)):
_config_create()
else:
_config_create()
@pytest.mark.parametrize(
"preset_dict",
[
{"nfo_tags": {"tags": {"key-1": "preset_0"}}},
{"output_directory_nfo_tags": {"nfo_root": "test"}},
{"output_options": {"file_name": "test"}},
{"output_options": {"keep_files_after": "today"}},
{"ytdl_options": {"format": "best"}},
{"overrides": {"a": "b"}},
],
)
def test_success(self, preset_dict: Dict):
self._partial_validate(preset_dict)
@pytest.mark.parametrize("key", ["output_options", "overrides", "ytdl_options"])
def test_success__empty_preset_keys(self, key):
self._partial_validate({key: {}})
@pytest.mark.parametrize("plugin", PluginMapping.plugins())
def test_success__empty_plugins(self, plugin: str):
self._partial_validate({plugin: {}})
@pytest.mark.parametrize("source", DownloadStrategyMapping.sources())
def test_success__empty_sources(self, source: str):
for download_strategy in DownloadStrategyMapping.source_download_strategies(source):
self._partial_validate({source: {"download_strategy": download_strategy}})
def test_error__bad_preset_section(self):
self._partial_validate(
preset_dict={"does_not_exist": "lol"},
expected_error_message="Validation error in partial_preset: "
"'partial_preset' contains the field 'does_not_exist' which is not allowed. "
f"Allowed fields: {', '.join(sorted(PRESET_KEYS))}",
)
def test_error__multiple_sources(self):
self._partial_validate(
preset_dict={"youtube": {}, "generic": {}},
expected_error_message="Validation error in partial_preset: "
"Contains the sources generic, youtube but can only have one",
)
def test_error__no_download_strategy(self):
self._partial_validate(
preset_dict={"generic": {}},
expected_error_message="Validation error in partial_preset.generic: "
"missing the required field 'download_strategy'",
)
def test_error__bad_download_strategy(self):
self._partial_validate(
preset_dict={"generic": {"download_strategy": "fail"}},
expected_error_message="Validation error in partial_preset.generic: "
"Tried to use download strategy 'fail' with source 'generic', "
"which does not exist. Available download strategies: collection, source",
)
def test_error__bad_download_strategy_args(self):
self._partial_validate(
preset_dict={"generic": {"download_strategy": "collection", "bad_key": "nope"}},
expected_error_message="Validation error in partial_preset.generic: "
"'partial_preset.generic' contains the field 'bad_key' which is not allowed. "
"Allowed fields: urls",
)
@pytest.mark.parametrize(
"preset_dict",
[
{"nfo_tags": {"tags": {"key-1": {"attributes": {"test": "2"}}}}},
{"generic": {"urls": [{"variables_to_set": {"name": "value"}}]}},
],
)
def test_partial_validate__incomplete_list_item(self, preset_dict):
with pytest.raises(ValidationException):
_ = ConfigFile(
name="test_partial_validate",
value={
"configuration": {"working_directory": "."},
"presets": {"partial_preset": preset_dict},
},
)
@pytest.mark.parametrize(
"preset_dict",
[
{"overrides": "not a dict"},
{"overrides": {"nested": {"dict": "value"}}},
{"overrides": ["list"]},
],
)
def test_error__bad_overrides(self, preset_dict):
self._partial_validate(
preset_dict=preset_dict,
expected_error_message="Validation error in partial_preset.overrides",
)
@pytest.mark.parametrize(
"preset_dict",
[
{"ytdl_options": "not a dict"},
{"ytdl_options": ["list"]},
],
)
def test_error__bad_ytdl_options(self, preset_dict):
self._partial_validate(
preset_dict=preset_dict,
expected_error_message="Validation error in partial_preset.ytdl_options",
)
@pytest.mark.parametrize(
"preset_dict",
[
{"preset": "DNE"},
{"preset": ["DNE"]},
],
)
def test_error__non_existent_preset(self, preset_dict):
self._partial_validate(
preset_dict=preset_dict,
expected_error_message="Validation error in partial_preset: "
"preset 'DNE' does not exist in the provided config.",
)

4
tests/unit/config/test_preset.py
View file

@ -1,5 +1,3 @@
from typing import Dict
import pytest
from ytdl_sub.config.preset import Preset
@ -102,7 +100,7 @@ class TestPreset:
"youtube": youtube_video,
"output_options": dict(
output_options,
**{"maintain_download_archive": True, "keep_files_after": "today-{ttl}"}
**{"maintain_download_archive": True, "keep_files_after": "today-{ttl}"},
),
"overrides": {"ttl": "2months"},
},