From d31c4313f2f217a88b00cd643810c3899b404c31 Mon Sep 17 00:00:00 2001 From: jbannon Date: Thu, 31 Mar 2022 00:20:06 +0000 Subject: [PATCH] begin parsing yaml better --- config.yaml | 39 ++++++ requirements.txt | 1 + subscriptions.yaml | 115 ++++++------------ ytdl_subscribe/enums.py | 15 ++- ytdl_subscribe/main.py | 2 +- ytdl_subscribe/parse.py | 10 +- ytdl_subscribe/subscriptions/soundcloud.py | 3 - ytdl_subscribe/subscriptions/subscription.py | 2 - ytdl_subscribe/subscriptions/youtube.py | 3 - ytdl_subscribe/validators/__init__.py | 0 ytdl_subscribe/validators/native/__init__.py | 0 .../validators/native/object_validator.py | 56 +++++++++ ytdl_subscribe/validators/native/validator.py | 10 ++ .../validators/subscriptions/__init__.py | 0 .../validators/subscriptions/sources.py | 66 ++++++++++ .../subscriptions/subscription_preset.py | 46 +++++++ 16 files changed, 276 insertions(+), 92 deletions(-) create mode 100644 config.yaml create mode 100644 ytdl_subscribe/validators/__init__.py create mode 100644 ytdl_subscribe/validators/native/__init__.py create mode 100644 ytdl_subscribe/validators/native/object_validator.py create mode 100644 ytdl_subscribe/validators/native/validator.py create mode 100644 ytdl_subscribe/validators/subscriptions/__init__.py create mode 100644 ytdl_subscribe/validators/subscriptions/sources.py create mode 100644 ytdl_subscribe/validators/subscriptions/subscription_preset.py diff --git a/config.yaml b/config.yaml new file mode 100644 index 00000000..3cb55c26 --- /dev/null +++ b/config.yaml @@ -0,0 +1,39 @@ +config: + working_directory: 'tmp' + +presets: + soundcloud_with_id3_tags: + soundcloud: + download_strategy: "albums_then_tracks" + skip_premiere_tracks: True + ytdl_opts: + format: 'bestaudio[ext=mp3]' + post_process: + file_name: "[{album_year}] {sanitized_album}/{track_number_padded} - {sanitized_title}.{ext}" + thumbnail_name: "[{album_year}] {sanitized_album}/folder.{thumbnail_ext}" + tagging: + artist: "{artist}" + albumartist: "{artist}" + title: "{title}" + album: "{album}" + tracknumber: "{track_number}" + year: "{album_year}" + genre: "{genre}" + + music_videos: + youtube: + download_strategy: "playlist" + ytdl_opts: + format: 'best' + ignoreerrors: True + post_process: + file_name: "{sanitized_artist} - {sanitized_title}.{ext}" + convert_thumbnail: "jpg" + thumbnail_name: "{sanitized_artist} - {sanitized_title}.jpg" + nfo_name: "{sanitized_artist} - {sanitized_title}.nfo" + nfo_root: "musicvideo" + nfo: + artist: "{artist}" + title: "{title}" + album: "Music Videos" + year: "{upload_year}" diff --git a/requirements.txt b/requirements.txt index 85281bdb..fc84dd08 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,6 +3,7 @@ black==21.11b1 coverage==6.3.2 dicttoxml==1.7.4 isort==5.10.1 +jsonschema==4.4.0 mergedeep==1.3.4 music-tag==0.4.3 pathvalidate==2.4.1 diff --git a/subscriptions.yaml b/subscriptions.yaml index e9662ffd..1da717f9 100644 --- a/subscriptions.yaml +++ b/subscriptions.yaml @@ -1,81 +1,42 @@ -config: - working_directory: 'tmp' +ALISON: + preset: "soundcloud_with_id3_tags" + soundcloud: + username: alis_on + output_path: "/mnt/nas/Downloads/ytdl/music/ALISON" + overrides: + artist: "A.L.I.S.O.N." + genre: "Synthwave / Electronic" -presets: - soundcloud_with_id3_tags: - soundcloud: - download_strategy: "albums_then_tracks" - skip_premiere_tracks: True - ytdl_opts: - format: 'bestaudio[ext=mp3]' - post_process: - file_name: "[{album_year}] {sanitized_album}/{track_number_padded} - {sanitized_title}.{ext}" - thumbnail_name: "[{album_year}] {sanitized_album}/folder.{thumbnail_ext}" - tagging: - artist: "{artist}" - albumartist: "{artist}" - title: "{title}" - album: "{album}" - tracknumber: "{track_number}" - year: "{album_year}" - genre: "{genre}" +delorra: + preset: "soundcloud_with_id3_tags" + soundcloud: + username: delorra + output_path: "/mnt/nas/Downloads/ytdl/music/DeLorra" + overrides: + artist: "DeLorra" + genre: "Synthwave / Electronic" - music_videos: - youtube: - noop: True - ytdl_opts: - format: 'best' - ignoreerrors: True - post_process: - file_name: "{sanitized_artist} - {sanitized_title}.{ext}" - convert_thumbnail: "jpg" - thumbnail_name: "{sanitized_artist} - {sanitized_title}.jpg" - nfo_name: "{sanitized_artist} - {sanitized_title}.nfo" - nfo_root: "musicvideo" - nfo: - artist: "{artist}" - title: "{title}" - album: "Music Videos" - year: "{upload_year}" +bl00dwave: + preset: "soundcloud_with_id3_tags" + soundcloud: + username: bl00dwave + output_path: "/tmp/bl00dwave" + overrides: + artist: "bl00dwave" + genre: "Synthwave / Electronic" -subscriptions: - ALISON: - preset: "soundcloud_with_id3_tags" - soundcloud: - username: alis_on - output_path: "/mnt/nas/Downloads/ytdl/music/ALISON" - overrides: - artist: "A.L.I.S.O.N." - genre: "Synthwave / Electronic" - delorra: - preset: "soundcloud_with_id3_tags" - soundcloud: - username: delorra - output_path: "/mnt/nas/Downloads/ytdl/music/DeLorra" - overrides: - artist: "DeLorra" - genre: "Synthwave / Electronic" - bl00dwave: - preset: "soundcloud_with_id3_tags" - soundcloud: - username: bl00dwave - output_path: "/tmp/bl00dwave" - overrides: - artist: "bl00dwave" - genre: "Synthwave / Electronic" +tom_petty: + youtube: + playlist_id: PLoopXDarluPBnuxs4PTC55Sc_2ShAXC0i + preset: "music_videos" + output_path: "/tmp/Tom Petty" + overrides: + artist: "Tom Petty and the Heartbreakers" - tom_petty: - youtube: - playlist_id: PLoopXDarluPBnuxs4PTC55Sc_2ShAXC0i - preset: "music_videos" - output_path: "/tmp/Tom Petty" - overrides: - artist: "Tom Petty and the Heartbreakers" - - rammstein: - youtube: - playlist_id: "PLVTLbc6i-h_iuhdwUfuPDLFLXG2QQnz-x" - preset: "music_videos" - output_path: "/mnt/nas/Downloads/ytdl/music_videos/Rammstein" - overrides: - artist: "Rammstein" +rammstein: + youtube: + playlist_id: "PLVTLbc6i-h_iuhdwUfuPDLFLXG2QQnz-x" + preset: "music_videos" + output_path: "/mnt/nas/Downloads/ytdl/music_videos/Rammstein" + overrides: + artist: "Rammstein" diff --git a/ytdl_subscribe/enums.py b/ytdl_subscribe/enums.py index 45e5a23d..78e996b5 100644 --- a/ytdl_subscribe/enums.py +++ b/ytdl_subscribe/enums.py @@ -1,7 +1,20 @@ -class SubscriptionSource(object): +from typing import List + + +class SubscriptionSourceName(object): YOUTUBE = "youtube" SOUNDCLOUD = "soundcloud" + @classmethod + def all(cls) -> List[str]: + """Returns all Subscription sources sorted as a list""" + return sorted([cls.YOUTUBE, cls.SOUNDCLOUD]) + + @classmethod + def pretty_all(cls) -> str: + """Returns all subscription sources pretty printed""" + return ", ".join(cls.all()) + class YAMLSection(object): CONFIG_KEY = "config" diff --git a/ytdl_subscribe/main.py b/ytdl_subscribe/main.py index 75b11c02..151f1d50 100644 --- a/ytdl_subscribe/main.py +++ b/ytdl_subscribe/main.py @@ -5,7 +5,7 @@ from ytdl_subscribe.parse import parse_subscriptions from ytdl_subscribe.parse import parse_subscriptions_file parser = argparse.ArgumentParser( - description="YoutubeDL-Subscribe: Download and organize your favorite media easily." + description="YoutubeDL-Subscribe: Download and organize your favorite media hassle-free." ) parser.add_argument( "-c", diff --git a/ytdl_subscribe/parse.py b/ytdl_subscribe/parse.py index 98eef50d..8b819eed 100644 --- a/ytdl_subscribe/parse.py +++ b/ytdl_subscribe/parse.py @@ -3,7 +3,7 @@ from typing import Optional import yaml from mergedeep import mergedeep -from ytdl_subscribe.enums import SubscriptionSource +from ytdl_subscribe.enums import SubscriptionSourceName from ytdl_subscribe.enums import YAMLSection from ytdl_subscribe.subscriptions.soundcloud import SoundcloudSubscription from ytdl_subscribe.subscriptions.subscription import Subscription @@ -84,11 +84,11 @@ def parse_subscriptions(yaml_dict: dict, presets: dict, subscriptions: Optional[ preset = presets[subscription["preset"]] subscription = mergedeep.merge({}, preset, subscription) - if SubscriptionSource.SOUNDCLOUD in subscription: - subscription_source = SubscriptionSource.SOUNDCLOUD + if SubscriptionSourceName.SOUNDCLOUD in subscription: + subscription_source = SubscriptionSourceName.SOUNDCLOUD subscription_class = SoundcloudSubscription - elif SubscriptionSource.YOUTUBE in subscription: - subscription_source = SubscriptionSource.YOUTUBE + elif SubscriptionSourceName.YOUTUBE in subscription: + subscription_source = SubscriptionSourceName.YOUTUBE subscription_class = YoutubeSubscription else: raise ValueError("dne") diff --git a/ytdl_subscribe/subscriptions/soundcloud.py b/ytdl_subscribe/subscriptions/soundcloud.py index 7212e985..2092d3c7 100644 --- a/ytdl_subscribe/subscriptions/soundcloud.py +++ b/ytdl_subscribe/subscriptions/soundcloud.py @@ -3,13 +3,10 @@ from typing import List from ytdl_subscribe.downloaders.soundcloud_downloader import SoundcloudDownloader from ytdl_subscribe.entries.soundcloud import SoundcloudAlbum from ytdl_subscribe.entries.soundcloud import SoundcloudTrack -from ytdl_subscribe.enums import SubscriptionSource from ytdl_subscribe.subscriptions.subscription import Subscription class SoundcloudSubscription(Subscription): - source = SubscriptionSource.SOUNDCLOUD - def extract_info(self): tracks: List[SoundcloudTrack] = [] soundcloud_downloader = SoundcloudDownloader( diff --git a/ytdl_subscribe/subscriptions/subscription.py b/ytdl_subscribe/subscriptions/subscription.py index b19a752f..9c264fcb 100644 --- a/ytdl_subscribe/subscriptions/subscription.py +++ b/ytdl_subscribe/subscriptions/subscription.py @@ -13,8 +13,6 @@ from ytdl_subscribe.entries.entry import Entry class Subscription(object): WORKING_DIRECTORY = "" - source = None - def __init__(self, name, options, ytdl_opts, post_process, overrides, output_path): """ Parameters diff --git a/ytdl_subscribe/subscriptions/youtube.py b/ytdl_subscribe/subscriptions/youtube.py index e4dff3f2..b83e1b9b 100644 --- a/ytdl_subscribe/subscriptions/youtube.py +++ b/ytdl_subscribe/subscriptions/youtube.py @@ -1,11 +1,8 @@ from ytdl_subscribe.downloaders.youtube_downloader import YoutubeDownloader -from ytdl_subscribe.enums import SubscriptionSource from ytdl_subscribe.subscriptions.subscription import Subscription class YoutubeSubscription(Subscription): - source = SubscriptionSource.YOUTUBE - def extract_info(self): youtube_downloader = YoutubeDownloader( output_directory=self.output_path, diff --git a/ytdl_subscribe/validators/__init__.py b/ytdl_subscribe/validators/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ytdl_subscribe/validators/native/__init__.py b/ytdl_subscribe/validators/native/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ytdl_subscribe/validators/native/object_validator.py b/ytdl_subscribe/validators/native/object_validator.py new file mode 100644 index 00000000..59dcaf88 --- /dev/null +++ b/ytdl_subscribe/validators/native/object_validator.py @@ -0,0 +1,56 @@ +from typing import Set + +from ytdl_subscribe.validators.native.validator import Validator + + +class ObjectValidator(Validator): + + required_fields: Set[str] = [] + optional_fields: Set[str] = [] + allow_extra_fields = False + + @property + def allowed_fields(self): + return sorted(self.required_fields.union(self.optional_fields)) + + @property + def object_keys(self): + return sorted(list(self.value.keys())) + + @property + def object_items(self): + return self.value.items() + + def validate(self) -> "ObjectValidator": + super().validate() + + # Ensure the value is an object + if not isinstance(self.value, dict): + error_msg = f"'{self.key}' must be an object" + if self.required_fields: + error_msg = f"{error_msg} with the required field: {', '.join(self.required_fields)}" + raise ValueError(error_msg) + + # Ensure the object contains all the required fields + for required_key in self.required_fields: + if required_key not in self.value: + error_msg = ( + f"'{self.key}' is missing the required field '{required_key}'" + ) + raise ValueError(error_msg) + + # If extra fields are not allowed, ensure we only have required or optional keys + if not self.allow_extra_fields: + for object_key in self.object_keys: + if ( + object_key not in self.required_fields + or object_key not in self.optional_fields + ): + + error_msg = ( + f"'{self.key}' contains the field '{object_key} which is not allowed. " + f"Allowed fields: {', '.join(self.allowed_fields)}" + ) + raise ValueError(error_msg) + + return self diff --git a/ytdl_subscribe/validators/native/validator.py b/ytdl_subscribe/validators/native/validator.py new file mode 100644 index 00000000..ac975185 --- /dev/null +++ b/ytdl_subscribe/validators/native/validator.py @@ -0,0 +1,10 @@ +from typing import Any + + +class Validator: + def __init__(self, key: str, value: Any): + self.key = key + self.value = value + + def validate(self) -> "Validator": + return self diff --git a/ytdl_subscribe/validators/subscriptions/__init__.py b/ytdl_subscribe/validators/subscriptions/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ytdl_subscribe/validators/subscriptions/sources.py b/ytdl_subscribe/validators/subscriptions/sources.py new file mode 100644 index 00000000..e670429b --- /dev/null +++ b/ytdl_subscribe/validators/subscriptions/sources.py @@ -0,0 +1,66 @@ +from typing import Any +from typing import Optional +from typing import Set + +from ytdl_subscribe.validators.native.object_validator import ObjectValidator + + +class SubscriptionSource(ObjectValidator): + required_fields = {"download_strategy"} + download_strategies: Set[str] = {} + + def __init__(self, key: str, value: Any): + super().__init__(key=key, value=value) + self.download_strategy: Optional[str] = None + + def validate(self) -> "SubscriptionSource": + super().validate() + + if self.value["download_strategy"] not in self.download_strategies: + raise ValueError( + f"'download_strategy' must be one of the following: {', '.join(self.download_strategies)}" + ) + + return self + + +class YoutubeSource(SubscriptionSource): + optional_fields = {"playlist_id"} + download_strategies = {"playlist"} + + def __init__(self, key: str, value: Any): + super().__init__(key=key, value=value) + self.playlist_id: Optional[str] = None + + def validate(self) -> "YoutubeSource": + super().validate() + + if self.download_strategy == "playlist": + self.playlist_id = self.value.get("playlist_id") + if not self.playlist_id or not isinstance(self.playlist_id, str): + raise ValueError( + "Youtube download_strategy 'playlist' requires the field 'playlist_id' to be a string" + ) + + return self + + +class SoundcloudSource(SubscriptionSource): + optional_fields = {"username"} + download_strategies = {"albums_and_singles"} + + def __init__(self, key: str, value: Any): + super().__init__(key=key, value=value) + self.username: Optional[str] = None + + def validate(self) -> "SoundcloudSource": + super().validate() + + if self.download_strategy == "albums_and_singles": + self.username = self.value.get("username") + if not self.username or not isinstance(self.username, str): + raise ValueError( + "Soundcloud download_strategy 'albums_and_singles' requires the field 'username'" + ) + + return self diff --git a/ytdl_subscribe/validators/subscriptions/subscription_preset.py b/ytdl_subscribe/validators/subscriptions/subscription_preset.py new file mode 100644 index 00000000..ba429ea7 --- /dev/null +++ b/ytdl_subscribe/validators/subscriptions/subscription_preset.py @@ -0,0 +1,46 @@ +from typing import Any +from typing import Optional + +from ytdl_subscribe.enums import SubscriptionSourceName +from ytdl_subscribe.validators.native.object_validator import ObjectValidator +from ytdl_subscribe.validators.subscriptions.sources import SoundcloudSource +from ytdl_subscribe.validators.subscriptions.sources import SubscriptionSource +from ytdl_subscribe.validators.subscriptions.sources import YoutubeSource + + +class SubscriptionPreset(ObjectValidator): + required_fields = {"post_process"} + optional_fields = {"ytdl_options", *SubscriptionSourceName.all()} + allow_extra_fields = False + + def __init__(self, key: str, value: Any): + super().__init__(key=key, value=value) + self.subscription_source: Optional[SubscriptionSource] = None + + def _validate_subscription_source(self): + # Make sure we only have a single subscription source + for object_key, object_value in self.object_items: + if object_key in SubscriptionSourceName.all() and self.subscription_source: + raise ValueError( + f"'{self.key}' can only have one of the following sources: {SubscriptionSourceName.pretty_all()}" + ) + + if object_key == SubscriptionSourceName.SOUNDCLOUD: + self.subscription_source = SoundcloudSource( + key=object_key, value=object_value + ).validate() + elif object_key == SubscriptionSourceName.YOUTUBE: + self.subscription_source = YoutubeSource( + key=object_key, value=object_value + ).validate() + + # If subscription source was not set, error + if not self.subscription_source: + raise ValueError( + f"'{self.key} must have one of the following sources: {SubscriptionSourceName.pretty_all()}" + ) + + def validate(self) -> "SubscriptionPreset": + super().validate() + self._validate_subscription_source() + return self