From 255805c871525c3c21719efef79a8fe57f18d51c Mon Sep 17 00:00:00 2001 From: Jesse Bannon Date: Sun, 6 Nov 2022 17:00:26 -0800 Subject: [PATCH] [REFACTOR] Rename `source`/`collection` to `url`/`multi_url` (#310) --- src/ytdl_sub/config/preset_class_mappings.py | 8 +++--- src/ytdl_sub/downloaders/downloader.py | 24 +++++++---------- .../generic/{collection.py => multi_url.py} | 12 ++++----- .../downloaders/generic/{source.py => url.py} | 14 +++++----- ...{collection_validator.py => validators.py} | 26 +++++++++---------- .../soundcloud/albums_and_singles.py | 6 ++--- src/ytdl_sub/downloaders/youtube/channel.py | 6 ++--- src/ytdl_sub/downloaders/youtube/playlist.py | 6 ++--- src/ytdl_sub/downloaders/youtube/video.py | 8 +++--- .../music_videos/kodi_music_videos.yaml | 2 +- .../prebuilt_presets/tv_show/tv_show.yaml | 4 +-- .../tv_show/tv_show_collection.yaml | 10 +++---- tests/unit/config/test_config_file.py | 4 +-- 13 files changed, 63 insertions(+), 67 deletions(-) rename src/ytdl_sub/downloaders/generic/{collection.py => multi_url.py} (77%) rename src/ytdl_sub/downloaders/generic/{source.py => url.py} (62%) rename src/ytdl_sub/downloaders/generic/{collection_validator.py => validators.py} (88%) diff --git a/src/ytdl_sub/config/preset_class_mappings.py b/src/ytdl_sub/config/preset_class_mappings.py index 7c8a8673..2c8e003b 100644 --- a/src/ytdl_sub/config/preset_class_mappings.py +++ b/src/ytdl_sub/config/preset_class_mappings.py @@ -3,8 +3,8 @@ from typing import List from typing import Type from ytdl_sub.downloaders.downloader import Downloader -from ytdl_sub.downloaders.generic.collection import CollectionDownloader -from ytdl_sub.downloaders.generic.source import SourceDownloader +from ytdl_sub.downloaders.generic.multi_url import MultiUrlDownloader +from ytdl_sub.downloaders.generic.url import UrlDownloader from ytdl_sub.downloaders.soundcloud.albums_and_singles import SoundcloudAlbumsAndSinglesDownloader from ytdl_sub.downloaders.youtube.channel import YoutubeChannelDownloader from ytdl_sub.downloaders.youtube.merge_playlist import YoutubeMergePlaylistDownloader @@ -40,8 +40,8 @@ class DownloadStrategyMapping: "albums_and_singles": SoundcloudAlbumsAndSinglesDownloader, }, "download": { - "collection": CollectionDownloader, - "source": SourceDownloader, + "multi_url": MultiUrlDownloader, + "url": UrlDownloader, }, } diff --git a/src/ytdl_sub/downloaders/downloader.py b/src/ytdl_sub/downloaders/downloader.py index 74dd097d..1a1da366 100644 --- a/src/ytdl_sub/downloaders/downloader.py +++ b/src/ytdl_sub/downloaders/downloader.py @@ -26,9 +26,9 @@ from yt_dlp.utils import RejectedVideoReached from ytdl_sub.config.preset_options import AddsVariablesMixin from ytdl_sub.config.preset_options import Overrides -from ytdl_sub.downloaders.generic.collection_validator import CollectionThumbnailListValidator -from ytdl_sub.downloaders.generic.collection_validator import CollectionUrlValidator -from ytdl_sub.downloaders.generic.collection_validator import CollectionValidator +from ytdl_sub.downloaders.generic.validators import MultiUrlValidator +from ytdl_sub.downloaders.generic.validators import UrlThumbnailListValidator +from ytdl_sub.downloaders.generic.validators import UrlValidator from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder from ytdl_sub.entries.base_entry import BaseEntry from ytdl_sub.entries.entry import Entry @@ -63,11 +63,11 @@ class DownloaderValidator(StrictDictValidator, AddsVariablesMixin, ABC): @property @abc.abstractmethod - def collection_validator(self) -> CollectionValidator: + def collection_validator(self) -> MultiUrlValidator: """ Returns ------- - CollectionValidator + MultiUrlValidator To determine how the entries are downloaded """ @@ -380,7 +380,7 @@ class Downloader(DownloadArchiver, Generic[DownloaderOptionsT], ABC): self.downloaded_entries[_entry_key(entry)] = entry @property - def collection(self) -> CollectionValidator: + def collection(self) -> MultiUrlValidator: """Return the download options collection""" return self.download_options.collection_validator @@ -480,9 +480,7 @@ class Downloader(DownloadArchiver, Generic[DownloaderOptionsT], ABC): for entry_child in self._download_parent_entry(parent=parent_child): yield entry_child - def _set_collection_variables( - self, collection_url: CollectionUrlValidator, entry: Entry | EntryParent - ): + def _set_collection_variables(self, collection_url: UrlValidator, entry: Entry | EntryParent): if isinstance(entry, EntryParent): for child in entry.parent_children(): self._set_collection_variables(collection_url, child) @@ -495,7 +493,7 @@ class Downloader(DownloadArchiver, Generic[DownloaderOptionsT], ABC): entry.add_variables(variables_to_add=collection_url.variables.dict_with_format_strings) def _download_url_metadata( - self, collection_url: CollectionUrlValidator + self, collection_url: UrlValidator ) -> Tuple[List[EntryParent], List[Entry]]: """ Downloads only info.json files and forms EntryParent trees @@ -594,7 +592,7 @@ class Downloader(DownloadArchiver, Generic[DownloaderOptionsT], ABC): def _download_parent_thumbnails( self, thumbnails_downloaded: Set[str], - thumbnail_list_info: CollectionThumbnailListValidator, + thumbnail_list_info: UrlThumbnailListValidator, entry: Entry, is_last_entry: bool, parent: EntryParent, @@ -637,9 +635,7 @@ class Downloader(DownloadArchiver, Generic[DownloaderOptionsT], ABC): return thumbnails_downloaded - def _download_url_thumbnails( - self, collection_url: CollectionUrlValidator, entries: List[Entry] - ): + def _download_url_thumbnails(self, collection_url: UrlValidator, entries: List[Entry]): """ After all media entries have been downloaded, post processed, and moved to the output directory, run this function. This lets the downloader add any extra files directly to the diff --git a/src/ytdl_sub/downloaders/generic/collection.py b/src/ytdl_sub/downloaders/generic/multi_url.py similarity index 77% rename from src/ytdl_sub/downloaders/generic/collection.py rename to src/ytdl_sub/downloaders/generic/multi_url.py index 368a06a8..a78d6f15 100644 --- a/src/ytdl_sub/downloaders/generic/collection.py +++ b/src/ytdl_sub/downloaders/generic/multi_url.py @@ -1,9 +1,9 @@ from ytdl_sub.downloaders.downloader import Downloader from ytdl_sub.downloaders.downloader import DownloaderValidator -from ytdl_sub.downloaders.generic.collection_validator import CollectionValidator +from ytdl_sub.downloaders.generic.validators import MultiUrlValidator -class CollectionDownloadOptions(CollectionValidator, DownloaderValidator): +class MultiUrlDownloadOptions(MultiUrlValidator, DownloaderValidator): """ Downloads from multiple URLs. If an entry is returned from more than one URL, it will resolve to the bottom-most URL settings. @@ -16,7 +16,7 @@ class CollectionDownloadOptions(CollectionValidator, DownloaderValidator): my_example_preset: download: # required - download_strategy: "collection" + download_strategy: "multi_url" urls: - url: "youtube.com/channel/UCsvn_Po0SmunchJYtttWpOxMg" variables: @@ -39,10 +39,10 @@ class CollectionDownloadOptions(CollectionValidator, DownloaderValidator): """ @property - def collection_validator(self) -> CollectionValidator: + def collection_validator(self) -> MultiUrlValidator: """Returns itself!""" return self -class CollectionDownloader(Downloader[CollectionDownloadOptions]): - downloader_options_type = CollectionDownloadOptions +class MultiUrlDownloader(Downloader[MultiUrlDownloadOptions]): + downloader_options_type = MultiUrlDownloadOptions diff --git a/src/ytdl_sub/downloaders/generic/source.py b/src/ytdl_sub/downloaders/generic/url.py similarity index 62% rename from src/ytdl_sub/downloaders/generic/source.py rename to src/ytdl_sub/downloaders/generic/url.py index f2b4f118..9a3b1735 100644 --- a/src/ytdl_sub/downloaders/generic/source.py +++ b/src/ytdl_sub/downloaders/generic/url.py @@ -1,10 +1,10 @@ from ytdl_sub.downloaders.downloader import Downloader from ytdl_sub.downloaders.downloader import DownloaderValidator -from ytdl_sub.downloaders.generic.collection_validator import CollectionUrlValidator -from ytdl_sub.downloaders.generic.collection_validator import CollectionValidator +from ytdl_sub.downloaders.generic.validators import MultiUrlValidator +from ytdl_sub.downloaders.generic.validators import UrlValidator -class SourceDownloadOptions(CollectionUrlValidator, DownloaderValidator): +class UrlDownloadOptions(UrlValidator, DownloaderValidator): """ Downloads from a single URL supported by yt-dlp. @@ -27,13 +27,13 @@ class SourceDownloadOptions(CollectionUrlValidator, DownloaderValidator): """ @property - def collection_validator(self) -> CollectionValidator: + def collection_validator(self) -> MultiUrlValidator: """Returns itself!""" - return CollectionValidator( + return MultiUrlValidator( name=self._name, value={"urls": [self._value]}, ) -class SourceDownloader(Downloader[SourceDownloadOptions]): - downloader_options_type = SourceDownloadOptions +class UrlDownloader(Downloader[UrlDownloadOptions]): + downloader_options_type = UrlDownloadOptions diff --git a/src/ytdl_sub/downloaders/generic/collection_validator.py b/src/ytdl_sub/downloaders/generic/validators.py similarity index 88% rename from src/ytdl_sub/downloaders/generic/collection_validator.py rename to src/ytdl_sub/downloaders/generic/validators.py index 3e310d83..d2f425f4 100644 --- a/src/ytdl_sub/downloaders/generic/collection_validator.py +++ b/src/ytdl_sub/downloaders/generic/validators.py @@ -11,7 +11,7 @@ from ytdl_sub.validators.string_formatter_validators import StringFormatterValid from ytdl_sub.validators.validators import ListValidator -class CollectionThumbnailValidator(StrictDictValidator): +class UrlThumbnailValidator(StrictDictValidator): _required_keys = {"name", "uid"} def __init__(self, name, value): @@ -35,11 +35,11 @@ class CollectionThumbnailValidator(StrictDictValidator): return self._uid -class CollectionThumbnailListValidator(ListValidator[CollectionThumbnailValidator]): - _inner_list_type = CollectionThumbnailValidator +class UrlThumbnailListValidator(ListValidator[UrlThumbnailValidator]): + _inner_list_type = UrlThumbnailValidator -class CollectionUrlValidator(StrictDictValidator): +class UrlValidator(StrictDictValidator): _required_keys = {"url"} _optional_keys = {"variables", "source_thumbnails", "playlist_thumbnails"} @@ -62,10 +62,10 @@ class CollectionUrlValidator(StrictDictValidator): ) self._source_thumbnails = self._validate_key_if_present( - key="source_thumbnails", validator=CollectionThumbnailListValidator, default=[] + key="source_thumbnails", validator=UrlThumbnailListValidator, default=[] ) self._playlist_thumbnails = self._validate_key_if_present( - key="playlist_thumbnails", validator=CollectionThumbnailListValidator, default=[] + key="playlist_thumbnails", validator=UrlThumbnailListValidator, default=[] ) @property @@ -86,7 +86,7 @@ class CollectionUrlValidator(StrictDictValidator): return self._variables @property - def source_thumbnails(self) -> Optional[CollectionThumbnailListValidator]: + def source_thumbnails(self) -> Optional[UrlThumbnailListValidator]: """ Thumbnails to download from the source, if any exist. The hierarchy is defined as source -> playlist -> entry. @@ -105,7 +105,7 @@ class CollectionUrlValidator(StrictDictValidator): return self._source_thumbnails @property - def playlist_thumbnails(self) -> Optional[CollectionThumbnailListValidator]: + def playlist_thumbnails(self) -> Optional[UrlThumbnailListValidator]: """ Thumbnails to download from the source, if any exist. The hierarchy is defined as source -> playlist -> entry. @@ -124,8 +124,8 @@ class CollectionUrlValidator(StrictDictValidator): return self._playlist_thumbnails -class CollectionUrlListValidator(ListValidator[CollectionUrlValidator]): - _inner_list_type = CollectionUrlValidator +class UrlListValidator(ListValidator[UrlValidator]): + _inner_list_type = UrlValidator _expected_value_type_name = "collection url list" def __init__(self, name, value): @@ -151,7 +151,7 @@ class CollectionUrlListValidator(ListValidator[CollectionUrlValidator]): collection_variables[var] = added_variables[var] -class CollectionValidator(StrictDictValidator, AddsVariablesMixin): +class MultiUrlValidator(StrictDictValidator, AddsVariablesMixin): """ Downloads from multiple URLs. If an entry is returned from more than one URL, it will resolve to the bottom-most URL settings. @@ -170,10 +170,10 @@ class CollectionValidator(StrictDictValidator, AddsVariablesMixin): def __init__(self, name, value): super().__init__(name, value) - self._urls = self._validate_key(key="urls", validator=CollectionUrlListValidator) + self._urls = self._validate_key(key="urls", validator=UrlListValidator) @property - def collection_urls(self) -> CollectionUrlListValidator: + def collection_urls(self) -> UrlListValidator: """ Required. The Soundcloud user's url, i.e. ``soundcloud.com/the_username`` """ diff --git a/src/ytdl_sub/downloaders/soundcloud/albums_and_singles.py b/src/ytdl_sub/downloaders/soundcloud/albums_and_singles.py index 70a89e97..74475ea6 100644 --- a/src/ytdl_sub/downloaders/soundcloud/albums_and_singles.py +++ b/src/ytdl_sub/downloaders/soundcloud/albums_and_singles.py @@ -4,7 +4,7 @@ from typing import Generator from ytdl_sub.downloaders.downloader import Downloader from ytdl_sub.downloaders.downloader import DownloaderValidator -from ytdl_sub.downloaders.generic.collection_validator import CollectionValidator +from ytdl_sub.downloaders.generic.validators import MultiUrlValidator from ytdl_sub.entries.entry import Entry from ytdl_sub.validators.url_validator import SoundcloudUsernameUrlValidator from ytdl_sub.validators.validators import BoolValidator @@ -52,9 +52,9 @@ class SoundcloudAlbumsAndSinglesDownloadOptions(DownloaderValidator): ) @property - def collection_validator(self) -> CollectionValidator: + def collection_validator(self) -> MultiUrlValidator: """Downloads the album tracks first, then the tracks""" - return CollectionValidator( + return MultiUrlValidator( name=self._name, value={ "urls": [ diff --git a/src/ytdl_sub/downloaders/youtube/channel.py b/src/ytdl_sub/downloaders/youtube/channel.py index ca019a64..6ae77994 100644 --- a/src/ytdl_sub/downloaders/youtube/channel.py +++ b/src/ytdl_sub/downloaders/youtube/channel.py @@ -5,7 +5,7 @@ from typing import Optional from ytdl_sub.downloaders.downloader import Downloader from ytdl_sub.downloaders.downloader import DownloaderValidator -from ytdl_sub.downloaders.generic.collection_validator import CollectionValidator +from ytdl_sub.downloaders.generic.validators import MultiUrlValidator from ytdl_sub.validators.string_formatter_validators import OverridesStringFormatterValidator from ytdl_sub.validators.url_validator import YoutubeChannelUrlValidator @@ -59,7 +59,7 @@ class YoutubeChannelDownloaderOptions(DownloaderValidator): ) @property - def collection_validator(self) -> CollectionValidator: + def collection_validator(self) -> MultiUrlValidator: """Download from the channel url""" playlist_thumbnails: List[Dict] = [] if self._channel_avatar_path: @@ -77,7 +77,7 @@ class YoutubeChannelDownloaderOptions(DownloaderValidator): } ) - return CollectionValidator( + return MultiUrlValidator( name=self._name, value={ "urls": [ diff --git a/src/ytdl_sub/downloaders/youtube/playlist.py b/src/ytdl_sub/downloaders/youtube/playlist.py index d8c42835..0adac218 100644 --- a/src/ytdl_sub/downloaders/youtube/playlist.py +++ b/src/ytdl_sub/downloaders/youtube/playlist.py @@ -5,7 +5,7 @@ from typing import Optional from ytdl_sub.downloaders.downloader import Downloader from ytdl_sub.downloaders.downloader import DownloaderValidator -from ytdl_sub.downloaders.generic.collection_validator import CollectionValidator +from ytdl_sub.downloaders.generic.validators import MultiUrlValidator from ytdl_sub.utils.thumbnail import ThumbnailTypes from ytdl_sub.validators.string_formatter_validators import OverridesStringFormatterValidator from ytdl_sub.validators.url_validator import YoutubePlaylistUrlValidator @@ -53,7 +53,7 @@ class YoutubePlaylistDownloaderOptions(DownloaderValidator): ) @property - def collection_validator(self) -> CollectionValidator: + def collection_validator(self) -> MultiUrlValidator: """Downloads the playlist url""" playlist_thumbnails: List[Dict] = [] if self.playlist_thumbnail_name: @@ -64,7 +64,7 @@ class YoutubePlaylistDownloaderOptions(DownloaderValidator): } ) - return CollectionValidator( + return MultiUrlValidator( name=self._name, value={ "urls": [ diff --git a/src/ytdl_sub/downloaders/youtube/video.py b/src/ytdl_sub/downloaders/youtube/video.py index 0c30c11e..1a7f59c5 100644 --- a/src/ytdl_sub/downloaders/youtube/video.py +++ b/src/ytdl_sub/downloaders/youtube/video.py @@ -3,8 +3,8 @@ from typing import Dict from ytdl_sub.downloaders.downloader import Downloader from ytdl_sub.downloaders.downloader import DownloaderValidator -from ytdl_sub.downloaders.generic.collection_validator import CollectionValidator -from ytdl_sub.downloaders.generic.source import SourceDownloadOptions +from ytdl_sub.downloaders.generic.url import UrlDownloadOptions +from ytdl_sub.downloaders.generic.validators import MultiUrlValidator from ytdl_sub.validators.url_validator import YoutubeVideoUrlValidator @@ -47,9 +47,9 @@ class YoutubeVideoDownloaderOptions(DownloaderValidator): self._video_url = self._validate_key("video_url", YoutubeVideoUrlValidator).video_url @property - def collection_validator(self) -> CollectionValidator: + def collection_validator(self) -> MultiUrlValidator: """Downloads the video url""" - return SourceDownloadOptions( + return UrlDownloadOptions( name=self._name, value={"url": self.video_url} ).collection_validator diff --git a/src/ytdl_sub/prebuilt_presets/music_videos/kodi_music_videos.yaml b/src/ytdl_sub/prebuilt_presets/music_videos/kodi_music_videos.yaml index fdd48694..308c7223 100644 --- a/src/ytdl_sub/prebuilt_presets/music_videos/kodi_music_videos.yaml +++ b/src/ytdl_sub/prebuilt_presets/music_videos/kodi_music_videos.yaml @@ -2,7 +2,7 @@ presets: kodi_music_video: download: - download_strategy: "source" + download_strategy: "url" url: "{music_video_url}" output_options: diff --git a/src/ytdl_sub/prebuilt_presets/tv_show/tv_show.yaml b/src/ytdl_sub/prebuilt_presets/tv_show/tv_show.yaml index 72fb74f0..8d16fc44 100644 --- a/src/ytdl_sub/prebuilt_presets/tv_show/tv_show.yaml +++ b/src/ytdl_sub/prebuilt_presets/tv_show/tv_show.yaml @@ -29,7 +29,7 @@ presets: # TV show from any single source. Uses latest entry's thumbnail as tv show poster _tv_show_by_date: download: - download_strategy: "source" + download_strategy: "url" url: "{url}" playlist_thumbnails: - name: "{tv_show_poster_file_name}" @@ -43,7 +43,7 @@ presets: # addition. Each season sets its own `collection_season_number/_padded` _tv_show_collection: download: - download_strategy: "collection" + download_strategy: "multi_url" overrides: season_number: "{collection_season_number}" diff --git a/src/ytdl_sub/prebuilt_presets/tv_show/tv_show_collection.yaml b/src/ytdl_sub/prebuilt_presets/tv_show/tv_show_collection.yaml index e9c2c72b..696b3a47 100644 --- a/src/ytdl_sub/prebuilt_presets/tv_show/tv_show_collection.yaml +++ b/src/ytdl_sub/prebuilt_presets/tv_show/tv_show_collection.yaml @@ -18,7 +18,7 @@ presets: collection_season_1: download: - download_strategy: "collection" + download_strategy: "multi_url" urls: - url: "{collection_season_1_url}" variables: @@ -43,7 +43,7 @@ presets: collection_season_2: download: - download_strategy: "collection" + download_strategy: "multi_url" urls: - url: "{collection_season_2_url}" variables: @@ -62,7 +62,7 @@ presets: collection_season_3: download: - download_strategy: "collection" + download_strategy: "multi_url" urls: - url: "{collection_season_3_url}" variables: @@ -81,7 +81,7 @@ presets: collection_season_4: download: - download_strategy: "collection" + download_strategy: "multi_url" urls: - url: "{collection_season_4_url}" variables: @@ -100,7 +100,7 @@ presets: collection_season_5: download: - download_strategy: "collection" + download_strategy: "multi_url" urls: - url: "{collection_season_5_url}" variables: diff --git a/tests/unit/config/test_config_file.py b/tests/unit/config/test_config_file.py index 542da16d..a08b35a9 100644 --- a/tests/unit/config/test_config_file.py +++ b/tests/unit/config/test_config_file.py @@ -85,12 +85,12 @@ class TestConfigFilePartiallyValidatesPresets: preset_dict={"download": {"download_strategy": "fail"}}, expected_error_message="Validation error in partial_preset.download: " "Tried to use download strategy 'fail' with source 'download', " - "which does not exist. Available download strategies: collection, source", + "which does not exist. Available download strategies: multi_url, url", ) def test_error__bad_download_strategy_args(self): self._partial_validate( - preset_dict={"download": {"download_strategy": "collection", "bad_key": "nope"}}, + preset_dict={"download": {"download_strategy": "multi_url", "bad_key": "nope"}}, expected_error_message="Validation error in partial_preset.download: " "'partial_preset.download' contains the field 'bad_key' which is not allowed. " "Allowed fields: urls",