From aeca125c2ec304560adf86ada1a1fcd6ecdd005a Mon Sep 17 00:00:00 2001 From: Jesse Bannon Date: Wed, 28 Sep 2022 12:32:50 -0700 Subject: [PATCH] [BACKEND] `generic.source` download strategy (#252) --- src/ytdl_sub/config/preset_class_mappings.py | 2 + src/ytdl_sub/downloaders/downloader.py | 6 +- .../downloaders/generic/collection.py | 32 ++++++----- .../generic/collection_validator.py | 57 ++++++++++++++----- src/ytdl_sub/downloaders/generic/source.py | 39 +++++++++++++ src/ytdl_sub/downloaders/youtube/video.py | 8 +-- 6 files changed, 110 insertions(+), 34 deletions(-) create mode 100644 src/ytdl_sub/downloaders/generic/source.py diff --git a/src/ytdl_sub/config/preset_class_mappings.py b/src/ytdl_sub/config/preset_class_mappings.py index 73be4059..52867a1f 100644 --- a/src/ytdl_sub/config/preset_class_mappings.py +++ b/src/ytdl_sub/config/preset_class_mappings.py @@ -4,6 +4,7 @@ from typing import Type from ytdl_sub.downloaders.downloader import Downloader from ytdl_sub.downloaders.generic.collection import CollectionDownloader +from ytdl_sub.downloaders.generic.source import SourceDownloader from ytdl_sub.downloaders.soundcloud.albums_and_singles import SoundcloudAlbumsAndSinglesDownloader from ytdl_sub.downloaders.youtube.channel import YoutubeChannelDownloader from ytdl_sub.downloaders.youtube.merge_playlist import YoutubeMergePlaylistDownloader @@ -39,6 +40,7 @@ class DownloadStrategyMapping: }, "generic": { "collection": CollectionDownloader, + "source": SourceDownloader, }, } diff --git a/src/ytdl_sub/downloaders/downloader.py b/src/ytdl_sub/downloaders/downloader.py index b07a9751..93b76614 100644 --- a/src/ytdl_sub/downloaders/downloader.py +++ b/src/ytdl_sub/downloaders/downloader.py @@ -415,10 +415,12 @@ class Downloader(DownloadArchiver, Generic[DownloaderOptionsT], ABC): for child in entry.parent_children(): self._set_collection_variables(collection_url, child) for child in entry.entry_children(): - child.add_variables(variables_to_add=collection_url.variables) + child.add_variables( + variables_to_add=collection_url.variables.dict_with_format_strings + ) elif isinstance(entry, Entry): - entry.add_variables(variables_to_add=collection_url.variables) + entry.add_variables(variables_to_add=collection_url.variables.dict_with_format_strings) def _download_url_metadata( self, collection_url: CollectionUrlValidator diff --git a/src/ytdl_sub/downloaders/generic/collection.py b/src/ytdl_sub/downloaders/generic/collection.py index d798f23b..7e079505 100644 --- a/src/ytdl_sub/downloaders/generic/collection.py +++ b/src/ytdl_sub/downloaders/generic/collection.py @@ -18,20 +18,24 @@ class CollectionDownloadOptions(CollectionValidator, DownloaderValidator): # required download_strategy: "collection" urls: - - url: "soundcloud.com/tracks" - variables: - season: "1" - album: "{title}" - playlist_thumbnails: - - name: - uid: "square" / "largest" / "last entry" / actual name - source_thumbnail: - - path: - type: " - - url: "soundcloud.com/albums" - variables: - season: "1" - album: "{playlist_title}" + - url: "youtube.com/channel/UCsvn_Po0SmunchJYtttWpOxMg" + variables: + season_index: "1" + season_name: "Uploads" + playlist_thumbnails: + - name: "poster.jpg" + uid: "avatar_uncropped" + - name: "fanart.jpg" + uid: "banner_uncropped" + - name: "season{season_index}-poster.jpg" + uid: "latest_entry" + - url: "https://www.youtube.com/playlist?list=UCsvn_Po0SmunchJYtttWpOxMg" + variables: + season_index: "2" + season_name: "Playlist as Season" + playlist_thumbnails: + - name: "season{season_index}-poster.jpg" + uid: "latest_entry" """ @property diff --git a/src/ytdl_sub/downloaders/generic/collection_validator.py b/src/ytdl_sub/downloaders/generic/collection_validator.py index ec2fb580..2772c6fc 100644 --- a/src/ytdl_sub/downloaders/generic/collection_validator.py +++ b/src/ytdl_sub/downloaders/generic/collection_validator.py @@ -48,8 +48,9 @@ class CollectionUrlValidator(StrictDictValidator): # TODO: url validate using yt-dlp IE self._url = self._validate_key(key="url", validator=StringValidator) - variables = self._validate_key_if_present(key="variables", validator=DictFormatterValidator) - self._variables = variables.dict_with_format_strings if variables else {} + self._variables = self._validate_key_if_present( + key="variables", validator=DictFormatterValidator, default={} + ) self._source_thumbnails = self._validate_key_if_present( key="source_thumbnails", validator=CollectionThumbnailListValidator, default=[] @@ -61,30 +62,55 @@ class CollectionUrlValidator(StrictDictValidator): @property def url(self) -> str: """ - Returns - ------- - URL to download from + URL to download from, listed in priority from lowest (top) to highest (bottom). If a + download exists in more than one URL, it will resolve to the bottom-most one and inherit + those variables. """ return self._url.value @property - def variables(self) -> Dict[str, str]: + def variables(self) -> DictFormatterValidator: """ - Variables to add to each entry + Source variables to add to each entry. The top-most collection must define all possible + variables. Collections below can redefine all of them or a subset of the top-most variables. """ return self._variables @property def source_thumbnails(self) -> Optional[CollectionThumbnailListValidator]: """ - TODO:docstring + Thumbnails to download from the source, if any exist. The hierarchy is defined as + source -> playlist -> entry. + + Usage: + + .. code-block:: yaml + + source_thumbnails: + - name: "poster.jpg" + uid: "avatar_uncropped" + + UID is the yt-dlp thumbnail ID or can specify ``latest_entry`` to use the last entry's + thumbnail that was downloaded. """ return self._source_thumbnails @property def playlist_thumbnails(self) -> Optional[CollectionThumbnailListValidator]: """ - TODO:docstring + Thumbnails to download from the source, if any exist. The hierarchy is defined as + source -> playlist -> entry. + + Usage: + + .. code-block:: yaml + + playlist_thumbnails: + - name: "poster.jpg" + uid: "avatar_uncropped" + + UID is the yt-dlp thumbnail ID or can specify ``latest_entry`` to use the last entry's + thumbnail that was downloaded. """ return self._playlist_thumbnails @@ -96,10 +122,10 @@ class CollectionUrlListValidator(ListValidator[CollectionUrlValidator]): def __init__(self, name, value): super().__init__(name, value) - added_variables: Dict[str, str] = self.list[0].variables + added_variables: Dict[str, str] = self.list[0].variables.dict_with_format_strings for idx, collection_url_validator in enumerate(self.list[1:]): - collection_variables = collection_url_validator.variables + collection_variables = collection_url_validator.variables.dict_with_format_strings # see if this collection contains new added vars (it should not) for var in collection_variables.keys(): @@ -112,7 +138,7 @@ class CollectionUrlListValidator(ListValidator[CollectionUrlValidator]): # see if this collection is missing any added vars (if so, inherit from the top) for var in added_variables.keys(): - if var not in collection_variables: + if var not in collection_variables.keys(): collection_variables[var] = added_variables[var] @@ -141,7 +167,7 @@ class CollectionValidator(StrictDictValidator, AddsVariablesMixin): ------- List of variables added. The first collection url always contains all the variables. """ - return list(self._urls.list[0].variables.keys()) + return list(self._urls.list[0].variables.keys) def validate_with_variables( self, source_variables: List[str], override_variables: Dict[str, str] @@ -162,7 +188,10 @@ class CollectionValidator(StrictDictValidator, AddsVariablesMixin): # Apply formatting to each new source variable, ensure it resolves for collection_url in self.collection_urls.list: - for source_var_name, source_var_formatter_str in collection_url.variables.items(): + for ( + source_var_name, + source_var_formatter_str, + ) in collection_url.variables.dict_with_format_strings.items(): _ = StringFormatterValidator( name=f"{self._name}.{source_var_name}", value=source_var_formatter_str ).apply_formatter(base_variables) diff --git a/src/ytdl_sub/downloaders/generic/source.py b/src/ytdl_sub/downloaders/generic/source.py new file mode 100644 index 00000000..a28134c6 --- /dev/null +++ b/src/ytdl_sub/downloaders/generic/source.py @@ -0,0 +1,39 @@ +from ytdl_sub.downloaders.downloader import Downloader +from ytdl_sub.downloaders.downloader import DownloaderValidator +from ytdl_sub.downloaders.generic.collection_validator import CollectionUrlValidator +from ytdl_sub.downloaders.generic.collection_validator import CollectionValidator + + +class SourceDownloadOptions(CollectionUrlValidator, DownloaderValidator): + """ + Downloads from a single URL supported by yt-dlp. + + Usage: + + .. code-block:: yaml + + presets: + my_example_preset: + generic: + # required + download_strategy: "source" + url: "youtube.com/channel/UCsvn_Po0SmunchJYtttWpOxMg" + # optional + playlist_thumbnails: + - name: "poster.jpg" + uid: "avatar_uncropped" + - name: "fanart.jpg" + uid: "banner_uncropped" + """ + + @property + def collection_validator(self) -> CollectionValidator: + """Returns itself!""" + return CollectionValidator( + name=self._name, + value={"urls": [self._value]}, + ) + + +class SourceDownloader(Downloader[SourceDownloadOptions]): + downloader_options_type = SourceDownloadOptions diff --git a/src/ytdl_sub/downloaders/youtube/video.py b/src/ytdl_sub/downloaders/youtube/video.py index d469cb06..c9b9f951 100644 --- a/src/ytdl_sub/downloaders/youtube/video.py +++ b/src/ytdl_sub/downloaders/youtube/video.py @@ -3,6 +3,7 @@ from typing import Dict from ytdl_sub.downloaders.downloader import Downloader from ytdl_sub.downloaders.downloader import DownloaderValidator from ytdl_sub.downloaders.generic.collection_validator import CollectionValidator +from ytdl_sub.downloaders.generic.source import SourceDownloadOptions from ytdl_sub.validators.url_validator import YoutubeVideoUrlValidator @@ -38,10 +39,9 @@ class YoutubeVideoDownloaderOptions(DownloaderValidator): @property def collection_validator(self) -> CollectionValidator: """Downloads the video url""" - return CollectionValidator( - name=self._name, - value={"urls": [{"url": self.video_url, "variables": {"playlist_size": "1"}}]}, - ) + return SourceDownloadOptions( + name=self._name, value={"url": self.video_url} + ).collection_validator @property def video_url(self) -> str: