From 6494f8199b309c7a25c537338d7f15b64a99ea63 Mon Sep 17 00:00:00 2001 From: Jesse Bannon Date: Sat, 17 Sep 2022 15:39:54 -0700 Subject: [PATCH] [BACKEND] Collection source/playlist thumbnail support --- src/ytdl_sub/downloaders/downloader.py | 125 ++++++++++++++++-- .../downloaders/generic/collection.py | 6 + .../generic/collection_validator.py | 53 +++++++- src/ytdl_sub/downloaders/youtube/channel.py | 82 +++--------- .../plugins/output_directory_nfo_tags.py | 11 +- 5 files changed, 203 insertions(+), 74 deletions(-) diff --git a/src/ytdl_sub/downloaders/downloader.py b/src/ytdl_sub/downloaders/downloader.py index 90ecbed9..47f2ee79 100644 --- a/src/ytdl_sub/downloaders/downloader.py +++ b/src/ytdl_sub/downloaders/downloader.py @@ -25,17 +25,21 @@ from yt_dlp.utils import RejectedVideoReached from ytdl_sub.config.preset_options import AddsVariablesMixin from ytdl_sub.config.preset_options import Overrides +from ytdl_sub.downloaders.generic.collection_validator import CollectionThumbnailListValidator from ytdl_sub.downloaders.generic.collection_validator import CollectionUrlValidator from ytdl_sub.downloaders.generic.collection_validator import CollectionValidator from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder from ytdl_sub.entries.base_entry import BaseEntry from ytdl_sub.entries.entry import Entry from ytdl_sub.entries.entry_parent import EntryParent +from ytdl_sub.entries.variables.kwargs import PLAYLIST_ENTRY +from ytdl_sub.entries.variables.kwargs import SOURCE_ENTRY from ytdl_sub.thread.log_entries_downloaded_listener import LogEntriesDownloadedListener from ytdl_sub.utils.exceptions import FileNotDownloadedException from ytdl_sub.utils.file_handler import FileHandler from ytdl_sub.utils.file_handler import FileMetadata from ytdl_sub.utils.logger import Logger +from ytdl_sub.utils.thumbnail import convert_url_thumbnail from ytdl_sub.validators.strict_dict_validator import StrictDictValidator from ytdl_sub.ytdl_additions.enhanced_download_archive import DownloadArchiver from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive @@ -139,7 +143,7 @@ class Downloader(DownloadArchiver, Generic[DownloaderOptionsT, DownloaderEntryT] ) self.parents: List[EntryParent] = [] - self.downloaded_entries: Set[str] = set() + self.downloaded_entries: Dict[str, Entry] = {} @contextmanager def ytdl_downloader(self, ytdl_options_overrides: Optional[Dict] = None) -> ytdl.YoutubeDL: @@ -331,6 +335,12 @@ class Downloader(DownloadArchiver, Generic[DownloaderOptionsT, DownloaderEntryT] ############################################################################################### # DOWNLOAD FUNCTIONS + def _is_downloaded(self, entry: Entry) -> bool: + return _entry_key(entry) in self.downloaded_entries + + def _mark_downloaded(self, entry: Entry) -> None: + self.downloaded_entries[_entry_key(entry)] = entry + @property def collection(self) -> CollectionValidator: """Return the download options collection""" @@ -380,14 +390,19 @@ class Downloader(DownloadArchiver, Generic[DownloaderOptionsT, DownloaderEntryT] return entry - def _download_parent_entry(self, parent: EntryParent) -> Generator[Entry, None, None]: - # Download the parent's entries first, in reverse order - for entry_child in reversed(parent.entry_children()): - if _entry_key(entry_child) in self.downloaded_entries: + def _download_entries(self, entries: List[Entry]) -> Generator[Entry, None, None]: + # Download entries in reverse order since they are scraped in the opposite direction. + # Helps deal with break_on_existing + for entry in reversed(entries): + if self._is_downloaded(entry): continue - yield self._download_entry(entry_child) - self.downloaded_entries.add(_entry_key(entry_child)) + yield self._download_entry(entry) + self._mark_downloaded(entry) + + def _download_parent_entry(self, parent: EntryParent) -> Generator[Entry, None, None]: + for entry_child in self._download_entries(parent.entry_children()): + yield entry_child # Recursion the parent's parent entries for parent_child in reversed(parent.parent_children()): @@ -453,7 +468,7 @@ class Downloader(DownloadArchiver, Generic[DownloaderOptionsT, DownloaderEntryT] for entry_child in self._download_parent_entry(parent=parent): yield entry_child - for orphan in orphans: + for orphan in self._download_entries(orphans): yield self._download_entry(orphan) def download( @@ -463,8 +478,102 @@ class Downloader(DownloadArchiver, Generic[DownloaderOptionsT, DownloaderEntryT] # download the bottom-most urls first since they are top-priority for collection_url in reversed(self.collection.collection_urls.list): parents, orphan_entries = self._download_url_metadata(collection_url=collection_url) + collection_url_entries: List[Entry] = [] + for entry in self._download(parents=parents, orphans=orphan_entries): yield entry + collection_url_entries.append(entry) + + self._download_url_thumbnails( + collection_url=collection_url, entries=collection_url_entries + ) + + @classmethod + def _download_thumbnail( + cls, + thumbnail_url: str, + output_thumbnail_path: str, + ) -> Optional[bool]: + """ + Downloads a thumbnail and stores it in the output directory + + Parameters + ---------- + thumbnail_url: + Url of the thumbnail + output_thumbnail_path: + Path to store the thumbnail after downloading + + Returns + ------- + True if the thumbnail converted. None if it is missing or failed. + """ + if not thumbnail_url: + return None + + return convert_url_thumbnail( + thumbnail_url=thumbnail_url, output_thumbnail_path=output_thumbnail_path + ) + + def _download_parent_thumbnails( + self, + thumbnails_downloaded: Set[str], + thumbnail_list_info: CollectionThumbnailListValidator, + entry: Entry, + parent: EntryParent, + ) -> Set[str]: + """ + Downloads and moves channel avatar and banner images to the output directory. + """ + for thumbnail_info in thumbnail_list_info.list: + thumbnail_name = self.overrides.apply_formatter(thumbnail_info.name, entry=entry) + thumbnail_id = self.overrides.apply_formatter(thumbnail_info.uid) + + if (thumbnail_url := parent.get_thumbnail_url(thumbnail_id=thumbnail_id)) is None: + download_logger.warning("TODO: Failed to download channel's avatar image") + continue + + if self._download_thumbnail( + thumbnail_url=thumbnail_url, + output_thumbnail_path=str(Path(self.working_directory) / thumbnail_name), + ): + self.save_file(file_name=thumbnail_name) + thumbnails_downloaded.add(thumbnail_name) + else: + download_logger.warning("TODO: Failed to download channel's avatar image") + + return thumbnails_downloaded + + def _download_url_thumbnails( + self, collection_url: CollectionUrlValidator, entries: List[Entry] + ): + """ + After all media entries have been downloaded, post processed, and moved to the output + directory, run this function. This lets the downloader add any extra files directly to the + output directory, for things like YT channel image, banner. + """ + thumbnails_downloaded: Set[str] = set() + + for entry in entries: + if entry.kwargs_contains(PLAYLIST_ENTRY): + thumbnails_downloaded = self._download_parent_thumbnails( + thumbnails_downloaded=thumbnails_downloaded, + thumbnail_list_info=collection_url.playlist_thumbnails, + entry=entry, + parent=EntryParent( + entry.kwargs(PLAYLIST_ENTRY), working_directory=self.working_directory + ), + ) + + if entry.kwargs_contains(SOURCE_ENTRY): + thumbnails_downloaded = self._download_parent_thumbnails( + thumbnails_downloaded=thumbnails_downloaded, + thumbnail_list_info=collection_url.source_thumbnails, + entry=entry, + parent=EntryParent( + entry.kwargs(SOURCE_ENTRY), working_directory=self.working_directory + ), + ) def post_download(self): """ diff --git a/src/ytdl_sub/downloaders/generic/collection.py b/src/ytdl_sub/downloaders/generic/collection.py index a982f684..16ac7d46 100644 --- a/src/ytdl_sub/downloaders/generic/collection.py +++ b/src/ytdl_sub/downloaders/generic/collection.py @@ -23,6 +23,12 @@ class CollectionDownloadOptions(CollectionValidator, DownloaderValidator): variables: season: "1" album: "{title}" + playlist_thumbnails: + - name: + uid: "square" / "largest" / "last entry" / actual name + source_thumbnail: + - path: + type: " - url: "soundcloud.com/albums" variables: season: "1" diff --git a/src/ytdl_sub/downloaders/generic/collection_validator.py b/src/ytdl_sub/downloaders/generic/collection_validator.py index fdb0744a..ec2fb580 100644 --- a/src/ytdl_sub/downloaders/generic/collection_validator.py +++ b/src/ytdl_sub/downloaders/generic/collection_validator.py @@ -1,17 +1,47 @@ from typing import Dict from typing import List +from typing import Optional from ytdl_sub.config.preset_options import AddsVariablesMixin from ytdl_sub.validators.strict_dict_validator import StrictDictValidator from ytdl_sub.validators.string_formatter_validators import DictFormatterValidator +from ytdl_sub.validators.string_formatter_validators import OverridesStringFormatterValidator from ytdl_sub.validators.string_formatter_validators import StringFormatterValidator from ytdl_sub.validators.validators import ListValidator from ytdl_sub.validators.validators import StringValidator +class CollectionThumbnailValidator(StrictDictValidator): + _required_keys = {"name", "uid"} + + def __init__(self, name, value): + super().__init__(name, value) + + self._name = self._validate_key(key="name", validator=StringFormatterValidator) + self._uid = self._validate_key(key="uid", validator=OverridesStringFormatterValidator) + + @property + def name(self) -> StringFormatterValidator: + """ + File name for the thumbnail + """ + return self._name + + @property + def uid(self) -> OverridesStringFormatterValidator: + """ + yt-dlp's unique ID of the thumbnail + """ + return self._uid + + +class CollectionThumbnailListValidator(ListValidator[CollectionThumbnailValidator]): + _inner_list_type = CollectionThumbnailValidator + + class CollectionUrlValidator(StrictDictValidator): _required_keys = {"url"} - _optional_keys = {"variables"} + _optional_keys = {"variables", "source_thumbnails", "playlist_thumbnails"} def __init__(self, name, value): super().__init__(name, value) @@ -21,6 +51,13 @@ class CollectionUrlValidator(StrictDictValidator): variables = self._validate_key_if_present(key="variables", validator=DictFormatterValidator) self._variables = variables.dict_with_format_strings if variables else {} + self._source_thumbnails = self._validate_key_if_present( + key="source_thumbnails", validator=CollectionThumbnailListValidator, default=[] + ) + self._playlist_thumbnails = self._validate_key_if_present( + key="playlist_thumbnails", validator=CollectionThumbnailListValidator, default=[] + ) + @property def url(self) -> str: """ @@ -37,6 +74,20 @@ class CollectionUrlValidator(StrictDictValidator): """ return self._variables + @property + def source_thumbnails(self) -> Optional[CollectionThumbnailListValidator]: + """ + TODO:docstring + """ + return self._source_thumbnails + + @property + def playlist_thumbnails(self) -> Optional[CollectionThumbnailListValidator]: + """ + TODO:docstring + """ + return self._playlist_thumbnails + class CollectionUrlListValidator(ListValidator[CollectionUrlValidator]): _inner_list_type = CollectionUrlValidator diff --git a/src/ytdl_sub/downloaders/youtube/channel.py b/src/ytdl_sub/downloaders/youtube/channel.py index 03d3d0aa..50a37b45 100644 --- a/src/ytdl_sub/downloaders/youtube/channel.py +++ b/src/ytdl_sub/downloaders/youtube/channel.py @@ -1,15 +1,12 @@ -from pathlib import Path -from typing import Dict +from typing import Dict, List from typing import Generator from typing import Optional -from ytdl_sub.downloaders.downloader import download_logger from ytdl_sub.downloaders.generic.collection_validator import CollectionValidator from ytdl_sub.downloaders.youtube.abc import YoutubeDownloader from ytdl_sub.downloaders.youtube.abc import YoutubeDownloaderOptions from ytdl_sub.entries.entry_parent import EntryParent from ytdl_sub.entries.youtube import YoutubeVideo -from ytdl_sub.utils.thumbnail import convert_url_thumbnail from ytdl_sub.validators.string_formatter_validators import OverridesStringFormatterValidator from ytdl_sub.validators.url_validator import YoutubeChannelUrlValidator @@ -54,9 +51,27 @@ class YoutubeChannelDownloaderOptions(YoutubeDownloaderOptions): @property def collection_validator(self) -> CollectionValidator: """Download from the channel url""" + playlist_thumbnails: List[Dict] = [] + if self._channel_avatar_path: + playlist_thumbnails.append({ + "name": self._channel_avatar_path.format_string, + "uid": "avatar_uncropped", + }) + if self._channel_banner_path: + playlist_thumbnails.append({ + "name": self._channel_banner_path.format_string, + "uid": "banner_uncropped", + }) + return CollectionValidator( name=self._name, - value={"urls": [{"url": self.channel_url}]}, + value={ + "urls": [ + { + "url": self.channel_url, + "playlist_thumbnails": playlist_thumbnails + } + ]}, ) @property @@ -121,59 +136,4 @@ class YoutubeChannelDownloader(YoutubeDownloader[YoutubeChannelDownloaderOptions Downloads all videos from a channel """ for entry in super().download(): - yield entry.to_type(YoutubeVideo) - - def _download_thumbnail( - self, - thumbnail_url: str, - output_thumbnail_path: str, - ) -> Optional[bool]: - """ - Downloads a thumbnail and stores it in the output directory - - Parameters - ---------- - thumbnail_url: - Url of the thumbnail - output_thumbnail_path: - Path to store the thumbnail after downloading - - Returns - ------- - True if the thumbnail converted. None if it is missing or failed. - """ - if not thumbnail_url: - download_logger.warning("Could not find a thumbnail for %s", self.channel.uid) - return None - - return convert_url_thumbnail( - thumbnail_url=thumbnail_url, output_thumbnail_path=output_thumbnail_path - ) - - def post_download(self): - """ - Downloads and moves channel avatar and banner images to the output directory. - """ - if self.download_options.channel_avatar_path: - avatar_thumbnail_name = self.overrides.apply_formatter( - self.download_options.channel_avatar_path - ) - if self._download_thumbnail( - thumbnail_url=self.channel.get_thumbnail_url("avatar_uncropped"), - output_thumbnail_path=str(Path(self.working_directory) / avatar_thumbnail_name), - ): - self.save_file(file_name=avatar_thumbnail_name) - else: - download_logger.warning("Failed to download channel's avatar image") - - if self.download_options.channel_banner_path: - banner_thumbnail_name = self.overrides.apply_formatter( - self.download_options.channel_banner_path - ) - if self._download_thumbnail( - thumbnail_url=self.channel.get_thumbnail_url("banner_uncropped"), - output_thumbnail_path=str(Path(self.working_directory) / banner_thumbnail_name), - ): - self.save_file(file_name=banner_thumbnail_name) - else: - download_logger.warning("Failed to download channel's banner image") + yield entry.to_type(YoutubeVideo) \ No newline at end of file diff --git a/src/ytdl_sub/plugins/output_directory_nfo_tags.py b/src/ytdl_sub/plugins/output_directory_nfo_tags.py index 4e1dda3b..a63f9684 100644 --- a/src/ytdl_sub/plugins/output_directory_nfo_tags.py +++ b/src/ytdl_sub/plugins/output_directory_nfo_tags.py @@ -61,11 +61,14 @@ class OutputDirectoryNfoTagsOptions(SharedNfoTagsOptions): .. code-block:: yaml tags: - title: - attributes: - year: "2022" - tag: "Sweet youtube TV show" + named_season: + - tag: "{source_title}" + attributes: + number: "{collection_index}" + behavior: "merge" genre: + - tag: "Comedy" + behavior: "overwrite" - "Comedy" - "Drama"