diff --git a/src/ytdl_sub/downloaders/downloader.py b/src/ytdl_sub/downloaders/downloader.py index 90ecbed9..e7e7453d 100644 --- a/src/ytdl_sub/downloaders/downloader.py +++ b/src/ytdl_sub/downloaders/downloader.py @@ -25,17 +25,21 @@ from yt_dlp.utils import RejectedVideoReached from ytdl_sub.config.preset_options import AddsVariablesMixin from ytdl_sub.config.preset_options import Overrides +from ytdl_sub.downloaders.generic.collection_validator import CollectionThumbnailListValidator from ytdl_sub.downloaders.generic.collection_validator import CollectionUrlValidator from ytdl_sub.downloaders.generic.collection_validator import CollectionValidator from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder from ytdl_sub.entries.base_entry import BaseEntry from ytdl_sub.entries.entry import Entry from ytdl_sub.entries.entry_parent import EntryParent +from ytdl_sub.entries.variables.kwargs import PLAYLIST_ENTRY +from ytdl_sub.entries.variables.kwargs import SOURCE_ENTRY from ytdl_sub.thread.log_entries_downloaded_listener import LogEntriesDownloadedListener from ytdl_sub.utils.exceptions import FileNotDownloadedException from ytdl_sub.utils.file_handler import FileHandler from ytdl_sub.utils.file_handler import FileMetadata from ytdl_sub.utils.logger import Logger +from ytdl_sub.utils.thumbnail import convert_url_thumbnail from ytdl_sub.validators.strict_dict_validator import StrictDictValidator from ytdl_sub.ytdl_additions.enhanced_download_archive import DownloadArchiver from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive @@ -139,7 +143,7 @@ class Downloader(DownloadArchiver, Generic[DownloaderOptionsT, DownloaderEntryT] ) self.parents: List[EntryParent] = [] - self.downloaded_entries: Set[str] = set() + self.downloaded_entries: Dict[str, Entry] = {} @contextmanager def ytdl_downloader(self, ytdl_options_overrides: Optional[Dict] = None) -> ytdl.YoutubeDL: @@ -331,6 +335,12 @@ class Downloader(DownloadArchiver, Generic[DownloaderOptionsT, DownloaderEntryT] ############################################################################################### # DOWNLOAD FUNCTIONS + def _is_downloaded(self, entry: Entry) -> bool: + return _entry_key(entry) in self.downloaded_entries + + def _mark_downloaded(self, entry: Entry) -> None: + self.downloaded_entries[_entry_key(entry)] = entry + @property def collection(self) -> CollectionValidator: """Return the download options collection""" @@ -380,14 +390,19 @@ class Downloader(DownloadArchiver, Generic[DownloaderOptionsT, DownloaderEntryT] return entry - def _download_parent_entry(self, parent: EntryParent) -> Generator[Entry, None, None]: - # Download the parent's entries first, in reverse order - for entry_child in reversed(parent.entry_children()): - if _entry_key(entry_child) in self.downloaded_entries: + def _download_entries(self, entries: List[Entry]) -> Generator[Entry, None, None]: + # Download entries in reverse order since they are scraped in the opposite direction. + # Helps deal with break_on_existing + for entry in reversed(entries): + if self._is_downloaded(entry): continue - yield self._download_entry(entry_child) - self.downloaded_entries.add(_entry_key(entry_child)) + yield self._download_entry(entry) + self._mark_downloaded(entry) + + def _download_parent_entry(self, parent: EntryParent) -> Generator[Entry, None, None]: + for entry_child in self._download_entries(parent.entry_children()): + yield entry_child # Recursion the parent's parent entries for parent_child in reversed(parent.parent_children()): @@ -453,7 +468,7 @@ class Downloader(DownloadArchiver, Generic[DownloaderOptionsT, DownloaderEntryT] for entry_child in self._download_parent_entry(parent=parent): yield entry_child - for orphan in orphans: + for orphan in self._download_entries(orphans): yield self._download_entry(orphan) def download( @@ -463,12 +478,103 @@ class Downloader(DownloadArchiver, Generic[DownloaderOptionsT, DownloaderEntryT] # download the bottom-most urls first since they are top-priority for collection_url in reversed(self.collection.collection_urls.list): parents, orphan_entries = self._download_url_metadata(collection_url=collection_url) + collection_url_entries: List[Entry] = [] + for entry in self._download(parents=parents, orphans=orphan_entries): yield entry + collection_url_entries.append(entry) - def post_download(self): + self._download_url_thumbnails( + collection_url=collection_url, entries=collection_url_entries + ) + + @classmethod + def _download_thumbnail( + cls, + thumbnail_url: str, + output_thumbnail_path: str, + ) -> Optional[bool]: + """ + Downloads a thumbnail and stores it in the output directory + + Parameters + ---------- + thumbnail_url: + Url of the thumbnail + output_thumbnail_path: + Path to store the thumbnail after downloading + + Returns + ------- + True if the thumbnail converted. None if it is missing or failed. + """ + if not thumbnail_url: + return None + + return convert_url_thumbnail( + thumbnail_url=thumbnail_url, output_thumbnail_path=output_thumbnail_path + ) + + def _download_parent_thumbnails( + self, + thumbnails_downloaded: Set[str], + thumbnail_list_info: CollectionThumbnailListValidator, + entry: Entry, + parent: EntryParent, + ) -> Set[str]: + """ + Downloads and moves channel avatar and banner images to the output directory. + """ + for thumbnail_info in thumbnail_list_info.list: + thumbnail_name = self.overrides.apply_formatter(thumbnail_info.name, entry=entry) + thumbnail_id = self.overrides.apply_formatter(thumbnail_info.uid) + + # alread downloaded + if thumbnail_name in thumbnails_downloaded: + continue + + if (thumbnail_url := parent.get_thumbnail_url(thumbnail_id=thumbnail_id)) is None: + download_logger.warning("TODO: Failed to download channel's avatar image") + continue + + if self._download_thumbnail( + thumbnail_url=thumbnail_url, + output_thumbnail_path=str(Path(self.working_directory) / thumbnail_name), + ): + self.save_file(file_name=thumbnail_name) + thumbnails_downloaded.add(thumbnail_name) + else: + download_logger.warning("TODO: Failed to download channel's avatar image") + + return thumbnails_downloaded + + def _download_url_thumbnails( + self, collection_url: CollectionUrlValidator, entries: List[Entry] + ): """ After all media entries have been downloaded, post processed, and moved to the output directory, run this function. This lets the downloader add any extra files directly to the output directory, for things like YT channel image, banner. """ + thumbnails_downloaded: Set[str] = set() + + for entry in entries: + if entry.kwargs_contains(PLAYLIST_ENTRY): + thumbnails_downloaded = self._download_parent_thumbnails( + thumbnails_downloaded=thumbnails_downloaded, + thumbnail_list_info=collection_url.playlist_thumbnails, + entry=entry, + parent=EntryParent( + entry.kwargs(PLAYLIST_ENTRY), working_directory=self.working_directory + ), + ) + + if entry.kwargs_contains(SOURCE_ENTRY): + thumbnails_downloaded = self._download_parent_thumbnails( + thumbnails_downloaded=thumbnails_downloaded, + thumbnail_list_info=collection_url.source_thumbnails, + entry=entry, + parent=EntryParent( + entry.kwargs(SOURCE_ENTRY), working_directory=self.working_directory + ), + ) diff --git a/src/ytdl_sub/downloaders/generic/collection.py b/src/ytdl_sub/downloaders/generic/collection.py index a982f684..16ac7d46 100644 --- a/src/ytdl_sub/downloaders/generic/collection.py +++ b/src/ytdl_sub/downloaders/generic/collection.py @@ -23,6 +23,12 @@ class CollectionDownloadOptions(CollectionValidator, DownloaderValidator): variables: season: "1" album: "{title}" + playlist_thumbnails: + - name: + uid: "square" / "largest" / "last entry" / actual name + source_thumbnail: + - path: + type: " - url: "soundcloud.com/albums" variables: season: "1" diff --git a/src/ytdl_sub/downloaders/generic/collection_validator.py b/src/ytdl_sub/downloaders/generic/collection_validator.py index fdb0744a..ec2fb580 100644 --- a/src/ytdl_sub/downloaders/generic/collection_validator.py +++ b/src/ytdl_sub/downloaders/generic/collection_validator.py @@ -1,17 +1,47 @@ from typing import Dict from typing import List +from typing import Optional from ytdl_sub.config.preset_options import AddsVariablesMixin from ytdl_sub.validators.strict_dict_validator import StrictDictValidator from ytdl_sub.validators.string_formatter_validators import DictFormatterValidator +from ytdl_sub.validators.string_formatter_validators import OverridesStringFormatterValidator from ytdl_sub.validators.string_formatter_validators import StringFormatterValidator from ytdl_sub.validators.validators import ListValidator from ytdl_sub.validators.validators import StringValidator +class CollectionThumbnailValidator(StrictDictValidator): + _required_keys = {"name", "uid"} + + def __init__(self, name, value): + super().__init__(name, value) + + self._name = self._validate_key(key="name", validator=StringFormatterValidator) + self._uid = self._validate_key(key="uid", validator=OverridesStringFormatterValidator) + + @property + def name(self) -> StringFormatterValidator: + """ + File name for the thumbnail + """ + return self._name + + @property + def uid(self) -> OverridesStringFormatterValidator: + """ + yt-dlp's unique ID of the thumbnail + """ + return self._uid + + +class CollectionThumbnailListValidator(ListValidator[CollectionThumbnailValidator]): + _inner_list_type = CollectionThumbnailValidator + + class CollectionUrlValidator(StrictDictValidator): _required_keys = {"url"} - _optional_keys = {"variables"} + _optional_keys = {"variables", "source_thumbnails", "playlist_thumbnails"} def __init__(self, name, value): super().__init__(name, value) @@ -21,6 +51,13 @@ class CollectionUrlValidator(StrictDictValidator): variables = self._validate_key_if_present(key="variables", validator=DictFormatterValidator) self._variables = variables.dict_with_format_strings if variables else {} + self._source_thumbnails = self._validate_key_if_present( + key="source_thumbnails", validator=CollectionThumbnailListValidator, default=[] + ) + self._playlist_thumbnails = self._validate_key_if_present( + key="playlist_thumbnails", validator=CollectionThumbnailListValidator, default=[] + ) + @property def url(self) -> str: """ @@ -37,6 +74,20 @@ class CollectionUrlValidator(StrictDictValidator): """ return self._variables + @property + def source_thumbnails(self) -> Optional[CollectionThumbnailListValidator]: + """ + TODO:docstring + """ + return self._source_thumbnails + + @property + def playlist_thumbnails(self) -> Optional[CollectionThumbnailListValidator]: + """ + TODO:docstring + """ + return self._playlist_thumbnails + class CollectionUrlListValidator(ListValidator[CollectionUrlValidator]): _inner_list_type = CollectionUrlValidator diff --git a/src/ytdl_sub/downloaders/youtube/channel.py b/src/ytdl_sub/downloaders/youtube/channel.py index 03d3d0aa..7cce1e50 100644 --- a/src/ytdl_sub/downloaders/youtube/channel.py +++ b/src/ytdl_sub/downloaders/youtube/channel.py @@ -1,15 +1,12 @@ -from pathlib import Path from typing import Dict from typing import Generator +from typing import List from typing import Optional -from ytdl_sub.downloaders.downloader import download_logger from ytdl_sub.downloaders.generic.collection_validator import CollectionValidator from ytdl_sub.downloaders.youtube.abc import YoutubeDownloader from ytdl_sub.downloaders.youtube.abc import YoutubeDownloaderOptions -from ytdl_sub.entries.entry_parent import EntryParent from ytdl_sub.entries.youtube import YoutubeVideo -from ytdl_sub.utils.thumbnail import convert_url_thumbnail from ytdl_sub.validators.string_formatter_validators import OverridesStringFormatterValidator from ytdl_sub.validators.url_validator import YoutubeChannelUrlValidator @@ -54,9 +51,25 @@ class YoutubeChannelDownloaderOptions(YoutubeDownloaderOptions): @property def collection_validator(self) -> CollectionValidator: """Download from the channel url""" + playlist_thumbnails: List[Dict] = [] + if self._channel_avatar_path: + playlist_thumbnails.append( + { + "name": self._channel_avatar_path.format_string, + "uid": "avatar_uncropped", + } + ) + if self._channel_banner_path: + playlist_thumbnails.append( + { + "name": self._channel_banner_path.format_string, + "uid": "banner_uncropped", + } + ) + return CollectionValidator( name=self._name, - value={"urls": [{"url": self.channel_url}]}, + value={"urls": [{"url": self.channel_url, "playlist_thumbnails": playlist_thumbnails}]}, ) @property @@ -110,70 +123,9 @@ class YoutubeChannelDownloader(YoutubeDownloader[YoutubeChannelDownloaderOptions # pylint: enable=line-too-long - @property - def channel(self) -> EntryParent: - """Gets the channel entry parent""" - assert len(self.parents) == 1, "Channel should be the only entry parent" - return self.parents[0] - def download(self) -> Generator[YoutubeVideo, None, None]: """ Downloads all videos from a channel """ for entry in super().download(): yield entry.to_type(YoutubeVideo) - - def _download_thumbnail( - self, - thumbnail_url: str, - output_thumbnail_path: str, - ) -> Optional[bool]: - """ - Downloads a thumbnail and stores it in the output directory - - Parameters - ---------- - thumbnail_url: - Url of the thumbnail - output_thumbnail_path: - Path to store the thumbnail after downloading - - Returns - ------- - True if the thumbnail converted. None if it is missing or failed. - """ - if not thumbnail_url: - download_logger.warning("Could not find a thumbnail for %s", self.channel.uid) - return None - - return convert_url_thumbnail( - thumbnail_url=thumbnail_url, output_thumbnail_path=output_thumbnail_path - ) - - def post_download(self): - """ - Downloads and moves channel avatar and banner images to the output directory. - """ - if self.download_options.channel_avatar_path: - avatar_thumbnail_name = self.overrides.apply_formatter( - self.download_options.channel_avatar_path - ) - if self._download_thumbnail( - thumbnail_url=self.channel.get_thumbnail_url("avatar_uncropped"), - output_thumbnail_path=str(Path(self.working_directory) / avatar_thumbnail_name), - ): - self.save_file(file_name=avatar_thumbnail_name) - else: - download_logger.warning("Failed to download channel's avatar image") - - if self.download_options.channel_banner_path: - banner_thumbnail_name = self.overrides.apply_formatter( - self.download_options.channel_banner_path - ) - if self._download_thumbnail( - thumbnail_url=self.channel.get_thumbnail_url("banner_uncropped"), - output_thumbnail_path=str(Path(self.working_directory) / banner_thumbnail_name), - ): - self.save_file(file_name=banner_thumbnail_name) - else: - download_logger.warning("Failed to download channel's banner image") diff --git a/src/ytdl_sub/plugins/output_directory_nfo_tags.py b/src/ytdl_sub/plugins/output_directory_nfo_tags.py index 4e1dda3b..a63f9684 100644 --- a/src/ytdl_sub/plugins/output_directory_nfo_tags.py +++ b/src/ytdl_sub/plugins/output_directory_nfo_tags.py @@ -61,11 +61,14 @@ class OutputDirectoryNfoTagsOptions(SharedNfoTagsOptions): .. code-block:: yaml tags: - title: - attributes: - year: "2022" - tag: "Sweet youtube TV show" + named_season: + - tag: "{source_title}" + attributes: + number: "{collection_index}" + behavior: "merge" genre: + - tag: "Comedy" + behavior: "overwrite" - "Comedy" - "Drama" diff --git a/src/ytdl_sub/subscriptions/subscription_download.py b/src/ytdl_sub/subscriptions/subscription_download.py index d04b1d32..bdbb15e3 100644 --- a/src/ytdl_sub/subscriptions/subscription_download.py +++ b/src/ytdl_sub/subscriptions/subscription_download.py @@ -265,7 +265,6 @@ class SubscriptionDownload(BaseSubscription, ABC): plugins=plugins, dry_run=dry_run, entry=entry, entry_metadata=entry_metadata ) - downloader.post_download() for plugin in plugins: plugin.post_process_subscription() diff --git a/tests/e2e/resources/expected_downloads_summaries/plugins/date_range/no_downloads.json b/tests/e2e/resources/expected_downloads_summaries/plugins/date_range/no_downloads.json index ccda5535..9e26dfee 100644 --- a/tests/e2e/resources/expected_downloads_summaries/plugins/date_range/no_downloads.json +++ b/tests/e2e/resources/expected_downloads_summaries/plugins/date_range/no_downloads.json @@ -1,5 +1 @@ -{ - ".ytdl-sub-recent-download-archive.json": "99914b932bd37a50b983c5e7c90ae93b", - "fanart.jpg": "129c6639b47299bc48062f0365e670ee", - "poster.jpg": "5de28eea5a921a041452ab3ce1041f73" -} \ No newline at end of file +{} \ No newline at end of file diff --git a/tests/e2e/resources/expected_downloads_summaries/plugins/date_range/test_channel_recent.json b/tests/e2e/resources/expected_downloads_summaries/plugins/date_range/test_channel_recent.json index 8acac411..5672df64 100644 --- a/tests/e2e/resources/expected_downloads_summaries/plugins/date_range/test_channel_recent.json +++ b/tests/e2e/resources/expected_downloads_summaries/plugins/date_range/test_channel_recent.json @@ -1,12 +1,12 @@ { ".ytdl-sub-recent-download-archive.json": "23520634f7908081f3d1333a1441a578", "Season 2018/s2018.e1029 - Jesse's Minecraft Server | Teaser Trailer-thumb.jpg": "705ca4e0d99b37e9ecdf6bfe4b90c59b", - "Season 2018/s2018.e1029 - Jesse's Minecraft Server | Teaser Trailer.info.json": "32994d191719a206c2d8b4e4bb529ce6", + "Season 2018/s2018.e1029 - Jesse's Minecraft Server | Teaser Trailer.info.json": "1f3a258a331dbec8513f07efb9311b4b", "Season 2018/s2018.e1029 - Jesse's Minecraft Server | Teaser Trailer.mp4": "82f6ee7253e1dbb83ae7215af08ffacc", "Season 2018/s2018.e1029 - Jesse's Minecraft Server | Teaser Trailer.nfo": "368d68db0cbe9eb4f43ece0517445e82", "Season 2018/s2018.e1102 - Jesse's Minecraft Server | IP mc.jesse.id-thumb.jpg": "28d852ede73b879b9ebf9a061cfc7d46", "Season 2018/s2018.e1102 - Jesse's Minecraft Server | IP mc.jesse.id.en.srt": "3d2c4e7f65d2ca5e96da38ce7eecfc4e", - "Season 2018/s2018.e1102 - Jesse's Minecraft Server | IP mc.jesse.id.info.json": "81bd514b7bc8c0b57e87683bf05fb9bc", + "Season 2018/s2018.e1102 - Jesse's Minecraft Server | IP mc.jesse.id.info.json": "444c5c772742074552ca48bf1ff37e02", "Season 2018/s2018.e1102 - Jesse's Minecraft Server | IP mc.jesse.id.mp4": "e733b4cc385b953b08c8eb0f47e03c1e", "Season 2018/s2018.e1102 - Jesse's Minecraft Server | IP mc.jesse.id.nfo": "d9114d43d87907b2afc06eb089a8ac0a", "fanart.jpg": "129c6639b47299bc48062f0365e670ee", diff --git a/tests/e2e/resources/expected_downloads_summaries/plugins/date_range/test_channel_rolling_recent.json b/tests/e2e/resources/expected_downloads_summaries/plugins/date_range/test_channel_rolling_recent.json index a5bdb3e1..54d7c652 100644 --- a/tests/e2e/resources/expected_downloads_summaries/plugins/date_range/test_channel_rolling_recent.json +++ b/tests/e2e/resources/expected_downloads_summaries/plugins/date_range/test_channel_rolling_recent.json @@ -2,7 +2,7 @@ ".ytdl-sub-recent-download-archive.json": "3bbf72c014d055ecf672c8ea603140f7", "Season 2018/s2018.e1102 - Jesse's Minecraft Server | IP mc.jesse.id-thumb.jpg": "28d852ede73b879b9ebf9a061cfc7d46", "Season 2018/s2018.e1102 - Jesse's Minecraft Server | IP mc.jesse.id.en.srt": "3d2c4e7f65d2ca5e96da38ce7eecfc4e", - "Season 2018/s2018.e1102 - Jesse's Minecraft Server | IP mc.jesse.id.info.json": "81bd514b7bc8c0b57e87683bf05fb9bc", + "Season 2018/s2018.e1102 - Jesse's Minecraft Server | IP mc.jesse.id.info.json": "444c5c772742074552ca48bf1ff37e02", "Season 2018/s2018.e1102 - Jesse's Minecraft Server | IP mc.jesse.id.mp4": "e733b4cc385b953b08c8eb0f47e03c1e", "Season 2018/s2018.e1102 - Jesse's Minecraft Server | IP mc.jesse.id.nfo": "d9114d43d87907b2afc06eb089a8ac0a", "fanart.jpg": "129c6639b47299bc48062f0365e670ee", diff --git a/tests/e2e/youtube/test_channel.py b/tests/e2e/youtube/test_channel.py index 1b43a53f..e355c0d6 100644 --- a/tests/e2e/youtube/test_channel.py +++ b/tests/e2e/youtube/test_channel.py @@ -63,20 +63,3 @@ class TestChannelAsKodiTvShow: dry_run=dry_run, expected_download_summary_file_name="youtube/test_channel_full.json", ) - - def test_channel_post_download(self, channel_as_tv_show_config, channel_preset_dict): - channel_preset_dict["ytdl_options"]["max_views"] = 1 # no downloads occur - full_channel_subscription = Subscription.from_dict( - config=channel_as_tv_show_config, preset_name="pz", preset_dict=channel_preset_dict - ) - - with assert_debug_log( # Ensure retry debug message is thrown - logger=retry_logger, - expected_message="Exception thrown when attempting to run %s, attempt %d of %d", - ), patch( # Make sleeps instant - "ytdl_sub.utils.retry.sleep" - ), patch( # Mock error when calling urlopen - "ytdl_sub.utils.thumbnail.urlopen" - ) as mock_urlopen: - mock_urlopen.side_effect = [Exception("error")] - full_channel_subscription.download(dry_run=True)