[BACKEND] Collection thumbnail support (#241)

src/ytdl_sub/downloaders/downloader.py

@@ -25,17 +25,21 @@ from yt_dlp.utils import RejectedVideoReached
 
 from ytdl_sub.config.preset_options import AddsVariablesMixin
 from ytdl_sub.config.preset_options import Overrides
+from ytdl_sub.downloaders.generic.collection_validator import CollectionThumbnailListValidator
 from ytdl_sub.downloaders.generic.collection_validator import CollectionUrlValidator
 from ytdl_sub.downloaders.generic.collection_validator import CollectionValidator
 from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder
 from ytdl_sub.entries.base_entry import BaseEntry
 from ytdl_sub.entries.entry import Entry
 from ytdl_sub.entries.entry_parent import EntryParent
+from ytdl_sub.entries.variables.kwargs import PLAYLIST_ENTRY
+from ytdl_sub.entries.variables.kwargs import SOURCE_ENTRY
 from ytdl_sub.thread.log_entries_downloaded_listener import LogEntriesDownloadedListener
 from ytdl_sub.utils.exceptions import FileNotDownloadedException
 from ytdl_sub.utils.file_handler import FileHandler
 from ytdl_sub.utils.file_handler import FileMetadata
 from ytdl_sub.utils.logger import Logger
+from ytdl_sub.utils.thumbnail import convert_url_thumbnail
 from ytdl_sub.validators.strict_dict_validator import StrictDictValidator
 from ytdl_sub.ytdl_additions.enhanced_download_archive import DownloadArchiver
 from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive
@@ -139,7 +143,7 @@ class Downloader(DownloadArchiver, Generic[DownloaderOptionsT, DownloaderEntryT]
         )
 
         self.parents: List[EntryParent] = []
-        self.downloaded_entries: Set[str] = set()
+        self.downloaded_entries: Dict[str, Entry] = {}
 
     @contextmanager
     def ytdl_downloader(self, ytdl_options_overrides: Optional[Dict] = None) -> ytdl.YoutubeDL:
@@ -331,6 +335,12 @@ class Downloader(DownloadArchiver, Generic[DownloaderOptionsT, DownloaderEntryT]
     ###############################################################################################
     # DOWNLOAD FUNCTIONS
 
+    def _is_downloaded(self, entry: Entry) -> bool:
+        return _entry_key(entry) in self.downloaded_entries
+
+    def _mark_downloaded(self, entry: Entry) -> None:
+        self.downloaded_entries[_entry_key(entry)] = entry
+
     @property
     def collection(self) -> CollectionValidator:
         """Return the download options collection"""
@@ -380,14 +390,19 @@ class Downloader(DownloadArchiver, Generic[DownloaderOptionsT, DownloaderEntryT]
 
         return entry
 
-    def _download_parent_entry(self, parent: EntryParent) -> Generator[Entry, None, None]:
+    def _download_entries(self, entries: List[Entry]) -> Generator[Entry, None, None]:
-        # Download the parent's entries first, in reverse order
+        # Download entries in reverse order since they are scraped in the opposite direction.
-        for entry_child in reversed(parent.entry_children()):
+        # Helps deal with break_on_existing
-            if _entry_key(entry_child) in self.downloaded_entries:
+        for entry in reversed(entries):
+            if self._is_downloaded(entry):
                 continue
 
-            yield self._download_entry(entry_child)
+            yield self._download_entry(entry)
-            self.downloaded_entries.add(_entry_key(entry_child))
+            self._mark_downloaded(entry)
+
+    def _download_parent_entry(self, parent: EntryParent) -> Generator[Entry, None, None]:
+        for entry_child in self._download_entries(parent.entry_children()):
+            yield entry_child
 
         # Recursion the parent's parent entries
         for parent_child in reversed(parent.parent_children()):
@@ -453,7 +468,7 @@ class Downloader(DownloadArchiver, Generic[DownloaderOptionsT, DownloaderEntryT]
                 for entry_child in self._download_parent_entry(parent=parent):
                     yield entry_child
 
-            for orphan in orphans:
+            for orphan in self._download_entries(orphans):
                 yield self._download_entry(orphan)
 
     def download(
@@ -463,12 +478,103 @@ class Downloader(DownloadArchiver, Generic[DownloaderOptionsT, DownloaderEntryT]
         # download the bottom-most urls first since they are top-priority
         for collection_url in reversed(self.collection.collection_urls.list):
             parents, orphan_entries = self._download_url_metadata(collection_url=collection_url)
+            collection_url_entries: List[Entry] = []
+
             for entry in self._download(parents=parents, orphans=orphan_entries):
                 yield entry
+                collection_url_entries.append(entry)
 
-    def post_download(self):
+            self._download_url_thumbnails(
+                collection_url=collection_url, entries=collection_url_entries
+            )
+
+    @classmethod
+    def _download_thumbnail(
+        cls,
+        thumbnail_url: str,
+        output_thumbnail_path: str,
+    ) -> Optional[bool]:
+        """
+        Downloads a thumbnail and stores it in the output directory
+
+        Parameters
+        ----------
+        thumbnail_url:
+            Url of the thumbnail
+        output_thumbnail_path:
+            Path to store the thumbnail after downloading
+
+        Returns
+        -------
+        True if the thumbnail converted. None if it is missing or failed.
+        """
+        if not thumbnail_url:
+            return None
+
+        return convert_url_thumbnail(
+            thumbnail_url=thumbnail_url, output_thumbnail_path=output_thumbnail_path
+        )
+
+    def _download_parent_thumbnails(
+        self,
+        thumbnails_downloaded: Set[str],
+        thumbnail_list_info: CollectionThumbnailListValidator,
+        entry: Entry,
+        parent: EntryParent,
+    ) -> Set[str]:
+        """
+        Downloads and moves channel avatar and banner images to the output directory.
+        """
+        for thumbnail_info in thumbnail_list_info.list:
+            thumbnail_name = self.overrides.apply_formatter(thumbnail_info.name, entry=entry)
+            thumbnail_id = self.overrides.apply_formatter(thumbnail_info.uid)
+
+            # alread downloaded
+            if thumbnail_name in thumbnails_downloaded:
+                continue
+
+            if (thumbnail_url := parent.get_thumbnail_url(thumbnail_id=thumbnail_id)) is None:
+                download_logger.warning("TODO: Failed to download channel's avatar image")
+                continue
+
+            if self._download_thumbnail(
+                thumbnail_url=thumbnail_url,
+                output_thumbnail_path=str(Path(self.working_directory) / thumbnail_name),
+            ):
+                self.save_file(file_name=thumbnail_name)
+                thumbnails_downloaded.add(thumbnail_name)
+            else:
+                download_logger.warning("TODO: Failed to download channel's avatar image")
+
+        return thumbnails_downloaded
+
+    def _download_url_thumbnails(
+        self, collection_url: CollectionUrlValidator, entries: List[Entry]
+    ):
         """
         After all media entries have been downloaded, post processed, and moved to the output
         directory, run this function. This lets the downloader add any extra files directly to the
         output directory, for things like YT channel image, banner.
         """
+        thumbnails_downloaded: Set[str] = set()
+
+        for entry in entries:
+            if entry.kwargs_contains(PLAYLIST_ENTRY):
+                thumbnails_downloaded = self._download_parent_thumbnails(
+                    thumbnails_downloaded=thumbnails_downloaded,
+                    thumbnail_list_info=collection_url.playlist_thumbnails,
+                    entry=entry,
+                    parent=EntryParent(
+                        entry.kwargs(PLAYLIST_ENTRY), working_directory=self.working_directory
+                    ),
+                )
+
+            if entry.kwargs_contains(SOURCE_ENTRY):
+                thumbnails_downloaded = self._download_parent_thumbnails(
+                    thumbnails_downloaded=thumbnails_downloaded,
+                    thumbnail_list_info=collection_url.source_thumbnails,
+                    entry=entry,
+                    parent=EntryParent(
+                        entry.kwargs(SOURCE_ENTRY), working_directory=self.working_directory
+                    ),
+                )

src/ytdl_sub/downloaders/generic/collection.py

@@ -23,6 +23,12 @@ class CollectionDownloadOptions(CollectionValidator, DownloaderValidator):
               variables:
                 season: "1"
                 album: "{title}"
+              playlist_thumbnails:
+                - name:
+                  uid: "square" / "largest" / "last entry" / actual name
+              source_thumbnail:
+                - path:
+                  type: "
             - url: "soundcloud.com/albums"
               variables:
                 season: "1"

src/ytdl_sub/downloaders/generic/collection_validator.py

@@ -1,17 +1,47 @@
 from typing import Dict
 from typing import List
+from typing import Optional
 
 from ytdl_sub.config.preset_options import AddsVariablesMixin
 from ytdl_sub.validators.strict_dict_validator import StrictDictValidator
 from ytdl_sub.validators.string_formatter_validators import DictFormatterValidator
+from ytdl_sub.validators.string_formatter_validators import OverridesStringFormatterValidator
 from ytdl_sub.validators.string_formatter_validators import StringFormatterValidator
 from ytdl_sub.validators.validators import ListValidator
 from ytdl_sub.validators.validators import StringValidator
 
 
+class CollectionThumbnailValidator(StrictDictValidator):
+    _required_keys = {"name", "uid"}
+
+    def __init__(self, name, value):
+        super().__init__(name, value)
+
+        self._name = self._validate_key(key="name", validator=StringFormatterValidator)
+        self._uid = self._validate_key(key="uid", validator=OverridesStringFormatterValidator)
+
+    @property
+    def name(self) -> StringFormatterValidator:
+        """
+        File name for the thumbnail
+        """
+        return self._name
+
+    @property
+    def uid(self) -> OverridesStringFormatterValidator:
+        """
+        yt-dlp's unique ID of the thumbnail
+        """
+        return self._uid
+
+
+class CollectionThumbnailListValidator(ListValidator[CollectionThumbnailValidator]):
+    _inner_list_type = CollectionThumbnailValidator
+
+
 class CollectionUrlValidator(StrictDictValidator):
     _required_keys = {"url"}
-    _optional_keys = {"variables"}
+    _optional_keys = {"variables", "source_thumbnails", "playlist_thumbnails"}
 
     def __init__(self, name, value):
         super().__init__(name, value)
@@ -21,6 +51,13 @@ class CollectionUrlValidator(StrictDictValidator):
         variables = self._validate_key_if_present(key="variables", validator=DictFormatterValidator)
         self._variables = variables.dict_with_format_strings if variables else {}
 
+        self._source_thumbnails = self._validate_key_if_present(
+            key="source_thumbnails", validator=CollectionThumbnailListValidator, default=[]
+        )
+        self._playlist_thumbnails = self._validate_key_if_present(
+            key="playlist_thumbnails", validator=CollectionThumbnailListValidator, default=[]
+        )
+
     @property
     def url(self) -> str:
         """
@@ -37,6 +74,20 @@ class CollectionUrlValidator(StrictDictValidator):
         """
         return self._variables
 
+    @property
+    def source_thumbnails(self) -> Optional[CollectionThumbnailListValidator]:
+        """
+        TODO:docstring
+        """
+        return self._source_thumbnails
+
+    @property
+    def playlist_thumbnails(self) -> Optional[CollectionThumbnailListValidator]:
+        """
+        TODO:docstring
+        """
+        return self._playlist_thumbnails
+
 
 class CollectionUrlListValidator(ListValidator[CollectionUrlValidator]):
     _inner_list_type = CollectionUrlValidator

src/ytdl_sub/downloaders/youtube/channel.py

@@ -1,15 +1,12 @@
-from pathlib import Path
 from typing import Dict
 from typing import Generator
+from typing import List
 from typing import Optional
 
-from ytdl_sub.downloaders.downloader import download_logger
 from ytdl_sub.downloaders.generic.collection_validator import CollectionValidator
 from ytdl_sub.downloaders.youtube.abc import YoutubeDownloader
 from ytdl_sub.downloaders.youtube.abc import YoutubeDownloaderOptions
-from ytdl_sub.entries.entry_parent import EntryParent
 from ytdl_sub.entries.youtube import YoutubeVideo
-from ytdl_sub.utils.thumbnail import convert_url_thumbnail
 from ytdl_sub.validators.string_formatter_validators import OverridesStringFormatterValidator
 from ytdl_sub.validators.url_validator import YoutubeChannelUrlValidator
 
@@ -54,9 +51,25 @@ class YoutubeChannelDownloaderOptions(YoutubeDownloaderOptions):
     @property
     def collection_validator(self) -> CollectionValidator:
         """Download from the channel url"""
+        playlist_thumbnails: List[Dict] = []
+        if self._channel_avatar_path:
+            playlist_thumbnails.append(
+                {
+                    "name": self._channel_avatar_path.format_string,
+                    "uid": "avatar_uncropped",
+                }
+            )
+        if self._channel_banner_path:
+            playlist_thumbnails.append(
+                {
+                    "name": self._channel_banner_path.format_string,
+                    "uid": "banner_uncropped",
+                }
+            )
+
         return CollectionValidator(
             name=self._name,
-            value={"urls": [{"url": self.channel_url}]},
+            value={"urls": [{"url": self.channel_url, "playlist_thumbnails": playlist_thumbnails}]},
         )
 
     @property
@@ -110,70 +123,9 @@ class YoutubeChannelDownloader(YoutubeDownloader[YoutubeChannelDownloaderOptions
 
     # pylint: enable=line-too-long
 
-    @property
-    def channel(self) -> EntryParent:
-        """Gets the channel entry parent"""
-        assert len(self.parents) == 1, "Channel should be the only entry parent"
-        return self.parents[0]
-
     def download(self) -> Generator[YoutubeVideo, None, None]:
         """
         Downloads all videos from a channel
         """
         for entry in super().download():
             yield entry.to_type(YoutubeVideo)
-
-    def _download_thumbnail(
-        self,
-        thumbnail_url: str,
-        output_thumbnail_path: str,
-    ) -> Optional[bool]:
-        """
-        Downloads a thumbnail and stores it in the output directory
-
-        Parameters
-        ----------
-        thumbnail_url:
-            Url of the thumbnail
-        output_thumbnail_path:
-            Path to store the thumbnail after downloading
-
-        Returns
-        -------
-        True if the thumbnail converted. None if it is missing or failed.
-        """
-        if not thumbnail_url:
-            download_logger.warning("Could not find a thumbnail for %s", self.channel.uid)
-            return None
-
-        return convert_url_thumbnail(
-            thumbnail_url=thumbnail_url, output_thumbnail_path=output_thumbnail_path
-        )
-
-    def post_download(self):
-        """
-        Downloads and moves channel avatar and banner images to the output directory.
-        """
-        if self.download_options.channel_avatar_path:
-            avatar_thumbnail_name = self.overrides.apply_formatter(
-                self.download_options.channel_avatar_path
-            )
-            if self._download_thumbnail(
-                thumbnail_url=self.channel.get_thumbnail_url("avatar_uncropped"),
-                output_thumbnail_path=str(Path(self.working_directory) / avatar_thumbnail_name),
-            ):
-                self.save_file(file_name=avatar_thumbnail_name)
-            else:
-                download_logger.warning("Failed to download channel's avatar image")
-
-        if self.download_options.channel_banner_path:
-            banner_thumbnail_name = self.overrides.apply_formatter(
-                self.download_options.channel_banner_path
-            )
-            if self._download_thumbnail(
-                thumbnail_url=self.channel.get_thumbnail_url("banner_uncropped"),
-                output_thumbnail_path=str(Path(self.working_directory) / banner_thumbnail_name),
-            ):
-                self.save_file(file_name=banner_thumbnail_name)
-            else:
-                download_logger.warning("Failed to download channel's banner image")

src/ytdl_sub/plugins/output_directory_nfo_tags.py

@@ -61,11 +61,14 @@ class OutputDirectoryNfoTagsOptions(SharedNfoTagsOptions):
         .. code-block:: yaml
 
            tags:
-             title:
+             named_season:
-               attributes:
+               - tag: "{source_title}"
-                 year: "2022"
+                 attributes:
-               tag: "Sweet youtube TV show"
+                   number: "{collection_index}"
+                 behavior: "merge"
              genre:
+               - tag: "Comedy"
+                 behavior: "overwrite"
                - "Comedy"
                - "Drama"
 

src/ytdl_sub/subscriptions/subscription_download.py

@@ -265,7 +265,6 @@ class SubscriptionDownload(BaseSubscription, ABC):
                         plugins=plugins, dry_run=dry_run, entry=entry, entry_metadata=entry_metadata
                     )
 
-            downloader.post_download()
             for plugin in plugins:
                 plugin.post_process_subscription()
 

tests/e2e/resources/expected_downloads_summaries/plugins/date_range/no_downloads.json

@@ -1,5 +1 @@
-{
+{}
-  ".ytdl-sub-recent-download-archive.json": "99914b932bd37a50b983c5e7c90ae93b",
-  "fanart.jpg": "129c6639b47299bc48062f0365e670ee",
-  "poster.jpg": "5de28eea5a921a041452ab3ce1041f73"
-}

tests/e2e/resources/expected_downloads_summaries/plugins/date_range/test_channel_recent.json

@@ -1,12 +1,12 @@
 {
   ".ytdl-sub-recent-download-archive.json": "23520634f7908081f3d1333a1441a578",
   "Season 2018/s2018.e1029 - Jesse's Minecraft Server ｜ Teaser Trailer-thumb.jpg": "705ca4e0d99b37e9ecdf6bfe4b90c59b",
-  "Season 2018/s2018.e1029 - Jesse's Minecraft Server ｜ Teaser Trailer.info.json": "32994d191719a206c2d8b4e4bb529ce6",
+  "Season 2018/s2018.e1029 - Jesse's Minecraft Server ｜ Teaser Trailer.info.json": "1f3a258a331dbec8513f07efb9311b4b",
   "Season 2018/s2018.e1029 - Jesse's Minecraft Server ｜ Teaser Trailer.mp4": "82f6ee7253e1dbb83ae7215af08ffacc",
   "Season 2018/s2018.e1029 - Jesse's Minecraft Server ｜ Teaser Trailer.nfo": "368d68db0cbe9eb4f43ece0517445e82",
   "Season 2018/s2018.e1102 - Jesse's Minecraft Server ｜ IP mc.jesse.id-thumb.jpg": "28d852ede73b879b9ebf9a061cfc7d46",
   "Season 2018/s2018.e1102 - Jesse's Minecraft Server ｜ IP mc.jesse.id.en.srt": "3d2c4e7f65d2ca5e96da38ce7eecfc4e",
-  "Season 2018/s2018.e1102 - Jesse's Minecraft Server ｜ IP mc.jesse.id.info.json": "81bd514b7bc8c0b57e87683bf05fb9bc",
+  "Season 2018/s2018.e1102 - Jesse's Minecraft Server ｜ IP mc.jesse.id.info.json": "444c5c772742074552ca48bf1ff37e02",
   "Season 2018/s2018.e1102 - Jesse's Minecraft Server ｜ IP mc.jesse.id.mp4": "e733b4cc385b953b08c8eb0f47e03c1e",
   "Season 2018/s2018.e1102 - Jesse's Minecraft Server ｜ IP mc.jesse.id.nfo": "d9114d43d87907b2afc06eb089a8ac0a",
   "fanart.jpg": "129c6639b47299bc48062f0365e670ee",

tests/e2e/resources/expected_downloads_summaries/plugins/date_range/test_channel_rolling_recent.json

@@ -2,7 +2,7 @@
   ".ytdl-sub-recent-download-archive.json": "3bbf72c014d055ecf672c8ea603140f7",
   "Season 2018/s2018.e1102 - Jesse's Minecraft Server ｜ IP mc.jesse.id-thumb.jpg": "28d852ede73b879b9ebf9a061cfc7d46",
   "Season 2018/s2018.e1102 - Jesse's Minecraft Server ｜ IP mc.jesse.id.en.srt": "3d2c4e7f65d2ca5e96da38ce7eecfc4e",
-  "Season 2018/s2018.e1102 - Jesse's Minecraft Server ｜ IP mc.jesse.id.info.json": "81bd514b7bc8c0b57e87683bf05fb9bc",
+  "Season 2018/s2018.e1102 - Jesse's Minecraft Server ｜ IP mc.jesse.id.info.json": "444c5c772742074552ca48bf1ff37e02",
   "Season 2018/s2018.e1102 - Jesse's Minecraft Server ｜ IP mc.jesse.id.mp4": "e733b4cc385b953b08c8eb0f47e03c1e",
   "Season 2018/s2018.e1102 - Jesse's Minecraft Server ｜ IP mc.jesse.id.nfo": "d9114d43d87907b2afc06eb089a8ac0a",
   "fanart.jpg": "129c6639b47299bc48062f0365e670ee",

tests/e2e/youtube/test_channel.py

@@ -63,20 +63,3 @@ class TestChannelAsKodiTvShow:
             dry_run=dry_run,
             expected_download_summary_file_name="youtube/test_channel_full.json",
         )
-
-    def test_channel_post_download(self, channel_as_tv_show_config, channel_preset_dict):
-        channel_preset_dict["ytdl_options"]["max_views"] = 1  # no downloads occur
-        full_channel_subscription = Subscription.from_dict(
-            config=channel_as_tv_show_config, preset_name="pz", preset_dict=channel_preset_dict
-        )
-
-        with assert_debug_log(  # Ensure retry debug message is thrown
-            logger=retry_logger,
-            expected_message="Exception thrown when attempting to run %s, attempt %d of %d",
-        ), patch(  # Make sleeps instant
-            "ytdl_sub.utils.retry.sleep"
-        ), patch(  # Mock error when calling urlopen
-            "ytdl_sub.utils.thumbnail.urlopen"
-        ) as mock_urlopen:
-            mock_urlopen.side_effect = [Exception("error")]
-            full_channel_subscription.download(dry_run=True)