[BACKEND] Collection thumbnail support (#241)

This commit is contained in:
Jesse Bannon 2022-09-18 11:16:25 -07:00 committed by GitHub
parent 3db6a543cf
commit 3d0ade1141
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
10 changed files with 202 additions and 106 deletions

View file

@ -25,17 +25,21 @@ from yt_dlp.utils import RejectedVideoReached
from ytdl_sub.config.preset_options import AddsVariablesMixin
from ytdl_sub.config.preset_options import Overrides
from ytdl_sub.downloaders.generic.collection_validator import CollectionThumbnailListValidator
from ytdl_sub.downloaders.generic.collection_validator import CollectionUrlValidator
from ytdl_sub.downloaders.generic.collection_validator import CollectionValidator
from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder
from ytdl_sub.entries.base_entry import BaseEntry
from ytdl_sub.entries.entry import Entry
from ytdl_sub.entries.entry_parent import EntryParent
from ytdl_sub.entries.variables.kwargs import PLAYLIST_ENTRY
from ytdl_sub.entries.variables.kwargs import SOURCE_ENTRY
from ytdl_sub.thread.log_entries_downloaded_listener import LogEntriesDownloadedListener
from ytdl_sub.utils.exceptions import FileNotDownloadedException
from ytdl_sub.utils.file_handler import FileHandler
from ytdl_sub.utils.file_handler import FileMetadata
from ytdl_sub.utils.logger import Logger
from ytdl_sub.utils.thumbnail import convert_url_thumbnail
from ytdl_sub.validators.strict_dict_validator import StrictDictValidator
from ytdl_sub.ytdl_additions.enhanced_download_archive import DownloadArchiver
from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive
@ -139,7 +143,7 @@ class Downloader(DownloadArchiver, Generic[DownloaderOptionsT, DownloaderEntryT]
)
self.parents: List[EntryParent] = []
self.downloaded_entries: Set[str] = set()
self.downloaded_entries: Dict[str, Entry] = {}
@contextmanager
def ytdl_downloader(self, ytdl_options_overrides: Optional[Dict] = None) -> ytdl.YoutubeDL:
@ -331,6 +335,12 @@ class Downloader(DownloadArchiver, Generic[DownloaderOptionsT, DownloaderEntryT]
###############################################################################################
# DOWNLOAD FUNCTIONS
def _is_downloaded(self, entry: Entry) -> bool:
return _entry_key(entry) in self.downloaded_entries
def _mark_downloaded(self, entry: Entry) -> None:
self.downloaded_entries[_entry_key(entry)] = entry
@property
def collection(self) -> CollectionValidator:
"""Return the download options collection"""
@ -380,14 +390,19 @@ class Downloader(DownloadArchiver, Generic[DownloaderOptionsT, DownloaderEntryT]
return entry
def _download_parent_entry(self, parent: EntryParent) -> Generator[Entry, None, None]:
# Download the parent's entries first, in reverse order
for entry_child in reversed(parent.entry_children()):
if _entry_key(entry_child) in self.downloaded_entries:
def _download_entries(self, entries: List[Entry]) -> Generator[Entry, None, None]:
# Download entries in reverse order since they are scraped in the opposite direction.
# Helps deal with break_on_existing
for entry in reversed(entries):
if self._is_downloaded(entry):
continue
yield self._download_entry(entry_child)
self.downloaded_entries.add(_entry_key(entry_child))
yield self._download_entry(entry)
self._mark_downloaded(entry)
def _download_parent_entry(self, parent: EntryParent) -> Generator[Entry, None, None]:
for entry_child in self._download_entries(parent.entry_children()):
yield entry_child
# Recursion the parent's parent entries
for parent_child in reversed(parent.parent_children()):
@ -453,7 +468,7 @@ class Downloader(DownloadArchiver, Generic[DownloaderOptionsT, DownloaderEntryT]
for entry_child in self._download_parent_entry(parent=parent):
yield entry_child
for orphan in orphans:
for orphan in self._download_entries(orphans):
yield self._download_entry(orphan)
def download(
@ -463,12 +478,103 @@ class Downloader(DownloadArchiver, Generic[DownloaderOptionsT, DownloaderEntryT]
# download the bottom-most urls first since they are top-priority
for collection_url in reversed(self.collection.collection_urls.list):
parents, orphan_entries = self._download_url_metadata(collection_url=collection_url)
collection_url_entries: List[Entry] = []
for entry in self._download(parents=parents, orphans=orphan_entries):
yield entry
collection_url_entries.append(entry)
def post_download(self):
self._download_url_thumbnails(
collection_url=collection_url, entries=collection_url_entries
)
@classmethod
def _download_thumbnail(
cls,
thumbnail_url: str,
output_thumbnail_path: str,
) -> Optional[bool]:
"""
Downloads a thumbnail and stores it in the output directory
Parameters
----------
thumbnail_url:
Url of the thumbnail
output_thumbnail_path:
Path to store the thumbnail after downloading
Returns
-------
True if the thumbnail converted. None if it is missing or failed.
"""
if not thumbnail_url:
return None
return convert_url_thumbnail(
thumbnail_url=thumbnail_url, output_thumbnail_path=output_thumbnail_path
)
def _download_parent_thumbnails(
self,
thumbnails_downloaded: Set[str],
thumbnail_list_info: CollectionThumbnailListValidator,
entry: Entry,
parent: EntryParent,
) -> Set[str]:
"""
Downloads and moves channel avatar and banner images to the output directory.
"""
for thumbnail_info in thumbnail_list_info.list:
thumbnail_name = self.overrides.apply_formatter(thumbnail_info.name, entry=entry)
thumbnail_id = self.overrides.apply_formatter(thumbnail_info.uid)
# alread downloaded
if thumbnail_name in thumbnails_downloaded:
continue
if (thumbnail_url := parent.get_thumbnail_url(thumbnail_id=thumbnail_id)) is None:
download_logger.warning("TODO: Failed to download channel's avatar image")
continue
if self._download_thumbnail(
thumbnail_url=thumbnail_url,
output_thumbnail_path=str(Path(self.working_directory) / thumbnail_name),
):
self.save_file(file_name=thumbnail_name)
thumbnails_downloaded.add(thumbnail_name)
else:
download_logger.warning("TODO: Failed to download channel's avatar image")
return thumbnails_downloaded
def _download_url_thumbnails(
self, collection_url: CollectionUrlValidator, entries: List[Entry]
):
"""
After all media entries have been downloaded, post processed, and moved to the output
directory, run this function. This lets the downloader add any extra files directly to the
output directory, for things like YT channel image, banner.
"""
thumbnails_downloaded: Set[str] = set()
for entry in entries:
if entry.kwargs_contains(PLAYLIST_ENTRY):
thumbnails_downloaded = self._download_parent_thumbnails(
thumbnails_downloaded=thumbnails_downloaded,
thumbnail_list_info=collection_url.playlist_thumbnails,
entry=entry,
parent=EntryParent(
entry.kwargs(PLAYLIST_ENTRY), working_directory=self.working_directory
),
)
if entry.kwargs_contains(SOURCE_ENTRY):
thumbnails_downloaded = self._download_parent_thumbnails(
thumbnails_downloaded=thumbnails_downloaded,
thumbnail_list_info=collection_url.source_thumbnails,
entry=entry,
parent=EntryParent(
entry.kwargs(SOURCE_ENTRY), working_directory=self.working_directory
),
)

View file

@ -23,6 +23,12 @@ class CollectionDownloadOptions(CollectionValidator, DownloaderValidator):
variables:
season: "1"
album: "{title}"
playlist_thumbnails:
- name:
uid: "square" / "largest" / "last entry" / actual name
source_thumbnail:
- path:
type: "
- url: "soundcloud.com/albums"
variables:
season: "1"

View file

@ -1,17 +1,47 @@
from typing import Dict
from typing import List
from typing import Optional
from ytdl_sub.config.preset_options import AddsVariablesMixin
from ytdl_sub.validators.strict_dict_validator import StrictDictValidator
from ytdl_sub.validators.string_formatter_validators import DictFormatterValidator
from ytdl_sub.validators.string_formatter_validators import OverridesStringFormatterValidator
from ytdl_sub.validators.string_formatter_validators import StringFormatterValidator
from ytdl_sub.validators.validators import ListValidator
from ytdl_sub.validators.validators import StringValidator
class CollectionThumbnailValidator(StrictDictValidator):
_required_keys = {"name", "uid"}
def __init__(self, name, value):
super().__init__(name, value)
self._name = self._validate_key(key="name", validator=StringFormatterValidator)
self._uid = self._validate_key(key="uid", validator=OverridesStringFormatterValidator)
@property
def name(self) -> StringFormatterValidator:
"""
File name for the thumbnail
"""
return self._name
@property
def uid(self) -> OverridesStringFormatterValidator:
"""
yt-dlp's unique ID of the thumbnail
"""
return self._uid
class CollectionThumbnailListValidator(ListValidator[CollectionThumbnailValidator]):
_inner_list_type = CollectionThumbnailValidator
class CollectionUrlValidator(StrictDictValidator):
_required_keys = {"url"}
_optional_keys = {"variables"}
_optional_keys = {"variables", "source_thumbnails", "playlist_thumbnails"}
def __init__(self, name, value):
super().__init__(name, value)
@ -21,6 +51,13 @@ class CollectionUrlValidator(StrictDictValidator):
variables = self._validate_key_if_present(key="variables", validator=DictFormatterValidator)
self._variables = variables.dict_with_format_strings if variables else {}
self._source_thumbnails = self._validate_key_if_present(
key="source_thumbnails", validator=CollectionThumbnailListValidator, default=[]
)
self._playlist_thumbnails = self._validate_key_if_present(
key="playlist_thumbnails", validator=CollectionThumbnailListValidator, default=[]
)
@property
def url(self) -> str:
"""
@ -37,6 +74,20 @@ class CollectionUrlValidator(StrictDictValidator):
"""
return self._variables
@property
def source_thumbnails(self) -> Optional[CollectionThumbnailListValidator]:
"""
TODO:docstring
"""
return self._source_thumbnails
@property
def playlist_thumbnails(self) -> Optional[CollectionThumbnailListValidator]:
"""
TODO:docstring
"""
return self._playlist_thumbnails
class CollectionUrlListValidator(ListValidator[CollectionUrlValidator]):
_inner_list_type = CollectionUrlValidator

View file

@ -1,15 +1,12 @@
from pathlib import Path
from typing import Dict
from typing import Generator
from typing import List
from typing import Optional
from ytdl_sub.downloaders.downloader import download_logger
from ytdl_sub.downloaders.generic.collection_validator import CollectionValidator
from ytdl_sub.downloaders.youtube.abc import YoutubeDownloader
from ytdl_sub.downloaders.youtube.abc import YoutubeDownloaderOptions
from ytdl_sub.entries.entry_parent import EntryParent
from ytdl_sub.entries.youtube import YoutubeVideo
from ytdl_sub.utils.thumbnail import convert_url_thumbnail
from ytdl_sub.validators.string_formatter_validators import OverridesStringFormatterValidator
from ytdl_sub.validators.url_validator import YoutubeChannelUrlValidator
@ -54,9 +51,25 @@ class YoutubeChannelDownloaderOptions(YoutubeDownloaderOptions):
@property
def collection_validator(self) -> CollectionValidator:
"""Download from the channel url"""
playlist_thumbnails: List[Dict] = []
if self._channel_avatar_path:
playlist_thumbnails.append(
{
"name": self._channel_avatar_path.format_string,
"uid": "avatar_uncropped",
}
)
if self._channel_banner_path:
playlist_thumbnails.append(
{
"name": self._channel_banner_path.format_string,
"uid": "banner_uncropped",
}
)
return CollectionValidator(
name=self._name,
value={"urls": [{"url": self.channel_url}]},
value={"urls": [{"url": self.channel_url, "playlist_thumbnails": playlist_thumbnails}]},
)
@property
@ -110,70 +123,9 @@ class YoutubeChannelDownloader(YoutubeDownloader[YoutubeChannelDownloaderOptions
# pylint: enable=line-too-long
@property
def channel(self) -> EntryParent:
"""Gets the channel entry parent"""
assert len(self.parents) == 1, "Channel should be the only entry parent"
return self.parents[0]
def download(self) -> Generator[YoutubeVideo, None, None]:
"""
Downloads all videos from a channel
"""
for entry in super().download():
yield entry.to_type(YoutubeVideo)
def _download_thumbnail(
self,
thumbnail_url: str,
output_thumbnail_path: str,
) -> Optional[bool]:
"""
Downloads a thumbnail and stores it in the output directory
Parameters
----------
thumbnail_url:
Url of the thumbnail
output_thumbnail_path:
Path to store the thumbnail after downloading
Returns
-------
True if the thumbnail converted. None if it is missing or failed.
"""
if not thumbnail_url:
download_logger.warning("Could not find a thumbnail for %s", self.channel.uid)
return None
return convert_url_thumbnail(
thumbnail_url=thumbnail_url, output_thumbnail_path=output_thumbnail_path
)
def post_download(self):
"""
Downloads and moves channel avatar and banner images to the output directory.
"""
if self.download_options.channel_avatar_path:
avatar_thumbnail_name = self.overrides.apply_formatter(
self.download_options.channel_avatar_path
)
if self._download_thumbnail(
thumbnail_url=self.channel.get_thumbnail_url("avatar_uncropped"),
output_thumbnail_path=str(Path(self.working_directory) / avatar_thumbnail_name),
):
self.save_file(file_name=avatar_thumbnail_name)
else:
download_logger.warning("Failed to download channel's avatar image")
if self.download_options.channel_banner_path:
banner_thumbnail_name = self.overrides.apply_formatter(
self.download_options.channel_banner_path
)
if self._download_thumbnail(
thumbnail_url=self.channel.get_thumbnail_url("banner_uncropped"),
output_thumbnail_path=str(Path(self.working_directory) / banner_thumbnail_name),
):
self.save_file(file_name=banner_thumbnail_name)
else:
download_logger.warning("Failed to download channel's banner image")

View file

@ -61,11 +61,14 @@ class OutputDirectoryNfoTagsOptions(SharedNfoTagsOptions):
.. code-block:: yaml
tags:
title:
attributes:
year: "2022"
tag: "Sweet youtube TV show"
named_season:
- tag: "{source_title}"
attributes:
number: "{collection_index}"
behavior: "merge"
genre:
- tag: "Comedy"
behavior: "overwrite"
- "Comedy"
- "Drama"

View file

@ -265,7 +265,6 @@ class SubscriptionDownload(BaseSubscription, ABC):
plugins=plugins, dry_run=dry_run, entry=entry, entry_metadata=entry_metadata
)
downloader.post_download()
for plugin in plugins:
plugin.post_process_subscription()

View file

@ -1,5 +1 @@
{
".ytdl-sub-recent-download-archive.json": "99914b932bd37a50b983c5e7c90ae93b",
"fanart.jpg": "129c6639b47299bc48062f0365e670ee",
"poster.jpg": "5de28eea5a921a041452ab3ce1041f73"
}
{}

View file

@ -1,12 +1,12 @@
{
".ytdl-sub-recent-download-archive.json": "23520634f7908081f3d1333a1441a578",
"Season 2018/s2018.e1029 - Jesse's Minecraft Server Teaser Trailer-thumb.jpg": "705ca4e0d99b37e9ecdf6bfe4b90c59b",
"Season 2018/s2018.e1029 - Jesse's Minecraft Server Teaser Trailer.info.json": "32994d191719a206c2d8b4e4bb529ce6",
"Season 2018/s2018.e1029 - Jesse's Minecraft Server Teaser Trailer.info.json": "1f3a258a331dbec8513f07efb9311b4b",
"Season 2018/s2018.e1029 - Jesse's Minecraft Server Teaser Trailer.mp4": "82f6ee7253e1dbb83ae7215af08ffacc",
"Season 2018/s2018.e1029 - Jesse's Minecraft Server Teaser Trailer.nfo": "368d68db0cbe9eb4f43ece0517445e82",
"Season 2018/s2018.e1102 - Jesse's Minecraft Server IP mc.jesse.id-thumb.jpg": "28d852ede73b879b9ebf9a061cfc7d46",
"Season 2018/s2018.e1102 - Jesse's Minecraft Server IP mc.jesse.id.en.srt": "3d2c4e7f65d2ca5e96da38ce7eecfc4e",
"Season 2018/s2018.e1102 - Jesse's Minecraft Server IP mc.jesse.id.info.json": "81bd514b7bc8c0b57e87683bf05fb9bc",
"Season 2018/s2018.e1102 - Jesse's Minecraft Server IP mc.jesse.id.info.json": "444c5c772742074552ca48bf1ff37e02",
"Season 2018/s2018.e1102 - Jesse's Minecraft Server IP mc.jesse.id.mp4": "e733b4cc385b953b08c8eb0f47e03c1e",
"Season 2018/s2018.e1102 - Jesse's Minecraft Server IP mc.jesse.id.nfo": "d9114d43d87907b2afc06eb089a8ac0a",
"fanart.jpg": "129c6639b47299bc48062f0365e670ee",

View file

@ -2,7 +2,7 @@
".ytdl-sub-recent-download-archive.json": "3bbf72c014d055ecf672c8ea603140f7",
"Season 2018/s2018.e1102 - Jesse's Minecraft Server IP mc.jesse.id-thumb.jpg": "28d852ede73b879b9ebf9a061cfc7d46",
"Season 2018/s2018.e1102 - Jesse's Minecraft Server IP mc.jesse.id.en.srt": "3d2c4e7f65d2ca5e96da38ce7eecfc4e",
"Season 2018/s2018.e1102 - Jesse's Minecraft Server IP mc.jesse.id.info.json": "81bd514b7bc8c0b57e87683bf05fb9bc",
"Season 2018/s2018.e1102 - Jesse's Minecraft Server IP mc.jesse.id.info.json": "444c5c772742074552ca48bf1ff37e02",
"Season 2018/s2018.e1102 - Jesse's Minecraft Server IP mc.jesse.id.mp4": "e733b4cc385b953b08c8eb0f47e03c1e",
"Season 2018/s2018.e1102 - Jesse's Minecraft Server IP mc.jesse.id.nfo": "d9114d43d87907b2afc06eb089a8ac0a",
"fanart.jpg": "129c6639b47299bc48062f0365e670ee",

View file

@ -63,20 +63,3 @@ class TestChannelAsKodiTvShow:
dry_run=dry_run,
expected_download_summary_file_name="youtube/test_channel_full.json",
)
def test_channel_post_download(self, channel_as_tv_show_config, channel_preset_dict):
channel_preset_dict["ytdl_options"]["max_views"] = 1 # no downloads occur
full_channel_subscription = Subscription.from_dict(
config=channel_as_tv_show_config, preset_name="pz", preset_dict=channel_preset_dict
)
with assert_debug_log( # Ensure retry debug message is thrown
logger=retry_logger,
expected_message="Exception thrown when attempting to run %s, attempt %d of %d",
), patch( # Make sleeps instant
"ytdl_sub.utils.retry.sleep"
), patch( # Mock error when calling urlopen
"ytdl_sub.utils.thumbnail.urlopen"
) as mock_urlopen:
mock_urlopen.side_effect = [Exception("error")]
full_channel_subscription.download(dry_run=True)