diff --git a/examples/kodi_music_videos_config.yaml b/examples/kodi_music_videos_config.yaml new file mode 100644 index 00000000..b0506938 --- /dev/null +++ b/examples/kodi_music_videos_config.yaml @@ -0,0 +1,67 @@ +# This example shows how to download and format a Youtube video OR playlist +# to display in Kodi as a music video. Kodi requires music videos to be in +# a shared directory, so we will configure this to make the output directory +# formatted as: +# +# /path/to/Music Videos +# Elton John - Rocketman.jpg +# Elton John - Rocketman.mp4 +# Elton John - Rocketman.nfo +# System of a Down - Chop Suey.jpg +# System of a Down - Chop Suey.mp4 +# System of a Down - Chop Suey.nfo +# ... +# +configuration: + working_directory: '.ytdl-sub-downloads' + +presets: + yt_music_video: + # Youtube playlists are our source/download strategy. However, this + # can be overwritten to download music videos from a 'channel' or a + # single 'video' + youtube: + download_strategy: "playlist" + + # For advanced YTDL users only. + # It is wise to leave ignoreerrors=True to avoid errors for things + # like age-restricted videos in case you have not set your cookie. + ytdl_options: + ignoreerrors: True + + # For each video downloaded, set the file and thumbnail name here. + # We set both with {music_video_name}, which is a variable we define in + # the overrides section further below to represent consistent naming format. + # + # Another field worth mentioning is maintain_download_archive=True. This + # is generally a good thing to enable with playlists because it will + # store previously downloaded video ids to tell YTDL not to re-download + # them on a successive invocation. + output_options: + output_directory: "path/to/Music Videos" + file_name: "{music_video_name}.{ext}" + thumbnail_name: "{music_video_name}.jpg" + maintain_download_archive: True + + # Always convert the video thumbnail to jpg + # TODO: always convert all thumbnails to jpg in code + convert_thumbnail: + to: "jpg" + + # For each video downloaded, add a music video NFO file for it. Populate it + # with tags that Kodi will read and use to display it in the music or music + # videos section. + nfo_tags: + nfo_name: "{music_video_name}.nfo" + nfo_root: "musicvideo" + tags: + artist: "{artist}" + title: "{title}" + album: "Music Videos" + year: "{upload_year}" + + # Overrides is a section where we can define our own variables, and use them in + # any other section. We define our episode file name here, which gets reused above + # for the video, thumbnail, and NFO file. + overrides: + music_video_name: "{sanitized_artist} - {sanitized_title}" diff --git a/examples/kodi_music_videos_subscriptions.yaml b/examples/kodi_music_videos_subscriptions.yaml new file mode 100644 index 00000000..3d718cd1 --- /dev/null +++ b/examples/kodi_music_videos_subscriptions.yaml @@ -0,0 +1,65 @@ +# This example shows how we can use the `kodi_music_videos_config.yaml` preset +# to download music videos in a few different ways. We will use made-up artists +# in each example + +############################################################################### +# LEVEL 1 - DOWNLOAD MUSIC VIDEO PLAYLIST + +# Subscription names are defined by you. We will call this one john_smith +# for simplicity, and it will download every single video in john_smith's music +# video playlist. Many artists maintain a playlist of all their music videos, +# which makes this an easy way to grab all of them. +john_smith: + # We must define a preset to use from our config. We named the one in the + # config example "yt_music_video", so set that here. + preset: "yt_music_video" + + # Since our preset download strategy is set to 'playlist', set the playlist id. + youtube: + playlist_id: "UCsvn_Po0SmunchJYtttWpOxMg" + + # Overrides can be defined per-subscription. If you noticed, we used {artist} + # and {sanitized_artist} in our "yt_music_video" preset. We intended to reserve + # that variable to be defined for each individual subscription. Each override + # defined here will create a 'sanitized_' version that is safe for file systems. + # + # A note for Kodi music videos, it is important to make sure your artist name + # exactly matches how it is formatted in Kodi itself, otherwise it will be + # read in as a new artist + overrides: + artist: "John Smith and the Instrument Players" + +############################################################################### +# LEVEL 2 - DOWNLOAD SINGLE MUSIC VIDEO + +# It is not always ideal to download all of an artist's music videos. +# Maybe you only like one song of theirs. We can reuse our preset +# to download a single video instead. +# +# The only difference between this example and the one above is +# - youtube.download_strategy +# We defined this as 'playlist' in the config. We must override it +# with 'video' to perform a single video download +# - youtube.video_id +# The id of the video to download +# +# Of course, defining yaml configuration to download a single video once +# and never again seems weird. Instead, we can perform this download via +# command: +# +# ytdl-sub dl \ +# --preset "yt_channel_as_tv" \ +# --youtube.download_strategy "video" \ +# --youtube.video_id "QhY6r6oAErg" \ +# --overrides.artist "John Smith and the Instrument Players" +# +# The --preset and --youtube.download_strategy args will always be the +# same. We will try to simplify the dl command to make it shorter. + +john_smith_one_hit_wonder: + preset: "yt_channel_as_tv" + youtube: + download_strategy: "video" + video_id: "QhY6r6oAErg" + overrides: + artist: "John Smith and the Instrument Players" diff --git a/src/ytdl_sub/downloaders/downloader.py b/src/ytdl_sub/downloaders/downloader.py index c9f12e4a..bb5c7148 100644 --- a/src/ytdl_sub/downloaders/downloader.py +++ b/src/ytdl_sub/downloaders/downloader.py @@ -1,4 +1,6 @@ import abc +import json +import os from abc import ABC from contextlib import contextmanager from pathlib import Path @@ -6,12 +8,16 @@ from typing import Dict from typing import Generic from typing import List from typing import Optional +from typing import Tuple from typing import Type from typing import TypeVar import yt_dlp as ytdl +from yt_dlp.utils import ExistingVideoReached +from yt_dlp.utils import RejectedVideoReached from ytdl_sub.config.preset_options import Overrides +from ytdl_sub.entries.base_entry import BaseEntry from ytdl_sub.entries.entry import Entry from ytdl_sub.validators.strict_dict_validator import StrictDictValidator @@ -24,6 +30,7 @@ class DownloaderValidator(StrictDictValidator, ABC): DownloaderOptionsT = TypeVar("DownloaderOptionsT", bound=DownloaderValidator) DownloaderEntryT = TypeVar("DownloaderEntryT", bound=Entry) +DownloaderParentEntryT = TypeVar("DownloaderParentEntryT", bound=BaseEntry) class Downloader(Generic[DownloaderOptionsT, DownloaderEntryT], ABC): @@ -32,8 +39,8 @@ class Downloader(Generic[DownloaderOptionsT, DownloaderEntryT], ABC): and should translate that to list of Entry objects. """ - downloader_options_type: Type[DownloaderOptionsT] = NotImplemented - downloader_entry_type: Type[DownloaderEntryT] = NotImplemented + downloader_options_type: Type[DownloaderValidator] = DownloaderValidator + downloader_entry_type: Type[Entry] = Entry @classmethod def ytdl_option_overrides(cls) -> Dict: @@ -113,10 +120,84 @@ class Downloader(Generic[DownloaderOptionsT, DownloaderEntryT], ABC): """ Wrapper around yt_dlp.YoutubeDL.YoutubeDL.extract_info All kwargs will passed to the extract_info function. + + Parameters + ---------- + ytdl_options_overrides + Optional. Dict containing ytdl args to override other predefined ytdl args + **kwargs + arguments passed directory to YoutubeDL extract_info + """ with self.ytdl_downloader(ytdl_options_overrides) as ytdl_downloader: return ytdl_downloader.extract_info(**kwargs) + def extract_info_json(self, ytdl_options_overrides: Optional[Dict] = None, **kwargs) -> None: + """ + Wrapper around yt_dlp.YoutubeDL.YoutubeDL.extract_info + + All kwargs will passed to the extract_info function. This also enables ytdl to write + .info.json files for all media downloaded. + + Catches RejectedVideoReached and ExistingVideoReached exceptions. + + Parameters + ---------- + ytdl_options_overrides + Optional. Dict containing ytdl args to override other predefined ytdl args + **kwargs + arguments passed directory to YoutubeDL extract_info + """ + if ytdl_options_overrides is None: + ytdl_options_overrides = {} + + ytdl_options_overrides = dict(ytdl_options_overrides, **{"writeinfojson": True}) + + try: + _ = self.extract_info(ytdl_options_overrides=ytdl_options_overrides, **kwargs) + except (RejectedVideoReached, ExistingVideoReached): + pass + + def extract_from_info_json( + self, parent_prefix: str, parent_entry_type: Type[DownloaderParentEntryT] + ) -> Tuple[DownloaderParentEntryT, List[DownloaderEntryT]]: + """ + Reads all .info.json files in the working directory, and casts them to the + parent_entry_type (i.e. YoutubeChannel) and downloader_entry_type (i.e. YoutubeVideo) + + Parameters + ---------- + parent_prefix + info.json file name prefix to indicate its the parent (i.e. the YT channel id) + parent_entry_type + Class type for the parent entry + + Returns + ------- + Tuple containing parent, list of videos belong to the parent + """ + # Load the entries from info.json + + parent_entry: Optional[DownloaderParentEntryT] = None + entries: List[DownloaderEntryT] = [] + + for file_name in os.listdir(self.working_directory): + if file_name.endswith(".info.json"): + with open(Path(self.working_directory) / file_name, "r", encoding="utf-8") as file: + + if file_name.startswith(parent_prefix): + parent_entry = parent_entry_type( + entry_dict=json.load(file), working_directory=self.working_directory + ) + else: + entries.append( + self.downloader_entry_type( + entry_dict=json.load(file), working_directory=self.working_directory + ) + ) + + return parent_entry, entries + @abc.abstractmethod def download(self) -> List[DownloaderEntryT]: """The function to perform the download of all media entries""" diff --git a/src/ytdl_sub/downloaders/youtube_downloader.py b/src/ytdl_sub/downloaders/youtube_downloader.py index 45bb057c..bffbb043 100644 --- a/src/ytdl_sub/downloaders/youtube_downloader.py +++ b/src/ytdl_sub/downloaders/youtube_downloader.py @@ -1,5 +1,3 @@ -import json -import os from abc import ABC from pathlib import Path from typing import Dict @@ -11,11 +9,14 @@ from urllib.request import urlopen from PIL.Image import Image from PIL.Image import open as pil_open -from yt_dlp.utils import RejectedVideoReached from ytdl_sub.config.preset_options import Overrides from ytdl_sub.downloaders.downloader import Downloader +from ytdl_sub.downloaders.downloader import DownloaderOptionsT from ytdl_sub.downloaders.downloader import DownloaderValidator +from ytdl_sub.entries.youtube import YoutubeChannel +from ytdl_sub.entries.youtube import YoutubePlaylist +from ytdl_sub.entries.youtube import YoutubePlaylistVideo from ytdl_sub.entries.youtube import YoutubeVideo from ytdl_sub.utils.logger import Logger from ytdl_sub.validators.date_range_validator import DateRangeValidator @@ -35,56 +36,20 @@ class YoutubeDownloaderOptions(DownloaderValidator, ABC): YoutubeDownloaderOptionsT = TypeVar("YoutubeDownloaderOptionsT", bound=YoutubeDownloaderOptions) +YoutubeVideoT = TypeVar("YoutubeVideoT", bound=YoutubeVideo) class YoutubeDownloader( - Downloader[YoutubeDownloaderOptionsT, YoutubeVideo], Generic[YoutubeDownloaderOptionsT], ABC + Downloader[YoutubeDownloaderOptionsT, YoutubeVideoT], + Generic[YoutubeDownloaderOptionsT, YoutubeVideoT], + ABC, ): """ Class that handles downloading youtube entries via ytdl and converting them into - YoutubeVideo objects + YoutubeVideo like objects. Reserved for any future logic that is shared amongst all YT + downloaders. """ - downloader_entry_type = YoutubeVideo - - def _download_using_metadata( - self, - url: str, - ignore_prefix: str, - ytdl_options_overrides: Optional[Dict] = None, - ) -> List[YoutubeVideo]: - """ - Do not get entries from the extract info, let it write to the info.json file and load - that instead. This is because if the video is already downloaded in a playlist, it will - not fetch the metadata (maybe there is a way??) - """ - entries: List[YoutubeVideo] = [] - - ytdl_overrides = { - "writeinfojson": True, - } - if ytdl_options_overrides: - ytdl_overrides = dict(ytdl_overrides, **ytdl_options_overrides) - - try: - _ = self.extract_info(ytdl_options_overrides=ytdl_overrides, url=url) - except RejectedVideoReached: - pass - - # Load the entries from info.json, ignore the playlist entry - for file_name in os.listdir(self.working_directory): - if file_name.startswith(ignore_prefix) or not file_name.endswith(".info.json"): - continue - - with open(Path(self.working_directory) / file_name, "r", encoding="utf-8") as file: - entries.append( - YoutubeVideo( - entry_dict=json.load(file), working_directory=self.working_directory - ) - ) - - return entries - ############################################################################### # Youtube single video downloader + options @@ -98,8 +63,9 @@ class YoutubeVideoDownloaderOptions(YoutubeDownloaderOptions): self.video_id = self._validate_key("video_id", StringValidator) -class YoutubeVideoDownloader(YoutubeDownloader[YoutubeVideoDownloaderOptions]): +class YoutubeVideoDownloader(YoutubeDownloader[YoutubeVideoDownloaderOptions, YoutubeVideo]): downloader_options_type = YoutubeVideoDownloaderOptions + downloader_entry_type = YoutubeVideo @classmethod def video_url(cls, video_id: str) -> str: @@ -111,8 +77,8 @@ class YoutubeVideoDownloader(YoutubeDownloader[YoutubeVideoDownloaderOptions]): video_id = self.download_options.video_id.value video_url = self.video_url(video_id=video_id) - entry = self.extract_info(url=video_url) - return [YoutubeVideo(entry_dict=entry, working_directory=self.working_directory)] + entry_dict = self.extract_info(url=video_url) + return [YoutubeVideo(entry_dict=entry_dict, working_directory=self.working_directory)] ############################################################################### @@ -127,22 +93,31 @@ class YoutubePlaylistDownloaderOptions(YoutubeDownloaderOptions): self.playlist_id = self._validate_key("playlist_id", StringValidator) -class YoutubePlaylistDownloader(YoutubeDownloader[YoutubePlaylistDownloaderOptions]): +class YoutubePlaylistDownloader( + YoutubeDownloader[YoutubePlaylistDownloaderOptions, YoutubePlaylistVideo] +): downloader_options_type = YoutubePlaylistDownloaderOptions + downloader_entry_type = YoutubePlaylistVideo @classmethod def playlist_url(cls, playlist_id: str) -> str: """Returns full playlist url""" return f"https://youtube.com/playlist?list={playlist_id}" - def download(self) -> List[YoutubeVideo]: + def download(self) -> List[YoutubePlaylistVideo]: """ Downloads all videos in a Youtube playlist """ playlist_id = self.download_options.playlist_id.value playlist_url = self.playlist_url(playlist_id=playlist_id) - return self._download_using_metadata(url=playlist_url, ignore_prefix=playlist_id) + self.extract_info_json(url=playlist_url) + + _, videos = self.extract_from_info_json( + parent_prefix=playlist_id, parent_entry_type=YoutubePlaylist + ) + + return videos ############################################################################### @@ -165,8 +140,24 @@ class YoutubeChannelDownloaderOptions(YoutubeDownloaderOptions, DateRangeValidat ) -class YoutubeChannelDownloader(YoutubeDownloader[YoutubeChannelDownloaderOptions]): +class YoutubeChannelDownloader(YoutubeDownloader[YoutubeChannelDownloaderOptions, YoutubeVideo]): downloader_options_type = YoutubeChannelDownloaderOptions + downloader_entry_type = YoutubeVideo + + def __init__( + self, + working_directory: str, + download_options: DownloaderOptionsT, + ytdl_options: Optional[Dict] = None, + download_archive_file_name: Optional[str] = None, + ): + super().__init__( + working_directory=working_directory, + download_options=download_options, + ytdl_options=ytdl_options, + download_archive_file_name=download_archive_file_name, + ) + self.channel: Optional[YoutubeChannel] = None @classmethod def channel_url(cls, channel_id: str) -> str: @@ -187,46 +178,38 @@ class YoutubeChannelDownloader(YoutubeDownloader[YoutubeChannelDownloaderOptions Downloads all videos from a channel """ channel_url = self.channel_url(channel_id=self.channel_id) + ytdl_options_overrides = {} # If a date range is specified when download a YT channel, add it into the ytdl options - ytdl_options_overrides = {} source_date_range = self.download_options.get_date_range() if source_date_range: ytdl_options_overrides["daterange"] = source_date_range - return self._download_using_metadata( - url=channel_url, - ignore_prefix=self.channel_id, - ytdl_options_overrides=ytdl_options_overrides, + self.extract_info_json(ytdl_options_overrides=ytdl_options_overrides, url=channel_url) + + self.channel, videos = self.extract_from_info_json( + parent_prefix=self.channel_id, parent_entry_type=YoutubeChannel ) - @classmethod - def __download_thumbnail( - cls, - entry_dict: dict, - thumbnail_id: str, + return videos + + def _download_thumbnail( + self, + thumbnail_url: str, output_thumbnail_path: str, ): """ - Downloads a specific thumbnail from a YTDL entry's thumbnail list + Downloads a thumbnail and stores it in the output directory Parameters ---------- - entry_dict: - YTDL entry dict - thumbnail_id: - Id of the thumbnail defined in the YTDL thumnail + thumbnail_url: + Url of the thumbnail output_thumbnail_path: Path to store the thumbnail after downloading """ - thumbnail_url = None - for thumbnail in entry_dict.get("thumbnails", []): - if thumbnail["id"] == thumbnail_id: - thumbnail_url = thumbnail["url"] - break - if not thumbnail_url: - logger.warning("Could not find a thumbnail for %s", entry_dict.get("id")) + logger.warning("Could not find a thumbnail for %s", self.channel.uid) return with urlopen(thumbnail_url) as file: @@ -245,22 +228,14 @@ class YoutubeChannelDownloader(YoutubeDownloader[YoutubeChannelDownloaderOptions output_directory Output directory path """ - channel_json_file_path = Path(self.working_directory) / f"{self.channel_id}.info.json" - with open(channel_json_file_path, "r", encoding="utf-8") as channel_json: - channel_entry = json.load(channel_json) + avatar_thumbnail_name = overrides.apply_formatter(self.download_options.channel_avatar_path) + self._download_thumbnail( + thumbnail_url=self.channel.avatar_thumbnail_url(), + output_thumbnail_path=str(Path(output_directory) / avatar_thumbnail_name), + ) - if self.download_options.channel_avatar_path: - thumbnail_name = overrides.apply_formatter(self.download_options.channel_avatar_path) - self.__download_thumbnail( - entry_dict=channel_entry, - thumbnail_id="avatar_uncropped", - output_thumbnail_path=str(Path(output_directory) / thumbnail_name), - ) - - if self.download_options.channel_banner_path: - thumbnail_name = overrides.apply_formatter(self.download_options.channel_banner_path) - self.__download_thumbnail( - entry_dict=channel_entry, - thumbnail_id="banner_uncropped", - output_thumbnail_path=str(Path(output_directory) / thumbnail_name), - ) + banner_thumbnail_name = overrides.apply_formatter(self.download_options.channel_banner_path) + self._download_thumbnail( + thumbnail_url=self.channel.banner_thumbnail_url(), + output_thumbnail_path=str(Path(output_directory) / banner_thumbnail_name), + ) diff --git a/src/ytdl_sub/entries/variables/youtube_variables.py b/src/ytdl_sub/entries/variables/youtube_variables.py index 49247ce7..bdace313 100644 --- a/src/ytdl_sub/entries/variables/youtube_variables.py +++ b/src/ytdl_sub/entries/variables/youtube_variables.py @@ -18,3 +18,22 @@ class YoutubeVideoVariables(EntryVariables): return self.kwargs("track") return super().title + + @property + def playlist_index(self) -> int: + """ + Returns + ------- + The index of the video in the playlist. For non-playlist download strategies, this will + always return 1. + """ + return 1 + + @property + def playlist_size(self) -> int: + """ + Returns + ------- + The size of the playlist. For non-playlist download strategies, this will always return 1. + """ + return 1 diff --git a/src/ytdl_sub/entries/youtube.py b/src/ytdl_sub/entries/youtube.py index da5b39e7..e9e74b2c 100644 --- a/src/ytdl_sub/entries/youtube.py +++ b/src/ytdl_sub/entries/youtube.py @@ -1,5 +1,7 @@ import os.path from pathlib import Path +from typing import List +from typing import Optional from ytdl_sub.entries.entry import Entry from ytdl_sub.entries.variables.youtube_variables import YoutubeVideoVariables @@ -31,3 +33,77 @@ class YoutubeVideo(YoutubeVideoVariables, Entry): return possible_thumbnail_path return super().get_download_thumbnail_path() + + +class YoutubePlaylistVideo(YoutubeVideo): + @property + def playlist_index(self) -> int: + """ + Returns + ------- + The playlist index + """ + return self.kwargs("playlist_index") + + @property + def playlist_size(self) -> int: + """ + Returns + ------- + The size of the playlist + """ + return self.kwargs("playlist_count") + + +class YoutubePlaylist(Entry): + """ + Class placeholder for youtube playlists + """ + + +class YoutubeChannel(Entry): + def videos(self) -> List[YoutubeVideo]: + """ + Returns + ------- + All videos in the playlist represented as YoutubePlaylistVideos. This updates + playlist-specific fields like playlist_index and playlist_size with its actual value. + """ + return [ + YoutubeVideo(entry_dict=entry, working_directory=self._working_directory) + for entry in self.kwargs("entries") + ] + + def _get_thumbnail_url(self, thumbnail_id: str) -> Optional[str]: + """ + Downloads a specific thumbnail from a YTDL entry's thumbnail list + + Parameters + ---------- + thumbnail_id: + Id of the thumbnail defined in the channel's thumbnail + + Returns + ------- + Desired thumbnail url if it exists. None if it does not. + """ + for thumbnail in self.kwargs("thumbnails"): + if thumbnail["id"] == thumbnail_id: + return thumbnail["url"] + return None + + def avatar_thumbnail_url(self) -> str: + """ + Returns + ------- + The channel's uncropped avatar image url + """ + return self._get_thumbnail_url(thumbnail_id="avatar_uncropped") + + def banner_thumbnail_url(self) -> str: + """ + Returns + ------- + The channel's uncropped banner image url + """ + return self._get_thumbnail_url(thumbnail_id="banner_uncropped") diff --git a/subscriptions.yaml b/subscriptions.yaml index bfc55eb6..cd29a4f3 100644 --- a/subscriptions.yaml +++ b/subscriptions.yaml @@ -18,23 +18,23 @@ # artist: "DeLorra" # genre: "Synthwave / Electronic" # -bl00dwave: - preset: "soundcloud_with_id3_tags" - soundcloud: - username: bl00dwave - overrides: - artist: "bl00dwave" - genre: "Synthwave / Electronic" -# -#tom_petty: -# preset: "music_videos" -# youtube: -# download_strategy: "playlist" -# playlist_id: PLoopXDarluPBnuxs4PTC55Sc_2ShAXC0i -# output_options: -# output_directory: "/tmp/Tom Petty" +#bl00dwave: +# preset: "soundcloud_with_id3_tags" +# soundcloud: +# username: bl00dwave # overrides: -# artist: "Tom Petty and the Heartbreakers" +# artist: "bl00dwave" +# genre: "Synthwave / Electronic" +# +tom_petty: + preset: "music_videos" + youtube: + download_strategy: "playlist" + playlist_id: PLoopXDarluPBnuxs4PTC55Sc_2ShAXC0i + output_options: + output_directory: "/tmp/Tom Petty" + overrides: + artist: "Tom Petty and the Heartbreakers" # #rammstein: # preset: "music_videos"