danvics/ytdl-sub
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Add playlist_index and playlist_size to Youtube playlist videos (#17)

Browse source
This commit is contained in:
Jesse Bannon 2022-04-30 19:24:44 -07:00 committed by GitHub
parent fd5c48b897
commit 9734d391a6
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
7 changed files with 393 additions and 110 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

67
examples/kodi_music_videos_config.yaml Normal file
View file

@ -0,0 +1,67 @@
# This example shows how to download and format a Youtube video OR playlist
# to display in Kodi as a music video. Kodi requires music videos to be in
# a shared directory, so we will configure this to make the output directory
# formatted as:
#
# /path/to/Music Videos
# Elton John - Rocketman.jpg
# Elton John - Rocketman.mp4
# Elton John - Rocketman.nfo
# System of a Down - Chop Suey.jpg
# System of a Down - Chop Suey.mp4
# System of a Down - Chop Suey.nfo
# ...
#
configuration:
working_directory: '.ytdl-sub-downloads'
presets:
yt_music_video:
# Youtube playlists are our source/download strategy. However, this
# can be overwritten to download music videos from a 'channel' or a
# single 'video'
youtube:
download_strategy: "playlist"
# For advanced YTDL users only.
# It is wise to leave ignoreerrors=True to avoid errors for things
# like age-restricted videos in case you have not set your cookie.
ytdl_options:
ignoreerrors: True
# For each video downloaded, set the file and thumbnail name here.
# We set both with {music_video_name}, which is a variable we define in
# the overrides section further below to represent consistent naming format.
#
# Another field worth mentioning is maintain_download_archive=True. This
# is generally a good thing to enable with playlists because it will
# store previously downloaded video ids to tell YTDL not to re-download
# them on a successive invocation.
output_options:
output_directory: "path/to/Music Videos"
file_name: "{music_video_name}.{ext}"
thumbnail_name: "{music_video_name}.jpg"
maintain_download_archive: True
# Always convert the video thumbnail to jpg
# TODO: always convert all thumbnails to jpg in code
convert_thumbnail:
to: "jpg"
# For each video downloaded, add a music video NFO file for it. Populate it
# with tags that Kodi will read and use to display it in the music or music
# videos section.
nfo_tags:
nfo_name: "{music_video_name}.nfo"
nfo_root: "musicvideo"
tags:
artist: "{artist}"
title: "{title}"
album: "Music Videos"
year: "{upload_year}"
# Overrides is a section where we can define our own variables, and use them in
# any other section. We define our episode file name here, which gets reused above
# for the video, thumbnail, and NFO file.
overrides:
music_video_name: "{sanitized_artist} - {sanitized_title}"

65
examples/kodi_music_videos_subscriptions.yaml Normal file
View file

@ -0,0 +1,65 @@
# This example shows how we can use the `kodi_music_videos_config.yaml` preset
# to download music videos in a few different ways. We will use made-up artists
# in each example
###############################################################################
# LEVEL 1 - DOWNLOAD MUSIC VIDEO PLAYLIST
# Subscription names are defined by you. We will call this one john_smith
# for simplicity, and it will download every single video in john_smith's music
# video playlist. Many artists maintain a playlist of all their music videos,
# which makes this an easy way to grab all of them.
john_smith:
# We must define a preset to use from our config. We named the one in the
# config example "yt_music_video", so set that here.
preset: "yt_music_video"
# Since our preset download strategy is set to 'playlist', set the playlist id.
youtube:
playlist_id: "UCsvn_Po0SmunchJYtttWpOxMg"
# Overrides can be defined per-subscription. If you noticed, we used {artist}
# and {sanitized_artist} in our "yt_music_video" preset. We intended to reserve
# that variable to be defined for each individual subscription. Each override
# defined here will create a 'sanitized_' version that is safe for file systems.
#
# A note for Kodi music videos, it is important to make sure your artist name
# exactly matches how it is formatted in Kodi itself, otherwise it will be
# read in as a new artist
overrides:
artist: "John Smith and the Instrument Players"
###############################################################################
# LEVEL 2 - DOWNLOAD SINGLE MUSIC VIDEO
# It is not always ideal to download all of an artist's music videos.
# Maybe you only like one song of theirs. We can reuse our preset
# to download a single video instead.
#
# The only difference between this example and the one above is
# - youtube.download_strategy
# We defined this as 'playlist' in the config. We must override it
# with 'video' to perform a single video download
# - youtube.video_id
# The id of the video to download
#
# Of course, defining yaml configuration to download a single video once
# and never again seems weird. Instead, we can perform this download via
# command:
#
# ytdl-sub dl \
# --preset "yt_channel_as_tv" \
# --youtube.download_strategy "video" \
# --youtube.video_id "QhY6r6oAErg" \
# --overrides.artist "John Smith and the Instrument Players"
#
# The --preset and --youtube.download_strategy args will always be the
# same. We will try to simplify the dl command to make it shorter.
john_smith_one_hit_wonder:
preset: "yt_channel_as_tv"
youtube:
download_strategy: "video"
video_id: "QhY6r6oAErg"
overrides:
artist: "John Smith and the Instrument Players"

85
src/ytdl_sub/downloaders/downloader.py
View file

@ -1,4 +1,6 @@
import abc
import json
import os
from abc import ABC
from contextlib import contextmanager
from pathlib import Path
@ -6,12 +8,16 @@ from typing import Dict
from typing import Generic
from typing import List
from typing import Optional
from typing import Tuple
from typing import Type
from typing import TypeVar
import yt_dlp as ytdl
from yt_dlp.utils import ExistingVideoReached
from yt_dlp.utils import RejectedVideoReached
from ytdl_sub.config.preset_options import Overrides
from ytdl_sub.entries.base_entry import BaseEntry
from ytdl_sub.entries.entry import Entry
from ytdl_sub.validators.strict_dict_validator import StrictDictValidator
@ -24,6 +30,7 @@ class DownloaderValidator(StrictDictValidator, ABC):
DownloaderOptionsT = TypeVar("DownloaderOptionsT", bound=DownloaderValidator)
DownloaderEntryT = TypeVar("DownloaderEntryT", bound=Entry)
DownloaderParentEntryT = TypeVar("DownloaderParentEntryT", bound=BaseEntry)
class Downloader(Generic[DownloaderOptionsT, DownloaderEntryT], ABC):
@ -32,8 +39,8 @@ class Downloader(Generic[DownloaderOptionsT, DownloaderEntryT], ABC):
and should translate that to list of Entry objects.
"""
downloader_options_type: Type[DownloaderOptionsT] = NotImplemented
downloader_entry_type: Type[DownloaderEntryT] = NotImplemented
downloader_options_type: Type[DownloaderValidator] = DownloaderValidator
downloader_entry_type: Type[Entry] = Entry
@classmethod
def ytdl_option_overrides(cls) -> Dict:
@ -113,10 +120,84 @@ class Downloader(Generic[DownloaderOptionsT, DownloaderEntryT], ABC):
"""
Wrapper around yt_dlp.YoutubeDL.YoutubeDL.extract_info
All kwargs will passed to the extract_info function.
Parameters
----------
ytdl_options_overrides
Optional. Dict containing ytdl args to override other predefined ytdl args
**kwargs
arguments passed directory to YoutubeDL extract_info
"""
with self.ytdl_downloader(ytdl_options_overrides) as ytdl_downloader:
return ytdl_downloader.extract_info(**kwargs)
def extract_info_json(self, ytdl_options_overrides: Optional[Dict] = None, **kwargs) -> None:
"""
Wrapper around yt_dlp.YoutubeDL.YoutubeDL.extract_info
All kwargs will passed to the extract_info function. This also enables ytdl to write
.info.json files for all media downloaded.
Catches RejectedVideoReached and ExistingVideoReached exceptions.
Parameters
----------
ytdl_options_overrides
Optional. Dict containing ytdl args to override other predefined ytdl args
**kwargs
arguments passed directory to YoutubeDL extract_info
"""
if ytdl_options_overrides is None:
ytdl_options_overrides = {}
ytdl_options_overrides = dict(ytdl_options_overrides, **{"writeinfojson": True})
try:
_ = self.extract_info(ytdl_options_overrides=ytdl_options_overrides, **kwargs)
except (RejectedVideoReached, ExistingVideoReached):
pass
def extract_from_info_json(
self, parent_prefix: str, parent_entry_type: Type[DownloaderParentEntryT]
) -> Tuple[DownloaderParentEntryT, List[DownloaderEntryT]]:
"""
Reads all .info.json files in the working directory, and casts them to the
parent_entry_type (i.e. YoutubeChannel) and downloader_entry_type (i.e. YoutubeVideo)
Parameters
----------
parent_prefix
info.json file name prefix to indicate its the parent (i.e. the YT channel id)
parent_entry_type
Class type for the parent entry
Returns
-------
Tuple containing parent, list of videos belong to the parent
"""
# Load the entries from info.json
parent_entry: Optional[DownloaderParentEntryT] = None
entries: List[DownloaderEntryT] = []
for file_name in os.listdir(self.working_directory):
if file_name.endswith(".info.json"):
with open(Path(self.working_directory) / file_name, "r", encoding="utf-8") as file:
if file_name.startswith(parent_prefix):
parent_entry = parent_entry_type(
entry_dict=json.load(file), working_directory=self.working_directory
)
else:
entries.append(
self.downloader_entry_type(
entry_dict=json.load(file), working_directory=self.working_directory
)
)
return parent_entry, entries
@abc.abstractmethod
def download(self) -> List[DownloaderEntryT]:
"""The function to perform the download of all media entries"""

159
src/ytdl_sub/downloaders/youtube_downloader.py
View file

@ -1,5 +1,3 @@
import json
import os
from abc import ABC
from pathlib import Path
from typing import Dict
@ -11,11 +9,14 @@ from urllib.request import urlopen
from PIL.Image import Image
from PIL.Image import open as pil_open
from yt_dlp.utils import RejectedVideoReached
from ytdl_sub.config.preset_options import Overrides
from ytdl_sub.downloaders.downloader import Downloader
from ytdl_sub.downloaders.downloader import DownloaderOptionsT
from ytdl_sub.downloaders.downloader import DownloaderValidator
from ytdl_sub.entries.youtube import YoutubeChannel
from ytdl_sub.entries.youtube import YoutubePlaylist
from ytdl_sub.entries.youtube import YoutubePlaylistVideo
from ytdl_sub.entries.youtube import YoutubeVideo
from ytdl_sub.utils.logger import Logger
from ytdl_sub.validators.date_range_validator import DateRangeValidator
@ -35,56 +36,20 @@ class YoutubeDownloaderOptions(DownloaderValidator, ABC):
YoutubeDownloaderOptionsT = TypeVar("YoutubeDownloaderOptionsT", bound=YoutubeDownloaderOptions)
YoutubeVideoT = TypeVar("YoutubeVideoT", bound=YoutubeVideo)
class YoutubeDownloader(
Downloader[YoutubeDownloaderOptionsT, YoutubeVideo], Generic[YoutubeDownloaderOptionsT], ABC
Downloader[YoutubeDownloaderOptionsT, YoutubeVideoT],
Generic[YoutubeDownloaderOptionsT, YoutubeVideoT],
ABC,
):
"""
Class that handles downloading youtube entries via ytdl and converting them into
YoutubeVideo objects
YoutubeVideo like objects. Reserved for any future logic that is shared amongst all YT
downloaders.
"""
downloader_entry_type = YoutubeVideo
def _download_using_metadata(
self,
url: str,
ignore_prefix: str,
ytdl_options_overrides: Optional[Dict] = None,
) -> List[YoutubeVideo]:
"""
Do not get entries from the extract info, let it write to the info.json file and load
that instead. This is because if the video is already downloaded in a playlist, it will
not fetch the metadata (maybe there is a way??)
"""
entries: List[YoutubeVideo] = []
ytdl_overrides = {
"writeinfojson": True,
}
if ytdl_options_overrides:
ytdl_overrides = dict(ytdl_overrides, **ytdl_options_overrides)
try:
_ = self.extract_info(ytdl_options_overrides=ytdl_overrides, url=url)
except RejectedVideoReached:
pass
# Load the entries from info.json, ignore the playlist entry
for file_name in os.listdir(self.working_directory):
if file_name.startswith(ignore_prefix) or not file_name.endswith(".info.json"):
continue
with open(Path(self.working_directory) / file_name, "r", encoding="utf-8") as file:
entries.append(
YoutubeVideo(
entry_dict=json.load(file), working_directory=self.working_directory
)
)
return entries
###############################################################################
# Youtube single video downloader + options
@ -98,8 +63,9 @@ class YoutubeVideoDownloaderOptions(YoutubeDownloaderOptions):
self.video_id = self._validate_key("video_id", StringValidator)
class YoutubeVideoDownloader(YoutubeDownloader[YoutubeVideoDownloaderOptions]):
class YoutubeVideoDownloader(YoutubeDownloader[YoutubeVideoDownloaderOptions, YoutubeVideo]):
downloader_options_type = YoutubeVideoDownloaderOptions
downloader_entry_type = YoutubeVideo
@classmethod
def video_url(cls, video_id: str) -> str:
@ -111,8 +77,8 @@ class YoutubeVideoDownloader(YoutubeDownloader[YoutubeVideoDownloaderOptions]):
video_id = self.download_options.video_id.value
video_url = self.video_url(video_id=video_id)
entry = self.extract_info(url=video_url)
return [YoutubeVideo(entry_dict=entry, working_directory=self.working_directory)]
entry_dict = self.extract_info(url=video_url)
return [YoutubeVideo(entry_dict=entry_dict, working_directory=self.working_directory)]
###############################################################################
@ -127,22 +93,31 @@ class YoutubePlaylistDownloaderOptions(YoutubeDownloaderOptions):
self.playlist_id = self._validate_key("playlist_id", StringValidator)
class YoutubePlaylistDownloader(YoutubeDownloader[YoutubePlaylistDownloaderOptions]):
class YoutubePlaylistDownloader(
YoutubeDownloader[YoutubePlaylistDownloaderOptions, YoutubePlaylistVideo]
):
downloader_options_type = YoutubePlaylistDownloaderOptions
downloader_entry_type = YoutubePlaylistVideo
@classmethod
def playlist_url(cls, playlist_id: str) -> str:
"""Returns full playlist url"""
return f"https://youtube.com/playlist?list={playlist_id}"
def download(self) -> List[YoutubeVideo]:
def download(self) -> List[YoutubePlaylistVideo]:
"""
Downloads all videos in a Youtube playlist
"""
playlist_id = self.download_options.playlist_id.value
playlist_url = self.playlist_url(playlist_id=playlist_id)
return self._download_using_metadata(url=playlist_url, ignore_prefix=playlist_id)
self.extract_info_json(url=playlist_url)
_, videos = self.extract_from_info_json(
parent_prefix=playlist_id, parent_entry_type=YoutubePlaylist
)
return videos
###############################################################################
@ -165,8 +140,24 @@ class YoutubeChannelDownloaderOptions(YoutubeDownloaderOptions, DateRangeValidat
)
class YoutubeChannelDownloader(YoutubeDownloader[YoutubeChannelDownloaderOptions]):
class YoutubeChannelDownloader(YoutubeDownloader[YoutubeChannelDownloaderOptions, YoutubeVideo]):
downloader_options_type = YoutubeChannelDownloaderOptions
downloader_entry_type = YoutubeVideo
def __init__(
self,
working_directory: str,
download_options: DownloaderOptionsT,
ytdl_options: Optional[Dict] = None,
download_archive_file_name: Optional[str] = None,
):
super().__init__(
working_directory=working_directory,
download_options=download_options,
ytdl_options=ytdl_options,
download_archive_file_name=download_archive_file_name,
)
self.channel: Optional[YoutubeChannel] = None
@classmethod
def channel_url(cls, channel_id: str) -> str:
@ -187,46 +178,38 @@ class YoutubeChannelDownloader(YoutubeDownloader[YoutubeChannelDownloaderOptions
Downloads all videos from a channel
"""
channel_url = self.channel_url(channel_id=self.channel_id)
ytdl_options_overrides = {}
# If a date range is specified when download a YT channel, add it into the ytdl options
ytdl_options_overrides = {}
source_date_range = self.download_options.get_date_range()
if source_date_range:
ytdl_options_overrides["daterange"] = source_date_range
return self._download_using_metadata(
url=channel_url,
ignore_prefix=self.channel_id,
ytdl_options_overrides=ytdl_options_overrides,
self.extract_info_json(ytdl_options_overrides=ytdl_options_overrides, url=channel_url)
self.channel, videos = self.extract_from_info_json(
parent_prefix=self.channel_id, parent_entry_type=YoutubeChannel
)
@classmethod
def __download_thumbnail(
cls,
entry_dict: dict,
thumbnail_id: str,
return videos
def _download_thumbnail(
self,
thumbnail_url: str,
output_thumbnail_path: str,
):
"""
Downloads a specific thumbnail from a YTDL entry's thumbnail list
Downloads a thumbnail and stores it in the output directory
Parameters
----------
entry_dict:
YTDL entry dict
thumbnail_id:
Id of the thumbnail defined in the YTDL thumnail
thumbnail_url:
Url of the thumbnail
output_thumbnail_path:
Path to store the thumbnail after downloading
"""
thumbnail_url = None
for thumbnail in entry_dict.get("thumbnails", []):
if thumbnail["id"] == thumbnail_id:
thumbnail_url = thumbnail["url"]
break
if not thumbnail_url:
logger.warning("Could not find a thumbnail for %s", entry_dict.get("id"))
logger.warning("Could not find a thumbnail for %s", self.channel.uid)
return
with urlopen(thumbnail_url) as file:
@ -245,22 +228,14 @@ class YoutubeChannelDownloader(YoutubeDownloader[YoutubeChannelDownloaderOptions
output_directory
Output directory path
"""
channel_json_file_path = Path(self.working_directory) / f"{self.channel_id}.info.json"
with open(channel_json_file_path, "r", encoding="utf-8") as channel_json:
channel_entry = json.load(channel_json)
avatar_thumbnail_name = overrides.apply_formatter(self.download_options.channel_avatar_path)
self._download_thumbnail(
thumbnail_url=self.channel.avatar_thumbnail_url(),
output_thumbnail_path=str(Path(output_directory) / avatar_thumbnail_name),
)
if self.download_options.channel_avatar_path:
thumbnail_name = overrides.apply_formatter(self.download_options.channel_avatar_path)
self.__download_thumbnail(
entry_dict=channel_entry,
thumbnail_id="avatar_uncropped",
output_thumbnail_path=str(Path(output_directory) / thumbnail_name),
)
if self.download_options.channel_banner_path:
thumbnail_name = overrides.apply_formatter(self.download_options.channel_banner_path)
self.__download_thumbnail(
entry_dict=channel_entry,
thumbnail_id="banner_uncropped",
output_thumbnail_path=str(Path(output_directory) / thumbnail_name),
)
banner_thumbnail_name = overrides.apply_formatter(self.download_options.channel_banner_path)
self._download_thumbnail(
thumbnail_url=self.channel.banner_thumbnail_url(),
output_thumbnail_path=str(Path(output_directory) / banner_thumbnail_name),
)

19
src/ytdl_sub/entries/variables/youtube_variables.py
View file

@ -18,3 +18,22 @@ class YoutubeVideoVariables(EntryVariables):
return self.kwargs("track")
return super().title
@property
def playlist_index(self) -> int:
"""
Returns
-------
The index of the video in the playlist. For non-playlist download strategies, this will
always return 1.
"""
return 1
@property
def playlist_size(self) -> int:
"""
Returns
-------
The size of the playlist. For non-playlist download strategies, this will always return 1.
"""
return 1

76
src/ytdl_sub/entries/youtube.py
View file

@ -1,5 +1,7 @@
import os.path
from pathlib import Path
from typing import List
from typing import Optional
from ytdl_sub.entries.entry import Entry
from ytdl_sub.entries.variables.youtube_variables import YoutubeVideoVariables
@ -31,3 +33,77 @@ class YoutubeVideo(YoutubeVideoVariables, Entry):
return possible_thumbnail_path
return super().get_download_thumbnail_path()
class YoutubePlaylistVideo(YoutubeVideo):
@property
def playlist_index(self) -> int:
"""
Returns
-------
The playlist index
"""
return self.kwargs("playlist_index")
@property
def playlist_size(self) -> int:
"""
Returns
-------
The size of the playlist
"""
return self.kwargs("playlist_count")
class YoutubePlaylist(Entry):
"""
Class placeholder for youtube playlists
"""
class YoutubeChannel(Entry):
def videos(self) -> List[YoutubeVideo]:
"""
Returns
-------
All videos in the playlist represented as YoutubePlaylistVideos. This updates
playlist-specific fields like playlist_index and playlist_size with its actual value.
"""
return [
YoutubeVideo(entry_dict=entry, working_directory=self._working_directory)
for entry in self.kwargs("entries")
]
def _get_thumbnail_url(self, thumbnail_id: str) -> Optional[str]:
"""
Downloads a specific thumbnail from a YTDL entry's thumbnail list
Parameters
----------
thumbnail_id:
Id of the thumbnail defined in the channel's thumbnail
Returns
-------
Desired thumbnail url if it exists. None if it does not.
"""
for thumbnail in self.kwargs("thumbnails"):
if thumbnail["id"] == thumbnail_id:
return thumbnail["url"]
return None
def avatar_thumbnail_url(self) -> str:
"""
Returns
-------
The channel's uncropped avatar image url
"""
return self._get_thumbnail_url(thumbnail_id="avatar_uncropped")
def banner_thumbnail_url(self) -> str:
"""
Returns
-------
The channel's uncropped banner image url
"""
return self._get_thumbnail_url(thumbnail_id="banner_uncropped")

32
subscriptions.yaml
View file

@ -18,23 +18,23 @@
# artist: "DeLorra"
# genre: "Synthwave / Electronic"
#
bl00dwave:
preset: "soundcloud_with_id3_tags"
soundcloud:
username: bl00dwave
overrides:
artist: "bl00dwave"
genre: "Synthwave / Electronic"
#
#tom_petty:
# preset: "music_videos"
# youtube:
# download_strategy: "playlist"
# playlist_id: PLoopXDarluPBnuxs4PTC55Sc_2ShAXC0i
# output_options:
# output_directory: "/tmp/Tom Petty"
#bl00dwave:
# preset: "soundcloud_with_id3_tags"
# soundcloud:
# username: bl00dwave
# overrides:
# artist: "Tom Petty and the Heartbreakers"
# artist: "bl00dwave"
# genre: "Synthwave / Electronic"
#
tom_petty:
preset: "music_videos"
youtube:
download_strategy: "playlist"
playlist_id: PLoopXDarluPBnuxs4PTC55Sc_2ShAXC0i
output_options:
output_directory: "/tmp/Tom Petty"
overrides:
artist: "Tom Petty and the Heartbreakers"
#
#rammstein:
# preset: "music_videos"