danvics/ytdl-sub
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

downloaders have access to overrides

Browse source
This commit is contained in:
Jesse Bannon 2022-08-31 22:12:55 -07:00
parent 74c9210448
commit 7ef5947fd7
8 changed files with 47 additions and 72 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

34
src/ytdl_sub/downloaders/downloader.py
View file

@ -88,6 +88,7 @@ class Downloader(DownloadArchiver, Generic[DownloaderOptionsT, DownloaderEntryT]
download_options: DownloaderOptionsT,
enhanced_download_archive: EnhancedDownloadArchive,
ytdl_options_builder: YTDLOptionsBuilder,
overrides: Overrides,
):
"""
Parameters
@ -98,14 +99,16 @@ class Downloader(DownloadArchiver, Generic[DownloaderOptionsT, DownloaderEntryT]
Download archive
ytdl_options_builder
YTDL options builder
overrides
Override variables
"""
DownloadArchiver.__init__(self=self, enhanced_download_archive=enhanced_download_archive)
self.download_options = download_options
self.overrides = overrides
self._ytdl_options_builder = ytdl_options_builder.clone().add(
self.ytdl_option_defaults(), before=True
)
self._added_override_variables: Dict[str, str] = {}
@contextmanager
def ytdl_downloader(self, ytdl_options_overrides: Optional[Dict] = None) -> ytdl.YoutubeDL:
@ -128,28 +131,6 @@ class Downloader(DownloadArchiver, Generic[DownloaderOptionsT, DownloaderEntryT]
"""
return self._ytdl_options_builder.to_dict().get("skip_download", False)
def add_override_variables(self, override_variables_to_add: Dict[str, str]) -> None:
"""
Override variables added from the downloader. Should be added before yielding
entries
Parameters
----------
override_variables_to_add
The override variables to add
"""
self._added_override_variables = dict(
self._added_override_variables, **override_variables_to_add
)
def get_added_override_variables(self) -> Dict[str, str]:
"""
Returns
-------
Added override variables
"""
return self._added_override_variables
def extract_info(self, ytdl_options_overrides: Optional[Dict] = None, **kwargs) -> Dict:
"""
Wrapper around yt_dlp.YoutubeDL.YoutubeDL.extract_info
@ -354,14 +335,9 @@ class Downloader(DownloadArchiver, Generic[DownloaderOptionsT, DownloaderEntryT]
) -> Iterable[DownloaderEntryT] | Iterable[Tuple[DownloaderEntryT, FileMetadata]]:
"""The function to perform the download of all media entries"""
def post_download(self, overrides: Overrides):
def post_download(self):
"""
After all media entries have been downloaded, post processed, and moved to the output
directory, run this function. This lets the downloader add any extra files directly to the
output directory, for things like YT channel image, banner.
Parameters
----------
overrides:
Subscription overrides
"""

38
src/ytdl_sub/downloaders/youtube/channel.py
View file

@ -137,12 +137,15 @@ class YoutubeChannelDownloader(YoutubeDownloader[YoutubeChannelDownloaderOptions
download_options: DownloaderOptionsT,
enhanced_download_archive: EnhancedDownloadArchive,
ytdl_options_builder: YTDLOptionsBuilder,
overrides: Overrides,
):
super().__init__(
download_options=download_options,
enhanced_download_archive=enhanced_download_archive,
ytdl_options_builder=ytdl_options_builder,
overrides=overrides,
)
self.channel: Optional[YoutubeChannel] = None
def _get_channel(self, entry_dicts: List[Dict]) -> YoutubeChannel:
@ -151,6 +154,12 @@ class YoutubeChannelDownloader(YoutubeDownloader[YoutubeChannelDownloaderOptions
working_directory=self.working_directory,
)
def _get_channel_videos(self, entry_dicts: List[Dict]) -> List[YoutubeVideo]:
return [
YoutubeVideo(entry_dict=entry_dict, working_directory=self.working_directory)
for entry_dict in self._filter_entry_dicts(entry_dicts, sort_by="playlist_index")
]
def download(self) -> Generator[YoutubeVideo, None, None]:
"""
Downloads all videos from a channel
@ -172,21 +181,19 @@ class YoutubeChannelDownloader(YoutubeDownloader[YoutubeChannelDownloaderOptions
log_prefix_on_info_json_dl="Downloading metadata for",
url=self.download_options.channel_url,
)
self.channel = self._get_channel(entry_dicts=entry_dicts)
self.add_override_variables(
override_variables_to_add={
"source_description": self.channel.kwargs_get("description", "")
}
)
channel_videos = self._filter_entry_dicts(entry_dicts, sort_by="playlist_index")
self.channel = self._get_channel(entry_dicts=entry_dicts)
entries_to_download = self._get_channel_videos(entry_dicts=entry_dicts)
self.overrides.add_override_variables(
variables_to_add={"source_description": self.channel.kwargs_get("description", "")}
)
# Iterate in descending order to process older videos first. In case an error occurs and a
# the channel must be redownloaded, it will fetch most recent metadata first, and break
# on the older video that's been processed and is in the download archive.
for idx, entry_dict in enumerate(reversed(channel_videos), start=1):
video = YoutubeVideo(entry_dict=entry_dict, working_directory=self.working_directory)
download_logger.info("Downloading %d/%d %s", idx, len(entry_dicts), video.title)
for idx, video in enumerate(reversed(entries_to_download), start=1):
download_logger.info("Downloading %d/%d %s", idx, len(entries_to_download), video.title)
# Re-download the contents even if it's a dry-run as a single video. At this time,
# channels do not download subtitles or subtitle metadata
@ -226,17 +233,12 @@ class YoutubeChannelDownloader(YoutubeDownloader[YoutubeChannelDownloaderOptions
thumbnail_url=thumbnail_url, output_thumbnail_path=output_thumbnail_path
)
def post_download(self, overrides: Overrides):
def post_download(self):
"""
Downloads and moves channel avatar and banner images to the output directory.
Parameters
----------
overrides
Overrides that can contain variables in the avatar or banner file path
"""
if self.download_options.channel_avatar_path:
avatar_thumbnail_name = overrides.apply_formatter(
avatar_thumbnail_name = self.overrides.apply_formatter(
self.download_options.channel_avatar_path
)
self._download_thumbnail(
@ -246,7 +248,7 @@ class YoutubeChannelDownloader(YoutubeDownloader[YoutubeChannelDownloaderOptions
self.save_file(file_name=avatar_thumbnail_name)
if self.download_options.channel_banner_path:
banner_thumbnail_name = overrides.apply_formatter(
banner_thumbnail_name = self.overrides.apply_formatter(
self.download_options.channel_banner_path
)
self._download_thumbnail(

4
src/ytdl_sub/downloaders/youtube/playlist.py
View file

@ -82,8 +82,8 @@ class YoutubePlaylistDownloader(
)
playlist = self._filter_entry_dicts(entry_dicts, extractor="youtube:tab")[0]
self.add_override_variables(
override_variables_to_add={"source_description": playlist.get("description", "")}
self.overrides.add_override_variables(
variables_to_add={"source_description": playlist.get("description", "")}
)
# Iterate in reverse order to process older videos first. In case an error occurs and a

2
src/ytdl_sub/downloaders/youtube/video.py
View file

@ -69,6 +69,6 @@ class YoutubeVideoDownloader(YoutubeDownloader[YoutubeVideoDownloaderOptions, Yo
"""Download a single Youtube video"""
entry_dict = self.extract_info(url=self.download_options.video_url)
video = YoutubeVideo(entry_dict=entry_dict, working_directory=self.working_directory)
self.add_override_variables({"source_description": video.description})
self.overrides.add_override_variables({"source_description": video.description})
return [video]

3
src/ytdl_sub/entries/base_entry.py
View file

@ -36,6 +36,9 @@ class BaseEntry(ABC):
return self._kwargs[key]
def kwargs_get(self, key: str, default: Optional[Any] = None) -> Any:
"""
Dict get on kwargs
"""
if not self.kwargs_contains(key):
return default
return self.kwargs(key)

11
src/ytdl_sub/subscriptions/subscription_download.py
View file

@ -234,7 +234,6 @@ class SubscriptionDownload(BaseSubscription, ABC):
"""
self._enhanced_download_archive.reinitialize(dry_run=dry_run)
plugins = self._initialize_plugins()
added_override_variables = False
ytdl_options_builder = SubscriptionYTDLOptions(
preset=self._preset_options,
@ -249,16 +248,10 @@ class SubscriptionDownload(BaseSubscription, ABC):
download_options=self.downloader_options,
enhanced_download_archive=self._enhanced_download_archive,
ytdl_options_builder=ytdl_options_builder,
overrides=self.overrides,
)
for entry in downloader.download():
# TODO: make this a step before download
if not added_override_variables:
self.overrides.add_override_variables(
variables_to_add=downloader.get_added_override_variables()
)
added_override_variables = True
entry_metadata = FileMetadata()
if isinstance(entry, tuple):
entry, entry_metadata = entry
@ -272,7 +265,7 @@ class SubscriptionDownload(BaseSubscription, ABC):
plugins=plugins, dry_run=dry_run, entry=entry, entry_metadata=entry_metadata
)
downloader.post_download(overrides=self.overrides)
downloader.post_download()
for plugin in plugins:
plugin.post_process_subscription()

26
tests/e2e/resources/expected_downloads_summaries/youtube/test_channel_full.json
View file

@ -1,55 +1,55 @@
{
".ytdl-sub-pz-download-archive.json": "70615451318cdb5e018e007c77893a39",
"Season 2010/s2010.e0813 - Oblivion Mod ＂Falcor＂ p.1-thumb.jpg": "fb95b510681676e81c321171fc23143e",
"Season 2010/s2010.e0813 - Oblivion Mod ＂Falcor＂ p.1.info.json": "5d16096bc4256239932db2c2d98161a3",
"Season 2010/s2010.e0813 - Oblivion Mod ＂Falcor＂ p.1.info.json": "a5e93b8087ba3e0fa4a8410b5929eded",
"Season 2010/s2010.e0813 - Oblivion Mod ＂Falcor＂ p.1.mp4": "931a705864c57d21d6fedebed4af6bbc",
"Season 2010/s2010.e0813 - Oblivion Mod ＂Falcor＂ p.1.nfo": "2d0738094d8e649eaebbab16fd647da1",
"Season 2010/s2010.e1202 - Oblivion Mod ＂Falcor＂ p.2-thumb.jpg": "8b32ee9c037fa669e444a0ac181525a1",
"Season 2010/s2010.e1202 - Oblivion Mod ＂Falcor＂ p.2.info.json": "514d2c3eab500f9541910d7573cb495e",
"Season 2010/s2010.e1202 - Oblivion Mod ＂Falcor＂ p.2.info.json": "d7112915f5d437e38f6b1ec633274fcb",
"Season 2010/s2010.e1202 - Oblivion Mod ＂Falcor＂ p.2.mp4": "d3469b4dca7139cb3dbc38712b6796bf",
"Season 2010/s2010.e1202 - Oblivion Mod ＂Falcor＂ p.2.nfo": "5c258f9e54854ef292ce3c58331da110",
"Season 2011/s2011.e0201 - Jesse's Minecraft Server [Trailer - Feb.1]-thumb.jpg": "b232d253df621aa770b780c1301d364d",
"Season 2011/s2011.e0201 - Jesse's Minecraft Server [Trailer - Feb.1].info.json": "f9ccc0b104551ecd7b737451051118aa",
"Season 2011/s2011.e0201 - Jesse's Minecraft Server [Trailer - Feb.1].info.json": "51bb8da215aa43a2824126002d4c9cb5",
"Season 2011/s2011.e0201 - Jesse's Minecraft Server [Trailer - Feb.1].mp4": "e66287b9832277b6a4d1554e29d9fdcc",
"Season 2011/s2011.e0201 - Jesse's Minecraft Server [Trailer - Feb.1].nfo": "abb3ac33366cc3b86d0467c8fb80a323",
"Season 2011/s2011.e0227 - Jesse's Minecraft Server [Trailer - Feb.27]-thumb.jpg": "d17c379ea8b362f5b97c6b213b0342cb",
"Season 2011/s2011.e0227 - Jesse's Minecraft Server [Trailer - Feb.27].info.json": "6ad149bea780198512d73f075722ef82",
"Season 2011/s2011.e0227 - Jesse's Minecraft Server [Trailer - Feb.27].info.json": "a7d62181e7e45640af423312039f9d0f",
"Season 2011/s2011.e0227 - Jesse's Minecraft Server [Trailer - Feb.27].mp4": "04ab5cb3cc12325d0c96a7cd04a8b91d",
"Season 2011/s2011.e0227 - Jesse's Minecraft Server [Trailer - Feb.27].nfo": "46190954652c9d9812e061fc0c9e1d92",
"Season 2011/s2011.e0321 - Jesse's Minecraft Server [Trailer - Mar.21]-thumb.jpg": "e7830aa8a64b0cde65ba3f7e5fc56530",
"Season 2011/s2011.e0321 - Jesse's Minecraft Server [Trailer - Mar.21].info.json": "d040165f8ebc346c202605b19ed8e46b",
"Season 2011/s2011.e0321 - Jesse's Minecraft Server [Trailer - Mar.21].info.json": "14e7e02d3b0fec685dcabd585256a7e0",
"Season 2011/s2011.e0321 - Jesse's Minecraft Server [Trailer - Mar.21].mp4": "025de6099a5c98e6397153c7a62d517d",
"Season 2011/s2011.e0321 - Jesse's Minecraft Server [Trailer - Mar.21].nfo": "30993fa8e00a0e370b4db244f3da8f7d",
"Season 2011/s2011.e0529 - Project Zombie ｜Official Trailer｜ (IP： mc.projectzombie.beastnode.net)-thumb.jpg": "c956192a379b3661595c9920972d4819",
"Season 2011/s2011.e0529 - Project Zombie ｜Official Trailer｜ (IP： mc.projectzombie.beastnode.net).info.json": "331af1c81b1bfa1df62179067a5f06b0",
"Season 2011/s2011.e0529 - Project Zombie ｜Official Trailer｜ (IP： mc.projectzombie.beastnode.net).info.json": "4b97b5e5450bcafaa1b439e05a4559db",
"Season 2011/s2011.e0529 - Project Zombie ｜Official Trailer｜ (IP： mc.projectzombie.beastnode.net).mp4": "3d9c19835b03355d6fd5d00cd59dbe5b",
"Season 2011/s2011.e0529 - Project Zombie ｜Official Trailer｜ (IP： mc.projectzombie.beastnode.net).nfo": "11a0e8754c414875bcd454358683da5f",
"Season 2011/s2011.e0630 - Project Zombie ｜Fin｜-thumb.jpg": "00ed383591779ffe98291de60f198fe9",
"Season 2011/s2011.e0630 - Project Zombie ｜Fin｜.info.json": "07d6b5e38b8e03cbe0f1ca795f266a43",
"Season 2011/s2011.e0630 - Project Zombie ｜Fin｜.info.json": "83bb5f10310a9a8c5f8f002a6017a40b",
"Season 2011/s2011.e0630 - Project Zombie ｜Fin｜.mp4": "4971cb2d4fa29460361031f3fa8e1ea9",
"Season 2011/s2011.e0630 - Project Zombie ｜Fin｜.nfo": "a464b9c8c48a9a5d4776436d8108f8f5",
"Season 2011/s2011.e1121 - Skyrim 'Ultra HD w⧸Mods' [PC]-thumb.jpg": "1718599d5189c65f7d8cf6acfa5ea851",
"Season 2011/s2011.e1121 - Skyrim 'Ultra HD w⧸Mods' [PC].info.json": "e804c82258849767d5f006627e66ac8f",
"Season 2011/s2011.e1121 - Skyrim 'Ultra HD w⧸Mods' [PC].info.json": "2a139950a598abb1c6d335e6fb5bbe5f",
"Season 2011/s2011.e1121 - Skyrim 'Ultra HD w⧸Mods' [PC].mp4": "55e9b0add08c48c9c66105da0def2426",
"Season 2011/s2011.e1121 - Skyrim 'Ultra HD w⧸Mods' [PC].nfo": "3562934ab9a5e802d955eda24ad355de",
"Season 2012/s2012.e0123 - Project Zombie ｜Map Trailer｜-thumb.jpg": "54ebe9df801b278fdd17b21afa8373a6",
"Season 2012/s2012.e0123 - Project Zombie ｜Map Trailer｜.info.json": "d81b260f37fbf8499d6ba716f5d1c42a",
"Season 2012/s2012.e0123 - Project Zombie ｜Map Trailer｜.info.json": "efc6100c795e73fd4ae4ccab791b4136",
"Season 2012/s2012.e0123 - Project Zombie ｜Map Trailer｜.mp4": "65e4ce53ed5ec4139995469f99477a50",
"Season 2012/s2012.e0123 - Project Zombie ｜Map Trailer｜.nfo": "5e810d839be90dab579400a6177f90b3",
"Season 2013/s2013.e0719 - Project Zombie Rewind ｜Trailer｜-thumb.jpg": "e29d49433175de8a761af35c5307791f",
"Season 2013/s2013.e0719 - Project Zombie Rewind ｜Trailer｜.info.json": "4e450b92e57f8c99033b49e3326cdf72",
"Season 2013/s2013.e0719 - Project Zombie Rewind ｜Trailer｜.info.json": "68b2b622896d086ba8237932725d7430",
"Season 2013/s2013.e0719 - Project Zombie Rewind ｜Trailer｜.mp4": "18620a8257a686beda65e54add4d4cd1",
"Season 2013/s2013.e0719 - Project Zombie Rewind ｜Trailer｜.nfo": "83772bec917bb5d71e1ca0c061c1ec78",
"Season 2018/s2018.e1029 - Jesse's Minecraft Server ｜ Teaser Trailer-thumb.jpg": "705ca4e0d99b37e9ecdf6bfe4b90c59b",
"Season 2018/s2018.e1029 - Jesse's Minecraft Server ｜ Teaser Trailer.info.json": "01a181a123d1b92d016dced5615ca67a",
"Season 2018/s2018.e1029 - Jesse's Minecraft Server ｜ Teaser Trailer.info.json": "3e174f2abeadfecfdcd40dd18ab6f2e1",
"Season 2018/s2018.e1029 - Jesse's Minecraft Server ｜ Teaser Trailer.mp4": "82f6ee7253e1dbb83ae7215af08ffacc",
"Season 2018/s2018.e1029 - Jesse's Minecraft Server ｜ Teaser Trailer.nfo": "368d68db0cbe9eb4f43ece0517445e82",
"Season 2018/s2018.e1102 - Jesse's Minecraft Server ｜ IP mc.jesse.id-thumb.jpg": "28d852ede73b879b9ebf9a061cfc7d46",
"Season 2018/s2018.e1102 - Jesse's Minecraft Server ｜ IP mc.jesse.id.en.srt": "3d2c4e7f65d2ca5e96da38ce7eecfc4e",
"Season 2018/s2018.e1102 - Jesse's Minecraft Server ｜ IP mc.jesse.id.info.json": "2ca573a5eb58e92df951be0a05e8e49c",
"Season 2018/s2018.e1102 - Jesse's Minecraft Server ｜ IP mc.jesse.id.info.json": "070fb6004aec97be575887224b93b229",
"Season 2018/s2018.e1102 - Jesse's Minecraft Server ｜ IP mc.jesse.id.mp4": "e733b4cc385b953b08c8eb0f47e03c1e",
"Season 2018/s2018.e1102 - Jesse's Minecraft Server ｜ IP mc.jesse.id.nfo": "d9114d43d87907b2afc06eb089a8ac0a",
"fanart.jpg": "129c6639b47299bc48062f0365e670ee",
"poster.jpg": "5de28eea5a921a041452ab3ce1041f73",
"tvshow.nfo": "85cef9db54e9afb7af8e4912b5262d3f"
"tvshow.nfo": "c1c888ff6691f36328d1fb9d8c43adff"
}

1
tests/e2e/resources/transaction_log_summaries/youtube/test_channel_full.txt
View file

@ -283,4 +283,5 @@ poster.jpg
tvshow.nfo
NFO tags:
tvshow:
plot: Plugin and map updates for the server Project Zombie.
title: Project / Zombie