From 7ef5947fd7cd64327192717ff705a5356f7675ac Mon Sep 17 00:00:00 2001 From: Jesse Bannon Date: Wed, 31 Aug 2022 22:12:55 -0700 Subject: [PATCH] downloaders have access to overrides --- src/ytdl_sub/downloaders/downloader.py | 34 +++-------------- src/ytdl_sub/downloaders/youtube/channel.py | 38 ++++++++++--------- src/ytdl_sub/downloaders/youtube/playlist.py | 4 +- src/ytdl_sub/downloaders/youtube/video.py | 2 +- src/ytdl_sub/entries/base_entry.py | 3 ++ .../subscriptions/subscription_download.py | 11 +----- .../youtube/test_channel_full.json | 26 ++++++------- .../youtube/test_channel_full.txt | 1 + 8 files changed, 47 insertions(+), 72 deletions(-) diff --git a/src/ytdl_sub/downloaders/downloader.py b/src/ytdl_sub/downloaders/downloader.py index 93792476..f7810c5b 100644 --- a/src/ytdl_sub/downloaders/downloader.py +++ b/src/ytdl_sub/downloaders/downloader.py @@ -88,6 +88,7 @@ class Downloader(DownloadArchiver, Generic[DownloaderOptionsT, DownloaderEntryT] download_options: DownloaderOptionsT, enhanced_download_archive: EnhancedDownloadArchive, ytdl_options_builder: YTDLOptionsBuilder, + overrides: Overrides, ): """ Parameters @@ -98,14 +99,16 @@ class Downloader(DownloadArchiver, Generic[DownloaderOptionsT, DownloaderEntryT] Download archive ytdl_options_builder YTDL options builder + overrides + Override variables """ DownloadArchiver.__init__(self=self, enhanced_download_archive=enhanced_download_archive) self.download_options = download_options + self.overrides = overrides self._ytdl_options_builder = ytdl_options_builder.clone().add( self.ytdl_option_defaults(), before=True ) - self._added_override_variables: Dict[str, str] = {} @contextmanager def ytdl_downloader(self, ytdl_options_overrides: Optional[Dict] = None) -> ytdl.YoutubeDL: @@ -128,28 +131,6 @@ class Downloader(DownloadArchiver, Generic[DownloaderOptionsT, DownloaderEntryT] """ return self._ytdl_options_builder.to_dict().get("skip_download", False) - def add_override_variables(self, override_variables_to_add: Dict[str, str]) -> None: - """ - Override variables added from the downloader. Should be added before yielding - entries - - Parameters - ---------- - override_variables_to_add - The override variables to add - """ - self._added_override_variables = dict( - self._added_override_variables, **override_variables_to_add - ) - - def get_added_override_variables(self) -> Dict[str, str]: - """ - Returns - ------- - Added override variables - """ - return self._added_override_variables - def extract_info(self, ytdl_options_overrides: Optional[Dict] = None, **kwargs) -> Dict: """ Wrapper around yt_dlp.YoutubeDL.YoutubeDL.extract_info @@ -354,14 +335,9 @@ class Downloader(DownloadArchiver, Generic[DownloaderOptionsT, DownloaderEntryT] ) -> Iterable[DownloaderEntryT] | Iterable[Tuple[DownloaderEntryT, FileMetadata]]: """The function to perform the download of all media entries""" - def post_download(self, overrides: Overrides): + def post_download(self): """ After all media entries have been downloaded, post processed, and moved to the output directory, run this function. This lets the downloader add any extra files directly to the output directory, for things like YT channel image, banner. - - Parameters - ---------- - overrides: - Subscription overrides """ diff --git a/src/ytdl_sub/downloaders/youtube/channel.py b/src/ytdl_sub/downloaders/youtube/channel.py index c0c0731a..7d0d8db5 100644 --- a/src/ytdl_sub/downloaders/youtube/channel.py +++ b/src/ytdl_sub/downloaders/youtube/channel.py @@ -137,12 +137,15 @@ class YoutubeChannelDownloader(YoutubeDownloader[YoutubeChannelDownloaderOptions download_options: DownloaderOptionsT, enhanced_download_archive: EnhancedDownloadArchive, ytdl_options_builder: YTDLOptionsBuilder, + overrides: Overrides, ): super().__init__( download_options=download_options, enhanced_download_archive=enhanced_download_archive, ytdl_options_builder=ytdl_options_builder, + overrides=overrides, ) + self.channel: Optional[YoutubeChannel] = None def _get_channel(self, entry_dicts: List[Dict]) -> YoutubeChannel: @@ -151,6 +154,12 @@ class YoutubeChannelDownloader(YoutubeDownloader[YoutubeChannelDownloaderOptions working_directory=self.working_directory, ) + def _get_channel_videos(self, entry_dicts: List[Dict]) -> List[YoutubeVideo]: + return [ + YoutubeVideo(entry_dict=entry_dict, working_directory=self.working_directory) + for entry_dict in self._filter_entry_dicts(entry_dicts, sort_by="playlist_index") + ] + def download(self) -> Generator[YoutubeVideo, None, None]: """ Downloads all videos from a channel @@ -172,21 +181,19 @@ class YoutubeChannelDownloader(YoutubeDownloader[YoutubeChannelDownloaderOptions log_prefix_on_info_json_dl="Downloading metadata for", url=self.download_options.channel_url, ) - self.channel = self._get_channel(entry_dicts=entry_dicts) - self.add_override_variables( - override_variables_to_add={ - "source_description": self.channel.kwargs_get("description", "") - } - ) - channel_videos = self._filter_entry_dicts(entry_dicts, sort_by="playlist_index") + self.channel = self._get_channel(entry_dicts=entry_dicts) + entries_to_download = self._get_channel_videos(entry_dicts=entry_dicts) + + self.overrides.add_override_variables( + variables_to_add={"source_description": self.channel.kwargs_get("description", "")} + ) # Iterate in descending order to process older videos first. In case an error occurs and a # the channel must be redownloaded, it will fetch most recent metadata first, and break # on the older video that's been processed and is in the download archive. - for idx, entry_dict in enumerate(reversed(channel_videos), start=1): - video = YoutubeVideo(entry_dict=entry_dict, working_directory=self.working_directory) - download_logger.info("Downloading %d/%d %s", idx, len(entry_dicts), video.title) + for idx, video in enumerate(reversed(entries_to_download), start=1): + download_logger.info("Downloading %d/%d %s", idx, len(entries_to_download), video.title) # Re-download the contents even if it's a dry-run as a single video. At this time, # channels do not download subtitles or subtitle metadata @@ -226,17 +233,12 @@ class YoutubeChannelDownloader(YoutubeDownloader[YoutubeChannelDownloaderOptions thumbnail_url=thumbnail_url, output_thumbnail_path=output_thumbnail_path ) - def post_download(self, overrides: Overrides): + def post_download(self): """ Downloads and moves channel avatar and banner images to the output directory. - - Parameters - ---------- - overrides - Overrides that can contain variables in the avatar or banner file path """ if self.download_options.channel_avatar_path: - avatar_thumbnail_name = overrides.apply_formatter( + avatar_thumbnail_name = self.overrides.apply_formatter( self.download_options.channel_avatar_path ) self._download_thumbnail( @@ -246,7 +248,7 @@ class YoutubeChannelDownloader(YoutubeDownloader[YoutubeChannelDownloaderOptions self.save_file(file_name=avatar_thumbnail_name) if self.download_options.channel_banner_path: - banner_thumbnail_name = overrides.apply_formatter( + banner_thumbnail_name = self.overrides.apply_formatter( self.download_options.channel_banner_path ) self._download_thumbnail( diff --git a/src/ytdl_sub/downloaders/youtube/playlist.py b/src/ytdl_sub/downloaders/youtube/playlist.py index f8a2e90f..b5394a2f 100644 --- a/src/ytdl_sub/downloaders/youtube/playlist.py +++ b/src/ytdl_sub/downloaders/youtube/playlist.py @@ -82,8 +82,8 @@ class YoutubePlaylistDownloader( ) playlist = self._filter_entry_dicts(entry_dicts, extractor="youtube:tab")[0] - self.add_override_variables( - override_variables_to_add={"source_description": playlist.get("description", "")} + self.overrides.add_override_variables( + variables_to_add={"source_description": playlist.get("description", "")} ) # Iterate in reverse order to process older videos first. In case an error occurs and a diff --git a/src/ytdl_sub/downloaders/youtube/video.py b/src/ytdl_sub/downloaders/youtube/video.py index 5d1930a6..a60a63d8 100644 --- a/src/ytdl_sub/downloaders/youtube/video.py +++ b/src/ytdl_sub/downloaders/youtube/video.py @@ -69,6 +69,6 @@ class YoutubeVideoDownloader(YoutubeDownloader[YoutubeVideoDownloaderOptions, Yo """Download a single Youtube video""" entry_dict = self.extract_info(url=self.download_options.video_url) video = YoutubeVideo(entry_dict=entry_dict, working_directory=self.working_directory) - self.add_override_variables({"source_description": video.description}) + self.overrides.add_override_variables({"source_description": video.description}) return [video] diff --git a/src/ytdl_sub/entries/base_entry.py b/src/ytdl_sub/entries/base_entry.py index 0129e3b5..8eb47b1e 100644 --- a/src/ytdl_sub/entries/base_entry.py +++ b/src/ytdl_sub/entries/base_entry.py @@ -36,6 +36,9 @@ class BaseEntry(ABC): return self._kwargs[key] def kwargs_get(self, key: str, default: Optional[Any] = None) -> Any: + """ + Dict get on kwargs + """ if not self.kwargs_contains(key): return default return self.kwargs(key) diff --git a/src/ytdl_sub/subscriptions/subscription_download.py b/src/ytdl_sub/subscriptions/subscription_download.py index 567a1e72..d04b1d32 100644 --- a/src/ytdl_sub/subscriptions/subscription_download.py +++ b/src/ytdl_sub/subscriptions/subscription_download.py @@ -234,7 +234,6 @@ class SubscriptionDownload(BaseSubscription, ABC): """ self._enhanced_download_archive.reinitialize(dry_run=dry_run) plugins = self._initialize_plugins() - added_override_variables = False ytdl_options_builder = SubscriptionYTDLOptions( preset=self._preset_options, @@ -249,16 +248,10 @@ class SubscriptionDownload(BaseSubscription, ABC): download_options=self.downloader_options, enhanced_download_archive=self._enhanced_download_archive, ytdl_options_builder=ytdl_options_builder, + overrides=self.overrides, ) for entry in downloader.download(): - # TODO: make this a step before download - if not added_override_variables: - self.overrides.add_override_variables( - variables_to_add=downloader.get_added_override_variables() - ) - added_override_variables = True - entry_metadata = FileMetadata() if isinstance(entry, tuple): entry, entry_metadata = entry @@ -272,7 +265,7 @@ class SubscriptionDownload(BaseSubscription, ABC): plugins=plugins, dry_run=dry_run, entry=entry, entry_metadata=entry_metadata ) - downloader.post_download(overrides=self.overrides) + downloader.post_download() for plugin in plugins: plugin.post_process_subscription() diff --git a/tests/e2e/resources/expected_downloads_summaries/youtube/test_channel_full.json b/tests/e2e/resources/expected_downloads_summaries/youtube/test_channel_full.json index a49364f2..93e32319 100644 --- a/tests/e2e/resources/expected_downloads_summaries/youtube/test_channel_full.json +++ b/tests/e2e/resources/expected_downloads_summaries/youtube/test_channel_full.json @@ -1,55 +1,55 @@ { ".ytdl-sub-pz-download-archive.json": "70615451318cdb5e018e007c77893a39", "Season 2010/s2010.e0813 - Oblivion Mod ＂Falcor＂ p.1-thumb.jpg": "fb95b510681676e81c321171fc23143e", - "Season 2010/s2010.e0813 - Oblivion Mod ＂Falcor＂ p.1.info.json": "5d16096bc4256239932db2c2d98161a3", + "Season 2010/s2010.e0813 - Oblivion Mod ＂Falcor＂ p.1.info.json": "a5e93b8087ba3e0fa4a8410b5929eded", "Season 2010/s2010.e0813 - Oblivion Mod ＂Falcor＂ p.1.mp4": "931a705864c57d21d6fedebed4af6bbc", "Season 2010/s2010.e0813 - Oblivion Mod ＂Falcor＂ p.1.nfo": "2d0738094d8e649eaebbab16fd647da1", "Season 2010/s2010.e1202 - Oblivion Mod ＂Falcor＂ p.2-thumb.jpg": "8b32ee9c037fa669e444a0ac181525a1", - "Season 2010/s2010.e1202 - Oblivion Mod ＂Falcor＂ p.2.info.json": "514d2c3eab500f9541910d7573cb495e", + "Season 2010/s2010.e1202 - Oblivion Mod ＂Falcor＂ p.2.info.json": "d7112915f5d437e38f6b1ec633274fcb", "Season 2010/s2010.e1202 - Oblivion Mod ＂Falcor＂ p.2.mp4": "d3469b4dca7139cb3dbc38712b6796bf", "Season 2010/s2010.e1202 - Oblivion Mod ＂Falcor＂ p.2.nfo": "5c258f9e54854ef292ce3c58331da110", "Season 2011/s2011.e0201 - Jesse's Minecraft Server [Trailer - Feb.1]-thumb.jpg": "b232d253df621aa770b780c1301d364d", - "Season 2011/s2011.e0201 - Jesse's Minecraft Server [Trailer - Feb.1].info.json": "f9ccc0b104551ecd7b737451051118aa", + "Season 2011/s2011.e0201 - Jesse's Minecraft Server [Trailer - Feb.1].info.json": "51bb8da215aa43a2824126002d4c9cb5", "Season 2011/s2011.e0201 - Jesse's Minecraft Server [Trailer - Feb.1].mp4": "e66287b9832277b6a4d1554e29d9fdcc", "Season 2011/s2011.e0201 - Jesse's Minecraft Server [Trailer - Feb.1].nfo": "abb3ac33366cc3b86d0467c8fb80a323", "Season 2011/s2011.e0227 - Jesse's Minecraft Server [Trailer - Feb.27]-thumb.jpg": "d17c379ea8b362f5b97c6b213b0342cb", - "Season 2011/s2011.e0227 - Jesse's Minecraft Server [Trailer - Feb.27].info.json": "6ad149bea780198512d73f075722ef82", + "Season 2011/s2011.e0227 - Jesse's Minecraft Server [Trailer - Feb.27].info.json": "a7d62181e7e45640af423312039f9d0f", "Season 2011/s2011.e0227 - Jesse's Minecraft Server [Trailer - Feb.27].mp4": "04ab5cb3cc12325d0c96a7cd04a8b91d", "Season 2011/s2011.e0227 - Jesse's Minecraft Server [Trailer - Feb.27].nfo": "46190954652c9d9812e061fc0c9e1d92", "Season 2011/s2011.e0321 - Jesse's Minecraft Server [Trailer - Mar.21]-thumb.jpg": "e7830aa8a64b0cde65ba3f7e5fc56530", - "Season 2011/s2011.e0321 - Jesse's Minecraft Server [Trailer - Mar.21].info.json": "d040165f8ebc346c202605b19ed8e46b", + "Season 2011/s2011.e0321 - Jesse's Minecraft Server [Trailer - Mar.21].info.json": "14e7e02d3b0fec685dcabd585256a7e0", "Season 2011/s2011.e0321 - Jesse's Minecraft Server [Trailer - Mar.21].mp4": "025de6099a5c98e6397153c7a62d517d", "Season 2011/s2011.e0321 - Jesse's Minecraft Server [Trailer - Mar.21].nfo": "30993fa8e00a0e370b4db244f3da8f7d", "Season 2011/s2011.e0529 - Project Zombie ｜Official Trailer｜ (IP： mc.projectzombie.beastnode.net)-thumb.jpg": "c956192a379b3661595c9920972d4819", - "Season 2011/s2011.e0529 - Project Zombie ｜Official Trailer｜ (IP： mc.projectzombie.beastnode.net).info.json": "331af1c81b1bfa1df62179067a5f06b0", + "Season 2011/s2011.e0529 - Project Zombie ｜Official Trailer｜ (IP： mc.projectzombie.beastnode.net).info.json": "4b97b5e5450bcafaa1b439e05a4559db", "Season 2011/s2011.e0529 - Project Zombie ｜Official Trailer｜ (IP： mc.projectzombie.beastnode.net).mp4": "3d9c19835b03355d6fd5d00cd59dbe5b", "Season 2011/s2011.e0529 - Project Zombie ｜Official Trailer｜ (IP： mc.projectzombie.beastnode.net).nfo": "11a0e8754c414875bcd454358683da5f", "Season 2011/s2011.e0630 - Project Zombie ｜Fin｜-thumb.jpg": "00ed383591779ffe98291de60f198fe9", - "Season 2011/s2011.e0630 - Project Zombie ｜Fin｜.info.json": "07d6b5e38b8e03cbe0f1ca795f266a43", + "Season 2011/s2011.e0630 - Project Zombie ｜Fin｜.info.json": "83bb5f10310a9a8c5f8f002a6017a40b", "Season 2011/s2011.e0630 - Project Zombie ｜Fin｜.mp4": "4971cb2d4fa29460361031f3fa8e1ea9", "Season 2011/s2011.e0630 - Project Zombie ｜Fin｜.nfo": "a464b9c8c48a9a5d4776436d8108f8f5", "Season 2011/s2011.e1121 - Skyrim 'Ultra HD w⧸Mods' [PC]-thumb.jpg": "1718599d5189c65f7d8cf6acfa5ea851", - "Season 2011/s2011.e1121 - Skyrim 'Ultra HD w⧸Mods' [PC].info.json": "e804c82258849767d5f006627e66ac8f", + "Season 2011/s2011.e1121 - Skyrim 'Ultra HD w⧸Mods' [PC].info.json": "2a139950a598abb1c6d335e6fb5bbe5f", "Season 2011/s2011.e1121 - Skyrim 'Ultra HD w⧸Mods' [PC].mp4": "55e9b0add08c48c9c66105da0def2426", "Season 2011/s2011.e1121 - Skyrim 'Ultra HD w⧸Mods' [PC].nfo": "3562934ab9a5e802d955eda24ad355de", "Season 2012/s2012.e0123 - Project Zombie ｜Map Trailer｜-thumb.jpg": "54ebe9df801b278fdd17b21afa8373a6", - "Season 2012/s2012.e0123 - Project Zombie ｜Map Trailer｜.info.json": "d81b260f37fbf8499d6ba716f5d1c42a", + "Season 2012/s2012.e0123 - Project Zombie ｜Map Trailer｜.info.json": "efc6100c795e73fd4ae4ccab791b4136", "Season 2012/s2012.e0123 - Project Zombie ｜Map Trailer｜.mp4": "65e4ce53ed5ec4139995469f99477a50", "Season 2012/s2012.e0123 - Project Zombie ｜Map Trailer｜.nfo": "5e810d839be90dab579400a6177f90b3", "Season 2013/s2013.e0719 - Project Zombie Rewind ｜Trailer｜-thumb.jpg": "e29d49433175de8a761af35c5307791f", - "Season 2013/s2013.e0719 - Project Zombie Rewind ｜Trailer｜.info.json": "4e450b92e57f8c99033b49e3326cdf72", + "Season 2013/s2013.e0719 - Project Zombie Rewind ｜Trailer｜.info.json": "68b2b622896d086ba8237932725d7430", "Season 2013/s2013.e0719 - Project Zombie Rewind ｜Trailer｜.mp4": "18620a8257a686beda65e54add4d4cd1", "Season 2013/s2013.e0719 - Project Zombie Rewind ｜Trailer｜.nfo": "83772bec917bb5d71e1ca0c061c1ec78", "Season 2018/s2018.e1029 - Jesse's Minecraft Server ｜ Teaser Trailer-thumb.jpg": "705ca4e0d99b37e9ecdf6bfe4b90c59b", - "Season 2018/s2018.e1029 - Jesse's Minecraft Server ｜ Teaser Trailer.info.json": "01a181a123d1b92d016dced5615ca67a", + "Season 2018/s2018.e1029 - Jesse's Minecraft Server ｜ Teaser Trailer.info.json": "3e174f2abeadfecfdcd40dd18ab6f2e1", "Season 2018/s2018.e1029 - Jesse's Minecraft Server ｜ Teaser Trailer.mp4": "82f6ee7253e1dbb83ae7215af08ffacc", "Season 2018/s2018.e1029 - Jesse's Minecraft Server ｜ Teaser Trailer.nfo": "368d68db0cbe9eb4f43ece0517445e82", "Season 2018/s2018.e1102 - Jesse's Minecraft Server ｜ IP mc.jesse.id-thumb.jpg": "28d852ede73b879b9ebf9a061cfc7d46", "Season 2018/s2018.e1102 - Jesse's Minecraft Server ｜ IP mc.jesse.id.en.srt": "3d2c4e7f65d2ca5e96da38ce7eecfc4e", - "Season 2018/s2018.e1102 - Jesse's Minecraft Server ｜ IP mc.jesse.id.info.json": "2ca573a5eb58e92df951be0a05e8e49c", + "Season 2018/s2018.e1102 - Jesse's Minecraft Server ｜ IP mc.jesse.id.info.json": "070fb6004aec97be575887224b93b229", "Season 2018/s2018.e1102 - Jesse's Minecraft Server ｜ IP mc.jesse.id.mp4": "e733b4cc385b953b08c8eb0f47e03c1e", "Season 2018/s2018.e1102 - Jesse's Minecraft Server ｜ IP mc.jesse.id.nfo": "d9114d43d87907b2afc06eb089a8ac0a", "fanart.jpg": "129c6639b47299bc48062f0365e670ee", "poster.jpg": "5de28eea5a921a041452ab3ce1041f73", - "tvshow.nfo": "85cef9db54e9afb7af8e4912b5262d3f" + "tvshow.nfo": "c1c888ff6691f36328d1fb9d8c43adff" } \ No newline at end of file diff --git a/tests/e2e/resources/transaction_log_summaries/youtube/test_channel_full.txt b/tests/e2e/resources/transaction_log_summaries/youtube/test_channel_full.txt index b339a15e..04a7ce3e 100644 --- a/tests/e2e/resources/transaction_log_summaries/youtube/test_channel_full.txt +++ b/tests/e2e/resources/transaction_log_summaries/youtube/test_channel_full.txt @@ -283,4 +283,5 @@ poster.jpg tvshow.nfo NFO tags: tvshow: + plot: Plugin and map updates for the server Project Zombie. title: Project / Zombie \ No newline at end of file