downloaders have access to overrides

src/ytdl_sub/downloaders/downloader.py

@@ -88,6 +88,7 @@ class Downloader(DownloadArchiver, Generic[DownloaderOptionsT, DownloaderEntryT]
         download_options: DownloaderOptionsT,
         enhanced_download_archive: EnhancedDownloadArchive,
         ytdl_options_builder: YTDLOptionsBuilder,
+        overrides: Overrides,
     ):
         """
         Parameters
@@ -98,14 +99,16 @@ class Downloader(DownloadArchiver, Generic[DownloaderOptionsT, DownloaderEntryT]
             Download archive
         ytdl_options_builder
             YTDL options builder
+        overrides
+            Override variables
         """
         DownloadArchiver.__init__(self=self, enhanced_download_archive=enhanced_download_archive)
         self.download_options = download_options
+        self.overrides = overrides
 
         self._ytdl_options_builder = ytdl_options_builder.clone().add(
             self.ytdl_option_defaults(), before=True
         )
-        self._added_override_variables: Dict[str, str] = {}
 
     @contextmanager
     def ytdl_downloader(self, ytdl_options_overrides: Optional[Dict] = None) -> ytdl.YoutubeDL:
@@ -128,28 +131,6 @@ class Downloader(DownloadArchiver, Generic[DownloaderOptionsT, DownloaderEntryT]
         """
         return self._ytdl_options_builder.to_dict().get("skip_download", False)
 
-    def add_override_variables(self, override_variables_to_add: Dict[str, str]) -> None:
-        """
-        Override variables added from the downloader. Should be added before yielding
-        entries
-
-        Parameters
-        ----------
-        override_variables_to_add
-            The override variables to add
-        """
-        self._added_override_variables = dict(
-            self._added_override_variables, **override_variables_to_add
-        )
-
-    def get_added_override_variables(self) -> Dict[str, str]:
-        """
-        Returns
-        -------
-        Added override variables
-        """
-        return self._added_override_variables
-
     def extract_info(self, ytdl_options_overrides: Optional[Dict] = None, **kwargs) -> Dict:
         """
         Wrapper around yt_dlp.YoutubeDL.YoutubeDL.extract_info
@@ -354,14 +335,9 @@ class Downloader(DownloadArchiver, Generic[DownloaderOptionsT, DownloaderEntryT]
     ) -> Iterable[DownloaderEntryT] | Iterable[Tuple[DownloaderEntryT, FileMetadata]]:
         """The function to perform the download of all media entries"""
 
-    def post_download(self, overrides: Overrides):
+    def post_download(self):
         """
         After all media entries have been downloaded, post processed, and moved to the output
         directory, run this function. This lets the downloader add any extra files directly to the
         output directory, for things like YT channel image, banner.
-
-        Parameters
-        ----------
-        overrides:
-            Subscription overrides
         """

src/ytdl_sub/downloaders/youtube/channel.py

@@ -137,12 +137,15 @@ class YoutubeChannelDownloader(YoutubeDownloader[YoutubeChannelDownloaderOptions
         download_options: DownloaderOptionsT,
         enhanced_download_archive: EnhancedDownloadArchive,
         ytdl_options_builder: YTDLOptionsBuilder,
+        overrides: Overrides,
     ):
         super().__init__(
             download_options=download_options,
             enhanced_download_archive=enhanced_download_archive,
             ytdl_options_builder=ytdl_options_builder,
+            overrides=overrides,
         )
+
         self.channel: Optional[YoutubeChannel] = None
 
     def _get_channel(self, entry_dicts: List[Dict]) -> YoutubeChannel:
@@ -151,6 +154,12 @@ class YoutubeChannelDownloader(YoutubeDownloader[YoutubeChannelDownloaderOptions
             working_directory=self.working_directory,
         )
 
+    def _get_channel_videos(self, entry_dicts: List[Dict]) -> List[YoutubeVideo]:
+        return [
+            YoutubeVideo(entry_dict=entry_dict, working_directory=self.working_directory)
+            for entry_dict in self._filter_entry_dicts(entry_dicts, sort_by="playlist_index")
+        ]
+
     def download(self) -> Generator[YoutubeVideo, None, None]:
         """
         Downloads all videos from a channel
@@ -172,21 +181,19 @@ class YoutubeChannelDownloader(YoutubeDownloader[YoutubeChannelDownloaderOptions
             log_prefix_on_info_json_dl="Downloading metadata for",
             url=self.download_options.channel_url,
         )
-        self.channel = self._get_channel(entry_dicts=entry_dicts)
-        self.add_override_variables(
-            override_variables_to_add={
-                "source_description": self.channel.kwargs_get("description", "")
-            }
-        )
 
-        channel_videos = self._filter_entry_dicts(entry_dicts, sort_by="playlist_index")
+        self.channel = self._get_channel(entry_dicts=entry_dicts)
+        entries_to_download = self._get_channel_videos(entry_dicts=entry_dicts)
+
+        self.overrides.add_override_variables(
+            variables_to_add={"source_description": self.channel.kwargs_get("description", "")}
+        )
 
         # Iterate in descending order to process older videos first. In case an error occurs and a
         # the channel must be redownloaded, it will fetch most recent metadata first, and break
         # on the older video that's been processed and is in the download archive.
-        for idx, entry_dict in enumerate(reversed(channel_videos), start=1):
+        for idx, video in enumerate(reversed(entries_to_download), start=1):
-            video = YoutubeVideo(entry_dict=entry_dict, working_directory=self.working_directory)
+            download_logger.info("Downloading %d/%d %s", idx, len(entries_to_download), video.title)
-            download_logger.info("Downloading %d/%d %s", idx, len(entry_dicts), video.title)
 
             # Re-download the contents even if it's a dry-run as a single video. At this time,
             # channels do not download subtitles or subtitle metadata
@@ -226,17 +233,12 @@ class YoutubeChannelDownloader(YoutubeDownloader[YoutubeChannelDownloaderOptions
             thumbnail_url=thumbnail_url, output_thumbnail_path=output_thumbnail_path
         )
 
-    def post_download(self, overrides: Overrides):
+    def post_download(self):
         """
         Downloads and moves channel avatar and banner images to the output directory.
-
-        Parameters
-        ----------
-        overrides
-            Overrides that can contain variables in the avatar or banner file path
         """
         if self.download_options.channel_avatar_path:
-            avatar_thumbnail_name = overrides.apply_formatter(
+            avatar_thumbnail_name = self.overrides.apply_formatter(
                 self.download_options.channel_avatar_path
             )
             self._download_thumbnail(
@@ -246,7 +248,7 @@ class YoutubeChannelDownloader(YoutubeDownloader[YoutubeChannelDownloaderOptions
             self.save_file(file_name=avatar_thumbnail_name)
 
         if self.download_options.channel_banner_path:
-            banner_thumbnail_name = overrides.apply_formatter(
+            banner_thumbnail_name = self.overrides.apply_formatter(
                 self.download_options.channel_banner_path
             )
             self._download_thumbnail(

src/ytdl_sub/downloaders/youtube/playlist.py

@@ -82,8 +82,8 @@ class YoutubePlaylistDownloader(
         )
 
         playlist = self._filter_entry_dicts(entry_dicts, extractor="youtube:tab")[0]
-        self.add_override_variables(
+        self.overrides.add_override_variables(
-            override_variables_to_add={"source_description": playlist.get("description", "")}
+            variables_to_add={"source_description": playlist.get("description", "")}
         )
 
         # Iterate in reverse order to process older videos first. In case an error occurs and a

src/ytdl_sub/downloaders/youtube/video.py

@@ -69,6 +69,6 @@ class YoutubeVideoDownloader(YoutubeDownloader[YoutubeVideoDownloaderOptions, Yo
         """Download a single Youtube video"""
         entry_dict = self.extract_info(url=self.download_options.video_url)
         video = YoutubeVideo(entry_dict=entry_dict, working_directory=self.working_directory)
-        self.add_override_variables({"source_description": video.description})
+        self.overrides.add_override_variables({"source_description": video.description})
 
         return [video]

src/ytdl_sub/entries/base_entry.py

@@ -36,6 +36,9 @@ class BaseEntry(ABC):
         return self._kwargs[key]
 
     def kwargs_get(self, key: str, default: Optional[Any] = None) -> Any:
+        """
+        Dict get on kwargs
+        """
         if not self.kwargs_contains(key):
             return default
         return self.kwargs(key)

src/ytdl_sub/subscriptions/subscription_download.py

@@ -234,7 +234,6 @@ class SubscriptionDownload(BaseSubscription, ABC):
         """
         self._enhanced_download_archive.reinitialize(dry_run=dry_run)
         plugins = self._initialize_plugins()
-        added_override_variables = False
 
         ytdl_options_builder = SubscriptionYTDLOptions(
             preset=self._preset_options,
@@ -249,16 +248,10 @@ class SubscriptionDownload(BaseSubscription, ABC):
                 download_options=self.downloader_options,
                 enhanced_download_archive=self._enhanced_download_archive,
                 ytdl_options_builder=ytdl_options_builder,
+                overrides=self.overrides,
             )
 
             for entry in downloader.download():
-                # TODO: make this a step before download
-                if not added_override_variables:
-                    self.overrides.add_override_variables(
-                        variables_to_add=downloader.get_added_override_variables()
-                    )
-                    added_override_variables = True
-
                 entry_metadata = FileMetadata()
                 if isinstance(entry, tuple):
                     entry, entry_metadata = entry
@@ -272,7 +265,7 @@ class SubscriptionDownload(BaseSubscription, ABC):
                         plugins=plugins, dry_run=dry_run, entry=entry, entry_metadata=entry_metadata
                     )
 
-            downloader.post_download(overrides=self.overrides)
+            downloader.post_download()
             for plugin in plugins:
                 plugin.post_process_subscription()
 

tests/e2e/resources/expected_downloads_summaries/youtube/test_channel_full.json

@@ -1,55 +1,55 @@
 {
   ".ytdl-sub-pz-download-archive.json": "70615451318cdb5e018e007c77893a39",
   "Season 2010/s2010.e0813 - Oblivion Mod ＂Falcor＂ p.1-thumb.jpg": "fb95b510681676e81c321171fc23143e",
-  "Season 2010/s2010.e0813 - Oblivion Mod ＂Falcor＂ p.1.info.json": "5d16096bc4256239932db2c2d98161a3",
+  "Season 2010/s2010.e0813 - Oblivion Mod ＂Falcor＂ p.1.info.json": "a5e93b8087ba3e0fa4a8410b5929eded",
   "Season 2010/s2010.e0813 - Oblivion Mod ＂Falcor＂ p.1.mp4": "931a705864c57d21d6fedebed4af6bbc",
   "Season 2010/s2010.e0813 - Oblivion Mod ＂Falcor＂ p.1.nfo": "2d0738094d8e649eaebbab16fd647da1",
   "Season 2010/s2010.e1202 - Oblivion Mod ＂Falcor＂ p.2-thumb.jpg": "8b32ee9c037fa669e444a0ac181525a1",
-  "Season 2010/s2010.e1202 - Oblivion Mod ＂Falcor＂ p.2.info.json": "514d2c3eab500f9541910d7573cb495e",
+  "Season 2010/s2010.e1202 - Oblivion Mod ＂Falcor＂ p.2.info.json": "d7112915f5d437e38f6b1ec633274fcb",
   "Season 2010/s2010.e1202 - Oblivion Mod ＂Falcor＂ p.2.mp4": "d3469b4dca7139cb3dbc38712b6796bf",
   "Season 2010/s2010.e1202 - Oblivion Mod ＂Falcor＂ p.2.nfo": "5c258f9e54854ef292ce3c58331da110",
   "Season 2011/s2011.e0201 - Jesse's Minecraft Server [Trailer - Feb.1]-thumb.jpg": "b232d253df621aa770b780c1301d364d",
-  "Season 2011/s2011.e0201 - Jesse's Minecraft Server [Trailer - Feb.1].info.json": "f9ccc0b104551ecd7b737451051118aa",
+  "Season 2011/s2011.e0201 - Jesse's Minecraft Server [Trailer - Feb.1].info.json": "51bb8da215aa43a2824126002d4c9cb5",
   "Season 2011/s2011.e0201 - Jesse's Minecraft Server [Trailer - Feb.1].mp4": "e66287b9832277b6a4d1554e29d9fdcc",
   "Season 2011/s2011.e0201 - Jesse's Minecraft Server [Trailer - Feb.1].nfo": "abb3ac33366cc3b86d0467c8fb80a323",
   "Season 2011/s2011.e0227 - Jesse's Minecraft Server [Trailer - Feb.27]-thumb.jpg": "d17c379ea8b362f5b97c6b213b0342cb",
-  "Season 2011/s2011.e0227 - Jesse's Minecraft Server [Trailer - Feb.27].info.json": "6ad149bea780198512d73f075722ef82",
+  "Season 2011/s2011.e0227 - Jesse's Minecraft Server [Trailer - Feb.27].info.json": "a7d62181e7e45640af423312039f9d0f",
   "Season 2011/s2011.e0227 - Jesse's Minecraft Server [Trailer - Feb.27].mp4": "04ab5cb3cc12325d0c96a7cd04a8b91d",
   "Season 2011/s2011.e0227 - Jesse's Minecraft Server [Trailer - Feb.27].nfo": "46190954652c9d9812e061fc0c9e1d92",
   "Season 2011/s2011.e0321 - Jesse's Minecraft Server [Trailer - Mar.21]-thumb.jpg": "e7830aa8a64b0cde65ba3f7e5fc56530",
-  "Season 2011/s2011.e0321 - Jesse's Minecraft Server [Trailer - Mar.21].info.json": "d040165f8ebc346c202605b19ed8e46b",
+  "Season 2011/s2011.e0321 - Jesse's Minecraft Server [Trailer - Mar.21].info.json": "14e7e02d3b0fec685dcabd585256a7e0",
   "Season 2011/s2011.e0321 - Jesse's Minecraft Server [Trailer - Mar.21].mp4": "025de6099a5c98e6397153c7a62d517d",
   "Season 2011/s2011.e0321 - Jesse's Minecraft Server [Trailer - Mar.21].nfo": "30993fa8e00a0e370b4db244f3da8f7d",
   "Season 2011/s2011.e0529 - Project Zombie ｜Official Trailer｜ (IP： mc.projectzombie.beastnode.net)-thumb.jpg": "c956192a379b3661595c9920972d4819",
-  "Season 2011/s2011.e0529 - Project Zombie ｜Official Trailer｜ (IP： mc.projectzombie.beastnode.net).info.json": "331af1c81b1bfa1df62179067a5f06b0",
+  "Season 2011/s2011.e0529 - Project Zombie ｜Official Trailer｜ (IP： mc.projectzombie.beastnode.net).info.json": "4b97b5e5450bcafaa1b439e05a4559db",
   "Season 2011/s2011.e0529 - Project Zombie ｜Official Trailer｜ (IP： mc.projectzombie.beastnode.net).mp4": "3d9c19835b03355d6fd5d00cd59dbe5b",
   "Season 2011/s2011.e0529 - Project Zombie ｜Official Trailer｜ (IP： mc.projectzombie.beastnode.net).nfo": "11a0e8754c414875bcd454358683da5f",
   "Season 2011/s2011.e0630 - Project Zombie ｜Fin｜-thumb.jpg": "00ed383591779ffe98291de60f198fe9",
-  "Season 2011/s2011.e0630 - Project Zombie ｜Fin｜.info.json": "07d6b5e38b8e03cbe0f1ca795f266a43",
+  "Season 2011/s2011.e0630 - Project Zombie ｜Fin｜.info.json": "83bb5f10310a9a8c5f8f002a6017a40b",
   "Season 2011/s2011.e0630 - Project Zombie ｜Fin｜.mp4": "4971cb2d4fa29460361031f3fa8e1ea9",
   "Season 2011/s2011.e0630 - Project Zombie ｜Fin｜.nfo": "a464b9c8c48a9a5d4776436d8108f8f5",
   "Season 2011/s2011.e1121 - Skyrim 'Ultra HD w⧸Mods' [PC]-thumb.jpg": "1718599d5189c65f7d8cf6acfa5ea851",
-  "Season 2011/s2011.e1121 - Skyrim 'Ultra HD w⧸Mods' [PC].info.json": "e804c82258849767d5f006627e66ac8f",
+  "Season 2011/s2011.e1121 - Skyrim 'Ultra HD w⧸Mods' [PC].info.json": "2a139950a598abb1c6d335e6fb5bbe5f",
   "Season 2011/s2011.e1121 - Skyrim 'Ultra HD w⧸Mods' [PC].mp4": "55e9b0add08c48c9c66105da0def2426",
   "Season 2011/s2011.e1121 - Skyrim 'Ultra HD w⧸Mods' [PC].nfo": "3562934ab9a5e802d955eda24ad355de",
   "Season 2012/s2012.e0123 - Project Zombie ｜Map Trailer｜-thumb.jpg": "54ebe9df801b278fdd17b21afa8373a6",
-  "Season 2012/s2012.e0123 - Project Zombie ｜Map Trailer｜.info.json": "d81b260f37fbf8499d6ba716f5d1c42a",
+  "Season 2012/s2012.e0123 - Project Zombie ｜Map Trailer｜.info.json": "efc6100c795e73fd4ae4ccab791b4136",
   "Season 2012/s2012.e0123 - Project Zombie ｜Map Trailer｜.mp4": "65e4ce53ed5ec4139995469f99477a50",
   "Season 2012/s2012.e0123 - Project Zombie ｜Map Trailer｜.nfo": "5e810d839be90dab579400a6177f90b3",
   "Season 2013/s2013.e0719 - Project Zombie Rewind ｜Trailer｜-thumb.jpg": "e29d49433175de8a761af35c5307791f",
-  "Season 2013/s2013.e0719 - Project Zombie Rewind ｜Trailer｜.info.json": "4e450b92e57f8c99033b49e3326cdf72",
+  "Season 2013/s2013.e0719 - Project Zombie Rewind ｜Trailer｜.info.json": "68b2b622896d086ba8237932725d7430",
   "Season 2013/s2013.e0719 - Project Zombie Rewind ｜Trailer｜.mp4": "18620a8257a686beda65e54add4d4cd1",
   "Season 2013/s2013.e0719 - Project Zombie Rewind ｜Trailer｜.nfo": "83772bec917bb5d71e1ca0c061c1ec78",
   "Season 2018/s2018.e1029 - Jesse's Minecraft Server ｜ Teaser Trailer-thumb.jpg": "705ca4e0d99b37e9ecdf6bfe4b90c59b",
-  "Season 2018/s2018.e1029 - Jesse's Minecraft Server ｜ Teaser Trailer.info.json": "01a181a123d1b92d016dced5615ca67a",
+  "Season 2018/s2018.e1029 - Jesse's Minecraft Server ｜ Teaser Trailer.info.json": "3e174f2abeadfecfdcd40dd18ab6f2e1",
   "Season 2018/s2018.e1029 - Jesse's Minecraft Server ｜ Teaser Trailer.mp4": "82f6ee7253e1dbb83ae7215af08ffacc",
   "Season 2018/s2018.e1029 - Jesse's Minecraft Server ｜ Teaser Trailer.nfo": "368d68db0cbe9eb4f43ece0517445e82",
   "Season 2018/s2018.e1102 - Jesse's Minecraft Server ｜ IP mc.jesse.id-thumb.jpg": "28d852ede73b879b9ebf9a061cfc7d46",
   "Season 2018/s2018.e1102 - Jesse's Minecraft Server ｜ IP mc.jesse.id.en.srt": "3d2c4e7f65d2ca5e96da38ce7eecfc4e",
-  "Season 2018/s2018.e1102 - Jesse's Minecraft Server ｜ IP mc.jesse.id.info.json": "2ca573a5eb58e92df951be0a05e8e49c",
+  "Season 2018/s2018.e1102 - Jesse's Minecraft Server ｜ IP mc.jesse.id.info.json": "070fb6004aec97be575887224b93b229",
   "Season 2018/s2018.e1102 - Jesse's Minecraft Server ｜ IP mc.jesse.id.mp4": "e733b4cc385b953b08c8eb0f47e03c1e",
   "Season 2018/s2018.e1102 - Jesse's Minecraft Server ｜ IP mc.jesse.id.nfo": "d9114d43d87907b2afc06eb089a8ac0a",
   "fanart.jpg": "129c6639b47299bc48062f0365e670ee",
   "poster.jpg": "5de28eea5a921a041452ab3ce1041f73",
-  "tvshow.nfo": "85cef9db54e9afb7af8e4912b5262d3f"
+  "tvshow.nfo": "c1c888ff6691f36328d1fb9d8c43adff"
 }

tests/e2e/resources/transaction_log_summaries/youtube/test_channel_full.txt

@@ -283,4 +283,5 @@ poster.jpg
 tvshow.nfo
   NFO tags:
     tvshow:
+      plot: Plugin and map updates for the server Project Zombie.
       title: Project / Zombie