[REFACTOR] SourcePluginExtension (#667)

* [REFACTOR] SourcePluginExtension

* param name

* priority to plugin extensions

* info json

src/ytdl_sub/config/plugin.py

@@ -28,7 +28,10 @@ class PluginPriority:
 
     MODIFY_ENTRY_FIRST = 0
 
-    def __init__(self, modify_entry: int = 5, post_process: int = 5):
+    def __init__(
+        self, modify_entry_metadata: int = 5, modify_entry: int = 5, post_process: int = 5
+    ):
+        self.modify_entry_metadata = modify_entry_metadata
         self.modify_entry = modify_entry
         self.post_process = post_process
 

src/ytdl_sub/downloaders/source_plugin.py

@@ -1,8 +1,12 @@
 import abc
 from abc import ABC
+from typing import Dict
 from typing import Generic
 from typing import Iterable
 from typing import List
+from typing import Optional
+from typing import Type
+from typing import final
 
 from ytdl_sub.config.plugin import BasePlugin
 from ytdl_sub.config.plugin import Plugin
@@ -19,8 +23,18 @@ class SourcePluginExtension(Plugin[TOptionsValidator], ABC):
     are the plugin options.
     """
 
+    @final
+    def ytdl_options(self) -> Optional[Dict]:
+        """
+        SourcePluginExtensions are intended to run after downloading. ytdl_options at that point
+        are not needed.
+        """
+        return None
+
 
 class SourcePlugin(BasePlugin[TOptionsValidator], Generic[TOptionsValidator], ABC):
+    plugin_extensions: List[Type[SourcePluginExtension]] = []
+
     def __init__(
         self,
         options: TOptionsValidator,
@@ -45,15 +59,14 @@ class SourcePlugin(BasePlugin[TOptionsValidator], Generic[TOptionsValidator], AB
     def download(self, entry: Entry) -> Entry:
         """The function to perform the download of all media entries"""
 
-    # pylint: disable=unused-argument
+    @final
-    @classmethod
+    def added_plugins(self) -> List[SourcePluginExtension]:
-    def added_plugins(
-        cls,
-        downloader_options: TOptionsValidator,
-        enhanced_download_archive: EnhancedDownloadArchive,
-        overrides: Overrides,
-    ) -> List[SourcePluginExtension]:
         """Add these plugins from the Downloader to the subscription"""
-        return []
+        return [
-
+            plugin_extension(
-    # pylint: enable=unused-argument
+                options=self.plugin_options,
+                overrides=self.overrides,
+                enhanced_download_archive=self._enhanced_download_archive,
+            )
+            for plugin_extension in self.plugin_extensions
+        ]

src/ytdl_sub/downloaders/url/downloader.py

@@ -12,7 +12,7 @@ from typing import Tuple
 
 from yt_dlp.utils import RejectedVideoReached
 
-from ytdl_sub.config.plugin import Plugin
+from ytdl_sub.config.plugin import PluginPriority
 from ytdl_sub.config.preset_options import Overrides
 from ytdl_sub.downloaders.source_plugin import SourcePlugin
 from ytdl_sub.downloaders.source_plugin import SourcePluginExtension
@@ -55,6 +55,8 @@ class URLDownloadState:
 
 
 class UrlDownloaderThumbnailPlugin(SourcePluginExtension):
+    priority = PluginPriority(modify_entry=0)
+
     def __init__(
         self,
         options: MultiUrlSourceOptionsValidator,
@@ -154,21 +156,23 @@ class UrlDownloaderThumbnailPlugin(SourcePluginExtension):
 
 
 class UrlDownloaderCollectionVariablePlugin(SourcePluginExtension):
+    priority = PluginPriority(modify_entry_metadata=0)
+
     def __init__(
         self,
-        downloader_options: MultiUrlSourceOptionsValidator,
+        options: MultiUrlSourceOptionsValidator,
         overrides: Overrides,
         enhanced_download_archive: EnhancedDownloadArchive,
     ):
         super().__init__(
-            options=downloader_options,
+            options=options,
             overrides=overrides,
             enhanced_download_archive=enhanced_download_archive,
         )
         self._thumbnails_downloaded: Set[str] = set()
         self._collection_url_mapping: Dict[str, UrlValidator] = {
             self.overrides.apply_formatter(collection_url.url): collection_url
-            for collection_url in downloader_options.collection_validator.urls.list
+            for collection_url in options.collection_validator.urls.list
         }
 
     def modify_entry_metadata(self, entry: Entry) -> Optional[Entry]:
@@ -196,30 +200,7 @@ class BaseUrlDownloader(SourcePlugin[TMultiURLSourceOptionsValidator], ABC):
     and should translate that to list of Entry objects.
     """
 
-    @classmethod
+    plugin_extensions = [UrlDownloaderThumbnailPlugin, UrlDownloaderCollectionVariablePlugin]
-    def added_plugins(
-        cls,
-        downloader_options: TMultiURLSourceOptionsValidator,
-        enhanced_download_archive: EnhancedDownloadArchive,
-        overrides: Overrides,
-    ) -> List[Plugin]:
-        """
-        Adds
-        1. URL thumbnail download plugin
-        2. Collection variable plugin to add to each entry
-        """
-        return [
-            UrlDownloaderThumbnailPlugin(
-                options=downloader_options,
-                overrides=overrides,
-                enhanced_download_archive=enhanced_download_archive,
-            ),
-            UrlDownloaderCollectionVariablePlugin(
-                downloader_options=downloader_options,
-                overrides=overrides,
-                enhanced_download_archive=enhanced_download_archive,
-            ),
-        ]
 
     @classmethod
     def ytdl_option_defaults(cls) -> Dict:

src/ytdl_sub/subscriptions/subscription_download.py

@@ -174,23 +174,14 @@ class SubscriptionDownload(BaseSubscription, ABC):
         -------
         List of plugins defined in the subscription, initialized and ready to use.
         """
-        # Always add plugins provided by the downloader
+        return [
-        plugins: List[Plugin] = self.downloader_class.added_plugins(
+            plugin_type(
-            downloader_options=self.downloader_options,
-            enhanced_download_archive=self._enhanced_download_archive,
-            overrides=self.overrides,
-        )
-
-        for plugin_type, plugin_options in self.plugins.zipped():
-            plugin = plugin_type(
                 options=plugin_options,
                 overrides=self.overrides,
                 enhanced_download_archive=self._enhanced_download_archive,
             )
-
+            for plugin_type, plugin_options in self.plugins.zipped()
-            plugins.append(plugin)
+        ]
-
-        return plugins
 
     @classmethod
     def _cleanup_entry_files(cls, entry: Entry):
@@ -201,7 +192,7 @@ class SubscriptionDownload(BaseSubscription, ABC):
     @classmethod
     def _preprocess_entry(cls, plugins: List[Plugin], entry: Entry) -> Optional[Entry]:
         maybe_entry: Optional[Entry] = entry
-        for plugin in plugins:
+        for plugin in sorted(plugins, key=lambda _plugin: _plugin.priority.modify_entry_metadata):
             if (maybe_entry := plugin.modify_entry_metadata(maybe_entry)) is None:
                 return None
 
@@ -347,6 +338,8 @@ class SubscriptionDownload(BaseSubscription, ABC):
             overrides=self.overrides,
         )
 
+        plugins.extend(downloader.added_plugins())
+
         return self._process_subscription(
             plugins=plugins,
             downloader=downloader,
@@ -365,6 +358,25 @@ class SubscriptionDownload(BaseSubscription, ABC):
         self._enhanced_download_archive.reinitialize(dry_run=dry_run)
         plugins = self._initialize_plugins()
 
+        subscription_ytdl_options = SubscriptionYTDLOptions(
+            preset=self._preset_options,
+            plugins=plugins,
+            enhanced_download_archive=self._enhanced_download_archive,
+            working_directory=self.working_directory,
+            dry_run=dry_run,
+        )
+
+        # Re-add the original downloader class' plugins
+        plugins.extend(
+            self.downloader_class(
+                options=self.downloader_options,
+                enhanced_download_archive=self._enhanced_download_archive,
+                download_ytdl_options=subscription_ytdl_options.download_builder(),
+                metadata_ytdl_options=subscription_ytdl_options.metadata_builder(),
+                overrides=self.overrides,
+            ).added_plugins()
+        )
+
         downloader = InfoJsonDownloader(
             options=InfoJsonDownloaderOptions(name="no-op", value={}),
             enhanced_download_archive=self._enhanced_download_archive,