[REFACTOR] Shared OptionsValidator, downloader plugins (#550)

src/ytdl_sub/config/preset.py

@@ -6,7 +6,6 @@ from typing import List
 from typing import Optional
 from typing import Tuple
 from typing import Type
-from typing import TypeVar
 from typing import Union
 
 from mergedeep import mergedeep
@@ -22,6 +21,7 @@ from ytdl_sub.downloaders.base_downloader import BaseDownloaderValidator
 from ytdl_sub.entries.entry import Entry
 from ytdl_sub.plugins.plugin import Plugin
 from ytdl_sub.plugins.plugin import PluginOptions
+from ytdl_sub.plugins.plugin import PluginOptionsT
 from ytdl_sub.prebuilt_presets import PREBUILT_PRESET_NAMES
 from ytdl_sub.prebuilt_presets import PUBLISHED_PRESET_NAMES
 from ytdl_sub.utils.exceptions import ValidationException
@@ -65,8 +65,6 @@ def _parent_preset_error_message(
 
 
 class PresetPlugins:
-    _TPluginOptions = TypeVar("_TPluginOptions", bound=PluginOptions)
-
     def __init__(self):
         self.plugin_types: List[Type[Plugin]] = []
         self.plugin_options: List[PluginOptions] = []
@@ -87,7 +85,7 @@ class PresetPlugins:
         """
         return zip(self.plugin_types, self.plugin_options)
 
-    def get(self, plugin_type: Type[_TPluginOptions]) -> Optional[_TPluginOptions]:
+    def get(self, plugin_type: Type[PluginOptionsT]) -> Optional[PluginOptionsT]:
         """
         Parameters
         ----------

src/ytdl_sub/config/preset_options.py

@@ -7,6 +7,7 @@ from typing import Optional
 from yt_dlp.utils import sanitize_filename
 
 from ytdl_sub.entries.entry import Entry
+from ytdl_sub.utils.exceptions import ValidationException
 from ytdl_sub.validators.file_path_validators import OverridesStringFormatterFilePathValidator
 from ytdl_sub.validators.file_path_validators import StringFormatterFileNameValidator
 from ytdl_sub.validators.strict_dict_validator import StrictDictValidator
@@ -20,11 +21,28 @@ from ytdl_sub.validators.validators import LiteralDictValidator
 
 # pylint: disable=no-self-use
 # pylint: disable=unused-argument
-class AddsVariablesMixin(ABC):
+class OptionsValidator(StrictDictValidator, ABC):
     """
-    Mixin for parts of the Preset that adds source variables
+    Abstract class that validates options for preset sections (plugins, downloaders)
     """
 
+    def validation_exception(
+        self,
+        error_message: str | Exception,
+    ) -> ValidationException:
+        """
+        Parameters
+        ----------
+        error_message
+            Error message to include in the validation exception
+
+        Returns
+        -------
+        Validation exception that points to the location in the config. To be used to throw good
+        validation exceptions at runtime from code outside this class.
+        """
+        return self._validation_exception(error_message=error_message)
+
     def added_source_variables(self) -> List[str]:
         """
         If the plugin adds source variables, list them here.

src/ytdl_sub/downloaders/base_downloader.py

@@ -6,38 +6,33 @@ from typing import List
 from typing import Type
 from typing import TypeVar
 
-from ytdl_sub.config.preset_options import AddsVariablesMixin
+from ytdl_sub.config.preset_options import OptionsValidator
 from ytdl_sub.config.preset_options import Overrides
 from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder
 from ytdl_sub.entries.entry import Entry
 from ytdl_sub.plugins.plugin import Plugin
-from ytdl_sub.plugins.plugin import PluginOptions
-from ytdl_sub.validators.strict_dict_validator import StrictDictValidator
 from ytdl_sub.ytdl_additions.enhanced_download_archive import DownloadArchiver
 from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive
 
-
+BaseDownloaderValidator = OptionsValidator
-class BaseDownloaderValidator(StrictDictValidator, AddsVariablesMixin, ABC):
-    pass
-
-
 BaseDownloaderOptionsT = TypeVar("BaseDownloaderOptionsT", bound=BaseDownloaderValidator)
 
 
-class BaseDownloaderPluginOptions(PluginOptions):
+class BaseDownloaderPlugin(Plugin[BaseDownloaderOptionsT], ABC):
-    _optional_keys = {"no-op"}
+    """
+    Plugins that get added automatically by using a downloader. Downloader options
+    are the plugin options.
+    """
 
-
-class BaseDownloaderPlugin(Plugin[BaseDownloaderPluginOptions], ABC):
     def __init__(
         self,
+        downloader_options: BaseDownloaderOptionsT,
         overrides: Overrides,
         enhanced_download_archive: EnhancedDownloadArchive,
     ):
         super().__init__(
-            # Downloader plugins do not have exposed YAML options, so keep it blank.
+            # Downloader plugins use download options as their plugin options
-            # Use init instead.
+            plugin_options=downloader_options,
-            plugin_options=BaseDownloaderPluginOptions(name=self.__class__.__name__, value={}),
             overrides=overrides,
             enhanced_download_archive=enhanced_download_archive,
         )
@@ -68,9 +63,15 @@ class BaseDownloader(DownloadArchiver, Generic[BaseDownloaderOptionsT], ABC):
     def download(self, entry: Entry) -> Entry:
         """The function to perform the download of all media entries"""
 
-    # pylint: disable=no-self-use
+    # pylint: disable=unused-argument
-    def added_plugins(self) -> List[BaseDownloaderPlugin]:
+    @classmethod
+    def added_plugins(
+        cls,
+        downloader_options: BaseDownloaderOptionsT,
+        enhanced_download_archive: EnhancedDownloadArchive,
+        overrides: Overrides,
+    ) -> List[BaseDownloaderPlugin]:
         """Add these plugins from the Downloader to the subscription"""
         return []
 
-    # pylint: enable=no-self-use
+    # pylint: enable=unused-argument

src/ytdl_sub/downloaders/url/downloader.py

@@ -87,18 +87,19 @@ class URLDownloadState:
 class UrlDownloaderThumbnailPlugin(BaseDownloaderPlugin):
     def __init__(
         self,
+        downloader_options: DownloaderValidator,
         overrides: Overrides,
         enhanced_download_archive: EnhancedDownloadArchive,
-        collection_urls: List[UrlValidator],
     ):
         super().__init__(
+            downloader_options=downloader_options,
             overrides=overrides,
             enhanced_download_archive=enhanced_download_archive,
         )
         self._thumbnails_downloaded: Set[str] = set()
         self._collection_url_mapping: Dict[str, UrlValidator] = {
             self.overrides.apply_formatter(collection_url.url): collection_url
-            for collection_url in collection_urls
+            for collection_url in downloader_options.collection_validator.urls.list
         }
 
     def _download_parent_thumbnails(
@@ -185,18 +186,19 @@ class UrlDownloaderThumbnailPlugin(BaseDownloaderPlugin):
 class UrlDownloaderCollectionVariablePlugin(BaseDownloaderPlugin):
     def __init__(
         self,
+        downloader_options: DownloaderValidator,
         overrides: Overrides,
         enhanced_download_archive: EnhancedDownloadArchive,
-        collection_urls: List[UrlValidator],
     ):
         super().__init__(
+            downloader_options=downloader_options,
             overrides=overrides,
             enhanced_download_archive=enhanced_download_archive,
         )
         self._thumbnails_downloaded: Set[str] = set()
         self._collection_url_mapping: Dict[str, UrlValidator] = {
             self.overrides.apply_formatter(collection_url.url): collection_url
-            for collection_url in collection_urls
+            for collection_url in downloader_options.collection_validator.urls.list
         }
 
     def modify_entry_metadata(self, entry: Entry) -> Optional[Entry]:
@@ -218,7 +220,13 @@ class BaseUrlDownloader(BaseDownloader[BaseDownloaderOptionsT], ABC):
     and should translate that to list of Entry objects.
     """
 
-    def added_plugins(self) -> List[Plugin]:
+    @classmethod
+    def added_plugins(
+        cls,
+        downloader_options: BaseDownloaderOptionsT,
+        enhanced_download_archive: EnhancedDownloadArchive,
+        overrides: Overrides,
+    ) -> List[Plugin]:
         """
         Adds
         1. URL thumbnail download plugin
@@ -226,14 +234,14 @@ class BaseUrlDownloader(BaseDownloader[BaseDownloaderOptionsT], ABC):
         """
         return [
             UrlDownloaderThumbnailPlugin(
-                overrides=self.overrides,
+                downloader_options=downloader_options,
-                enhanced_download_archive=self._enhanced_download_archive,
+                overrides=overrides,
-                collection_urls=self.collection.urls.list,
+                enhanced_download_archive=enhanced_download_archive,
             ),
             UrlDownloaderCollectionVariablePlugin(
-                overrides=self.overrides,
+                downloader_options=downloader_options,
-                enhanced_download_archive=self._enhanced_download_archive,
+                overrides=overrides,
-                collection_urls=self.collection.urls.list,
+                enhanced_download_archive=enhanced_download_archive,
             ),
         ]
 

src/ytdl_sub/downloaders/url/validators.py

@@ -3,7 +3,7 @@ from typing import Dict
 from typing import List
 from typing import Optional
 
-from ytdl_sub.config.preset_options import AddsVariablesMixin
+from ytdl_sub.config.preset_options import OptionsValidator
 from ytdl_sub.validators.strict_dict_validator import StrictDictValidator
 from ytdl_sub.validators.string_formatter_validators import DictFormatterValidator
 from ytdl_sub.validators.string_formatter_validators import OverridesStringFormatterValidator
@@ -174,7 +174,7 @@ class UrlListValidator(ListValidator[UrlValidator]):
                     collection_variables[var] = added_variables[var]
 
 
-class MultiUrlValidator(StrictDictValidator, AddsVariablesMixin):
+class MultiUrlValidator(OptionsValidator):
     """
     Downloads from multiple URLs. If an entry is returned from more than one URL, it will
     resolve to the bottom-most URL settings.

src/ytdl_sub/plugins/plugin.py

@@ -7,13 +7,11 @@ from typing import Tuple
 from typing import Type
 from typing import TypeVar
 
-from ytdl_sub.config.preset_options import AddsVariablesMixin
+from ytdl_sub.config.preset_options import OptionsValidator
 from ytdl_sub.config.preset_options import Overrides
 from ytdl_sub.entries.entry import Entry
-from ytdl_sub.utils.exceptions import ValidationException
 from ytdl_sub.utils.file_handler import FileMetadata
 from ytdl_sub.utils.logger import Logger
-from ytdl_sub.validators.strict_dict_validator import StrictDictValidator
 from ytdl_sub.ytdl_additions.enhanced_download_archive import DownloadArchiver
 from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive
 
@@ -40,29 +38,7 @@ class PluginPriority:
         return self.modify_entry >= PluginPriority.MODIFY_ENTRY_AFTER_SPLIT
 
 
-class PluginOptions(StrictDictValidator, AddsVariablesMixin, ABC):
+PluginOptions = OptionsValidator
-    """
-    Class that defines the parameters to a plugin
-    """
-
-    def validation_exception(
-        self,
-        error_message: str | Exception,
-    ) -> ValidationException:
-        """
-        Parameters
-        ----------
-        error_message
-            Error message to include in the validation exception
-
-        Returns
-        -------
-        Validation exception that points to the location in the config. To be used for plugins
-        to throw good validation exceptions at runtime.
-        """
-        return self._validation_exception(error_message=error_message)
-
-
 PluginOptionsT = TypeVar("PluginOptionsT", bound=PluginOptions)
 
 

src/ytdl_sub/subscriptions/subscription_download.py

@@ -156,7 +156,13 @@ class SubscriptionDownload(BaseSubscription, ABC):
         -------
         List of plugins defined in the subscription, initialized and ready to use.
         """
-        plugins: List[Plugin] = []
+        # Always add plugins provided by the downloader
+        plugins: List[Plugin] = self.downloader_class.added_plugins(
+            downloader_options=self.downloader_options,
+            enhanced_download_archive=self._enhanced_download_archive,
+            overrides=self.overrides,
+        )
+
         for plugin_type, plugin_options in self.plugins.zipped():
             plugin = plugin_type(
                 plugin_options=plugin_options,
@@ -292,8 +298,6 @@ class SubscriptionDownload(BaseSubscription, ABC):
             metadata_ytdl_options=subscription_ytdl_options.metadata_builder(),
             overrides=self.overrides,
         )
-        # This could be cleaned up....
-        plugins.extend(downloader.added_plugins())
 
         with self._subscription_download_context_managers():
             for entry in downloader.download_metadata():