From f46a1f3e98170fd1ea637aed8a87a159eef8cbcf Mon Sep 17 00:00:00 2001 From: Jesse Bannon Date: Wed, 15 Mar 2023 22:37:53 -0700 Subject: [PATCH 1/4] [REFACTOR] Shared plugin and download options class --- src/ytdl_sub/config/preset.py | 6 ++--- src/ytdl_sub/config/preset_options.py | 22 ++++++++++++++-- src/ytdl_sub/downloaders/base_downloader.py | 11 +++----- src/ytdl_sub/downloaders/url/validators.py | 4 +-- src/ytdl_sub/plugins/plugin.py | 28 ++------------------- 5 files changed, 29 insertions(+), 42 deletions(-) diff --git a/src/ytdl_sub/config/preset.py b/src/ytdl_sub/config/preset.py index 7cb5fcec..81f9a381 100644 --- a/src/ytdl_sub/config/preset.py +++ b/src/ytdl_sub/config/preset.py @@ -6,7 +6,6 @@ from typing import List from typing import Optional from typing import Tuple from typing import Type -from typing import TypeVar from typing import Union from mergedeep import mergedeep @@ -22,6 +21,7 @@ from ytdl_sub.downloaders.base_downloader import BaseDownloaderValidator from ytdl_sub.entries.entry import Entry from ytdl_sub.plugins.plugin import Plugin from ytdl_sub.plugins.plugin import PluginOptions +from ytdl_sub.plugins.plugin import PluginOptionsT from ytdl_sub.prebuilt_presets import PREBUILT_PRESET_NAMES from ytdl_sub.prebuilt_presets import PUBLISHED_PRESET_NAMES from ytdl_sub.utils.exceptions import ValidationException @@ -65,8 +65,6 @@ def _parent_preset_error_message( class PresetPlugins: - _TPluginOptions = TypeVar("_TPluginOptions", bound=PluginOptions) - def __init__(self): self.plugin_types: List[Type[Plugin]] = [] self.plugin_options: List[PluginOptions] = [] @@ -87,7 +85,7 @@ class PresetPlugins: """ return zip(self.plugin_types, self.plugin_options) - def get(self, plugin_type: Type[_TPluginOptions]) -> Optional[_TPluginOptions]: + def get(self, plugin_type: Type[PluginOptionsT]) -> Optional[PluginOptionsT]: """ Parameters ---------- diff --git a/src/ytdl_sub/config/preset_options.py b/src/ytdl_sub/config/preset_options.py index c2e8767c..21839ee3 100644 --- a/src/ytdl_sub/config/preset_options.py +++ b/src/ytdl_sub/config/preset_options.py @@ -7,6 +7,7 @@ from typing import Optional from yt_dlp.utils import sanitize_filename from ytdl_sub.entries.entry import Entry +from ytdl_sub.utils.exceptions import ValidationException from ytdl_sub.validators.file_path_validators import OverridesStringFormatterFilePathValidator from ytdl_sub.validators.file_path_validators import StringFormatterFileNameValidator from ytdl_sub.validators.strict_dict_validator import StrictDictValidator @@ -20,11 +21,28 @@ from ytdl_sub.validators.validators import LiteralDictValidator # pylint: disable=no-self-use # pylint: disable=unused-argument -class AddsVariablesMixin(ABC): +class OptionsValidator(StrictDictValidator, ABC): """ - Mixin for parts of the Preset that adds source variables + Abstract class that validates options for preset sections (plugins, downloaders) """ + def validation_exception( + self, + error_message: str | Exception, + ) -> ValidationException: + """ + Parameters + ---------- + error_message + Error message to include in the validation exception + + Returns + ------- + Validation exception that points to the location in the config. To be used to throw good + validation exceptions at runtime from code outside this class. + """ + return self._validation_exception(error_message=error_message) + def added_source_variables(self) -> List[str]: """ If the plugin adds source variables, list them here. diff --git a/src/ytdl_sub/downloaders/base_downloader.py b/src/ytdl_sub/downloaders/base_downloader.py index 5df45a5c..deb9ec6e 100644 --- a/src/ytdl_sub/downloaders/base_downloader.py +++ b/src/ytdl_sub/downloaders/base_downloader.py @@ -6,21 +6,16 @@ from typing import List from typing import Type from typing import TypeVar -from ytdl_sub.config.preset_options import AddsVariablesMixin +from ytdl_sub.config.preset_options import OptionsValidator from ytdl_sub.config.preset_options import Overrides from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder from ytdl_sub.entries.entry import Entry from ytdl_sub.plugins.plugin import Plugin from ytdl_sub.plugins.plugin import PluginOptions -from ytdl_sub.validators.strict_dict_validator import StrictDictValidator from ytdl_sub.ytdl_additions.enhanced_download_archive import DownloadArchiver from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive - -class BaseDownloaderValidator(StrictDictValidator, AddsVariablesMixin, ABC): - pass - - +BaseDownloaderValidator = OptionsValidator BaseDownloaderOptionsT = TypeVar("BaseDownloaderOptionsT", bound=BaseDownloaderValidator) @@ -28,7 +23,7 @@ class BaseDownloaderPluginOptions(PluginOptions): _optional_keys = {"no-op"} -class BaseDownloaderPlugin(Plugin[BaseDownloaderPluginOptions], ABC): +class BaseDownloaderPlugin(Plugin[BaseDownloaderOptionsT], Generic[BaseDownloaderOptionsT], ABC): def __init__( self, overrides: Overrides, diff --git a/src/ytdl_sub/downloaders/url/validators.py b/src/ytdl_sub/downloaders/url/validators.py index 5f94975c..5694a7ba 100644 --- a/src/ytdl_sub/downloaders/url/validators.py +++ b/src/ytdl_sub/downloaders/url/validators.py @@ -3,7 +3,7 @@ from typing import Dict from typing import List from typing import Optional -from ytdl_sub.config.preset_options import AddsVariablesMixin +from ytdl_sub.config.preset_options import OptionsValidator from ytdl_sub.validators.strict_dict_validator import StrictDictValidator from ytdl_sub.validators.string_formatter_validators import DictFormatterValidator from ytdl_sub.validators.string_formatter_validators import OverridesStringFormatterValidator @@ -174,7 +174,7 @@ class UrlListValidator(ListValidator[UrlValidator]): collection_variables[var] = added_variables[var] -class MultiUrlValidator(StrictDictValidator, AddsVariablesMixin): +class MultiUrlValidator(OptionsValidator): """ Downloads from multiple URLs. If an entry is returned from more than one URL, it will resolve to the bottom-most URL settings. diff --git a/src/ytdl_sub/plugins/plugin.py b/src/ytdl_sub/plugins/plugin.py index 0d1113c9..ccf21acb 100644 --- a/src/ytdl_sub/plugins/plugin.py +++ b/src/ytdl_sub/plugins/plugin.py @@ -7,13 +7,11 @@ from typing import Tuple from typing import Type from typing import TypeVar -from ytdl_sub.config.preset_options import AddsVariablesMixin +from ytdl_sub.config.preset_options import OptionsValidator from ytdl_sub.config.preset_options import Overrides from ytdl_sub.entries.entry import Entry -from ytdl_sub.utils.exceptions import ValidationException from ytdl_sub.utils.file_handler import FileMetadata from ytdl_sub.utils.logger import Logger -from ytdl_sub.validators.strict_dict_validator import StrictDictValidator from ytdl_sub.ytdl_additions.enhanced_download_archive import DownloadArchiver from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive @@ -40,29 +38,7 @@ class PluginPriority: return self.modify_entry >= PluginPriority.MODIFY_ENTRY_AFTER_SPLIT -class PluginOptions(StrictDictValidator, AddsVariablesMixin, ABC): - """ - Class that defines the parameters to a plugin - """ - - def validation_exception( - self, - error_message: str | Exception, - ) -> ValidationException: - """ - Parameters - ---------- - error_message - Error message to include in the validation exception - - Returns - ------- - Validation exception that points to the location in the config. To be used for plugins - to throw good validation exceptions at runtime. - """ - return self._validation_exception(error_message=error_message) - - +PluginOptions = OptionsValidator PluginOptionsT = TypeVar("PluginOptionsT", bound=PluginOptions) From b97fb3b7f93a5271fa2f85658b1437c517e29048 Mon Sep 17 00:00:00 2001 From: Jesse Bannon Date: Wed, 15 Mar 2023 22:52:58 -0700 Subject: [PATCH 2/4] more refactoring --- src/ytdl_sub/downloaders/base_downloader.py | 30 +++++++++++-------- src/ytdl_sub/downloaders/url/downloader.py | 30 ++++++++++++------- .../subscriptions/subscription_download.py | 8 +++-- 3 files changed, 41 insertions(+), 27 deletions(-) diff --git a/src/ytdl_sub/downloaders/base_downloader.py b/src/ytdl_sub/downloaders/base_downloader.py index deb9ec6e..656eeb82 100644 --- a/src/ytdl_sub/downloaders/base_downloader.py +++ b/src/ytdl_sub/downloaders/base_downloader.py @@ -11,7 +11,6 @@ from ytdl_sub.config.preset_options import Overrides from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder from ytdl_sub.entries.entry import Entry from ytdl_sub.plugins.plugin import Plugin -from ytdl_sub.plugins.plugin import PluginOptions from ytdl_sub.ytdl_additions.enhanced_download_archive import DownloadArchiver from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive @@ -19,20 +18,20 @@ BaseDownloaderValidator = OptionsValidator BaseDownloaderOptionsT = TypeVar("BaseDownloaderOptionsT", bound=BaseDownloaderValidator) -class BaseDownloaderPluginOptions(PluginOptions): - _optional_keys = {"no-op"} - - -class BaseDownloaderPlugin(Plugin[BaseDownloaderOptionsT], Generic[BaseDownloaderOptionsT], ABC): +class BaseDownloaderPlugin(Plugin[BaseDownloaderOptionsT], ABC): + """ + Plugins that get added automatically by using a downloader. Downloader options + are the plugin options. + """ def __init__( self, + downloader_options: BaseDownloaderOptionsT, overrides: Overrides, enhanced_download_archive: EnhancedDownloadArchive, ): super().__init__( - # Downloader plugins do not have exposed YAML options, so keep it blank. - # Use init instead. - plugin_options=BaseDownloaderPluginOptions(name=self.__class__.__name__, value={}), + # Downloader plugins use download options as their plugin options + plugin_options=downloader_options, overrides=overrides, enhanced_download_archive=enhanced_download_archive, ) @@ -63,9 +62,14 @@ class BaseDownloader(DownloadArchiver, Generic[BaseDownloaderOptionsT], ABC): def download(self, entry: Entry) -> Entry: """The function to perform the download of all media entries""" - # pylint: disable=no-self-use - def added_plugins(self) -> List[BaseDownloaderPlugin]: + # pylint: disable=unused-argument + @classmethod + def added_plugins( + cls, + downloader_options: BaseDownloaderOptionsT, + enhanced_download_archive: EnhancedDownloadArchive, + overrides: Overrides, + ) -> List[BaseDownloaderPlugin]: """Add these plugins from the Downloader to the subscription""" return [] - - # pylint: enable=no-self-use + # pylint: enable=unused-argument diff --git a/src/ytdl_sub/downloaders/url/downloader.py b/src/ytdl_sub/downloaders/url/downloader.py index 9372308f..a54c848d 100644 --- a/src/ytdl_sub/downloaders/url/downloader.py +++ b/src/ytdl_sub/downloaders/url/downloader.py @@ -87,18 +87,19 @@ class URLDownloadState: class UrlDownloaderThumbnailPlugin(BaseDownloaderPlugin): def __init__( self, + downloader_options: DownloaderValidator, overrides: Overrides, enhanced_download_archive: EnhancedDownloadArchive, - collection_urls: List[UrlValidator], ): super().__init__( + downloader_options=downloader_options, overrides=overrides, enhanced_download_archive=enhanced_download_archive, ) self._thumbnails_downloaded: Set[str] = set() self._collection_url_mapping: Dict[str, UrlValidator] = { self.overrides.apply_formatter(collection_url.url): collection_url - for collection_url in collection_urls + for collection_url in downloader_options.collection_validator.urls.list } def _download_parent_thumbnails( @@ -185,18 +186,19 @@ class UrlDownloaderThumbnailPlugin(BaseDownloaderPlugin): class UrlDownloaderCollectionVariablePlugin(BaseDownloaderPlugin): def __init__( self, + downloader_options: DownloaderValidator, overrides: Overrides, enhanced_download_archive: EnhancedDownloadArchive, - collection_urls: List[UrlValidator], ): super().__init__( + downloader_options=downloader_options, overrides=overrides, enhanced_download_archive=enhanced_download_archive, ) self._thumbnails_downloaded: Set[str] = set() self._collection_url_mapping: Dict[str, UrlValidator] = { self.overrides.apply_formatter(collection_url.url): collection_url - for collection_url in collection_urls + for collection_url in downloader_options.collection_validator.urls.list } def modify_entry_metadata(self, entry: Entry) -> Optional[Entry]: @@ -218,7 +220,13 @@ class BaseUrlDownloader(BaseDownloader[BaseDownloaderOptionsT], ABC): and should translate that to list of Entry objects. """ - def added_plugins(self) -> List[Plugin]: + @classmethod + def added_plugins( + cls, + downloader_options: BaseDownloaderOptionsT, + enhanced_download_archive: EnhancedDownloadArchive, + overrides: Overrides, + ) -> List[Plugin]: """ Adds 1. URL thumbnail download plugin @@ -226,14 +234,14 @@ class BaseUrlDownloader(BaseDownloader[BaseDownloaderOptionsT], ABC): """ return [ UrlDownloaderThumbnailPlugin( - overrides=self.overrides, - enhanced_download_archive=self._enhanced_download_archive, - collection_urls=self.collection.urls.list, + downloader_options=downloader_options, + overrides=overrides, + enhanced_download_archive=enhanced_download_archive, ), UrlDownloaderCollectionVariablePlugin( - overrides=self.overrides, - enhanced_download_archive=self._enhanced_download_archive, - collection_urls=self.collection.urls.list, + downloader_options=downloader_options, + overrides=overrides, + enhanced_download_archive=enhanced_download_archive, ), ] diff --git a/src/ytdl_sub/subscriptions/subscription_download.py b/src/ytdl_sub/subscriptions/subscription_download.py index 68e3f171..6be225d5 100644 --- a/src/ytdl_sub/subscriptions/subscription_download.py +++ b/src/ytdl_sub/subscriptions/subscription_download.py @@ -275,7 +275,11 @@ class SubscriptionDownload(BaseSubscription, ABC): directory. """ self._enhanced_download_archive.reinitialize(dry_run=dry_run) - plugins = self._initialize_plugins() + plugins = self._initialize_plugins() + self.downloader_class.added_plugins( + downloader_options=self.downloader_options, + enhanced_download_archive=self._enhanced_download_archive, + overrides=self.overrides, + ) subscription_ytdl_options = SubscriptionYTDLOptions( preset=self._preset_options, @@ -292,8 +296,6 @@ class SubscriptionDownload(BaseSubscription, ABC): metadata_ytdl_options=subscription_ytdl_options.metadata_builder(), overrides=self.overrides, ) - # This could be cleaned up.... - plugins.extend(downloader.added_plugins()) with self._subscription_download_context_managers(): for entry in downloader.download_metadata(): From b95ca506f6e2225cd2b548cccb1dc3769d78baec Mon Sep 17 00:00:00 2001 From: Jesse Bannon Date: Wed, 15 Mar 2023 22:53:26 -0700 Subject: [PATCH 3/4] lint --- src/ytdl_sub/downloaders/base_downloader.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/ytdl_sub/downloaders/base_downloader.py b/src/ytdl_sub/downloaders/base_downloader.py index 656eeb82..73759af0 100644 --- a/src/ytdl_sub/downloaders/base_downloader.py +++ b/src/ytdl_sub/downloaders/base_downloader.py @@ -23,6 +23,7 @@ class BaseDownloaderPlugin(Plugin[BaseDownloaderOptionsT], ABC): Plugins that get added automatically by using a downloader. Downloader options are the plugin options. """ + def __init__( self, downloader_options: BaseDownloaderOptionsT, @@ -72,4 +73,5 @@ class BaseDownloader(DownloadArchiver, Generic[BaseDownloaderOptionsT], ABC): ) -> List[BaseDownloaderPlugin]: """Add these plugins from the Downloader to the subscription""" return [] + # pylint: enable=unused-argument From 05a9098065909d59d48e63786a1b2880896a72bd Mon Sep 17 00:00:00 2001 From: Jesse Bannon Date: Wed, 15 Mar 2023 22:56:15 -0700 Subject: [PATCH 4/4] put in initialize plugins --- .../subscriptions/subscription_download.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/ytdl_sub/subscriptions/subscription_download.py b/src/ytdl_sub/subscriptions/subscription_download.py index 6be225d5..7d28fd94 100644 --- a/src/ytdl_sub/subscriptions/subscription_download.py +++ b/src/ytdl_sub/subscriptions/subscription_download.py @@ -156,7 +156,13 @@ class SubscriptionDownload(BaseSubscription, ABC): ------- List of plugins defined in the subscription, initialized and ready to use. """ - plugins: List[Plugin] = [] + # Always add plugins provided by the downloader + plugins: List[Plugin] = self.downloader_class.added_plugins( + downloader_options=self.downloader_options, + enhanced_download_archive=self._enhanced_download_archive, + overrides=self.overrides, + ) + for plugin_type, plugin_options in self.plugins.zipped(): plugin = plugin_type( plugin_options=plugin_options, @@ -275,11 +281,7 @@ class SubscriptionDownload(BaseSubscription, ABC): directory. """ self._enhanced_download_archive.reinitialize(dry_run=dry_run) - plugins = self._initialize_plugins() + self.downloader_class.added_plugins( - downloader_options=self.downloader_options, - enhanced_download_archive=self._enhanced_download_archive, - overrides=self.overrides, - ) + plugins = self._initialize_plugins() subscription_ytdl_options = SubscriptionYTDLOptions( preset=self._preset_options,