danvics/ytdl-sub
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

[REFACTOR] Shared OptionsValidator, downloader plugins (#550)

Browse source
This commit is contained in:
Jesse Bannon 2023-03-15 23:06:55 -07:00 committed by GitHub
parent a96fcff55f
commit 14ddd750d9
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
7 changed files with 71 additions and 66 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
src/ytdl_sub/config/preset.py
View file

@ -6,7 +6,6 @@ from typing import List
from typing import Optional
from typing import Tuple
from typing import Type
from typing import TypeVar
from typing import Union
from mergedeep import mergedeep
@ -22,6 +21,7 @@ from ytdl_sub.downloaders.base_downloader import BaseDownloaderValidator
from ytdl_sub.entries.entry import Entry
from ytdl_sub.plugins.plugin import Plugin
from ytdl_sub.plugins.plugin import PluginOptions
from ytdl_sub.plugins.plugin import PluginOptionsT
from ytdl_sub.prebuilt_presets import PREBUILT_PRESET_NAMES
from ytdl_sub.prebuilt_presets import PUBLISHED_PRESET_NAMES
from ytdl_sub.utils.exceptions import ValidationException
@ -65,8 +65,6 @@ def _parent_preset_error_message(
class PresetPlugins:
_TPluginOptions = TypeVar("_TPluginOptions", bound=PluginOptions)
def __init__(self):
self.plugin_types: List[Type[Plugin]] = []
self.plugin_options: List[PluginOptions] = []
@ -87,7 +85,7 @@ class PresetPlugins:
"""
return zip(self.plugin_types, self.plugin_options)
def get(self, plugin_type: Type[_TPluginOptions]) -> Optional[_TPluginOptions]:
def get(self, plugin_type: Type[PluginOptionsT]) -> Optional[PluginOptionsT]:
"""
Parameters
----------

22
src/ytdl_sub/config/preset_options.py
View file

@ -7,6 +7,7 @@ from typing import Optional
from yt_dlp.utils import sanitize_filename
from ytdl_sub.entries.entry import Entry
from ytdl_sub.utils.exceptions import ValidationException
from ytdl_sub.validators.file_path_validators import OverridesStringFormatterFilePathValidator
from ytdl_sub.validators.file_path_validators import StringFormatterFileNameValidator
from ytdl_sub.validators.strict_dict_validator import StrictDictValidator
@ -20,11 +21,28 @@ from ytdl_sub.validators.validators import LiteralDictValidator
# pylint: disable=no-self-use
# pylint: disable=unused-argument
class AddsVariablesMixin(ABC):
class OptionsValidator(StrictDictValidator, ABC):
"""
Mixin for parts of the Preset that adds source variables
Abstract class that validates options for preset sections (plugins, downloaders)
"""
def validation_exception(
self,
error_message: str | Exception,
) -> ValidationException:
"""
Parameters
----------
error_message
Error message to include in the validation exception
Returns
-------
Validation exception that points to the location in the config. To be used to throw good
validation exceptions at runtime from code outside this class.
"""
return self._validation_exception(error_message=error_message)
def added_source_variables(self) -> List[str]:
"""
If the plugin adds source variables, list them here.

37
src/ytdl_sub/downloaders/base_downloader.py
View file

@ -6,38 +6,33 @@ from typing import List
from typing import Type
from typing import TypeVar
from ytdl_sub.config.preset_options import AddsVariablesMixin
from ytdl_sub.config.preset_options import OptionsValidator
from ytdl_sub.config.preset_options import Overrides
from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder
from ytdl_sub.entries.entry import Entry
from ytdl_sub.plugins.plugin import Plugin
from ytdl_sub.plugins.plugin import PluginOptions
from ytdl_sub.validators.strict_dict_validator import StrictDictValidator
from ytdl_sub.ytdl_additions.enhanced_download_archive import DownloadArchiver
from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive
class BaseDownloaderValidator(StrictDictValidator, AddsVariablesMixin, ABC):
pass
BaseDownloaderValidator = OptionsValidator
BaseDownloaderOptionsT = TypeVar("BaseDownloaderOptionsT", bound=BaseDownloaderValidator)
class BaseDownloaderPluginOptions(PluginOptions):
_optional_keys = {"no-op"}
class BaseDownloaderPlugin(Plugin[BaseDownloaderOptionsT], ABC):
"""
Plugins that get added automatically by using a downloader. Downloader options
are the plugin options.
"""
class BaseDownloaderPlugin(Plugin[BaseDownloaderPluginOptions], ABC):
def __init__(
self,
downloader_options: BaseDownloaderOptionsT,
overrides: Overrides,
enhanced_download_archive: EnhancedDownloadArchive,
):
super().__init__(
# Downloader plugins do not have exposed YAML options, so keep it blank.
# Use init instead.
plugin_options=BaseDownloaderPluginOptions(name=self.__class__.__name__, value={}),
# Downloader plugins use download options as their plugin options
plugin_options=downloader_options,
overrides=overrides,
enhanced_download_archive=enhanced_download_archive,
)
@ -68,9 +63,15 @@ class BaseDownloader(DownloadArchiver, Generic[BaseDownloaderOptionsT], ABC):
def download(self, entry: Entry) -> Entry:
"""The function to perform the download of all media entries"""
# pylint: disable=no-self-use
def added_plugins(self) -> List[BaseDownloaderPlugin]:
# pylint: disable=unused-argument
@classmethod
def added_plugins(
cls,
downloader_options: BaseDownloaderOptionsT,
enhanced_download_archive: EnhancedDownloadArchive,
overrides: Overrides,
) -> List[BaseDownloaderPlugin]:
"""Add these plugins from the Downloader to the subscription"""
return []
# pylint: enable=no-self-use
# pylint: enable=unused-argument

30
src/ytdl_sub/downloaders/url/downloader.py
View file

@ -87,18 +87,19 @@ class URLDownloadState:
class UrlDownloaderThumbnailPlugin(BaseDownloaderPlugin):
def __init__(
self,
downloader_options: DownloaderValidator,
overrides: Overrides,
enhanced_download_archive: EnhancedDownloadArchive,
collection_urls: List[UrlValidator],
):
super().__init__(
downloader_options=downloader_options,
overrides=overrides,
enhanced_download_archive=enhanced_download_archive,
)
self._thumbnails_downloaded: Set[str] = set()
self._collection_url_mapping: Dict[str, UrlValidator] = {
self.overrides.apply_formatter(collection_url.url): collection_url
for collection_url in collection_urls
for collection_url in downloader_options.collection_validator.urls.list
}
def _download_parent_thumbnails(
@ -185,18 +186,19 @@ class UrlDownloaderThumbnailPlugin(BaseDownloaderPlugin):
class UrlDownloaderCollectionVariablePlugin(BaseDownloaderPlugin):
def __init__(
self,
downloader_options: DownloaderValidator,
overrides: Overrides,
enhanced_download_archive: EnhancedDownloadArchive,
collection_urls: List[UrlValidator],
):
super().__init__(
downloader_options=downloader_options,
overrides=overrides,
enhanced_download_archive=enhanced_download_archive,
)
self._thumbnails_downloaded: Set[str] = set()
self._collection_url_mapping: Dict[str, UrlValidator] = {
self.overrides.apply_formatter(collection_url.url): collection_url
for collection_url in collection_urls
for collection_url in downloader_options.collection_validator.urls.list
}
def modify_entry_metadata(self, entry: Entry) -> Optional[Entry]:
@ -218,7 +220,13 @@ class BaseUrlDownloader(BaseDownloader[BaseDownloaderOptionsT], ABC):
and should translate that to list of Entry objects.
"""
def added_plugins(self) -> List[Plugin]:
@classmethod
def added_plugins(
cls,
downloader_options: BaseDownloaderOptionsT,
enhanced_download_archive: EnhancedDownloadArchive,
overrides: Overrides,
) -> List[Plugin]:
"""
Adds
1. URL thumbnail download plugin
@ -226,14 +234,14 @@ class BaseUrlDownloader(BaseDownloader[BaseDownloaderOptionsT], ABC):
"""
return [
UrlDownloaderThumbnailPlugin(
overrides=self.overrides,
enhanced_download_archive=self._enhanced_download_archive,
collection_urls=self.collection.urls.list,
downloader_options=downloader_options,
overrides=overrides,
enhanced_download_archive=enhanced_download_archive,
),
UrlDownloaderCollectionVariablePlugin(
overrides=self.overrides,
enhanced_download_archive=self._enhanced_download_archive,
collection_urls=self.collection.urls.list,
downloader_options=downloader_options,
overrides=overrides,
enhanced_download_archive=enhanced_download_archive,
),
]

4
src/ytdl_sub/downloaders/url/validators.py
View file

@ -3,7 +3,7 @@ from typing import Dict
from typing import List
from typing import Optional
from ytdl_sub.config.preset_options import AddsVariablesMixin
from ytdl_sub.config.preset_options import OptionsValidator
from ytdl_sub.validators.strict_dict_validator import StrictDictValidator
from ytdl_sub.validators.string_formatter_validators import DictFormatterValidator
from ytdl_sub.validators.string_formatter_validators import OverridesStringFormatterValidator
@ -174,7 +174,7 @@ class UrlListValidator(ListValidator[UrlValidator]):
collection_variables[var] = added_variables[var]
class MultiUrlValidator(StrictDictValidator, AddsVariablesMixin):
class MultiUrlValidator(OptionsValidator):
"""
Downloads from multiple URLs. If an entry is returned from more than one URL, it will
resolve to the bottom-most URL settings.

28
src/ytdl_sub/plugins/plugin.py
View file

@ -7,13 +7,11 @@ from typing import Tuple
from typing import Type
from typing import TypeVar
from ytdl_sub.config.preset_options import AddsVariablesMixin
from ytdl_sub.config.preset_options import OptionsValidator
from ytdl_sub.config.preset_options import Overrides
from ytdl_sub.entries.entry import Entry
from ytdl_sub.utils.exceptions import ValidationException
from ytdl_sub.utils.file_handler import FileMetadata
from ytdl_sub.utils.logger import Logger
from ytdl_sub.validators.strict_dict_validator import StrictDictValidator
from ytdl_sub.ytdl_additions.enhanced_download_archive import DownloadArchiver
from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive
@ -40,29 +38,7 @@ class PluginPriority:
return self.modify_entry >= PluginPriority.MODIFY_ENTRY_AFTER_SPLIT
class PluginOptions(StrictDictValidator, AddsVariablesMixin, ABC):
"""
Class that defines the parameters to a plugin
"""
def validation_exception(
self,
error_message: str | Exception,
) -> ValidationException:
"""
Parameters
----------
error_message
Error message to include in the validation exception
Returns
-------
Validation exception that points to the location in the config. To be used for plugins
to throw good validation exceptions at runtime.
"""
return self._validation_exception(error_message=error_message)
PluginOptions = OptionsValidator
PluginOptionsT = TypeVar("PluginOptionsT", bound=PluginOptions)

10
src/ytdl_sub/subscriptions/subscription_download.py
View file

@ -156,7 +156,13 @@ class SubscriptionDownload(BaseSubscription, ABC):
-------
List of plugins defined in the subscription, initialized and ready to use.
"""
plugins: List[Plugin] = []
# Always add plugins provided by the downloader
plugins: List[Plugin] = self.downloader_class.added_plugins(
downloader_options=self.downloader_options,
enhanced_download_archive=self._enhanced_download_archive,
overrides=self.overrides,
)
for plugin_type, plugin_options in self.plugins.zipped():
plugin = plugin_type(
plugin_options=plugin_options,
@ -292,8 +298,6 @@ class SubscriptionDownload(BaseSubscription, ABC):
metadata_ytdl_options=subscription_ytdl_options.metadata_builder(),
overrides=self.overrides,
)
# This could be cleaned up....
plugins.extend(downloader.added_plugins())
with self._subscription_download_context_managers():
for entry in downloader.download_metadata():