danvics/ytdl-sub
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

[REFACTOR] SourcePluginExtension (#667)

Browse source 
* [REFACTOR] SourcePluginExtension

* param name

* priority to plugin extensions

* info json
This commit is contained in:
Jesse Bannon 2023-07-25 14:04:18 -07:00 committed by GitHub
parent 1236af270d
commit bf3c61213b
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
4 changed files with 63 additions and 54 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
src/ytdl_sub/config/plugin.py
View file

@ -28,7 +28,10 @@ class PluginPriority:
MODIFY_ENTRY_FIRST = 0
def __init__(self, modify_entry: int = 5, post_process: int = 5):
def __init__(
self, modify_entry_metadata: int = 5, modify_entry: int = 5, post_process: int = 5
):
self.modify_entry_metadata = modify_entry_metadata
self.modify_entry = modify_entry
self.post_process = post_process

35
src/ytdl_sub/downloaders/source_plugin.py
View file

@ -1,8 +1,12 @@
import abc
from abc import ABC
from typing import Dict
from typing import Generic
from typing import Iterable
from typing import List
from typing import Optional
from typing import Type
from typing import final
from ytdl_sub.config.plugin import BasePlugin
from ytdl_sub.config.plugin import Plugin
@ -19,8 +23,18 @@ class SourcePluginExtension(Plugin[TOptionsValidator], ABC):
are the plugin options.
"""
@final
def ytdl_options(self) -> Optional[Dict]:
"""
SourcePluginExtensions are intended to run after downloading. ytdl_options at that point
are not needed.
"""
return None
class SourcePlugin(BasePlugin[TOptionsValidator], Generic[TOptionsValidator], ABC):
plugin_extensions: List[Type[SourcePluginExtension]] = []
def __init__(
self,
options: TOptionsValidator,
@ -45,15 +59,14 @@ class SourcePlugin(BasePlugin[TOptionsValidator], Generic[TOptionsValidator], AB
def download(self, entry: Entry) -> Entry:
"""The function to perform the download of all media entries"""
# pylint: disable=unused-argument
@classmethod
def added_plugins(
cls,
downloader_options: TOptionsValidator,
enhanced_download_archive: EnhancedDownloadArchive,
overrides: Overrides,
) -> List[SourcePluginExtension]:
@final
def added_plugins(self) -> List[SourcePluginExtension]:
"""Add these plugins from the Downloader to the subscription"""
return []
# pylint: enable=unused-argument
return [
plugin_extension(
options=self.plugin_options,
overrides=self.overrides,
enhanced_download_archive=self._enhanced_download_archive,
)
for plugin_extension in self.plugin_extensions
]

37
src/ytdl_sub/downloaders/url/downloader.py
View file

@ -12,7 +12,7 @@ from typing import Tuple
from yt_dlp.utils import RejectedVideoReached
from ytdl_sub.config.plugin import Plugin
from ytdl_sub.config.plugin import PluginPriority
from ytdl_sub.config.preset_options import Overrides
from ytdl_sub.downloaders.source_plugin import SourcePlugin
from ytdl_sub.downloaders.source_plugin import SourcePluginExtension
@ -55,6 +55,8 @@ class URLDownloadState:
class UrlDownloaderThumbnailPlugin(SourcePluginExtension):
priority = PluginPriority(modify_entry=0)
def __init__(
self,
options: MultiUrlSourceOptionsValidator,
@ -154,21 +156,23 @@ class UrlDownloaderThumbnailPlugin(SourcePluginExtension):
class UrlDownloaderCollectionVariablePlugin(SourcePluginExtension):
priority = PluginPriority(modify_entry_metadata=0)
def __init__(
self,
downloader_options: MultiUrlSourceOptionsValidator,
options: MultiUrlSourceOptionsValidator,
overrides: Overrides,
enhanced_download_archive: EnhancedDownloadArchive,
):
super().__init__(
options=downloader_options,
options=options,
overrides=overrides,
enhanced_download_archive=enhanced_download_archive,
)
self._thumbnails_downloaded: Set[str] = set()
self._collection_url_mapping: Dict[str, UrlValidator] = {
self.overrides.apply_formatter(collection_url.url): collection_url
for collection_url in downloader_options.collection_validator.urls.list
for collection_url in options.collection_validator.urls.list
}
def modify_entry_metadata(self, entry: Entry) -> Optional[Entry]:
@ -196,30 +200,7 @@ class BaseUrlDownloader(SourcePlugin[TMultiURLSourceOptionsValidator], ABC):
and should translate that to list of Entry objects.
"""
@classmethod
def added_plugins(
cls,
downloader_options: TMultiURLSourceOptionsValidator,
enhanced_download_archive: EnhancedDownloadArchive,
overrides: Overrides,
) -> List[Plugin]:
"""
Adds
1. URL thumbnail download plugin
2. Collection variable plugin to add to each entry
"""
return [
UrlDownloaderThumbnailPlugin(
options=downloader_options,
overrides=overrides,
enhanced_download_archive=enhanced_download_archive,
),
UrlDownloaderCollectionVariablePlugin(
downloader_options=downloader_options,
overrides=overrides,
enhanced_download_archive=enhanced_download_archive,
),
]
plugin_extensions = [UrlDownloaderThumbnailPlugin, UrlDownloaderCollectionVariablePlugin]
@classmethod
def ytdl_option_defaults(cls) -> Dict:

40
src/ytdl_sub/subscriptions/subscription_download.py
View file

@ -174,23 +174,14 @@ class SubscriptionDownload(BaseSubscription, ABC):
-------
List of plugins defined in the subscription, initialized and ready to use.
"""
# Always add plugins provided by the downloader
plugins: List[Plugin] = self.downloader_class.added_plugins(
downloader_options=self.downloader_options,
enhanced_download_archive=self._enhanced_download_archive,
overrides=self.overrides,
)
for plugin_type, plugin_options in self.plugins.zipped():
plugin = plugin_type(
return [
plugin_type(
options=plugin_options,
overrides=self.overrides,
enhanced_download_archive=self._enhanced_download_archive,
)
plugins.append(plugin)
return plugins
for plugin_type, plugin_options in self.plugins.zipped()
]
@classmethod
def _cleanup_entry_files(cls, entry: Entry):
@ -201,7 +192,7 @@ class SubscriptionDownload(BaseSubscription, ABC):
@classmethod
def _preprocess_entry(cls, plugins: List[Plugin], entry: Entry) -> Optional[Entry]:
maybe_entry: Optional[Entry] = entry
for plugin in plugins:
for plugin in sorted(plugins, key=lambda _plugin: _plugin.priority.modify_entry_metadata):
if (maybe_entry := plugin.modify_entry_metadata(maybe_entry)) is None:
return None
@ -347,6 +338,8 @@ class SubscriptionDownload(BaseSubscription, ABC):
overrides=self.overrides,
)
plugins.extend(downloader.added_plugins())
return self._process_subscription(
plugins=plugins,
downloader=downloader,
@ -365,6 +358,25 @@ class SubscriptionDownload(BaseSubscription, ABC):
self._enhanced_download_archive.reinitialize(dry_run=dry_run)
plugins = self._initialize_plugins()
subscription_ytdl_options = SubscriptionYTDLOptions(
preset=self._preset_options,
plugins=plugins,
enhanced_download_archive=self._enhanced_download_archive,
working_directory=self.working_directory,
dry_run=dry_run,
)
# Re-add the original downloader class' plugins
plugins.extend(
self.downloader_class(
options=self.downloader_options,
enhanced_download_archive=self._enhanced_download_archive,
download_ytdl_options=subscription_ytdl_options.download_builder(),
metadata_ytdl_options=subscription_ytdl_options.metadata_builder(),
overrides=self.overrides,
).added_plugins()
)
downloader = InfoJsonDownloader(
options=InfoJsonDownloaderOptions(name="no-op", value={}),
enhanced_download_archive=self._enhanced_download_archive,