danvics/ytdl-sub
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

goodbye pluginpriority

Browse source
This commit is contained in:
Jesse Bannon 2023-12-12 17:57:30 -08:00
parent 5c031acd9e
commit acdcd4c3c2
15 changed files with 135 additions and 92 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

37
src/ytdl_sub/config/plugin.py
View file

@ -1,6 +1,5 @@
from abc import ABC
from abc import abstractmethod
from enum import Enum
from typing import Dict
from typing import Generic
from typing import List
@ -15,39 +14,6 @@ from ytdl_sub.utils.file_handler import FileMetadata
from ytdl_sub.ytdl_additions.enhanced_download_archive import DownloadArchiver
from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive
class PluginPriority:
"""
Defines priority for plugins, 0 is highest priority
"""
MODIFY_ENTRY_SPLIT = 9
# If modify_entry priority is >= to this value, run after split
MODIFY_ENTRY_AFTER_SPLIT = 10
# if post_process is >= to this value, run after file_convert
POST_PROCESS_AFTER_FILE_CONVERT = 10
MODIFY_ENTRY_FIRST = 0
def __init__(
self, modify_entry_metadata: int = 5, modify_entry: int = 5, post_process: int = 5
):
self.modify_entry_metadata = modify_entry_metadata
self.modify_entry = modify_entry
self.post_process = post_process
@property
def modify_entry_after_split(self) -> bool:
"""
Returns
-------
True if the plugin should modify an entry after a potential split. False otherwise.
"""
return self.modify_entry >= PluginPriority.MODIFY_ENTRY_AFTER_SPLIT
# pylint: disable=no-self-use,unused-argument
@ -56,7 +22,6 @@ class BasePlugin(DownloadArchiver, Generic[TOptionsValidator], ABC):
Shared code amongst all SourcePlugins (downloaders) and Plugins (post-download modification)
"""
priority: PluginPriority = PluginPriority()
plugin_options_type: Type[TOptionsValidator]
def __init__(
@ -147,8 +112,6 @@ class SplitPlugin(Plugin[TOptionsValidator], Generic[TOptionsValidator], ABC):
Plugin that splits entries into zero or more entries
"""
priority = PluginPriority(modify_entry_metadata=PluginPriority.MODIFY_ENTRY_SPLIT)
@abstractmethod
def split(self, entry: Entry) -> List[Tuple[Entry, FileMetadata]]:
"""

99
src/ytdl_sub/config/plugin_mapping.py
View file

@ -1,8 +1,15 @@
import copy
from typing import Dict
from typing import Iterable
from typing import List
from typing import Optional
from typing import Tuple
from typing import Type
from ytdl_sub.config.plugin import Plugin
from ytdl_sub.config.plugin import SplitPlugin
from ytdl_sub.config.plugin_operation import PluginOperation
from ytdl_sub.config.preset_options import OptionsValidator
from ytdl_sub.plugins.audio_extract import AudioExtractPlugin
from ytdl_sub.plugins.chapters import ChaptersPlugin
from ytdl_sub.plugins.date_range import DateRangePlugin
@ -45,6 +52,98 @@ class PluginMapping:
"throttle_protection": ThrottleProtectionPlugin,
}
# All other plugins are added after the defined ordered ones
_ORDER_MODIFY_ENTRY_METADATA: List[Type[Plugin]] = [
ThrottleProtectionPlugin,
SubtitlesPlugin,
# add all others
]
_ORDER_MODIFY_ENTRY: List[Type[Plugin]] = [
AudioExtractPlugin,
FileConvertPlugin,
SplitByChaptersPlugin,
RegexPlugin,
# add all others
]
_ORDER_POST_PROCESS: List[Type[Plugin]] = [
EmbedThumbnailPlugin,
]
@classmethod
def _order_by(
cls, plugin_types: List[Type[Plugin]], operation: PluginOperation
) -> List[Type[Plugin]]:
if operation == PluginOperation.MODIFY_ENTRY_METADATA:
ordering = cls._ORDER_MODIFY_ENTRY_METADATA
elif operation == PluginOperation.MODIFY_ENTRY:
ordering = cls._ORDER_MODIFY_ENTRY
elif operation == PluginOperation.POST_PROCESS:
ordering = cls._ORDER_POST_PROCESS
else:
raise ValueError("PluginOperation does not support ordering")
ordered_plugin_operations: List[Type[Plugin]] = []
for pl_type in reversed(ordering):
for plugin_type in plugin_types:
if plugin_type == pl_type:
ordered_plugin_operations.insert(0, plugin_type)
else:
ordered_plugin_operations.append(plugin_type)
return ordered_plugin_operations
@classmethod
def order_options_by(
cls, zipped: List[Tuple[Type[Plugin], OptionsValidator]], operation: PluginOperation
) -> List[OptionsValidator]:
ordered_types: List[Type[Plugin]] = cls._order_by(
plugin_types=[val[0] for val in zipped], operation=operation
)
ordered_options: List[OptionsValidator] = []
for plugin_type, plugin_options in zipped:
sorted_idx = ordered_types.index(plugin_type)
ordered_options.insert(sorted_idx, plugin_options)
return ordered_options
@classmethod
def _is_modified_after_split(cls, plugin: Plugin) -> bool:
if type(plugin) not in cls._ORDER_MODIFY_ENTRY:
return True
return cls._ORDER_MODIFY_ENTRY.index(type(plugin)) > cls._ORDER_MODIFY_ENTRY.index(
SplitByChaptersPlugin
)
@classmethod
def order_plugins_by(
cls, plugins: List[Plugin], operation: PluginOperation, before_split: Optional[bool] = None
) -> List[Plugin]:
ordered_types: List[Type[Plugin]] = cls._order_by(
plugin_types=[type(plugin) for plugin in plugins], operation=operation
)
ordered_plugins: List[Plugin] = []
for plugin in plugins:
sorted_idx = ordered_types.index(type(plugin))
ordered_plugins.insert(sorted_idx, plugin)
if before_split is None:
return ordered_plugins
# Remove the split plugin if differentiating
ordered_plugins = [
plugin for plugin in ordered_plugins if not isinstance(plugin, SplitPlugin)
]
if before_split is False:
return [
plugin for plugin in ordered_plugins if not cls._is_modified_after_split(plugin)
]
else: # before_split is True
return [plugin for plugin in ordered_plugins if cls._is_modified_after_split(plugin)]
@classmethod
def plugins(cls) -> List[str]:
"""

8
src/ytdl_sub/config/plugin_operation.py Normal file
View file

@ -0,0 +1,8 @@
from enum import Enum
class PluginOperation(Enum):
DOWNLOADER = -1
MODIFY_ENTRY_METADATA = 0
MODIFY_ENTRY = 1
POST_PROCESS = 2

14
src/ytdl_sub/config/preset.py
View file

@ -16,9 +16,9 @@ from ytdl_sub.config.config_validator import ConfigValidator
from ytdl_sub.config.overrides import Overrides
from ytdl_sub.config.plugin import Plugin
from ytdl_sub.config.plugin_mapping import PluginMapping
from ytdl_sub.config.plugin_operation import PluginOperation
from ytdl_sub.config.preset_options import OptionsValidator
from ytdl_sub.config.preset_options import OutputOptions
from ytdl_sub.config.preset_options import PluginOperation
from ytdl_sub.config.preset_options import TOptionsValidator
from ytdl_sub.config.preset_options import YTDLOptions
from ytdl_sub.downloaders.url.validators import MultiUrlValidator
@ -83,13 +83,13 @@ class PresetPlugins:
self.plugin_options.append(plugin_options)
return self
def zipped(self) -> Iterable[Tuple[Type[Plugin], OptionsValidator]]:
def zipped(self) -> List[Tuple[Type[Plugin], OptionsValidator]]:
"""
Returns
-------
Plugin and PluginOptions zipped
"""
return zip(self.plugin_types, self.plugin_options)
return list(zip(self.plugin_types, self.plugin_options))
def get(self, plugin_type: Type[TOptionsValidator]) -> Optional[TOptionsValidator]:
"""
@ -211,8 +211,8 @@ class Preset(_PresetShell):
script.add(ScriptUtils.add_dummy_variables(added_variables))
unresolved_variables -= added_variables
for _, plugin_options in sorted(
self.plugins.zipped(), key=lambda pl: pl[0].priority.modify_entry_metadata
for plugin_options in PluginMapping.order_options_by(
self.plugins.zipped(), PluginOperation.MODIFY_ENTRY_METADATA
):
added_variables = plugin_options.added_source_variables(
unresolved_variables=unresolved_variables
@ -223,8 +223,8 @@ class Preset(_PresetShell):
unresolved_variables -= added_variables
_ = script.resolve(unresolvable=unresolved_variables, update=True)
for _, plugin_options in sorted(
self.plugins.zipped(), key=lambda pl: pl[0].priority.modify_entry
for plugin_options in PluginMapping.order_options_by(
self.plugins.zipped(), PluginOperation.MODIFY_ENTRY
):
added_variables = plugin_options.added_source_variables(
unresolved_variables=unresolved_variables

9
src/ytdl_sub/config/preset_options.py
View file

@ -1,13 +1,12 @@
from abc import ABC
from enum import Enum
from typing import Any
from typing import Dict
from typing import List
from typing import Optional
from typing import Set
from typing import TypeVar
from ytdl_sub.config.defaults import DEFAULT_DOWNLOAD_ARCHIVE_NAME
from ytdl_sub.config.plugin_operation import PluginOperation
from ytdl_sub.utils.exceptions import ValidationException
from ytdl_sub.validators.file_path_validators import OverridesStringFormatterFilePathValidator
from ytdl_sub.validators.file_path_validators import StringFormatterFileNameValidator
@ -21,12 +20,6 @@ from ytdl_sub.validators.validators import LiteralDictValidator
from ytdl_sub.validators.validators import Validator
class PluginOperation(Enum):
DOWNLOADER = -1
MODIFY_ENTRY_METADATA = 0
MODIFY_ENTRY = 1
# pylint: disable=no-self-use
# pylint: disable=unused-argument
class OptionsValidator(Validator, ABC):

5
src/ytdl_sub/downloaders/url/downloader.py
View file

@ -12,7 +12,6 @@ from typing import Tuple
from yt_dlp.utils import RejectedVideoReached
from ytdl_sub.config.overrides import Overrides
from ytdl_sub.config.plugin import PluginPriority
from ytdl_sub.downloaders.source_plugin import SourcePlugin
from ytdl_sub.downloaders.source_plugin import SourcePluginExtension
from ytdl_sub.downloaders.url.validators import MultiUrlValidator
@ -40,8 +39,6 @@ class URLDownloadState:
class UrlDownloaderThumbnailPlugin(SourcePluginExtension):
priority = PluginPriority(modify_entry=0)
def __init__(
self,
options: MultiUrlValidator,
@ -145,8 +142,6 @@ class UrlDownloaderThumbnailPlugin(SourcePluginExtension):
class UrlDownloaderCollectionVariablePlugin(SourcePluginExtension):
priority = PluginPriority(modify_entry_metadata=0)
def __init__(
self,
options: MultiUrlValidator,

2
src/ytdl_sub/downloaders/url/validators.py
View file

@ -4,8 +4,8 @@ from typing import Dict
from typing import Optional
from typing import Set
from ytdl_sub.config.plugin_operation import PluginOperation
from ytdl_sub.config.preset_options import OptionsValidator
from ytdl_sub.config.preset_options import PluginOperation
from ytdl_sub.validators.strict_dict_validator import StrictDictValidator
from ytdl_sub.validators.string_formatter_validators import DictFormatterValidator
from ytdl_sub.validators.string_formatter_validators import OverridesStringFormatterValidator

2
src/ytdl_sub/plugins/audio_extract.py
View file

@ -5,8 +5,8 @@ from typing import Optional
from typing import Set
from ytdl_sub.config.plugin import Plugin
from ytdl_sub.config.plugin_operation import PluginOperation
from ytdl_sub.config.preset_options import OptionsDictValidator
from ytdl_sub.config.preset_options import PluginOperation
from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder
from ytdl_sub.entries.entry import Entry
from ytdl_sub.entries.script.variable_definitions import VARIABLES as v

2
src/ytdl_sub/plugins/embed_thumbnail.py
View file

@ -4,7 +4,6 @@ from typing import Optional
import mediafile
from ytdl_sub.config.plugin import Plugin
from ytdl_sub.config.plugin import PluginPriority
from ytdl_sub.config.preset_options import OptionsValidator
from ytdl_sub.entries.entry import Entry
from ytdl_sub.utils.ffmpeg import FFMPEG
@ -33,7 +32,6 @@ class EmbedThumbnailOptions(BoolValidator, OptionsValidator):
class EmbedThumbnailPlugin(Plugin[EmbedThumbnailOptions]):
plugin_options_type = EmbedThumbnailOptions
priority = PluginPriority(post_process=PluginPriority.POST_PROCESS_AFTER_FILE_CONVERT)
@property
def _embed_thumbnail(self) -> bool:

7
src/ytdl_sub/plugins/file_convert.py
View file

@ -6,9 +6,8 @@ from typing import Set
from ytdl_sub.config.overrides import Overrides
from ytdl_sub.config.plugin import Plugin
from ytdl_sub.config.plugin import PluginPriority
from ytdl_sub.config.plugin_operation import PluginOperation
from ytdl_sub.config.preset_options import OptionsDictValidator
from ytdl_sub.config.preset_options import PluginOperation
from ytdl_sub.entries.entry import Entry
from ytdl_sub.entries.script.variable_definitions import VARIABLES as v
from ytdl_sub.utils.exceptions import FileNotDownloadedException
@ -127,10 +126,6 @@ class FileConvertOptions(OptionsDictValidator):
class FileConvertPlugin(Plugin[FileConvertOptions]):
plugin_options_type = FileConvertOptions
# Perform this after regex
priority: PluginPriority = PluginPriority(
modify_entry=PluginPriority.MODIFY_ENTRY_AFTER_SPLIT + 1
)
def __init__(
self,

2
src/ytdl_sub/plugins/internal/view.py
View file

@ -3,7 +3,6 @@ from typing import Optional
from ytdl_sub.config.overrides import Overrides
from ytdl_sub.config.plugin import Plugin
from ytdl_sub.config.plugin import PluginPriority
from ytdl_sub.config.preset_options import OptionsDictValidator
from ytdl_sub.entries.entry import Entry
from ytdl_sub.utils.file_handler import FileMetadata
@ -20,7 +19,6 @@ class ViewOptions(OptionsDictValidator):
class ViewPlugin(Plugin[ViewOptions]):
plugin_options_type = ViewOptions
priority: PluginPriority = PluginPriority(modify_entry=PluginPriority.MODIFY_ENTRY_AFTER_SPLIT)
_MAX_LINE_WIDTH: int = 80

8
src/ytdl_sub/plugins/regex.py
View file

@ -7,9 +7,8 @@ from typing import Set
from ytdl_sub.config.overrides import Overrides
from ytdl_sub.config.plugin import Plugin
from ytdl_sub.config.plugin import PluginPriority
from ytdl_sub.config.plugin_operation import PluginOperation
from ytdl_sub.config.preset_options import OptionsDictValidator
from ytdl_sub.config.preset_options import PluginOperation
from ytdl_sub.entries.entry import Entry
from ytdl_sub.script.parser import parse
from ytdl_sub.script.utils.exceptions import ScriptVariableNotResolved
@ -227,7 +226,7 @@ class RegexOptions(OptionsDictValidator):
) -> bool:
if input_variable_name in unresolved_variables:
return False
for capture_group_default in regex_options.capture_group_defaults:
for capture_group_default in regex_options.capture_group_defaults or []:
parsed_default = parse(capture_group_default.format_string)
if parsed_default.variables.issubset(unresolved_variables):
return False
@ -261,9 +260,6 @@ class RegexOptions(OptionsDictValidator):
class RegexPlugin(Plugin[RegexOptions]):
plugin_options_type = RegexOptions
priority = PluginPriority(
modify_entry=PluginPriority.MODIFY_ENTRY_AFTER_SPLIT + 0,
)
def __init__(
self,

2
src/ytdl_sub/plugins/split_by_chapters.py
View file

@ -10,8 +10,8 @@ from typing import Tuple
from yt_dlp.utils import sanitize_filename
from ytdl_sub.config.plugin import SplitPlugin
from ytdl_sub.config.plugin_operation import PluginOperation
from ytdl_sub.config.preset_options import OptionsDictValidator
from ytdl_sub.config.preset_options import PluginOperation
from ytdl_sub.entries.entry import Entry
from ytdl_sub.entries.variables.kwargs import CHAPTERS
from ytdl_sub.entries.variables.kwargs import SPLIT_BY_CHAPTERS_PARENT_ENTRY

2
src/ytdl_sub/plugins/subtitles.py
View file

@ -5,8 +5,8 @@ from typing import Optional
from typing import Set
from ytdl_sub.config.plugin import Plugin
from ytdl_sub.config.plugin_operation import PluginOperation
from ytdl_sub.config.preset_options import OptionsDictValidator
from ytdl_sub.config.preset_options import PluginOperation
from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder
from ytdl_sub.entries.entry import Entry
from ytdl_sub.entries.script.variable_definitions import VARIABLES as v

28
src/ytdl_sub/subscriptions/subscription_download.py
View file

@ -9,6 +9,8 @@ from typing import Optional
from ytdl_sub.config.plugin import Plugin
from ytdl_sub.config.plugin import SplitPlugin
from ytdl_sub.config.plugin_mapping import PluginMapping
from ytdl_sub.config.plugin_operation import PluginOperation
from ytdl_sub.downloaders.info_json.info_json_downloader import InfoJsonDownloader
from ytdl_sub.downloaders.info_json.info_json_downloader import InfoJsonDownloaderOptions
from ytdl_sub.downloaders.source_plugin import SourcePlugin
@ -206,7 +208,9 @@ class SubscriptionDownload(BaseSubscription, ABC):
@classmethod
def _preprocess_entry(cls, plugins: List[Plugin], entry: Entry) -> Optional[Entry]:
maybe_entry: Optional[Entry] = entry
for plugin in sorted(plugins, key=lambda _plugin: _plugin.priority.modify_entry_metadata):
for plugin in PluginMapping.order_plugins_by(
plugins, PluginOperation.MODIFY_ENTRY_METADATA
):
if (maybe_entry := plugin.modify_entry_metadata(maybe_entry)) is None:
return None
@ -216,7 +220,7 @@ class SubscriptionDownload(BaseSubscription, ABC):
self, plugins: List[Plugin], dry_run: bool, entry: Entry, entry_metadata: FileMetadata
):
# Post-process the entry with all plugins
for plugin in sorted(plugins, key=lambda _plugin: _plugin.priority.post_process):
for plugin in PluginMapping.order_plugins_by(plugins, PluginOperation.POST_PROCESS):
optional_plugin_entry_metadata = plugin.post_process_entry(entry)
if optional_plugin_entry_metadata:
entry_metadata.extend(optional_plugin_entry_metadata)
@ -236,7 +240,7 @@ class SubscriptionDownload(BaseSubscription, ABC):
entry_: Optional[Entry] = entry
# First, modify the entry with all plugins
for plugin in sorted(plugins, key=lambda _plugin: _plugin.priority.modify_entry):
for plugin in PluginMapping.order_plugins_by(plugins, PluginOperation.MODIFY_ENTRY):
# Break if it is None, it is indicated to not process any further
if (entry_ := plugin.modify_entry(entry_)) is None:
break
@ -253,18 +257,10 @@ class SubscriptionDownload(BaseSubscription, ABC):
) -> None:
entry_: Optional[Entry] = entry
plugins_pre_split = sorted(
[plugin for plugin in plugins if not plugin.priority.modify_entry_after_split],
key=lambda _plugin: _plugin.priority.modify_entry,
)
plugins_post_split = sorted(
[plugin for plugin in plugins if plugin.priority.modify_entry_after_split],
key=lambda _plugin: _plugin.priority.modify_entry,
)
# First, modify the entry with pre_split plugins
for plugin in plugins_pre_split:
for plugin in PluginMapping.order_plugins_by(
plugins, PluginOperation.MODIFY_ENTRY, before_split=True
):
# Break if it is None, it is indicated to not process any further
if (entry_ := plugin.modify_entry(entry_)) is None:
break
@ -274,7 +270,9 @@ class SubscriptionDownload(BaseSubscription, ABC):
for split_entry, split_entry_metadata in split_plugin.split(entry=entry_):
split_entry_: Optional[Entry] = split_entry
for plugin in plugins_post_split:
for plugin in PluginMapping.order_plugins_by(
plugins, PluginOperation.MODIFY_ENTRY, before_split=False
):
# Return if it is None, it is indicated to not process any further.
# Break out of the plugin loop
if (split_entry_ := plugin.modify_entry(split_entry_)) is None: