[FEATURE] Toggleable plugin field enable for all dict-based plugins (#897)

For key/value-based plugins, add an `enable` field to make it easier for child presets to disable their functionality
This commit is contained in:
Jesse Bannon 2024-01-11 00:40:07 -08:00 committed by GitHub
parent 2076de3074
commit e3158583ca
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
22 changed files with 257 additions and 35 deletions

View file

@ -22,6 +22,15 @@ Extracts audio from a video file.
opus, vorbis, wav, and best to grab the best possible format at runtime.
``enable``
:expected type: Optional[OverridesFormatter]
:description:
Can typically be left undefined to always default to enable. For preset convenience,
this field can be set using an override variable to easily toggle whether this plugin
is enabled or not via Boolean.
``quality``
:expected type: Float
@ -76,6 +85,15 @@ chapters and remove specific ones. Can also remove chapters using regex.
Defaults to True. Embed chapters into the file.
``enable``
:expected type: Optional[OverridesFormatter]
:description:
Can typically be left undefined to always default to enable. For preset convenience,
this field can be set using an override variable to easily toggle whether this plugin
is enabled or not via Boolean.
``force_key_frames``
:expected type: Optional[Boolean]
@ -148,6 +166,15 @@ granularity possible.
Only download videos before this datetime.
``enable``
:expected type: Optional[OverridesFormatter]
:description:
Can typically be left undefined to always default to enable. For preset convenience,
this field can be set using an override variable to easily toggle whether this plugin
is enabled or not via Boolean.
----------------------------------------------------------------------------------------------------
download
@ -259,6 +286,15 @@ Also supports custom ffmpeg conversions:
with ``ffmpeg_post_process_args``. Defaults to ``yt-dlp``.
``enable``
:expected type: Optional[OverridesFormatter]
:description:
Can typically be left undefined to always default to enable. For preset convenience,
this field can be set using an override variable to easily toggle whether this plugin
is enabled or not via Boolean.
``ffmpeg_post_process_args``
:expected type: Optional[OverridesFormatter]
@ -395,6 +431,15 @@ with a ``.nfo`` extension. You can add any values into the NFO.
episode: "{upload_month}{upload_day_padded}"
kodi_safe: False
``enable``
:expected type: Optional[OverridesFormatter]
:description:
Can typically be left undefined to always default to enable. For preset convenience,
this field can be set using an override variable to easily toggle whether this plugin
is enabled or not via Boolean.
``kodi_safe``
:expected type: Optional[Boolean]
@ -484,6 +529,15 @@ Usage:
# optional
kodi_safe: False
``enable``
:expected type: Optional[OverridesFormatter]
:description:
Can typically be left undefined to always default to enable. For preset convenience,
this field can be set using an override variable to easily toggle whether this plugin
is enabled or not via Boolean.
``kodi_safe``
:expected type: Optional[Boolean]
@ -800,6 +854,15 @@ and using ``title_and_description`` can regex match/exclude from either ``title`
- "{upload_month}"
- "{upload_day}"
``enable``
:expected type: Optional[OverridesFormatter]
:description:
Can typically be left undefined to always default to enable. For preset convenience,
this field can be set using an override variable to easily toggle whether this plugin
is enabled or not via Boolean.
``skip_if_match_fails``
:expected type: Optional[Boolean]
@ -880,6 +943,15 @@ It will set the respective language to the correct subtitle file.
webm files can only embed "vtt" subtitle types.
``enable``
:expected type: Optional[OverridesFormatter]
:description:
Can typically be left undefined to always default to enable. For preset convenience,
this field can be set using an override variable to easily toggle whether this plugin
is enabled or not via Boolean.
``languages``
:expected type: Optional[List[String]]
@ -931,6 +1003,15 @@ scripted.
max: 36
subscription_download_probability: 1.0
``enable``
:expected type: Optional[OverridesFormatter]
:description:
Can typically be left undefined to always default to enable. For preset convenience,
this field can be set using an override variable to easily toggle whether this plugin
is enabled or not via Boolean.
``max_downloads_per_subscription``
:expected type: Optional[Range]

View file

@ -1,5 +1,6 @@
from abc import ABC
from abc import abstractmethod
from functools import cached_property
from typing import Dict
from typing import Generic
from typing import List
@ -8,9 +9,11 @@ from typing import Tuple
from typing import Type
from ytdl_sub.config.overrides import Overrides
from ytdl_sub.config.validators.options import ToggleableOptionsDictValidator
from ytdl_sub.config.validators.options import TOptionsValidator
from ytdl_sub.entries.entry import Entry
from ytdl_sub.utils.file_handler import FileMetadata
from ytdl_sub.utils.script import ScriptUtils
from ytdl_sub.ytdl_additions.enhanced_download_archive import DownloadArchiver
from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive
@ -40,6 +43,17 @@ class Plugin(BasePlugin[TOptionsValidator], Generic[TOptionsValidator], ABC):
Class to define the new plugin functionality
"""
@cached_property
def is_enabled(self) -> bool:
"""
Returns True if enabled, False if disabled.
"""
if isinstance(self.plugin_options, ToggleableOptionsDictValidator):
return ScriptUtils.bool_formatter_output(
self.overrides.apply_formatter(self.plugin_options.enable)
)
return True
def ytdl_options_match_filters(self) -> Tuple[List[str], List[str]]:
"""
Returns

View file

@ -6,6 +6,7 @@ from typing import TypeVar
from ytdl_sub.config.plugin.plugin_operation import PluginOperation
from ytdl_sub.utils.exceptions import ValidationException
from ytdl_sub.validators.strict_dict_validator import StrictDictValidator
from ytdl_sub.validators.string_formatter_validators import OverridesBooleanFormatterValidator
from ytdl_sub.validators.validators import Validator
# pylint: disable=no-self-use
@ -57,3 +58,28 @@ TOptionsValidator = TypeVar("TOptionsValidator", bound=OptionsValidator)
class OptionsDictValidator(StrictDictValidator, OptionsValidator, ABC):
pass
class ToggleableOptionsDictValidator(OptionsDictValidator):
_optional_keys = {"enable"}
def __init__(self, name, value):
assert (
"enable" in self._optional_keys
), f"{self.__class__.__name__} does not have enable as an optional field"
super().__init__(name, value)
self._enable = self._validate_key(
key="enable", validator=OverridesBooleanFormatterValidator, default="True"
)
@property
def enable(self) -> OverridesBooleanFormatterValidator:
"""
:expected type: Optional[OverridesFormatter]
:description:
Can typically be left undefined to always default to enable. For preset convenience,
this field can be set using an override variable to easily toggle whether this plugin
is enabled or not via Boolean.
"""
return self._enable

View file

@ -6,7 +6,7 @@ from typing import Set
from ytdl_sub.config.plugin.plugin import Plugin
from ytdl_sub.config.plugin.plugin_operation import PluginOperation
from ytdl_sub.config.validators.options import OptionsDictValidator
from ytdl_sub.config.validators.options import ToggleableOptionsDictValidator
from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder
from ytdl_sub.entries.entry import Entry
from ytdl_sub.entries.script.variable_definitions import VARIABLES
@ -21,7 +21,7 @@ from ytdl_sub.validators.validators import FloatValidator
v: VariableDefinitions = VARIABLES
class AudioExtractOptions(OptionsDictValidator):
class AudioExtractOptions(ToggleableOptionsDictValidator):
"""
Extracts audio from a video file.
@ -35,7 +35,7 @@ class AudioExtractOptions(OptionsDictValidator):
"""
_required_keys = {"codec"}
_optional_keys = {"quality"}
_optional_keys = {"enable", "quality"}
@classmethod
def partial_validate(cls, name: str, value: Any) -> None:

View file

@ -7,7 +7,7 @@ from typing import Set
from ytdl_sub.config.plugin.plugin import Plugin
from ytdl_sub.config.plugin.plugin_operation import PluginOperation
from ytdl_sub.config.validators.options import OptionsDictValidator
from ytdl_sub.config.validators.options import ToggleableOptionsDictValidator
from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder
from ytdl_sub.entries.entry import Entry
from ytdl_sub.entries.entry import ytdl_sub_chapters_from_comments
@ -59,7 +59,7 @@ class SponsorBlockCategoryListValidator(ListValidator[SponsorBlockCategoriesVali
_inner_list_type = SponsorBlockCategoriesValidator
class ChaptersOptions(OptionsDictValidator):
class ChaptersOptions(ToggleableOptionsDictValidator):
"""
Embeds chapters to video files if they are present. Additional options to add SponsorBlock
chapters and remove specific ones. Can also remove chapters using regex.
@ -90,6 +90,7 @@ class ChaptersOptions(OptionsDictValidator):
"""
_optional_keys = {
"enable",
"embed_chapters",
"allow_chapters_from_comments",
"sponsorblock_categories",

View file

@ -3,12 +3,12 @@ from typing import Optional
from typing import Tuple
from ytdl_sub.config.plugin.plugin import Plugin
from ytdl_sub.config.validators.options import OptionsDictValidator
from ytdl_sub.config.validators.options import ToggleableOptionsDictValidator
from ytdl_sub.utils.datetime import to_date_str
from ytdl_sub.validators.string_datetime import StringDatetimeValidator
class DateRangeOptions(OptionsDictValidator):
class DateRangeOptions(ToggleableOptionsDictValidator):
"""
Only download files uploaded within the specified date range.
Dates must adhere to a yt-dlp datetime. From their docs:
@ -31,7 +31,7 @@ class DateRangeOptions(OptionsDictValidator):
after: "today-2weeks"
"""
_optional_keys = {"before", "after"}
_optional_keys = {"enable", "before", "after"}
def __init__(self, name, value):
super().__init__(name, value)

View file

@ -7,7 +7,7 @@ from typing import Set
from ytdl_sub.config.overrides import Overrides
from ytdl_sub.config.plugin.plugin import Plugin
from ytdl_sub.config.plugin.plugin_operation import PluginOperation
from ytdl_sub.config.validators.options import OptionsDictValidator
from ytdl_sub.config.validators.options import ToggleableOptionsDictValidator
from ytdl_sub.entries.entry import Entry
from ytdl_sub.entries.script.variable_definitions import VARIABLES
from ytdl_sub.entries.script.variable_definitions import VariableDefinitions
@ -28,7 +28,7 @@ class FileConvertWithValidator(StringSelectValidator):
_select_values = {"yt-dlp", "ffmpeg"}
class FileConvertOptions(OptionsDictValidator):
class FileConvertOptions(ToggleableOptionsDictValidator):
"""
Converts video files from one extension to another.
@ -56,7 +56,7 @@ class FileConvertOptions(OptionsDictValidator):
"""
_required_keys = {"convert_to"}
_optional_keys = {"convert_with", "ffmpeg_post_process_args"}
_optional_keys = {"enable", "convert_with", "ffmpeg_post_process_args"}
@classmethod
def partial_validate(cls, name: str, value: Any) -> None:

View file

@ -1,4 +1,3 @@
import json
from typing import Dict
from typing import Optional
@ -8,6 +7,7 @@ from ytdl_sub.config.validators.options import OptionsValidator
from ytdl_sub.entries.entry import Entry
from ytdl_sub.utils.exceptions import StringFormattingException
from ytdl_sub.utils.logger import Logger
from ytdl_sub.utils.script import ScriptUtils
from ytdl_sub.validators.string_formatter_validators import ListFormatterValidator
from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive
@ -53,7 +53,9 @@ class FilterExcludePlugin(Plugin[FilterExcludeOptions]):
return entry
for formatter in self.plugin_options.list:
out = json.loads(self.overrides.apply_formatter(formatter=formatter, entry=entry))
out = ScriptUtils.bool_formatter_output(
self.overrides.apply_formatter(formatter=formatter, entry=entry)
)
if bool(out):
logger.info(
"Filtering '%s' from the filter %s evaluating to True",

View file

@ -1,4 +1,3 @@
import json
from typing import Dict
from typing import Optional
@ -8,6 +7,7 @@ from ytdl_sub.config.validators.options import OptionsValidator
from ytdl_sub.entries.entry import Entry
from ytdl_sub.utils.exceptions import StringFormattingException
from ytdl_sub.utils.logger import Logger
from ytdl_sub.utils.script import ScriptUtils
from ytdl_sub.validators.string_formatter_validators import ListFormatterValidator
from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive
@ -61,7 +61,9 @@ class FilterIncludePlugin(Plugin[FilterIncludeOptions]):
return entry
for formatter in self.plugin_options.list:
out = json.loads(self.overrides.apply_formatter(formatter=formatter, entry=entry))
out = ScriptUtils.bool_formatter_output(
self.overrides.apply_formatter(formatter=formatter, entry=entry)
)
if not bool(out):
logger.info(
"Filtering '%s' from the filter %s evaluating to False",

View file

@ -4,7 +4,7 @@ from typing import List
from typing import Tuple
from ytdl_sub.config.plugin.plugin import Plugin
from ytdl_sub.config.validators.options import OptionsDictValidator
from ytdl_sub.config.validators.options import ToggleableOptionsDictValidator
from ytdl_sub.utils.logger import Logger
from ytdl_sub.validators.validators import StringListValidator
@ -56,7 +56,7 @@ def combine_filters(filters: List[str], to_combine: List[str]) -> List[str]:
return output_filters
class MatchFiltersOptions(OptionsDictValidator):
class MatchFiltersOptions(ToggleableOptionsDictValidator):
"""
Set ``--match-filters`` to pass into yt-dlp to filter entries from being downloaded.
Uses the same syntax as yt-dlp. An entry will be downloaded if any one of the filters are met.
@ -74,7 +74,7 @@ class MatchFiltersOptions(OptionsDictValidator):
# - "availability=?public"
"""
_optional_keys = {"filters"}
_optional_keys = {"enable", "filters"}
@classmethod
def partial_validate(cls, name: str, value: Any) -> None:

View file

@ -8,7 +8,7 @@ from typing import List
from typing import Optional
from ytdl_sub.config.plugin.plugin import Plugin
from ytdl_sub.config.validators.options import OptionsDictValidator
from ytdl_sub.config.validators.options import ToggleableOptionsDictValidator
from ytdl_sub.entries.entry import Entry
from ytdl_sub.utils.file_handler import FileHandler
from ytdl_sub.utils.file_handler import FileMetadata
@ -23,13 +23,13 @@ from ytdl_sub.validators.string_formatter_validators import StringFormatterValid
from ytdl_sub.validators.validators import BoolValidator
class SharedNfoTagsOptions(OptionsDictValidator):
class SharedNfoTagsOptions(ToggleableOptionsDictValidator):
"""
Shared code between NFO tags and Ouptut Directory NFO Tags
"""
_required_keys = {"nfo_name", "nfo_root", "tags"}
_optional_keys = {"kodi_safe"}
_optional_keys = {"enable", "kodi_safe"}
@classmethod
def partial_validate(cls, name: str, value: Any) -> None:

View file

@ -32,7 +32,7 @@ class OutputDirectoryNfoTagsOptions(SharedNfoTagsOptions):
# Hack to make it so collection named seasons do not error
# when adding output_directory_nfo info for plex
_required_keys = set()
_optional_keys = {"kodi_safe", "nfo_name", "nfo_root", "tags"}
_optional_keys = {"enable", "kodi_safe", "nfo_name", "nfo_root", "tags"}
@property
def nfo_root(self) -> StringFormatterValidator:

View file

@ -8,7 +8,7 @@ from typing import Set
from ytdl_sub.config.overrides import Overrides
from ytdl_sub.config.plugin.plugin import Plugin
from ytdl_sub.config.plugin.plugin_operation import PluginOperation
from ytdl_sub.config.validators.options import OptionsDictValidator
from ytdl_sub.config.validators.options import ToggleableOptionsDictValidator
from ytdl_sub.entries.entry import Entry
from ytdl_sub.script.parser import parse
from ytdl_sub.script.utils.exceptions import RuntimeException
@ -120,7 +120,7 @@ class FromSourceVariablesRegex(DictValidator):
}
class RegexOptions(OptionsDictValidator):
class RegexOptions(ToggleableOptionsDictValidator):
r"""
.. attention::
@ -218,7 +218,7 @@ class RegexOptions(OptionsDictValidator):
"""
_required_keys = {"from"}
_optional_keys = {"skip_if_match_fails"}
_optional_keys = {"enable", "skip_if_match_fails"}
@classmethod
def partial_validate(cls, name: str, value: Any) -> None:

View file

@ -6,7 +6,7 @@ from typing import Set
from ytdl_sub.config.plugin.plugin import Plugin
from ytdl_sub.config.plugin.plugin_operation import PluginOperation
from ytdl_sub.config.validators.options import OptionsDictValidator
from ytdl_sub.config.validators.options import ToggleableOptionsDictValidator
from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder
from ytdl_sub.entries.entry import Entry
from ytdl_sub.entries.script.variable_definitions import VARIABLES
@ -31,7 +31,7 @@ class SubtitlesTypeValidator(StringSelectValidator):
_select_values = SUBTITLE_EXTENSIONS
class SubtitleOptions(OptionsDictValidator):
class SubtitleOptions(ToggleableOptionsDictValidator):
"""
Defines how to download and store subtitles. Using this plugin creates two new variables:
``lang`` and ``subtitles_ext``. ``lang`` is dynamic since you can download multiple subtitles.
@ -52,6 +52,7 @@ class SubtitleOptions(OptionsDictValidator):
"""
_optional_keys = {
"enable",
"subtitles_name",
"subtitles_type",
"embed_subtitles",

View file

@ -6,7 +6,7 @@ from typing import Tuple
from ytdl_sub.config.overrides import Overrides
from ytdl_sub.config.plugin.plugin import Plugin
from ytdl_sub.config.validators.options import OptionsDictValidator
from ytdl_sub.config.validators.options import ToggleableOptionsDictValidator
from ytdl_sub.entries.entry import Entry
from ytdl_sub.utils.file_handler import FileMetadata
from ytdl_sub.utils.logger import Logger
@ -59,7 +59,7 @@ class RandomizedRangeValidator(StrictDictValidator):
return int(self.randomized_float())
class ThrottleProtectionOptions(OptionsDictValidator):
class ThrottleProtectionOptions(ToggleableOptionsDictValidator):
"""
Provides options to make ytdl-sub look more 'human-like' to protect from throttling. For
range-based values, a random number will be chosen within the range to avoid sleeps looking
@ -85,6 +85,7 @@ class ThrottleProtectionOptions(OptionsDictValidator):
"""
_optional_keys = {
"enable",
"sleep_per_download_s",
"sleep_per_subscription_s",
"max_downloads_per_subscription",

View file

@ -3,7 +3,7 @@ from typing import Any
from typing import Dict
from ytdl_sub.config.plugin.plugin import Plugin
from ytdl_sub.config.validators.options import OptionsDictValidator
from ytdl_sub.config.validators.options import ToggleableOptionsDictValidator
from ytdl_sub.entries.entry import Entry
from ytdl_sub.utils.ffmpeg import add_ffmpeg_metadata_key_values
from ytdl_sub.utils.file_handler import FileMetadata
@ -13,7 +13,7 @@ from ytdl_sub.validators.string_formatter_validators import DictFormatterValidat
logger = Logger.get("video-tags")
class VideoTagsOptions(OptionsDictValidator):
class VideoTagsOptions(ToggleableOptionsDictValidator):
"""
Adds tags to every downloaded video file using ffmpeg ``-metadata key=value`` args.
@ -27,7 +27,7 @@ class VideoTagsOptions(OptionsDictValidator):
description: "{description}"
"""
_optional_keys = {"tags"}
_optional_keys = {"enable", "tags"}
_allow_extra_keys = True
@classmethod

View file

@ -190,7 +190,7 @@ class SubscriptionDownload(BaseSubscription, ABC):
-------
List of plugins defined in the subscription, initialized and ready to use.
"""
return [
plugins = [
plugin_type(
options=plugin_options,
overrides=self.overrides,
@ -198,6 +198,7 @@ class SubscriptionDownload(BaseSubscription, ABC):
)
for plugin_type, plugin_options in self.plugins.zipped()
]
return [plugin for plugin in plugins if plugin.is_enabled]
@classmethod
def _cleanup_entry_files(cls, entry: Entry):

View file

@ -38,3 +38,15 @@ class ScriptUtils:
out = f"{{%from_json('''{dumped_json}''')}}"
return out
@classmethod
def bool_formatter_output(cls, output: str) -> bool:
"""
Translate formatter output to a boolean
"""
if not output or output.lower() == "false":
return False
try:
return bool(json.loads(output))
except Exception: # pylint: disable=broad-except
return True

View file

@ -92,7 +92,7 @@ class OverridesStringFormatterValidator(StringFormatterValidator):
# pylint: enable=line-too-long
class OverridesIntegerFormatterValidator(StringFormatterValidator):
class OverridesIntegerFormatterValidator(OverridesStringFormatterValidator):
_expected_value_type_name = "integer"
def post_process(self, resolved: str) -> str:
@ -106,6 +106,10 @@ class OverridesIntegerFormatterValidator(StringFormatterValidator):
return resolved
class OverridesBooleanFormatterValidator(OverridesStringFormatterValidator):
_expected_value_type_name = "boolean"
class ListFormatterValidator(ListValidator[StringFormatterValidator]):
_inner_list_type = StringFormatterValidator

View file

@ -113,7 +113,7 @@ def assert_logs(
for call_args in patched_debug.call_args_list:
occurrences += int(expected_message in call_args.args[0])
if expected_occurrences:
if expected_occurrences is not None:
assert (
occurrences == expected_occurrences
), f"{expected_message} was expected {expected_occurrences} times, got {occurrences}"

View file

@ -1,3 +1,4 @@
import pytest
from conftest import assert_logs
from ytdl_sub.plugins.throttle_protection import logger as throttle_protection_logger
@ -57,3 +58,59 @@ class TestThrottleProtectionPlugin:
expected_occurrences=1,
):
_ = subscription.download(dry_run=False)
@pytest.mark.parametrize(
"disable_value",
[
"",
False,
"{tp_bool_string}",
"{tp_empty_string}",
],
)
def test_disabled(
self,
config,
subscription_name,
output_directory,
mock_download_collection_entries,
disable_value,
):
preset_dict = {
"preset": [
"Kodi Music Videos",
],
"overrides": {
"url": "https://your.name.here",
"music_video_directory": output_directory,
"tp_bool_string": "{ %bool(False) }",
"tp_empty_string": "",
},
"throttle_protection": {
"enable": disable_value,
"sleep_per_download_s": {
"min": 0.01,
"max": 0.01,
},
"sleep_per_subscription_s": {
"min": 0.02,
"max": 0.02,
},
},
}
subscription = Subscription.from_dict(
config=config,
preset_name=subscription_name,
preset_dict=preset_dict,
)
with mock_download_collection_entries(
is_youtube_channel=False, num_urls=1, is_extracted_audio=False
), assert_logs(
logger=throttle_protection_logger,
expected_message="Sleeping between downloads for %0.2f seconds",
log_level="debug",
expected_occurrences=0,
):
_ = subscription.download(dry_run=False)

View file

@ -1,5 +1,6 @@
import copy
import pytest
from unit.script.conftest import single_variable_output
from ytdl_sub.utils.script import ScriptUtils
@ -31,3 +32,22 @@ class TestScriptUtils:
output = single_variable_output(ScriptUtils.to_script(json_dict))
assert output == expected_output
@pytest.mark.parametrize(
"input_str, expected_output",
[
("", False),
("true", True),
("false", False),
("[ ]", False),
("{ }", False),
("True", True),
("False", False),
("lol not False", True),
("0", False),
("-1", True),
("1", True),
],
)
def test_bool_formatter_output(self, input_str: str, expected_output: bool):
assert ScriptUtils.bool_formatter_output(input_str) == expected_output