From df8b4a1df80c0ee6e91bc3842f1683daa90adcb2 Mon Sep 17 00:00:00 2001 From: Jesse Bannon Date: Thu, 28 Aug 2025 23:55:57 -0700 Subject: [PATCH] [FEATURE] Override variable support for throttle protection ranges (#1315) Adds support for setting throttle_protection range values using static override variables. `sleep_per_download_s` has additional support for entry variables since it's used per entry. We can now experiment with scaling this value based on entry attributes, such as duration. Example: ``` throttle_protection: sleep_per_download_s: min: >- { %mul(5.5, %pow( duration, 0.4 )) } max: >- { %mul(6.5, %pow( duration, 0.6 )) } ``` --- docs/source/config_reference/plugins.rst | 3 + src/ytdl_sub/plugins/throttle_protection.py | 175 +++++++++++++----- .../validators/string_formatter_validators.py | 20 ++ .../plugins/test_throttle_protection.py | 58 +++++- 4 files changed, 211 insertions(+), 45 deletions(-) diff --git a/docs/source/config_reference/plugins.rst b/docs/source/config_reference/plugins.rst index 97c88886..f2213937 100644 --- a/docs/source/config_reference/plugins.rst +++ b/docs/source/config_reference/plugins.rst @@ -1005,6 +1005,9 @@ Provides options to make ytdl-sub look more 'human-like' to protect from throttl range-based values, a random number will be chosen within the range to avoid sleeps looking scripted. +Range min and max values support static override variables within their definitions. +``sleep_per_download_s`` supports both static and override variables. + :Usage: .. code-block:: yaml diff --git a/src/ytdl_sub/plugins/throttle_protection.py b/src/ytdl_sub/plugins/throttle_protection.py index 27cad374..cf01e537 100644 --- a/src/ytdl_sub/plugins/throttle_protection.py +++ b/src/ytdl_sub/plugins/throttle_protection.py @@ -1,7 +1,10 @@ import random import time +from abc import ABC from typing import Dict from typing import Optional +from typing import Type +from typing import TypeVar from ytdl_sub.config.overrides import Overrides from ytdl_sub.config.plugin.plugin import Plugin @@ -10,68 +13,135 @@ from ytdl_sub.entries.entry import Entry from ytdl_sub.utils.file_handler import FileMetadata from ytdl_sub.utils.logger import Logger from ytdl_sub.validators.strict_dict_validator import StrictDictValidator -from ytdl_sub.validators.validators import FloatValidator +from ytdl_sub.validators.string_formatter_validators import FloatFormatterValidator +from ytdl_sub.validators.string_formatter_validators import OverridesFloatFormatterValidator from ytdl_sub.validators.validators import ProbabilityValidator from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive logger = Logger.get("throttle-protection") +FloatValidatorT = TypeVar("FloatValidatorT", bound=FloatFormatterValidator) -class RandomizedRangeValidator(StrictDictValidator): + +class _RandomizedRangeValidator(StrictDictValidator, ABC): """ - Validator to specify a float range between [min, max) + Base class for range validation, to support both entry and static overrides. """ + _float_validator: Type[FloatValidatorT] + _required_keys = {"max"} _optional_keys = {"min"} def __init__(self, name, value): super().__init__(name, value) - self._max = self._validate_key(key="max", validator=FloatValidator).value + self._max = self._validate_key(key="max", validator=self._float_validator) self._min = self._validate_key_if_present( - key="min", validator=FloatValidator, default=0.0 - ).value + key="min", validator=self._float_validator, default=0.0 + ) - if self._min < 0: - raise self._validation_exception("min must be greater than zero") + def _randomized_float(self, overrides: Overrides, entry: Optional[Entry] = None) -> float: + actualized_min = float(overrides.apply_formatter(self._min, entry=entry)) + actualized_max = float(overrides.apply_formatter(self._max, entry=entry)) - if self._max < self._min: + if actualized_min < 0: raise self._validation_exception( - f"max ({self._max}) must be greater than or equal to min ({self._min})" + f"min must be greater than zero, received {actualized_min}" + ) + if actualized_max < actualized_min: + raise self._validation_exception( + f"max ({actualized_max}) must be greater than or equal to min ({actualized_min})" ) - def min_value(self) -> float: - """ - Returns - ------- - Minimum value - """ - return self._min + return random.uniform(actualized_min, actualized_max) - def max_value(self) -> float: - """ - Returns - ------- - Maximum value - """ - return self._max - - def randomized_float(self) -> float: - """ - Returns - ------- - A random float within the range - """ - return random.uniform(self._min, self._max) - - def randomized_int(self) -> int: + def _randomized_int(self, overrides: Overrides, entry: Optional[Entry] = None) -> int: """ Returns ------- A random float within the range, then cast to an integer (floored) """ - return int(self.randomized_float()) + return int(self._randomized_float(overrides, entry=entry)) + + def _max_value(self, overrides: Overrides, entry: Optional[Entry] = None) -> float: + """ + Returns + ------- + Max possible value + """ + actualized_max = float(overrides.apply_formatter(self._max, entry=entry)) + if actualized_max < 0: + raise self._validation_exception( + f"max must be greater than zero, received {actualized_max}" + ) + return actualized_max + + +class RandomizedRangeValidator(_RandomizedRangeValidator): + """ + Validator to specify a float range between [min, max) with both + override and entry variable support. + """ + + _float_validator = FloatFormatterValidator + + def randomized_float(self, overrides: Overrides, entry: Entry) -> float: + """ + Returns + ------- + A random float within the range + """ + return self._randomized_float(overrides=overrides, entry=entry) + + def randomized_int(self, overrides: Overrides, entry: Entry) -> int: + """ + Returns + ------- + A random float within the range, then cast to an integer (floored) + """ + return self._randomized_int(overrides=overrides, entry=entry) + + def max_value(self, overrides: Overrides, entry: Entry) -> float: + """ + Returns + ------- + Max possible value + """ + return self._max_value(overrides=overrides, entry=entry) + + +class RandomizedRangeOverridesValidator(_RandomizedRangeValidator): + """ + Validator to specify a float range between [min, max) with + static variable support. + """ + + _float_validator = OverridesFloatFormatterValidator + + def randomized_float(self, overrides: Overrides) -> float: + """ + Returns + ------- + A random float within the range + """ + return self._randomized_float(overrides=overrides) + + def randomized_int(self, overrides: Overrides) -> int: + """ + Returns + ------- + A random float within the range, then cast to an integer (floored) + """ + return self._randomized_int(overrides=overrides) + + def max_value(self, overrides: Overrides) -> float: + """ + Returns + ------- + Max possible value + """ + return self._max_value(overrides=overrides) class ThrottleProtectionOptions(ToggleableOptionsDictValidator): @@ -80,6 +150,9 @@ class ThrottleProtectionOptions(ToggleableOptionsDictValidator): range-based values, a random number will be chosen within the range to avoid sleeps looking scripted. + Range min and max values support static override variables within their definitions. + ``sleep_per_download_s`` supports both static and override variables. + :Usage: .. code-block:: yaml @@ -115,16 +188,16 @@ class ThrottleProtectionOptions(ToggleableOptionsDictValidator): super().__init__(name, value) self._sleep_per_request_s = self._validate_key_if_present( - key="sleep_per_request_s", validator=RandomizedRangeValidator + key="sleep_per_request_s", validator=RandomizedRangeOverridesValidator ) self._sleep_per_download_s = self._validate_key_if_present( key="sleep_per_download_s", validator=RandomizedRangeValidator ) self._sleep_per_subscription_s = self._validate_key_if_present( - key="sleep_per_subscription_s", validator=RandomizedRangeValidator + key="sleep_per_subscription_s", validator=RandomizedRangeOverridesValidator ) self._max_downloads_per_subscription = self._validate_key_if_present( - key="max_downloads_per_subscription", validator=RandomizedRangeValidator + key="max_downloads_per_subscription", validator=RandomizedRangeOverridesValidator ) self._subscription_download_probability = self._validate_key_if_present( key="subscription_download_probability", validator=ProbabilityValidator @@ -205,12 +278,18 @@ class ThrottleProtectionPlugin(Plugin[ThrottleProtectionOptions]): # If subscriptions have a max download limit, set it here for the first subscription if self.plugin_options.max_downloads_per_subscription: self._subscription_max_downloads = ( - self.plugin_options.max_downloads_per_subscription.randomized_int() + self.plugin_options.max_downloads_per_subscription.randomized_int( + overrides=self.overrides + ) ) def ytdl_options(self) -> Optional[Dict]: if self.plugin_options.sleep_per_request_s is not None: - return {"sleep_interval_requests": self.plugin_options.sleep_per_request_s.max_value()} + return { + "sleep_interval_requests": self.plugin_options.sleep_per_request_s.max_value( + overrides=self.overrides + ) + } return {} def initialize_subscription(self) -> bool: @@ -254,8 +333,12 @@ class ThrottleProtectionPlugin(Plugin[ThrottleProtectionOptions]): self._subscription_download_counter += 1 if self.plugin_options.sleep_per_download_s: - sleep_time = self.plugin_options.sleep_per_download_s.randomized_float() - logger.info("Sleeping between downloads for %0.2f seconds", sleep_time) + sleep_time = self.plugin_options.sleep_per_download_s.randomized_float( + overrides=self.overrides, entry=entry + ) + # pylint: disable=logging-fstring-interpolation) + # needed to test logs in unit test + logger.info(f"Sleeping between downloads for {sleep_time:.2f} seconds") self.perform_sleep(sleep_time) return None @@ -267,10 +350,14 @@ class ThrottleProtectionPlugin(Plugin[ThrottleProtectionOptions]): # If present, reset max downloads for the next subscription if self.plugin_options.max_downloads_per_subscription: self._subscription_max_downloads = ( - self.plugin_options.max_downloads_per_subscription.randomized_int + self.plugin_options.max_downloads_per_subscription.randomized_int( + overrides=self.overrides + ) ) if self.plugin_options.sleep_per_subscription_s: - sleep_time = self.plugin_options.sleep_per_subscription_s.randomized_float() + sleep_time = self.plugin_options.sleep_per_subscription_s.randomized_float( + overrides=self.overrides + ) logger.info("Sleeping between subscriptions for %0.2f seconds", sleep_time) self.perform_sleep(sleep_time) diff --git a/src/ytdl_sub/validators/string_formatter_validators.py b/src/ytdl_sub/validators/string_formatter_validators.py index 855a607f..17436700 100644 --- a/src/ytdl_sub/validators/string_formatter_validators.py +++ b/src/ytdl_sub/validators/string_formatter_validators.py @@ -74,6 +74,20 @@ class StringFormatterValidator(StringValidator): return resolved +class FloatFormatterValidator(StringFormatterValidator): + _expected_value_type_name = "float" + + def post_process(self, resolved: str) -> str: + try: + float(resolved) + except Exception as exc: + raise self._validation_exception( + f"Expected a float, but received '{resolved}'" + ) from exc + + return resolved + + class StandardizedDateValidator(StringFormatterValidator): _expected_value_type_name = "standardized_date" @@ -118,6 +132,12 @@ class OverridesIntegerFormatterValidator(OverridesStringFormatterValidator): return resolved +class OverridesFloatFormatterValidator(FloatFormatterValidator, OverridesStringFormatterValidator): + """ + Float validator but static + """ + + class OverridesBooleanFormatterValidator(OverridesStringFormatterValidator): _expected_value_type_name = "boolean" diff --git a/tests/integration/plugins/test_throttle_protection.py b/tests/integration/plugins/test_throttle_protection.py index b5007ec5..d2275365 100644 --- a/tests/integration/plugins/test_throttle_protection.py +++ b/tests/integration/plugins/test_throttle_protection.py @@ -8,6 +8,7 @@ from ytdl_sub.plugins.throttle_protection import logger as throttle_protection_l from ytdl_sub.script.functions.print_functions import logger as script_print_logger from ytdl_sub.script.utils.exceptions import UserThrownRuntimeError from ytdl_sub.subscriptions.subscription import Subscription +from ytdl_sub.utils.exceptions import StringFormattingVariableNotFoundException @pytest.fixture @@ -57,7 +58,7 @@ class TestThrottleProtectionPlugin: ), assert_logs( logger=throttle_protection_logger, - expected_message="Sleeping between downloads for %0.2f seconds", + expected_message="Sleeping between downloads for 0.01 seconds", log_level="info", expected_occurrences=4, ), @@ -288,3 +289,58 @@ class TestResolutionAssert: pytest.raises(UserThrownRuntimeError, match=re.escape(expected_message)), ): _ = subscription.download(dry_run=True) + + def test_sleep_per_download_supports_entry_variables( + self, + config, + subscription_name, + throttle_subscription_dict, + output_directory, + mock_download_collection_entries, + ): + throttle_subscription_dict["throttle_protection"]["sleep_per_download_s"] = { + "min": "{%mul(3.14, duration)}", + "max": "{%mul(3.14, duration)}", + } + + subscription = Subscription.from_dict( + config=config, + preset_name=subscription_name, + preset_dict=throttle_subscription_dict, + ) + + with ( + mock_download_collection_entries( + is_youtube_channel=False, + num_urls=1, + is_extracted_audio=False, + mock_entry_kwargs={"duration": 1}, + ), + assert_logs( + logger=throttle_protection_logger, + expected_message="Sleeping between downloads for 3.14 seconds", + log_level="info", + expected_occurrences=4, + ), + ): + _ = subscription.download(dry_run=False) + + def test_sleep_per_subscription_does_not_support_entry_variables( + self, + config, + subscription_name, + throttle_subscription_dict, + output_directory, + mock_download_collection_entries, + ): + throttle_subscription_dict["throttle_protection"]["sleep_per_subscription_s"] = { + "min": "{%mul(3.14, duration)}", + "max": "{%mul(3.14, duration)}", + } + + with pytest.raises(StringFormattingVariableNotFoundException): + _ = Subscription.from_dict( + config=config, + preset_name=subscription_name, + preset_dict=throttle_subscription_dict, + )