[FEATURE] Override variable support for throttle protection ranges (#1315)

Adds support for setting throttle_protection range values using static override variables. `sleep_per_download_s` has additional support for entry variables since it's used per entry.

We can now experiment with scaling this value based on entry attributes, such as duration. Example:

```
throttle_protection:
  sleep_per_download_s:
    min: >-
      {
        %mul(5.5, %pow( duration, 0.4 ))
      }
    max: >-
      {
        %mul(6.5, %pow( duration, 0.6 ))
      }
```

docs/source/config_reference/plugins.rst

@@ -1005,6 +1005,9 @@ Provides options to make ytdl-sub look more 'human-like' to protect from throttl
 range-based values, a random number will be chosen within the range to avoid sleeps looking
 scripted.
 
+Range min and max values support static override variables within their definitions.
+``sleep_per_download_s`` supports both static and override variables.
+
 :Usage:
 
 .. code-block:: yaml

src/ytdl_sub/plugins/throttle_protection.py

@@ -1,7 +1,10 @@
 import random
 import time
+from abc import ABC
 from typing import Dict
 from typing import Optional
+from typing import Type
+from typing import TypeVar
 
 from ytdl_sub.config.overrides import Overrides
 from ytdl_sub.config.plugin.plugin import Plugin
@@ -10,68 +13,135 @@ from ytdl_sub.entries.entry import Entry
 from ytdl_sub.utils.file_handler import FileMetadata
 from ytdl_sub.utils.logger import Logger
 from ytdl_sub.validators.strict_dict_validator import StrictDictValidator
-from ytdl_sub.validators.validators import FloatValidator
+from ytdl_sub.validators.string_formatter_validators import FloatFormatterValidator
+from ytdl_sub.validators.string_formatter_validators import OverridesFloatFormatterValidator
 from ytdl_sub.validators.validators import ProbabilityValidator
 from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive
 
 logger = Logger.get("throttle-protection")
 
+FloatValidatorT = TypeVar("FloatValidatorT", bound=FloatFormatterValidator)
 
-class RandomizedRangeValidator(StrictDictValidator):
+
+class _RandomizedRangeValidator(StrictDictValidator, ABC):
     """
-    Validator to specify a float range between [min, max)
+    Base class for range validation, to support both entry and static overrides.
     """
 
+    _float_validator: Type[FloatValidatorT]
+
     _required_keys = {"max"}
     _optional_keys = {"min"}
 
     def __init__(self, name, value):
         super().__init__(name, value)
 
-        self._max = self._validate_key(key="max", validator=FloatValidator).value
+        self._max = self._validate_key(key="max", validator=self._float_validator)
         self._min = self._validate_key_if_present(
-            key="min", validator=FloatValidator, default=0.0
-        ).value
+            key="min", validator=self._float_validator, default=0.0
+        )
 
-        if self._min < 0:
-            raise self._validation_exception("min must be greater than zero")
+    def _randomized_float(self, overrides: Overrides, entry: Optional[Entry] = None) -> float:
+        actualized_min = float(overrides.apply_formatter(self._min, entry=entry))
+        actualized_max = float(overrides.apply_formatter(self._max, entry=entry))
 
-        if self._max < self._min:
+        if actualized_min < 0:
             raise self._validation_exception(
-                f"max ({self._max}) must be greater than or equal to min ({self._min})"
+                f"min must be greater than zero, received {actualized_min}"
+            )
+        if actualized_max < actualized_min:
+            raise self._validation_exception(
+                f"max ({actualized_max}) must be greater than or equal to min ({actualized_min})"
             )
 
-    def min_value(self) -> float:
-        """
-        Returns
-        -------
-        Minimum value
-        """
-        return self._min
+        return random.uniform(actualized_min, actualized_max)
 
-    def max_value(self) -> float:
-        """
-        Returns
-        -------
-        Maximum value
-        """
-        return self._max
-
-    def randomized_float(self) -> float:
-        """
-        Returns
-        -------
-        A random float within the range
-        """
-        return random.uniform(self._min, self._max)
-
-    def randomized_int(self) -> int:
+    def _randomized_int(self, overrides: Overrides, entry: Optional[Entry] = None) -> int:
         """
         Returns
         -------
         A random float within the range, then cast to an integer (floored)
         """
-        return int(self.randomized_float())
+        return int(self._randomized_float(overrides, entry=entry))
+
+    def _max_value(self, overrides: Overrides, entry: Optional[Entry] = None) -> float:
+        """
+        Returns
+        -------
+        Max possible value
+        """
+        actualized_max = float(overrides.apply_formatter(self._max, entry=entry))
+        if actualized_max < 0:
+            raise self._validation_exception(
+                f"max must be greater than zero, received {actualized_max}"
+            )
+        return actualized_max
+
+
+class RandomizedRangeValidator(_RandomizedRangeValidator):
+    """
+    Validator to specify a float range between [min, max) with both
+    override and entry variable support.
+    """
+
+    _float_validator = FloatFormatterValidator
+
+    def randomized_float(self, overrides: Overrides, entry: Entry) -> float:
+        """
+        Returns
+        -------
+        A random float within the range
+        """
+        return self._randomized_float(overrides=overrides, entry=entry)
+
+    def randomized_int(self, overrides: Overrides, entry: Entry) -> int:
+        """
+        Returns
+        -------
+        A random float within the range, then cast to an integer (floored)
+        """
+        return self._randomized_int(overrides=overrides, entry=entry)
+
+    def max_value(self, overrides: Overrides, entry: Entry) -> float:
+        """
+        Returns
+        -------
+        Max possible value
+        """
+        return self._max_value(overrides=overrides, entry=entry)
+
+
+class RandomizedRangeOverridesValidator(_RandomizedRangeValidator):
+    """
+    Validator to specify a float range between [min, max) with
+    static variable support.
+    """
+
+    _float_validator = OverridesFloatFormatterValidator
+
+    def randomized_float(self, overrides: Overrides) -> float:
+        """
+        Returns
+        -------
+        A random float within the range
+        """
+        return self._randomized_float(overrides=overrides)
+
+    def randomized_int(self, overrides: Overrides) -> int:
+        """
+        Returns
+        -------
+        A random float within the range, then cast to an integer (floored)
+        """
+        return self._randomized_int(overrides=overrides)
+
+    def max_value(self, overrides: Overrides) -> float:
+        """
+        Returns
+        -------
+        Max possible value
+        """
+        return self._max_value(overrides=overrides)
 
 
 class ThrottleProtectionOptions(ToggleableOptionsDictValidator):
@@ -80,6 +150,9 @@ class ThrottleProtectionOptions(ToggleableOptionsDictValidator):
     range-based values, a random number will be chosen within the range to avoid sleeps looking
     scripted.
 
+    Range min and max values support static override variables within their definitions.
+    ``sleep_per_download_s`` supports both static and override variables.
+
     :Usage:
 
     .. code-block:: yaml
@@ -115,16 +188,16 @@ class ThrottleProtectionOptions(ToggleableOptionsDictValidator):
         super().__init__(name, value)
 
         self._sleep_per_request_s = self._validate_key_if_present(
-            key="sleep_per_request_s", validator=RandomizedRangeValidator
+            key="sleep_per_request_s", validator=RandomizedRangeOverridesValidator
         )
         self._sleep_per_download_s = self._validate_key_if_present(
             key="sleep_per_download_s", validator=RandomizedRangeValidator
         )
         self._sleep_per_subscription_s = self._validate_key_if_present(
-            key="sleep_per_subscription_s", validator=RandomizedRangeValidator
+            key="sleep_per_subscription_s", validator=RandomizedRangeOverridesValidator
         )
         self._max_downloads_per_subscription = self._validate_key_if_present(
-            key="max_downloads_per_subscription", validator=RandomizedRangeValidator
+            key="max_downloads_per_subscription", validator=RandomizedRangeOverridesValidator
         )
         self._subscription_download_probability = self._validate_key_if_present(
             key="subscription_download_probability", validator=ProbabilityValidator
@@ -205,12 +278,18 @@ class ThrottleProtectionPlugin(Plugin[ThrottleProtectionOptions]):
         # If subscriptions have a max download limit, set it here for the first subscription
         if self.plugin_options.max_downloads_per_subscription:
             self._subscription_max_downloads = (
-                self.plugin_options.max_downloads_per_subscription.randomized_int()
+                self.plugin_options.max_downloads_per_subscription.randomized_int(
+                    overrides=self.overrides
+                )
             )
 
     def ytdl_options(self) -> Optional[Dict]:
         if self.plugin_options.sleep_per_request_s is not None:
-            return {"sleep_interval_requests": self.plugin_options.sleep_per_request_s.max_value()}
+            return {
+                "sleep_interval_requests": self.plugin_options.sleep_per_request_s.max_value(
+                    overrides=self.overrides
+                )
+            }
         return {}
 
     def initialize_subscription(self) -> bool:
@@ -254,8 +333,12 @@ class ThrottleProtectionPlugin(Plugin[ThrottleProtectionOptions]):
         self._subscription_download_counter += 1
 
         if self.plugin_options.sleep_per_download_s:
-            sleep_time = self.plugin_options.sleep_per_download_s.randomized_float()
-            logger.info("Sleeping between downloads for %0.2f seconds", sleep_time)
+            sleep_time = self.plugin_options.sleep_per_download_s.randomized_float(
+                overrides=self.overrides, entry=entry
+            )
+            # pylint: disable=logging-fstring-interpolation)
+            # needed to test logs in unit test
+            logger.info(f"Sleeping between downloads for {sleep_time:.2f} seconds")
             self.perform_sleep(sleep_time)
 
         return None
@@ -267,10 +350,14 @@ class ThrottleProtectionPlugin(Plugin[ThrottleProtectionOptions]):
         # If present, reset max downloads for the next subscription
         if self.plugin_options.max_downloads_per_subscription:
             self._subscription_max_downloads = (
-                self.plugin_options.max_downloads_per_subscription.randomized_int
+                self.plugin_options.max_downloads_per_subscription.randomized_int(
+                    overrides=self.overrides
+                )
             )
 
         if self.plugin_options.sleep_per_subscription_s:
-            sleep_time = self.plugin_options.sleep_per_subscription_s.randomized_float()
+            sleep_time = self.plugin_options.sleep_per_subscription_s.randomized_float(
+                overrides=self.overrides
+            )
             logger.info("Sleeping between subscriptions for %0.2f seconds", sleep_time)
             self.perform_sleep(sleep_time)

src/ytdl_sub/validators/string_formatter_validators.py

@@ -74,6 +74,20 @@ class StringFormatterValidator(StringValidator):
         return resolved
 
 
+class FloatFormatterValidator(StringFormatterValidator):
+    _expected_value_type_name = "float"
+
+    def post_process(self, resolved: str) -> str:
+        try:
+            float(resolved)
+        except Exception as exc:
+            raise self._validation_exception(
+                f"Expected a float, but received '{resolved}'"
+            ) from exc
+
+        return resolved
+
+
 class StandardizedDateValidator(StringFormatterValidator):
     _expected_value_type_name = "standardized_date"
 
@@ -118,6 +132,12 @@ class OverridesIntegerFormatterValidator(OverridesStringFormatterValidator):
         return resolved
 
 
+class OverridesFloatFormatterValidator(FloatFormatterValidator, OverridesStringFormatterValidator):
+    """
+    Float validator but static
+    """
+
+
 class OverridesBooleanFormatterValidator(OverridesStringFormatterValidator):
     _expected_value_type_name = "boolean"
 

tests/integration/plugins/test_throttle_protection.py

@@ -8,6 +8,7 @@ from ytdl_sub.plugins.throttle_protection import logger as throttle_protection_l
 from ytdl_sub.script.functions.print_functions import logger as script_print_logger
 from ytdl_sub.script.utils.exceptions import UserThrownRuntimeError
 from ytdl_sub.subscriptions.subscription import Subscription
+from ytdl_sub.utils.exceptions import StringFormattingVariableNotFoundException
 
 
 @pytest.fixture
@@ -57,7 +58,7 @@ class TestThrottleProtectionPlugin:
             ),
             assert_logs(
                 logger=throttle_protection_logger,
-                expected_message="Sleeping between downloads for %0.2f seconds",
+                expected_message="Sleeping between downloads for 0.01 seconds",
                 log_level="info",
                 expected_occurrences=4,
             ),
@@ -288,3 +289,58 @@ class TestResolutionAssert:
             pytest.raises(UserThrownRuntimeError, match=re.escape(expected_message)),
         ):
             _ = subscription.download(dry_run=True)
+
+    def test_sleep_per_download_supports_entry_variables(
+        self,
+        config,
+        subscription_name,
+        throttle_subscription_dict,
+        output_directory,
+        mock_download_collection_entries,
+    ):
+        throttle_subscription_dict["throttle_protection"]["sleep_per_download_s"] = {
+            "min": "{%mul(3.14, duration)}",
+            "max": "{%mul(3.14, duration)}",
+        }
+
+        subscription = Subscription.from_dict(
+            config=config,
+            preset_name=subscription_name,
+            preset_dict=throttle_subscription_dict,
+        )
+
+        with (
+            mock_download_collection_entries(
+                is_youtube_channel=False,
+                num_urls=1,
+                is_extracted_audio=False,
+                mock_entry_kwargs={"duration": 1},
+            ),
+            assert_logs(
+                logger=throttle_protection_logger,
+                expected_message="Sleeping between downloads for 3.14 seconds",
+                log_level="info",
+                expected_occurrences=4,
+            ),
+        ):
+            _ = subscription.download(dry_run=False)
+
+    def test_sleep_per_subscription_does_not_support_entry_variables(
+        self,
+        config,
+        subscription_name,
+        throttle_subscription_dict,
+        output_directory,
+        mock_download_collection_entries,
+    ):
+        throttle_subscription_dict["throttle_protection"]["sleep_per_subscription_s"] = {
+            "min": "{%mul(3.14, duration)}",
+            "max": "{%mul(3.14, duration)}",
+        }
+
+        with pytest.raises(StringFormattingVariableNotFoundException):
+            _ = Subscription.from_dict(
+                config=config,
+                preset_name=subscription_name,
+                preset_dict=throttle_subscription_dict,
+            )