From bc15fe09bd7be371f9d9543fb80d934c081fc59c Mon Sep 17 00:00:00 2001 From: Jesse Bannon Date: Sat, 31 May 2025 09:41:14 -0700 Subject: [PATCH] [FEATURE] Throttle protection per request (#1229) Allows adding a sleep inbetween every request during metadata scrape. ``` throttle_protection: sleep_per_request_s: min: 4 max: 6 ``` Closes https://github.com/jmbannon/ytdl-sub/issues/1222 --- docs/source/config_reference/plugins.rst | 13 ++++++ src/ytdl_sub/plugins/subtitles.py | 7 +++- src/ytdl_sub/plugins/throttle_protection.py | 41 +++++++++++++++++++ .../subscription_ytdl_options.py | 2 + tests/e2e/presets/test_soundcloud.py | 1 + 5 files changed, 63 insertions(+), 1 deletion(-) diff --git a/docs/source/config_reference/plugins.rst b/docs/source/config_reference/plugins.rst index 8e63e291..4825f0f7 100644 --- a/docs/source/config_reference/plugins.rst +++ b/docs/source/config_reference/plugins.rst @@ -903,6 +903,9 @@ scripted. presets: my_example_preset: throttle_protection: + sleep_per_request_s: + min: 5.5 + max: 10.4 sleep_per_download_s: min: 2.2 max: 10.8 @@ -938,6 +941,16 @@ scripted. ytdl-sub to perform post-processing. +``sleep_per_request_s`` + +:expected type: Optional[Range] +:description: + Number in seconds to sleep between each request during metadata download. Note that + metadata download refers to the initial info.json download, not the actual audio/video + download for the entry. Also, yt-dlp only supports a single value at this time for this, + so will always use the max value. + + ``sleep_per_subscription_s`` :expected type: Optional[Range] diff --git a/src/ytdl_sub/plugins/subtitles.py b/src/ytdl_sub/plugins/subtitles.py index ecf32a1d..74aee5a8 100644 --- a/src/ytdl_sub/plugins/subtitles.py +++ b/src/ytdl_sub/plugins/subtitles.py @@ -146,7 +146,12 @@ class SubtitlesPlugin(Plugin[SubtitleOptions]): plugin_options_type = SubtitleOptions def ytdl_options(self) -> Optional[Dict]: - builder = YTDLOptionsBuilder().add({"writesubtitles": True}) + builder = YTDLOptionsBuilder().add( + { + "writesubtitles": True, + "sleep_interval_subtitles": 5.367, # for safe measure + } + ) if self.plugin_options.embed_subtitles: builder.add( diff --git a/src/ytdl_sub/plugins/throttle_protection.py b/src/ytdl_sub/plugins/throttle_protection.py index 5bd45f10..77ec4fa8 100644 --- a/src/ytdl_sub/plugins/throttle_protection.py +++ b/src/ytdl_sub/plugins/throttle_protection.py @@ -1,5 +1,6 @@ import random import time +from typing import Dict from typing import Optional from ytdl_sub.config.overrides import Overrides @@ -40,6 +41,22 @@ class RandomizedRangeValidator(StrictDictValidator): f"max ({self._max}) must be greater than or equal to min ({self._min})" ) + def min_value(self) -> float: + """ + Returns + ------- + Minimum value + """ + return self._min + + def max_value(self) -> float: + """ + Returns + ------- + Maximum value + """ + return self._max + def randomized_float(self) -> float: """ Returns @@ -70,6 +87,9 @@ class ThrottleProtectionOptions(ToggleableOptionsDictValidator): presets: my_example_preset: throttle_protection: + sleep_per_request_s: + min: 5.5 + max: 10.4 sleep_per_download_s: min: 2.2 max: 10.8 @@ -84,6 +104,7 @@ class ThrottleProtectionOptions(ToggleableOptionsDictValidator): _optional_keys = { "enable", + "sleep_per_request_s", "sleep_per_download_s", "sleep_per_subscription_s", "max_downloads_per_subscription", @@ -93,6 +114,9 @@ class ThrottleProtectionOptions(ToggleableOptionsDictValidator): def __init__(self, name, value): super().__init__(name, value) + self._sleep_per_request_s = self._validate_key_if_present( + key="sleep_per_request_s", validator=RandomizedRangeValidator + ) self._sleep_per_download_s = self._validate_key_if_present( key="sleep_per_download_s", validator=RandomizedRangeValidator ) @@ -106,6 +130,18 @@ class ThrottleProtectionOptions(ToggleableOptionsDictValidator): key="subscription_download_probability", validator=ProbabilityValidator ) + @property + def sleep_per_request_s(self) -> Optional[RandomizedRangeValidator]: + """ + :expected type: Optional[Range] + :description: + Number in seconds to sleep between each request during metadata download. Note that + metadata download refers to the initial info.json download, not the actual audio/video + download for the entry. Also, yt-dlp only supports a single value at this time for this, + so will always use the max value. + """ + return self._sleep_per_request_s + @property def sleep_per_download_s(self) -> Optional[RandomizedRangeValidator]: """ @@ -165,6 +201,11 @@ class ThrottleProtectionPlugin(Plugin[ThrottleProtectionOptions]): self.plugin_options.max_downloads_per_subscription.randomized_int() ) + def ytdl_options(self) -> Optional[Dict]: + if self.plugin_options.sleep_per_request_s is not None: + return {"sleep_interval_requests": self.plugin_options.sleep_per_request_s.max_value()} + return {} + def initialize_subscription(self) -> bool: if self.plugin_options.subscription_download_probability: proba = self.plugin_options.subscription_download_probability.value diff --git a/src/ytdl_sub/subscriptions/subscription_ytdl_options.py b/src/ytdl_sub/subscriptions/subscription_ytdl_options.py index 12efb951..b5772197 100644 --- a/src/ytdl_sub/subscriptions/subscription_ytdl_options.py +++ b/src/ytdl_sub/subscriptions/subscription_ytdl_options.py @@ -19,6 +19,7 @@ from ytdl_sub.plugins.match_filters import MatchFiltersPlugin from ytdl_sub.plugins.match_filters import combine_filters from ytdl_sub.plugins.match_filters import default_filters from ytdl_sub.plugins.subtitles import SubtitlesPlugin +from ytdl_sub.plugins.throttle_protection import ThrottleProtectionPlugin from ytdl_sub.utils.ffmpeg import FFMPEG from ytdl_sub.utils.logger import Logger from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive @@ -175,6 +176,7 @@ class SubscriptionYTDLOptions: self._global_options, self._output_options, self._plugin_match_filters, + self._plugin_ytdl_options(ThrottleProtectionPlugin), self._plugin_ytdl_options(FormatPlugin), self._plugin_ytdl_options(AudioExtractPlugin), # will override format self._user_ytdl_options, # user ytdl options... diff --git a/tests/e2e/presets/test_soundcloud.py b/tests/e2e/presets/test_soundcloud.py index 9dbe6481..15c6d7d6 100644 --- a/tests/e2e/presets/test_soundcloud.py +++ b/tests/e2e/presets/test_soundcloud.py @@ -19,6 +19,7 @@ def subscription_dict(output_directory): "subscription_indent_1": "Acoustic", "music_directory": output_directory, }, + "throttle_protection": {"sleep_per_request_s": {"min": 0.1, "max": 0.5}}, }