From 288f1068780d5b2eaaf9eb1926564b1ca20fa4f3 Mon Sep 17 00:00:00 2001 From: Jesse Bannon Date: Tue, 19 Aug 2025 17:42:56 -0700 Subject: [PATCH] [FEATURE] Resolution assert built into throttle protection (#1288) Closes https://github.com/jmbannon/ytdl-sub/issues/1241 YouTube is upping their throttling by serving 360p videos. Most likely to give us and genAI scraping a bad time on purpose. Resolution assertion will be enabled by default. It will: - Require > 360 height (pixels) - Can configure the width by setting `resolution_assert_height_gte: 1080` to require 1080p, or any desired value - Can disable resolution assert by setting `enable_resolution_assert: False` --- .../helpers/throttle_protection.yaml | 34 ++++++ tests/integration/conftest.py | 19 ++- .../plugins/test_throttle_protection.py | 111 ++++++++++++++++++ 3 files changed, 163 insertions(+), 1 deletion(-) diff --git a/src/ytdl_sub/prebuilt_presets/helpers/throttle_protection.yaml b/src/ytdl_sub/prebuilt_presets/helpers/throttle_protection.yaml index f9c6b93a..1f68ddfb 100644 --- a/src/ytdl_sub/prebuilt_presets/helpers/throttle_protection.yaml +++ b/src/ytdl_sub/prebuilt_presets/helpers/throttle_protection.yaml @@ -31,3 +31,37 @@ presets: max: 26.1 overrides: enable_throttle_protection: True + + ##### Resolution Assert + enable_resolution_assert: True + resolution_assert_height_gte: 361 + resolution_assert_print: >- + { + %print( + %if( + enable_resolution_assert, + "Resolution assert is enabled, will fail on low-quality video downloads and presume throttle. Disable using the override variable `enable_resolution_assert: False`", + "Resolution assert is disabled. Use at your own risk!" + ), + enable_resolution_assert + ) + } + resolution_readable: "{width}x{height}" + # height is not a guaranteed populated variable, do not assert if it's missing (which defaults to 0) + resolution_assert: >- + { + %if( + %and( + enable_resolution_assert, + %ne( height, 0 ) + ), + %assert( + %gte( height, resolution_assert_height_gte ), + %concat( + "Entry ", title, " downloaded at a low resolution (", resolution_readable, ") which is classified as throttle. ", + "Stopping additional downloads. Disable using the override variable `enable_resolution_assert: False`" + ) + ), + "false is no-op" + ) + } diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index b1d40434..b0aab494 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -1,6 +1,7 @@ import contextlib import os from pathlib import Path +from typing import Any from typing import Callable from typing import Dict from typing import List @@ -55,6 +56,7 @@ def mock_entry_dict_factory(mock_downloaded_file_path) -> Callable: mock_download_to_working_dir: bool = True, is_extracted_audio: bool = False, release_date: Optional[str] = None, + mock_entry_kwargs: Optional[Dict[str, Any]] = None, ) -> Dict: entry_dict = { v.uid.metadata_key: uid, @@ -104,6 +106,9 @@ def mock_entry_dict_factory(mock_downloaded_file_path) -> Callable: copy_file_fixture( fixture_name="thumb.jpg", output_file_path=mock_downloaded_file_path(f"{uid}.jpg") ) + + if mock_entry_kwargs: + return dict(entry_dict, **mock_entry_kwargs) return entry_dict return _mock_entry_dict_factory @@ -137,7 +142,9 @@ def mock_download_collection_thumbnail(mock_downloaded_file_path): @pytest.fixture def mock_download_collection_entries( - mock_download_collection_thumbnail, mock_entry_dict_factory: Callable, working_directory: str + mock_download_collection_thumbnail, + mock_entry_dict_factory: Callable, + working_directory: str, ): @contextlib.contextmanager def _mock_download_collection_entries_factory( @@ -145,6 +152,7 @@ def mock_download_collection_entries( num_urls: int = 1, is_extracted_audio: bool = False, is_dry_run: bool = False, + mock_entry_kwargs: Optional[Dict[str, Any]] = None, ): is_real_run = not is_dry_run @@ -166,6 +174,7 @@ def mock_download_collection_entries( is_youtube_channel=is_youtube_channel, is_extracted_audio=is_extracted_audio, mock_download_to_working_dir=is_real_run, + mock_entry_kwargs=mock_entry_kwargs, ), # 1 mock_entry_dict_factory( uid="20-1", @@ -177,6 +186,7 @@ def mock_download_collection_entries( is_youtube_channel=is_youtube_channel, is_extracted_audio=is_extracted_audio, mock_download_to_working_dir=is_real_run, + mock_entry_kwargs=mock_entry_kwargs, ), # 2 98 mock_entry_dict_factory( uid="20-2", @@ -188,6 +198,7 @@ def mock_download_collection_entries( is_youtube_channel=is_youtube_channel, is_extracted_audio=is_extracted_audio, mock_download_to_working_dir=is_real_run, + mock_entry_kwargs=mock_entry_kwargs, ), # 1 99 mock_entry_dict_factory( uid="20-3", @@ -199,6 +210,7 @@ def mock_download_collection_entries( is_youtube_channel=is_youtube_channel, is_extracted_audio=is_extracted_audio, mock_download_to_working_dir=is_real_run, + mock_entry_kwargs=mock_entry_kwargs, ), ] return [ @@ -213,6 +225,7 @@ def mock_download_collection_entries( is_youtube_channel=is_youtube_channel, is_extracted_audio=is_extracted_audio, mock_download_to_working_dir=False, + mock_entry_kwargs=mock_entry_kwargs, ), mock_entry_dict_factory( uid="20-4", @@ -224,6 +237,7 @@ def mock_download_collection_entries( is_youtube_channel=is_youtube_channel, is_extracted_audio=is_extracted_audio, mock_download_to_working_dir=is_real_run, + mock_entry_kwargs=mock_entry_kwargs, ), mock_entry_dict_factory( uid="20-5", @@ -235,6 +249,7 @@ def mock_download_collection_entries( is_youtube_channel=is_youtube_channel, is_extracted_audio=is_extracted_audio, mock_download_to_working_dir=is_real_run, + mock_entry_kwargs=mock_entry_kwargs, ), mock_entry_dict_factory( uid="20-6", @@ -246,6 +261,7 @@ def mock_download_collection_entries( is_youtube_channel=is_youtube_channel, is_extracted_audio=is_extracted_audio, mock_download_to_working_dir=is_real_run, + mock_entry_kwargs=mock_entry_kwargs, ), mock_entry_dict_factory( uid="20-7", @@ -257,6 +273,7 @@ def mock_download_collection_entries( is_youtube_channel=is_youtube_channel, is_extracted_audio=is_extracted_audio, mock_download_to_working_dir=is_real_run, + mock_entry_kwargs=mock_entry_kwargs, ), ] diff --git a/tests/integration/plugins/test_throttle_protection.py b/tests/integration/plugins/test_throttle_protection.py index 51f3cc23..f28744ab 100644 --- a/tests/integration/plugins/test_throttle_protection.py +++ b/tests/integration/plugins/test_throttle_protection.py @@ -1,9 +1,12 @@ +import re from typing import Dict import pytest from conftest import assert_logs from ytdl_sub.plugins.throttle_protection import logger as throttle_protection_logger +from ytdl_sub.script.functions.print_functions import logger as script_print_logger +from ytdl_sub.script.utils.exceptions import UserThrownRuntimeError from ytdl_sub.subscriptions.subscription import Subscription @@ -175,3 +178,111 @@ class TestThrottleProtectionPlugin: transaction_log = subscription.download(dry_run=True) assert transaction_log.is_empty + + +class TestResolutionAssert: + @pytest.mark.parametrize( + "disable_value", + [ + "", + False, + "{bool_false_variable}", + "{empty_string_variable}", + ], + ) + def test_disabled( + self, + config, + subscription_name, + throttle_subscription_dict, + output_directory, + mock_download_collection_entries, + disable_value, + ): + throttle_subscription_dict["overrides"]["enable_resolution_assert"] = disable_value + + with assert_logs( + logger=script_print_logger, + expected_message="Resolution assert is disabled. Use at your own risk!", + log_level="info", + expected_occurrences=1, + ): + _ = Subscription.from_dict( + config=config, + preset_name=subscription_name, + preset_dict=throttle_subscription_dict, + ) + + @pytest.mark.parametrize( + "width, height", + [ + (0, 0), # missing + (361, 361), # min value + ], + ) + def test_runs_successfully( + self, + config, + subscription_name, + throttle_subscription_dict, + output_directory, + mock_download_collection_entries, + width, + height, + ): + with assert_logs( + logger=script_print_logger, + expected_message=( + "Resolution assert is enabled, will fail on low-quality video downloads and presume throttle. " + "Disable using the override variable `enable_resolution_assert: False`" + ), + log_level="info", + expected_occurrences=1, + ): + subscription = Subscription.from_dict( + config=config, + preset_name=subscription_name, + preset_dict=throttle_subscription_dict, + ) + + with ( + mock_download_collection_entries( + is_youtube_channel=False, + num_urls=1, + is_extracted_audio=False, + is_dry_run=True, + mock_entry_kwargs={"height": height, "width": width}, + ), + ): + _ = subscription.download(dry_run=True) + + def test_fails_low_resolution( + self, + config, + subscription_name, + throttle_subscription_dict, + output_directory, + mock_download_collection_entries, + ): + subscription = Subscription.from_dict( + config=config, + preset_name=subscription_name, + preset_dict=throttle_subscription_dict, + ) + + expected_message = ( + "Entry Mock Entry 20-3 downloaded at a low resolution (640x360) which is classified as throttle. " + "Stopping additional downloads. Disable using the override variable `enable_resolution_assert: False`" + ) + + with ( + mock_download_collection_entries( + is_youtube_channel=False, + num_urls=1, + is_extracted_audio=False, + is_dry_run=True, + mock_entry_kwargs={"height": 360, "width": 640}, + ), + pytest.raises(UserThrownRuntimeError, match=re.escape(expected_message)), + ): + _ = subscription.download(dry_run=True)