[FEATURE] Resolution assert built into throttle protection (#1288)

Closes https://github.com/jmbannon/ytdl-sub/issues/1241

YouTube is upping their throttling by serving 360p videos. Most likely to give us and genAI scraping a bad time on purpose.

Resolution assertion will be enabled by default. It will:

- Require > 360 height (pixels)
- Can configure the width by setting `resolution_assert_height_gte: 1080` to require 1080p, or any desired value
- Can disable resolution assert by setting `enable_resolution_assert: False`

src/ytdl_sub/prebuilt_presets/helpers/throttle_protection.yaml

@@ -31,3 +31,37 @@ presets:
         max: 26.1
     overrides:
       enable_throttle_protection: True
+
+      ##### Resolution Assert
+      enable_resolution_assert: True
+      resolution_assert_height_gte: 361
+      resolution_assert_print: >-
+        {
+          %print(
+            %if(
+              enable_resolution_assert,
+              "Resolution assert is enabled, will fail on low-quality video downloads and presume throttle. Disable using the override variable `enable_resolution_assert: False`",
+              "Resolution assert is disabled. Use at your own risk!"
+            ),
+            enable_resolution_assert
+          )
+        }
+      resolution_readable: "{width}x{height}"
+      # height is not a guaranteed populated variable, do not assert if it's missing (which defaults to 0)
+      resolution_assert: >-
+        {
+          %if(
+            %and(
+              enable_resolution_assert,
+              %ne( height, 0 )
+            ),
+            %assert(
+              %gte( height, resolution_assert_height_gte ),
+              %concat(
+                "Entry ", title, " downloaded at a low resolution (", resolution_readable, ") which is classified as throttle. ",
+                "Stopping additional downloads. Disable using the override variable `enable_resolution_assert: False`"
+              )
+            ),
+            "false is no-op"
+          )
+        }

tests/integration/conftest.py

@@ -1,6 +1,7 @@
 import contextlib
 import os
 from pathlib import Path
+from typing import Any
 from typing import Callable
 from typing import Dict
 from typing import List
@@ -55,6 +56,7 @@ def mock_entry_dict_factory(mock_downloaded_file_path) -> Callable:
         mock_download_to_working_dir: bool = True,
         is_extracted_audio: bool = False,
         release_date: Optional[str] = None,
+        mock_entry_kwargs: Optional[Dict[str, Any]] = None,
     ) -> Dict:
         entry_dict = {
             v.uid.metadata_key: uid,
@@ -104,6 +106,9 @@ def mock_entry_dict_factory(mock_downloaded_file_path) -> Callable:
             copy_file_fixture(
                 fixture_name="thumb.jpg", output_file_path=mock_downloaded_file_path(f"{uid}.jpg")
             )
+
+        if mock_entry_kwargs:
+            return dict(entry_dict, **mock_entry_kwargs)
         return entry_dict
 
     return _mock_entry_dict_factory
@@ -137,7 +142,9 @@ def mock_download_collection_thumbnail(mock_downloaded_file_path):
 
 @pytest.fixture
 def mock_download_collection_entries(
-    mock_download_collection_thumbnail, mock_entry_dict_factory: Callable, working_directory: str
+    mock_download_collection_thumbnail,
+    mock_entry_dict_factory: Callable,
+    working_directory: str,
 ):
     @contextlib.contextmanager
     def _mock_download_collection_entries_factory(
@@ -145,6 +152,7 @@ def mock_download_collection_entries(
         num_urls: int = 1,
         is_extracted_audio: bool = False,
         is_dry_run: bool = False,
+        mock_entry_kwargs: Optional[Dict[str, Any]] = None,
     ):
         is_real_run = not is_dry_run
 
@@ -166,6 +174,7 @@ def mock_download_collection_entries(
                         is_youtube_channel=is_youtube_channel,
                         is_extracted_audio=is_extracted_audio,
                         mock_download_to_working_dir=is_real_run,
+                        mock_entry_kwargs=mock_entry_kwargs,
                     ),  # 1
                     mock_entry_dict_factory(
                         uid="20-1",
@@ -177,6 +186,7 @@ def mock_download_collection_entries(
                         is_youtube_channel=is_youtube_channel,
                         is_extracted_audio=is_extracted_audio,
                         mock_download_to_working_dir=is_real_run,
+                        mock_entry_kwargs=mock_entry_kwargs,
                     ),  # 2  98
                     mock_entry_dict_factory(
                         uid="20-2",
@@ -188,6 +198,7 @@ def mock_download_collection_entries(
                         is_youtube_channel=is_youtube_channel,
                         is_extracted_audio=is_extracted_audio,
                         mock_download_to_working_dir=is_real_run,
+                        mock_entry_kwargs=mock_entry_kwargs,
                     ),  # 1  99
                     mock_entry_dict_factory(
                         uid="20-3",
@@ -199,6 +210,7 @@ def mock_download_collection_entries(
                         is_youtube_channel=is_youtube_channel,
                         is_extracted_audio=is_extracted_audio,
                         mock_download_to_working_dir=is_real_run,
+                        mock_entry_kwargs=mock_entry_kwargs,
                     ),
                 ]
             return [
@@ -213,6 +225,7 @@ def mock_download_collection_entries(
                     is_youtube_channel=is_youtube_channel,
                     is_extracted_audio=is_extracted_audio,
                     mock_download_to_working_dir=False,
+                    mock_entry_kwargs=mock_entry_kwargs,
                 ),
                 mock_entry_dict_factory(
                     uid="20-4",
@@ -224,6 +237,7 @@ def mock_download_collection_entries(
                     is_youtube_channel=is_youtube_channel,
                     is_extracted_audio=is_extracted_audio,
                     mock_download_to_working_dir=is_real_run,
+                    mock_entry_kwargs=mock_entry_kwargs,
                 ),
                 mock_entry_dict_factory(
                     uid="20-5",
@@ -235,6 +249,7 @@ def mock_download_collection_entries(
                     is_youtube_channel=is_youtube_channel,
                     is_extracted_audio=is_extracted_audio,
                     mock_download_to_working_dir=is_real_run,
+                    mock_entry_kwargs=mock_entry_kwargs,
                 ),
                 mock_entry_dict_factory(
                     uid="20-6",
@@ -246,6 +261,7 @@ def mock_download_collection_entries(
                     is_youtube_channel=is_youtube_channel,
                     is_extracted_audio=is_extracted_audio,
                     mock_download_to_working_dir=is_real_run,
+                    mock_entry_kwargs=mock_entry_kwargs,
                 ),
                 mock_entry_dict_factory(
                     uid="20-7",
@@ -257,6 +273,7 @@ def mock_download_collection_entries(
                     is_youtube_channel=is_youtube_channel,
                     is_extracted_audio=is_extracted_audio,
                     mock_download_to_working_dir=is_real_run,
+                    mock_entry_kwargs=mock_entry_kwargs,
                 ),
             ]
 

tests/integration/plugins/test_throttle_protection.py

@@ -1,9 +1,12 @@
+import re
 from typing import Dict
 
 import pytest
 from conftest import assert_logs
 
 from ytdl_sub.plugins.throttle_protection import logger as throttle_protection_logger
+from ytdl_sub.script.functions.print_functions import logger as script_print_logger
+from ytdl_sub.script.utils.exceptions import UserThrownRuntimeError
 from ytdl_sub.subscriptions.subscription import Subscription
 
 
@@ -175,3 +178,111 @@ class TestThrottleProtectionPlugin:
             transaction_log = subscription.download(dry_run=True)
 
         assert transaction_log.is_empty
+
+
+class TestResolutionAssert:
+    @pytest.mark.parametrize(
+        "disable_value",
+        [
+            "",
+            False,
+            "{bool_false_variable}",
+            "{empty_string_variable}",
+        ],
+    )
+    def test_disabled(
+        self,
+        config,
+        subscription_name,
+        throttle_subscription_dict,
+        output_directory,
+        mock_download_collection_entries,
+        disable_value,
+    ):
+        throttle_subscription_dict["overrides"]["enable_resolution_assert"] = disable_value
+
+        with assert_logs(
+            logger=script_print_logger,
+            expected_message="Resolution assert is disabled. Use at your own risk!",
+            log_level="info",
+            expected_occurrences=1,
+        ):
+            _ = Subscription.from_dict(
+                config=config,
+                preset_name=subscription_name,
+                preset_dict=throttle_subscription_dict,
+            )
+
+    @pytest.mark.parametrize(
+        "width, height",
+        [
+            (0, 0),  # missing
+            (361, 361),  # min value
+        ],
+    )
+    def test_runs_successfully(
+        self,
+        config,
+        subscription_name,
+        throttle_subscription_dict,
+        output_directory,
+        mock_download_collection_entries,
+        width,
+        height,
+    ):
+        with assert_logs(
+            logger=script_print_logger,
+            expected_message=(
+                "Resolution assert is enabled, will fail on low-quality video downloads and presume throttle. "
+                "Disable using the override variable `enable_resolution_assert: False`"
+            ),
+            log_level="info",
+            expected_occurrences=1,
+        ):
+            subscription = Subscription.from_dict(
+                config=config,
+                preset_name=subscription_name,
+                preset_dict=throttle_subscription_dict,
+            )
+
+        with (
+            mock_download_collection_entries(
+                is_youtube_channel=False,
+                num_urls=1,
+                is_extracted_audio=False,
+                is_dry_run=True,
+                mock_entry_kwargs={"height": height, "width": width},
+            ),
+        ):
+            _ = subscription.download(dry_run=True)
+
+    def test_fails_low_resolution(
+        self,
+        config,
+        subscription_name,
+        throttle_subscription_dict,
+        output_directory,
+        mock_download_collection_entries,
+    ):
+        subscription = Subscription.from_dict(
+            config=config,
+            preset_name=subscription_name,
+            preset_dict=throttle_subscription_dict,
+        )
+
+        expected_message = (
+            "Entry Mock Entry 20-3 downloaded at a low resolution (640x360) which is classified as throttle. "
+            "Stopping additional downloads. Disable using the override variable `enable_resolution_assert: False`"
+        )
+
+        with (
+            mock_download_collection_entries(
+                is_youtube_channel=False,
+                num_urls=1,
+                is_extracted_audio=False,
+                is_dry_run=True,
+                mock_entry_kwargs={"height": 360, "width": 640},
+            ),
+            pytest.raises(UserThrownRuntimeError, match=re.escape(expected_message)),
+        ):
+            _ = subscription.download(dry_run=True)