danvics/ytdl-sub
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

[FEATURE] Resolution assert built into throttle protection (#1288)

Browse source 
Closes https://github.com/jmbannon/ytdl-sub/issues/1241

YouTube is upping their throttling by serving 360p videos. Most likely to give us and genAI scraping a bad time on purpose.

Resolution assertion will be enabled by default. It will:

- Require > 360 height (pixels)
- Can configure the width by setting `resolution_assert_height_gte: 1080` to require 1080p, or any desired value
- Can disable resolution assert by setting `enable_resolution_assert: False`
This commit is contained in:
Jesse Bannon 2025-08-19 17:42:56 -07:00 committed by GitHub
parent e1f81e5274
commit 288f106878
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
3 changed files with 163 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

34
src/ytdl_sub/prebuilt_presets/helpers/throttle_protection.yaml
View file

@ -31,3 +31,37 @@ presets:
max: 26.1
overrides:
enable_throttle_protection: True
##### Resolution Assert
enable_resolution_assert: True
resolution_assert_height_gte: 361
resolution_assert_print: >-
{
%print(
%if(
enable_resolution_assert,
"Resolution assert is enabled, will fail on low-quality video downloads and presume throttle. Disable using the override variable `enable_resolution_assert: False`",
"Resolution assert is disabled. Use at your own risk!"
),
enable_resolution_assert
)
}
resolution_readable: "{width}x{height}"
# height is not a guaranteed populated variable, do not assert if it's missing (which defaults to 0)
resolution_assert: >-
{
%if(
%and(
enable_resolution_assert,
%ne( height, 0 )
),
%assert(
%gte( height, resolution_assert_height_gte ),
%concat(
"Entry ", title, " downloaded at a low resolution (", resolution_readable, ") which is classified as throttle. ",
"Stopping additional downloads. Disable using the override variable `enable_resolution_assert: False`"
)
),
"false is no-op"
)
}

19
tests/integration/conftest.py
View file

@ -1,6 +1,7 @@
import contextlib
import os
from pathlib import Path
from typing import Any
from typing import Callable
from typing import Dict
from typing import List
@ -55,6 +56,7 @@ def mock_entry_dict_factory(mock_downloaded_file_path) -> Callable:
mock_download_to_working_dir: bool = True,
is_extracted_audio: bool = False,
release_date: Optional[str] = None,
mock_entry_kwargs: Optional[Dict[str, Any]] = None,
) -> Dict:
entry_dict = {
v.uid.metadata_key: uid,
@ -104,6 +106,9 @@ def mock_entry_dict_factory(mock_downloaded_file_path) -> Callable:
copy_file_fixture(
fixture_name="thumb.jpg", output_file_path=mock_downloaded_file_path(f"{uid}.jpg")
)
if mock_entry_kwargs:
return dict(entry_dict, **mock_entry_kwargs)
return entry_dict
return _mock_entry_dict_factory
@ -137,7 +142,9 @@ def mock_download_collection_thumbnail(mock_downloaded_file_path):
@pytest.fixture
def mock_download_collection_entries(
mock_download_collection_thumbnail, mock_entry_dict_factory: Callable, working_directory: str
mock_download_collection_thumbnail,
mock_entry_dict_factory: Callable,
working_directory: str,
):
@contextlib.contextmanager
def _mock_download_collection_entries_factory(
@ -145,6 +152,7 @@ def mock_download_collection_entries(
num_urls: int = 1,
is_extracted_audio: bool = False,
is_dry_run: bool = False,
mock_entry_kwargs: Optional[Dict[str, Any]] = None,
):
is_real_run = not is_dry_run
@ -166,6 +174,7 @@ def mock_download_collection_entries(
is_youtube_channel=is_youtube_channel,
is_extracted_audio=is_extracted_audio,
mock_download_to_working_dir=is_real_run,
mock_entry_kwargs=mock_entry_kwargs,
), # 1
mock_entry_dict_factory(
uid="20-1",
@ -177,6 +186,7 @@ def mock_download_collection_entries(
is_youtube_channel=is_youtube_channel,
is_extracted_audio=is_extracted_audio,
mock_download_to_working_dir=is_real_run,
mock_entry_kwargs=mock_entry_kwargs,
), # 2 98
mock_entry_dict_factory(
uid="20-2",
@ -188,6 +198,7 @@ def mock_download_collection_entries(
is_youtube_channel=is_youtube_channel,
is_extracted_audio=is_extracted_audio,
mock_download_to_working_dir=is_real_run,
mock_entry_kwargs=mock_entry_kwargs,
), # 1 99
mock_entry_dict_factory(
uid="20-3",
@ -199,6 +210,7 @@ def mock_download_collection_entries(
is_youtube_channel=is_youtube_channel,
is_extracted_audio=is_extracted_audio,
mock_download_to_working_dir=is_real_run,
mock_entry_kwargs=mock_entry_kwargs,
),
]
return [
@ -213,6 +225,7 @@ def mock_download_collection_entries(
is_youtube_channel=is_youtube_channel,
is_extracted_audio=is_extracted_audio,
mock_download_to_working_dir=False,
mock_entry_kwargs=mock_entry_kwargs,
),
mock_entry_dict_factory(
uid="20-4",
@ -224,6 +237,7 @@ def mock_download_collection_entries(
is_youtube_channel=is_youtube_channel,
is_extracted_audio=is_extracted_audio,
mock_download_to_working_dir=is_real_run,
mock_entry_kwargs=mock_entry_kwargs,
),
mock_entry_dict_factory(
uid="20-5",
@ -235,6 +249,7 @@ def mock_download_collection_entries(
is_youtube_channel=is_youtube_channel,
is_extracted_audio=is_extracted_audio,
mock_download_to_working_dir=is_real_run,
mock_entry_kwargs=mock_entry_kwargs,
),
mock_entry_dict_factory(
uid="20-6",
@ -246,6 +261,7 @@ def mock_download_collection_entries(
is_youtube_channel=is_youtube_channel,
is_extracted_audio=is_extracted_audio,
mock_download_to_working_dir=is_real_run,
mock_entry_kwargs=mock_entry_kwargs,
),
mock_entry_dict_factory(
uid="20-7",
@ -257,6 +273,7 @@ def mock_download_collection_entries(
is_youtube_channel=is_youtube_channel,
is_extracted_audio=is_extracted_audio,
mock_download_to_working_dir=is_real_run,
mock_entry_kwargs=mock_entry_kwargs,
),
]

111
tests/integration/plugins/test_throttle_protection.py
View file

@ -1,9 +1,12 @@
import re
from typing import Dict
import pytest
from conftest import assert_logs
from ytdl_sub.plugins.throttle_protection import logger as throttle_protection_logger
from ytdl_sub.script.functions.print_functions import logger as script_print_logger
from ytdl_sub.script.utils.exceptions import UserThrownRuntimeError
from ytdl_sub.subscriptions.subscription import Subscription
@ -175,3 +178,111 @@ class TestThrottleProtectionPlugin:
transaction_log = subscription.download(dry_run=True)
assert transaction_log.is_empty
class TestResolutionAssert:
@pytest.mark.parametrize(
"disable_value",
[
"",
False,
"{bool_false_variable}",
"{empty_string_variable}",
],
)
def test_disabled(
self,
config,
subscription_name,
throttle_subscription_dict,
output_directory,
mock_download_collection_entries,
disable_value,
):
throttle_subscription_dict["overrides"]["enable_resolution_assert"] = disable_value
with assert_logs(
logger=script_print_logger,
expected_message="Resolution assert is disabled. Use at your own risk!",
log_level="info",
expected_occurrences=1,
):
_ = Subscription.from_dict(
config=config,
preset_name=subscription_name,
preset_dict=throttle_subscription_dict,
)
@pytest.mark.parametrize(
"width, height",
[
(0, 0), # missing
(361, 361), # min value
],
)
def test_runs_successfully(
self,
config,
subscription_name,
throttle_subscription_dict,
output_directory,
mock_download_collection_entries,
width,
height,
):
with assert_logs(
logger=script_print_logger,
expected_message=(
"Resolution assert is enabled, will fail on low-quality video downloads and presume throttle. "
"Disable using the override variable `enable_resolution_assert: False`"
),
log_level="info",
expected_occurrences=1,
):
subscription = Subscription.from_dict(
config=config,
preset_name=subscription_name,
preset_dict=throttle_subscription_dict,
)
with (
mock_download_collection_entries(
is_youtube_channel=False,
num_urls=1,
is_extracted_audio=False,
is_dry_run=True,
mock_entry_kwargs={"height": height, "width": width},
),
):
_ = subscription.download(dry_run=True)
def test_fails_low_resolution(
self,
config,
subscription_name,
throttle_subscription_dict,
output_directory,
mock_download_collection_entries,
):
subscription = Subscription.from_dict(
config=config,
preset_name=subscription_name,
preset_dict=throttle_subscription_dict,
)
expected_message = (
"Entry Mock Entry 20-3 downloaded at a low resolution (640x360) which is classified as throttle. "
"Stopping additional downloads. Disable using the override variable `enable_resolution_assert: False`"
)
with (
mock_download_collection_entries(
is_youtube_channel=False,
num_urls=1,
is_extracted_audio=False,
is_dry_run=True,
mock_entry_kwargs={"height": 360, "width": 640},
),
pytest.raises(UserThrownRuntimeError, match=re.escape(expected_message)),
):
_ = subscription.download(dry_run=True)