diff --git a/docs/source/config_reference/scripting/scripting_functions.rst b/docs/source/config_reference/scripting/scripting_functions.rst index a12b0ebd..cc42c52c 100644 --- a/docs/source/config_reference/scripting/scripting_functions.rst +++ b/docs/source/config_reference/scripting/scripting_functions.rst @@ -646,6 +646,13 @@ contains :description: Returns True if ``contains`` is in ``string``. False otherwise. +contains_all +~~~~~~~~~~~~ +:spec: ``contains_all(string: String, contains_array: Array) -> Boolean`` + +:description: + Returns true if all elements in ``contains_array`` are in ``string``. False otherwise. + contains_any ~~~~~~~~~~~~ :spec: ``contains_any(string: String, contains_array: Array) -> Boolean`` diff --git a/docs/source/prebuilt_presets/helpers.rst b/docs/source/prebuilt_presets/helpers.rst index a4b141c6..e8e37c31 100644 --- a/docs/source/prebuilt_presets/helpers.rst +++ b/docs/source/prebuilt_presets/helpers.rst @@ -4,21 +4,7 @@ Helper Presets .. hint:: - Apply presets to your subscriptions using pipes: - - .. code-block:: yaml - :caption: Pipes can separate presets and values to apply them to all subscriptions below them. - - Plex TV Show by Date | best_video_quality: - - = Documentaries | chunk_initial_download: - "NOVA PBS": "https://www.youtube.com/@novapbs" - "National Geographic": "https://www.youtube.com/@NatGeo" - - = Documentaries: - "Cosmos - What If": "https://www.youtube.com/playlist?list=PLZdXRHYAVxTJno6oFF9nLGuwXNGYHmE8U" - -Common presets are not usable by themselves- setting one of these as the sole preset of your subscription and attempting to download will not work. But you can add these presets to quickly modify an existing preset to better suit your needs. + See how to apply helper presets :doc:`here ` Only Recent ----------- @@ -42,32 +28,19 @@ upload date is outside of the range, or you hit max files, older videos will be To prevent deletion of files, use the preset ``Only Recent Archive`` instead. -Best A/V Quality ----------------- - -Add the following preset to download the best available video and audio quality, and remux it into an MP4 container: - -``best_video_quality`` - - -Max 1080p Video ---------------- - -Add the following preset to download the best available audio and video quality, with the video not greater than 1080p, and remux it into an MP4 container: - -``max_1080p`` - Filter Keywords --------------- ``Filter Keywords`` can include or exclude media with any of the listed keywords. Both keywords and title/description are lower-cased before filtering. +Default behavior for Keyword evaluation is ANY, meaning the filter will succeed if any of the keywords are present. This can be set to ANY or ALL using the respective ``_eval`` variable. + Supports the following override variables: -* ``title_include_keywords`` -* ``title_exclude_keywords`` -* ``description_include_keywords`` -* ``description_exclude_keywords`` +* ``title_include_keywords``, ``title_include_eval`` +* ``title_exclude_keywords``, ``title_exclude_eval`` +* ``description_include_keywords``, ``title_exclude_eval`` +* ``description_exclude_keywords``, ``title_exclude_eval`` .. tip:: @@ -90,6 +63,13 @@ Supports the following override variables: title_include_keywords: - "To Catch a Smuggler" + = Sports: + "~Maple Leafs Highlights": + url: "https://www.youtube.com/@NHL" + title_include_eval: "ALL" + title_include_keywords: + - "maple leafs" + - "highlights" Chunk Downloads --------------- diff --git a/docs/source/prebuilt_presets/index.rst b/docs/source/prebuilt_presets/index.rst index 744428ce..bb95ac0c 100644 --- a/docs/source/prebuilt_presets/index.rst +++ b/docs/source/prebuilt_presets/index.rst @@ -3,13 +3,34 @@ Prebuilt Presets ================ ``ytdl-sub`` offers a number of built-in presets using best practices for formatting -media in various players. For advanced users, you can review the prebuilt preset +media in various players. + +.. hint:: + + Apply multiple presets to your subscriptions using pipes. Pipes can define multiple presets and values + on the same line to apply to all subscriptions nested below them. + + .. code-block:: yaml + :caption: Applies Max Video Quality preset to all TV shows, and Chunk Downloads preset to some + + Plex TV Show by Date | Max Video Quality: + + = Documentaries | Chunk Downloads: + "NOVA PBS": "https://www.youtube.com/@novapbs" + "National Geographic": "https://www.youtube.com/@NatGeo" + + = Documentaries: + "Cosmos - What If": "https://www.youtube.com/playlist?list=PLZdXRHYAVxTJno6oFF9nLGuwXNGYHmE8U" + + +For advanced users, you can review the prebuilt preset definitions :doc:`here `. .. toctree:: :titlesonly: - - helpers + tv_shows music - music_videos \ No newline at end of file + music_videos + media_quality + helpers \ No newline at end of file diff --git a/docs/source/prebuilt_presets/media_quality.rst b/docs/source/prebuilt_presets/media_quality.rst new file mode 100644 index 00000000..ceb92479 --- /dev/null +++ b/docs/source/prebuilt_presets/media_quality.rst @@ -0,0 +1,29 @@ +====================== +Media Quality Presets +====================== + +.. hint:: + + See how to apply media quality presets :doc:`here ` + +Video +----- +The following presets set video quality specifications to yt-dlp. + +- ``Max Video Quality`` +- ``Max 2160p`` +- ``Max 1440p`` +- ``Max 1080p`` +- ``Max 720p`` +- ``Max 480p`` + +Audio +----- +The following presets set audio quality specifications to yt-dlp. +These assume you are only extracting audio (no video). + +- ``Max Audio Quality``, format is determined by the source +- ``Max MP3 Quality`` +- ``Max Opus Quality`` +- ``MP3 320k`` +- ``MP3 128k`` diff --git a/pyproject.toml b/pyproject.toml index dd14f164..8ad1494a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,7 +15,7 @@ classifiers = [ "Programming Language :: Python :: 3.11", ] dependencies = [ - "yt-dlp[default]==2024.11.4", + "yt-dlp[default]==2024.12.13", "colorama~=0.4", "mergedeep~=1.3", "mediafile~=0.12", @@ -44,12 +44,12 @@ where = ["src"] test = [ "coverage[toml]>=6.3,<8.0", "pytest>=7.2,<9.0", - "pytest-rerunfailures~=14.0", + "pytest-rerunfailures>=14,<16", ] lint = [ "black==24.10.0", "isort==5.13.2", - "pylint==3.3.1", + "pylint==3.3.2", ] docs = [ "sphinx>=7,<9", @@ -60,7 +60,7 @@ docs = [ ] build = [ "build~=1.2", - "twine~=5.0", + "twine>=5,<7", "pyinstaller~=6.5", ] [project.scripts] diff --git a/src/ytdl_sub/prebuilt_presets/helpers/filtering.yaml b/src/ytdl_sub/prebuilt_presets/helpers/filtering.yaml index b964cc8c..4d99e93e 100644 --- a/src/ytdl_sub/prebuilt_presets/helpers/filtering.yaml +++ b/src/ytdl_sub/prebuilt_presets/helpers/filtering.yaml @@ -12,6 +12,11 @@ presets: description_include_keywords: "{ [] }" description_exclude_keywords: "{ [] }" + title_include_eval: "ANY" + title_exclude_eval: "ANY" + description_include_eval: "ANY" + description_exclude_eval: "ANY" + "%ensure_string": >- { %assert_then( @@ -32,21 +37,36 @@ presets: ) } + # $0 - var to evaluate + # $1 - keyword list + # $2 - eval type + "%contains_keywords_inner": >- + { + %elif( + %eq(%ensure_string($2), 'any'), + %contains_any( $0, $1 ), + %eq(%ensure_string($2), 'all'), + %contains_all( $0, $1 ), + %throw('Keyword eval must be either ANY or ALL') + ) + } + # $0 - var to evaluate # $1 - keyword list # $2 - variable name for error messages - # $3 - default return if keyword list is empty + # $3 - keyword eval + # $4 - default return if keyword list is empty "%contains_keywords": >- { %if( %bool( $1 ), - %contains_any( %lower($0), %ensure_lower_array($1, $2) ), - $3 + %contains_keywords_inner( %lower($0), %ensure_lower_array($1, $2), $3 ), + $4 ) } filter_exclude: - - "{ %not( %contains_keywords(title, title_include_keywords, 'title_include_keywords', true) ) }" - - "{ %not( %contains_keywords(description, description_include_keywords, 'description_include_keywords', true) ) }" - - "{ %contains_keywords(title, title_exclude_keywords, 'title_exclude_keywords', false) }" - - "{ %contains_keywords(description, description_exclude_keywords, 'description_exclude_keywords',false) }" \ No newline at end of file + - "{ %not( %contains_keywords(title, title_include_keywords, 'title_include_keywords', title_include_eval, true) ) }" + - "{ %not( %contains_keywords(description, description_include_keywords, 'description_include_keywords', description_include_eval, true) ) }" + - "{ %contains_keywords(title, title_exclude_keywords, 'title_exclude_keywords', title_exclude_eval, false) }" + - "{ %contains_keywords(description, description_exclude_keywords, 'description_exclude_keywords', description_exclude_eval, false) }" \ No newline at end of file diff --git a/src/ytdl_sub/prebuilt_presets/helpers/media_quality.yaml b/src/ytdl_sub/prebuilt_presets/helpers/media_quality.yaml index a4c35dc9..78b99ca4 100644 --- a/src/ytdl_sub/prebuilt_presets/helpers/media_quality.yaml +++ b/src/ytdl_sub/prebuilt_presets/helpers/media_quality.yaml @@ -13,6 +13,10 @@ presets: preset: - best_video_quality + "Max Video Quality": + preset: + - best_video_quality + ############################################################################# # Max 2160p @@ -55,4 +59,32 @@ presets: "Max 480p": format: "(bv*[height<=480]+bestaudio/best[height<=480])" ytdl_options: - merge_output_format: "mp4" \ No newline at end of file + merge_output_format: "mp4" + + ############################################################################# + # Audio Quality Presets + + "Max Audio Quality": + audio_extract: + codec: "best" + quality: 0 + + "Max MP3 Quality": + audio_extract: + codec: "mp3" + quality: 0 + + "Max Opus Quality": + audio_extract: + codec: "opus" + quality: 0 + + "MP3 320k": + audio_extract: + codec: "mp3" + quality: 320 + + "MP3 128k": + audio_extract: + codec: "mp3" + quality: 128 diff --git a/src/ytdl_sub/script/functions/string_functions.py b/src/ytdl_sub/script/functions/string_functions.py index c2977db0..e9af7a54 100644 --- a/src/ytdl_sub/script/functions/string_functions.py +++ b/src/ytdl_sub/script/functions/string_functions.py @@ -40,6 +40,20 @@ class StringFunctions: ) ) + @staticmethod + def contains_all(string: String, contains_array: Array) -> Boolean: + """ + :description: + Returns true if all elements in ``contains_array`` are in ``string``. False otherwise. + """ + return Boolean( + all( + str(val) in string.value + for val in contains_array.value + if isinstance(val, (String, Integer, Boolean, Float)) + ) + ) + @staticmethod def slice(string: String, start: Integer, end: Optional[Integer] = None) -> String: """ diff --git a/src/ytdl_sub/utils/file_path.py b/src/ytdl_sub/utils/file_path.py index af8246f1..ff50c991 100644 --- a/src/ytdl_sub/utils/file_path.py +++ b/src/ytdl_sub/utils/file_path.py @@ -1,4 +1,5 @@ import os +import posixpath from pathlib import Path from typing import Tuple @@ -59,3 +60,8 @@ class FilePathTruncater: return str(Path(file_directory) / cls._truncate_file_name(file_name)) return str(file_path) + + @classmethod + def to_native_filepath(cls, file_path: str) -> str: + """Ensures file paths use the correct separator""" + return os.path.expanduser(file_path.replace(posixpath.sep, os.sep)) diff --git a/src/ytdl_sub/validators/file_path_validators.py b/src/ytdl_sub/validators/file_path_validators.py index c255ed1a..f5a69f61 100644 --- a/src/ytdl_sub/validators/file_path_validators.py +++ b/src/ytdl_sub/validators/file_path_validators.py @@ -2,7 +2,7 @@ import os from pathlib import Path from typing import Any -from ytdl_sub.script.script import Script +from ytdl_sub.utils.file_path import FilePathTruncater from ytdl_sub.validators.string_formatter_validators import OverridesStringFormatterValidator from ytdl_sub.validators.string_formatter_validators import StringFormatterValidator from ytdl_sub.validators.validators import StringValidator @@ -45,15 +45,8 @@ class StringFormatterFileNameValidator(StringFormatterValidator): _expected_value_type_name = "filepath" def post_process(self, resolved: str) -> str: - return ( - Script( - { - "tmp_var_1": resolved, - "tmp_var_2": "{%to_native_filepath(%truncate_filepath_if_too_long(tmp_var_1))}", - } - ) - .resolve() - .get_str("tmp_var_2") + return FilePathTruncater.to_native_filepath( + FilePathTruncater.maybe_truncate_file_path(resolved) ) @@ -61,13 +54,6 @@ class OverridesStringFormatterFilePathValidator(OverridesStringFormatterValidato _expected_value_type_name = "static filepath" def post_process(self, resolved: str) -> str: - return ( - Script( - { - "tmp_var_1": resolved, - "tmp_var_2": "{%to_native_filepath(%truncate_filepath_if_too_long(tmp_var_1))}", - } - ) - .resolve() - .get_str("tmp_var_2") + return FilePathTruncater.to_native_filepath( + FilePathTruncater.maybe_truncate_file_path(resolved) ) diff --git a/tests/e2e/test_debug_repro.py b/tests/e2e/test_debug_repro.py index b96f530b..8587dd5c 100644 --- a/tests/e2e/test_debug_repro.py +++ b/tests/e2e/test_debug_repro.py @@ -1,4 +1,3 @@ -import json from typing import Dict import pytest @@ -44,13 +43,13 @@ class TestReproduce: def test_debug_log_repro( self, default_config, - repro_preset_dict, + debug_log_rerpo, output_directory, ): sub = Subscription.from_dict( config=default_config, preset_name="repro", - preset_dict=repro_preset_dict, + preset_dict=debug_log_rerpo, ) transaction_log = sub.download(dry_run=False) diff --git a/tests/integration/prebuilt_presets/test_filter_keywords.py b/tests/integration/prebuilt_presets/test_filter_keywords.py index 41761b03..733f27b6 100644 --- a/tests/integration/prebuilt_presets/test_filter_keywords.py +++ b/tests/integration/prebuilt_presets/test_filter_keywords.py @@ -1,11 +1,8 @@ -import re - import pytest from expected_transaction_log import assert_transaction_log_matches from ytdl_sub.script.utils.exceptions import UserThrownRuntimeError from ytdl_sub.subscriptions.subscription import Subscription -from ytdl_sub.utils.exceptions import ValidationException @pytest.fixture @@ -77,6 +74,39 @@ class TestFilterKeywords: transaction_log_summary_file_name=f"integration/prebuilt_presets/title_filter_keywords_{filter_mode}.txt", ) + @pytest.mark.parametrize("filter_mode", ["include", "exclude"]) + def test_title_all( + self, + config, + filter_subscription_dict, + output_directory, + subscription_name, + mock_download_collection_entries, + filter_mode: str, + ): + filter_subscription_dict["overrides"][f"title_{filter_mode}_eval"] = "all" + filter_subscription_dict["overrides"][f"title_{filter_mode}_keywords"] = [ + "MOCK", + "ENTRY", + "20-3", + ] + subscription = Subscription.from_dict( + config=config, + preset_name=subscription_name, + preset_dict=filter_subscription_dict, + ) + + with mock_download_collection_entries( + is_youtube_channel=False, num_urls=1, is_dry_run=True + ): + transaction_log = subscription.download(dry_run=True) + + assert_transaction_log_matches( + output_directory=output_directory, + transaction_log=transaction_log, + transaction_log_summary_file_name=f"integration/prebuilt_presets/title_filter_keywords_{filter_mode}.txt", + ) + @pytest.mark.parametrize("filter_mode", ["include", "exclude"]) def test_description( self, @@ -108,6 +138,38 @@ class TestFilterKeywords: transaction_log_summary_file_name=f"integration/prebuilt_presets/description_filter_keywords_{filter_mode}.txt", ) + @pytest.mark.parametrize("filter_mode", ["include", "exclude"]) + def test_description_all( + self, + config, + filter_subscription_dict, + output_directory, + subscription_name, + mock_download_collection_entries, + filter_mode: str, + ): + filter_subscription_dict["overrides"][f"description_{filter_mode}_eval"] = "ALL" + filter_subscription_dict["overrides"][f"description_{filter_mode}_keywords"] = [ + "descr", + "iption", + ] + subscription = Subscription.from_dict( + config=config, + preset_name=subscription_name, + preset_dict=filter_subscription_dict, + ) + + with mock_download_collection_entries( + is_youtube_channel=False, num_urls=1, is_dry_run=True + ): + transaction_log = subscription.download(dry_run=True) + + assert_transaction_log_matches( + output_directory=output_directory, + transaction_log=transaction_log, + transaction_log_summary_file_name=f"integration/prebuilt_presets/description_filter_keywords_{filter_mode}.txt", + ) + @pytest.mark.parametrize( "keyword_variable", [ @@ -169,3 +231,35 @@ class TestFilterKeywords: pytest.raises(UserThrownRuntimeError, match="filter keywords must be strings"), ): _ = subscription.download(dry_run=True) + + @pytest.mark.parametrize( + "keyword_variable", + [ + "title_include", + "title_exclude", + "description_include", + "description_exclude", + ], + ) + def test_error_not_correct_eval( + self, + config, + filter_subscription_dict, + output_directory, + subscription_name, + mock_download_collection_entries, + keyword_variable, + ): + filter_subscription_dict["overrides"][f"{keyword_variable}_keywords"] = ["hmm"] + filter_subscription_dict["overrides"][f"{keyword_variable}_eval"] = "LOL" + subscription = Subscription.from_dict( + config=config, + preset_name=subscription_name, + preset_dict=filter_subscription_dict, + ) + + with ( + mock_download_collection_entries(is_youtube_channel=False, num_urls=1, is_dry_run=True), + pytest.raises(UserThrownRuntimeError, match="Keyword eval must be either ANY or ALL"), + ): + _ = subscription.download(dry_run=True) diff --git a/tests/unit/script/functions/test_string_functions.py b/tests/unit/script/functions/test_string_functions.py index a5a2961d..f5c85741 100644 --- a/tests/unit/script/functions/test_string_functions.py +++ b/tests/unit/script/functions/test_string_functions.py @@ -126,6 +126,14 @@ class TestNumericFunctions: output = single_variable_output(f"{{%contains_any('a brown dog', {value})}}") assert output == expected_output + @pytest.mark.parametrize( + "value, expected_output", + [("['a', 'b', 'c']", False), ("['nope', [], {}]", False), ("['a', 'dog']", True)], + ) + def test_contains_all(self, value, expected_output): + output = single_variable_output(f"{{%contains_all('a brown dog', {value})}}") + assert output == expected_output + @pytest.mark.parametrize( "input_string, split, max_split, expected_output", [ diff --git a/tests/unit/validators/test_file_path_validators.py b/tests/unit/validators/test_file_path_validators.py index 7ff62143..467f7b80 100644 --- a/tests/unit/validators/test_file_path_validators.py +++ b/tests/unit/validators/test_file_path_validators.py @@ -54,19 +54,17 @@ class TestStringFormatterFilePathValidator: if "thumb" not in ext: # do not put . in front of -thumb ext = f".{ext}" # pytest args with . in the beginning act weird - base_file_name = "s2023.e031701 - 𝗪𝗔𝗥𝗡𝗜𝗡𝗚： LG Secretly Overhaul This OLED Feature on C３ & G３… Should You Buy C２ Instead？" + base_file_name = "s2023.e031701 - 𝗪𝗔𝗥𝗡𝗜𝗡𝗚： {LG} Secretly Overhaul This OLED Feature on C３ & G３… Should You Buy C２ Instead？" with tempfile.TemporaryDirectory() as temp_dir: file_path = str(Path(temp_dir) / f"{base_file_name}{ext}") - formatter = StringFormatterFileNameValidator(name="test", value=str(file_path)) - truncated_file_path = formatter.post_process( - Script({"file_name": formatter.format_string}).resolve().get_str("file_name") - ) + formatter = StringFormatterFileNameValidator(name="test", value="") + truncated_file_path = formatter.post_process(file_path) assert truncated_file_path == str( Path(temp_dir) - / f"s2023.e031701 - 𝗪𝗔𝗥𝗡𝗜𝗡𝗚： LG Secretly Overhaul This OLED Feature on C３ & G３… Should You Buy C２ Instead？{ext}" + / f"s2023.e031701 - 𝗪𝗔𝗥𝗡𝗜𝗡𝗚： {{LG}} Secretly Overhaul This OLED Feature on C３ & G３… Should You Buy C２ Instead？{ext}" ) # Ensure it can actually open the file