Merge branch 'master' into j/airplance/getting-started-overhaul

docs/source/config_reference/scripting/scripting_functions.rst

@@ -646,6 +646,13 @@ contains
 :description:
   Returns True if ``contains`` is in ``string``. False otherwise.
 
+contains_all
+~~~~~~~~~~~~
+:spec: ``contains_all(string: String, contains_array: Array) -> Boolean``
+
+:description:
+    Returns true if all elements in ``contains_array`` are in ``string``. False otherwise.
+
 contains_any
 ~~~~~~~~~~~~
 :spec: ``contains_any(string: String, contains_array: Array) -> Boolean``

docs/source/prebuilt_presets/helpers.rst

@@ -4,21 +4,7 @@ Helper Presets
 
 .. hint::
 
-   Apply presets to your subscriptions using pipes:
+   See how to apply helper presets :doc:`here </prebuilt_presets/index>`
-
-   .. code-block:: yaml
-     :caption: Pipes can separate presets and values to apply them to all subscriptions below them.
-
-     Plex TV Show by Date | best_video_quality:
-
-       = Documentaries | chunk_initial_download:
-         "NOVA PBS": "https://www.youtube.com/@novapbs"
-         "National Geographic": "https://www.youtube.com/@NatGeo"
-
-       = Documentaries:
-         "Cosmos - What If": "https://www.youtube.com/playlist?list=PLZdXRHYAVxTJno6oFF9nLGuwXNGYHmE8U"
-
-Common presets are not usable by themselves- setting one of these as the sole preset of your subscription and attempting to download will not work. But you can add these presets to quickly modify an existing preset to better suit your needs.
 
 Only Recent
 -----------
@@ -42,32 +28,19 @@ upload date is outside of the range, or you hit max files, older videos will be
 To prevent deletion of files, use the preset ``Only Recent Archive`` instead.
 
 
-Best A/V Quality
-----------------
-
-Add the following preset to download the best available video and audio quality, and remux it into an MP4 container:
-
-``best_video_quality``
-
-
-Max 1080p Video
----------------
-
-Add the following preset to download the best available audio and video quality, with the video not greater than 1080p, and remux it into an MP4 container:
-
-``max_1080p``
-
 Filter Keywords
 ---------------
 
 ``Filter Keywords`` can include or exclude media with any of the listed keywords. Both keywords and title/description are lower-cased before filtering.
 
+Default behavior for Keyword evaluation is ANY, meaning the filter will succeed if any of the keywords are present. This can be set to ANY or ALL using the respective ``_eval`` variable.
+
 Supports the following override variables:
 
-* ``title_include_keywords``
+* ``title_include_keywords``, ``title_include_eval``
-* ``title_exclude_keywords``
+* ``title_exclude_keywords``, ``title_exclude_eval``
-* ``description_include_keywords``
+* ``description_include_keywords``, ``title_exclude_eval``
-* ``description_exclude_keywords``
+* ``description_exclude_keywords``, ``title_exclude_eval``
 
 .. tip::
 
@@ -90,6 +63,13 @@ Supports the following override variables:
             title_include_keywords:
               - "To Catch a Smuggler"
 
+        = Sports:
+          "~Maple Leafs Highlights":
+            url: "https://www.youtube.com/@NHL"
+            title_include_eval: "ALL"
+            title_include_keywords:
+              - "maple leafs"
+              - "highlights"
 
 Chunk Downloads
 ---------------

docs/source/prebuilt_presets/index.rst

@@ -3,13 +3,34 @@ Prebuilt Presets
 ================
 
 ``ytdl-sub`` offers a number of built-in presets using best practices for formatting
-media in various players. For advanced users, you can review the prebuilt preset
+media in various players.
+
+.. hint::
+
+   Apply multiple presets to your subscriptions using pipes. Pipes can define multiple presets and values
+   on the same line to apply to all subscriptions nested below them.
+
+   .. code-block:: yaml
+     :caption: Applies Max Video Quality preset to all TV shows, and Chunk Downloads preset to some
+
+     Plex TV Show by Date | Max Video Quality:
+
+       = Documentaries | Chunk Downloads:
+         "NOVA PBS": "https://www.youtube.com/@novapbs"
+         "National Geographic": "https://www.youtube.com/@NatGeo"
+
+       = Documentaries:
+         "Cosmos - What If": "https://www.youtube.com/playlist?list=PLZdXRHYAVxTJno6oFF9nLGuwXNGYHmE8U"
+
+
+For advanced users, you can review the prebuilt preset
 definitions :doc:`here </config_reference/prebuilt_presets/index>`.
 
 .. toctree:: 
   :titlesonly:
-  
+
-  helpers
   tv_shows
   music
-  music_videos
+  music_videos
+  media_quality
+  helpers

docs/source/prebuilt_presets/media_quality.rst

@@ -0,0 +1,29 @@
+======================
+Media Quality Presets
+======================
+
+.. hint::
+
+   See how to apply media quality presets :doc:`here </prebuilt_presets/index>`
+
+Video
+-----
+The following presets set video quality specifications to yt-dlp.
+
+- ``Max Video Quality``
+- ``Max 2160p``
+- ``Max 1440p``
+- ``Max 1080p``
+- ``Max 720p``
+- ``Max 480p``
+
+Audio
+-----
+The following presets set audio quality specifications to yt-dlp.
+These assume you are only extracting audio (no video).
+
+- ``Max Audio Quality``, format is determined by the source
+- ``Max MP3 Quality``
+- ``Max Opus Quality``
+- ``MP3 320k``
+- ``MP3 128k``

pyproject.toml

@@ -15,7 +15,7 @@ classifiers = [
     "Programming Language :: Python :: 3.11",
 ]
 dependencies = [
-    "yt-dlp[default]==2024.11.4",
+    "yt-dlp[default]==2024.12.13",
     "colorama~=0.4",
     "mergedeep~=1.3",
     "mediafile~=0.12",
@@ -44,12 +44,12 @@ where = ["src"]
 test = [
     "coverage[toml]>=6.3,<8.0",
     "pytest>=7.2,<9.0",
-    "pytest-rerunfailures~=14.0",
+    "pytest-rerunfailures>=14,<16",
 ]
 lint = [
     "black==24.10.0",
     "isort==5.13.2",
-    "pylint==3.3.1",
+    "pylint==3.3.2",
 ]
 docs = [
     "sphinx>=7,<9",
@@ -60,7 +60,7 @@ docs = [
 ]
 build = [
     "build~=1.2",
-    "twine~=5.0",
+    "twine>=5,<7",
     "pyinstaller~=6.5",
 ]
 [project.scripts]

src/ytdl_sub/prebuilt_presets/helpers/filtering.yaml

@@ -12,6 +12,11 @@ presets:
       description_include_keywords: "{ [] }"
       description_exclude_keywords: "{ [] }"
 
+      title_include_eval: "ANY"
+      title_exclude_eval: "ANY"
+      description_include_eval: "ANY"
+      description_exclude_eval: "ANY"
+
       "%ensure_string": >-
         {
           %assert_then(
@@ -32,21 +37,36 @@ presets:
           )
         }
 
+      # $0 - var to evaluate
+      # $1 - keyword list
+      # $2 - eval type
+      "%contains_keywords_inner": >-
+        {
+          %elif(
+            %eq(%ensure_string($2), 'any'),
+            %contains_any( $0, $1 ),
+            %eq(%ensure_string($2), 'all'),
+            %contains_all( $0, $1 ),
+            %throw('Keyword eval must be either ANY or ALL')
+          )
+        }
+
       # $0 - var to evaluate
       # $1 - keyword list
       # $2 - variable name for error messages
-      # $3 - default return if keyword list is empty
+      # $3 - keyword eval
+      # $4 - default return if keyword list is empty
       "%contains_keywords": >-
         { 
           %if(
             %bool( $1 ),
-            %contains_any( %lower($0), %ensure_lower_array($1, $2) ),
+            %contains_keywords_inner( %lower($0), %ensure_lower_array($1, $2), $3 ),
-            $3
+            $4
           )
         }
 
     filter_exclude:
-        - "{ %not( %contains_keywords(title, title_include_keywords, 'title_include_keywords', true) ) }"
+        - "{ %not( %contains_keywords(title, title_include_keywords, 'title_include_keywords', title_include_eval, true) ) }"
-        - "{ %not( %contains_keywords(description, description_include_keywords, 'description_include_keywords', true) ) }"
+        - "{ %not( %contains_keywords(description, description_include_keywords, 'description_include_keywords', description_include_eval, true) ) }"
-        - "{ %contains_keywords(title, title_exclude_keywords, 'title_exclude_keywords', false) }"
+        - "{ %contains_keywords(title, title_exclude_keywords, 'title_exclude_keywords', title_exclude_eval, false) }"
-        - "{ %contains_keywords(description, description_exclude_keywords, 'description_exclude_keywords',false) }"
+        - "{ %contains_keywords(description, description_exclude_keywords, 'description_exclude_keywords', description_exclude_eval, false) }"

src/ytdl_sub/prebuilt_presets/helpers/media_quality.yaml

@@ -13,6 +13,10 @@ presets:
     preset: 
       - best_video_quality
 
+  "Max Video Quality":
+    preset:
+      - best_video_quality
+
   #############################################################################
   # Max 2160p
 
@@ -55,4 +59,32 @@ presets:
   "Max 480p":
     format: "(bv*[height<=480]+bestaudio/best[height<=480])"
     ytdl_options:
-      merge_output_format: "mp4"
+      merge_output_format: "mp4"
+
+  #############################################################################
+  # Audio Quality Presets
+
+  "Max Audio Quality":
+    audio_extract:
+      codec: "best"
+      quality: 0
+
+  "Max MP3 Quality":
+    audio_extract:
+      codec: "mp3"
+      quality: 0
+
+  "Max Opus Quality":
+    audio_extract:
+      codec: "opus"
+      quality: 0
+
+  "MP3 320k":
+    audio_extract:
+      codec: "mp3"
+      quality: 320
+
+  "MP3 128k":
+    audio_extract:
+      codec: "mp3"
+      quality: 128

src/ytdl_sub/script/functions/string_functions.py

@@ -40,6 +40,20 @@ class StringFunctions:
             )
         )
 
+    @staticmethod
+    def contains_all(string: String, contains_array: Array) -> Boolean:
+        """
+        :description:
+            Returns true if all elements in ``contains_array`` are in ``string``. False otherwise.
+        """
+        return Boolean(
+            all(
+                str(val) in string.value
+                for val in contains_array.value
+                if isinstance(val, (String, Integer, Boolean, Float))
+            )
+        )
+
     @staticmethod
     def slice(string: String, start: Integer, end: Optional[Integer] = None) -> String:
         """

src/ytdl_sub/utils/file_path.py

@@ -1,4 +1,5 @@
 import os
+import posixpath
 from pathlib import Path
 from typing import Tuple
 
@@ -59,3 +60,8 @@ class FilePathTruncater:
             return str(Path(file_directory) / cls._truncate_file_name(file_name))
 
         return str(file_path)
+
+    @classmethod
+    def to_native_filepath(cls, file_path: str) -> str:
+        """Ensures file paths use the correct separator"""
+        return os.path.expanduser(file_path.replace(posixpath.sep, os.sep))

src/ytdl_sub/validators/file_path_validators.py

@@ -2,7 +2,7 @@ import os
 from pathlib import Path
 from typing import Any
 
-from ytdl_sub.script.script import Script
+from ytdl_sub.utils.file_path import FilePathTruncater
 from ytdl_sub.validators.string_formatter_validators import OverridesStringFormatterValidator
 from ytdl_sub.validators.string_formatter_validators import StringFormatterValidator
 from ytdl_sub.validators.validators import StringValidator
@@ -45,15 +45,8 @@ class StringFormatterFileNameValidator(StringFormatterValidator):
     _expected_value_type_name = "filepath"
 
     def post_process(self, resolved: str) -> str:
-        return (
+        return FilePathTruncater.to_native_filepath(
-            Script(
+            FilePathTruncater.maybe_truncate_file_path(resolved)
-                {
-                    "tmp_var_1": resolved,
-                    "tmp_var_2": "{%to_native_filepath(%truncate_filepath_if_too_long(tmp_var_1))}",
-                }
-            )
-            .resolve()
-            .get_str("tmp_var_2")
         )
 
 
@@ -61,13 +54,6 @@ class OverridesStringFormatterFilePathValidator(OverridesStringFormatterValidato
     _expected_value_type_name = "static filepath"
 
     def post_process(self, resolved: str) -> str:
-        return (
+        return FilePathTruncater.to_native_filepath(
-            Script(
+            FilePathTruncater.maybe_truncate_file_path(resolved)
-                {
-                    "tmp_var_1": resolved,
-                    "tmp_var_2": "{%to_native_filepath(%truncate_filepath_if_too_long(tmp_var_1))}",
-                }
-            )
-            .resolve()
-            .get_str("tmp_var_2")
         )

tests/e2e/test_debug_repro.py

@@ -1,4 +1,3 @@
-import json
 from typing import Dict
 
 import pytest
@@ -44,13 +43,13 @@ class TestReproduce:
     def test_debug_log_repro(
         self,
         default_config,
-        repro_preset_dict,
+        debug_log_rerpo,
         output_directory,
     ):
         sub = Subscription.from_dict(
             config=default_config,
             preset_name="repro",
-            preset_dict=repro_preset_dict,
+            preset_dict=debug_log_rerpo,
         )
 
         transaction_log = sub.download(dry_run=False)

tests/integration/prebuilt_presets/test_filter_keywords.py

@@ -1,11 +1,8 @@
-import re
-
 import pytest
 from expected_transaction_log import assert_transaction_log_matches
 
 from ytdl_sub.script.utils.exceptions import UserThrownRuntimeError
 from ytdl_sub.subscriptions.subscription import Subscription
-from ytdl_sub.utils.exceptions import ValidationException
 
 
 @pytest.fixture
@@ -77,6 +74,39 @@ class TestFilterKeywords:
             transaction_log_summary_file_name=f"integration/prebuilt_presets/title_filter_keywords_{filter_mode}.txt",
         )
 
+    @pytest.mark.parametrize("filter_mode", ["include", "exclude"])
+    def test_title_all(
+        self,
+        config,
+        filter_subscription_dict,
+        output_directory,
+        subscription_name,
+        mock_download_collection_entries,
+        filter_mode: str,
+    ):
+        filter_subscription_dict["overrides"][f"title_{filter_mode}_eval"] = "all"
+        filter_subscription_dict["overrides"][f"title_{filter_mode}_keywords"] = [
+            "MOCK",
+            "ENTRY",
+            "20-3",
+        ]
+        subscription = Subscription.from_dict(
+            config=config,
+            preset_name=subscription_name,
+            preset_dict=filter_subscription_dict,
+        )
+
+        with mock_download_collection_entries(
+            is_youtube_channel=False, num_urls=1, is_dry_run=True
+        ):
+            transaction_log = subscription.download(dry_run=True)
+
+        assert_transaction_log_matches(
+            output_directory=output_directory,
+            transaction_log=transaction_log,
+            transaction_log_summary_file_name=f"integration/prebuilt_presets/title_filter_keywords_{filter_mode}.txt",
+        )
+
     @pytest.mark.parametrize("filter_mode", ["include", "exclude"])
     def test_description(
         self,
@@ -108,6 +138,38 @@ class TestFilterKeywords:
             transaction_log_summary_file_name=f"integration/prebuilt_presets/description_filter_keywords_{filter_mode}.txt",
         )
 
+    @pytest.mark.parametrize("filter_mode", ["include", "exclude"])
+    def test_description_all(
+        self,
+        config,
+        filter_subscription_dict,
+        output_directory,
+        subscription_name,
+        mock_download_collection_entries,
+        filter_mode: str,
+    ):
+        filter_subscription_dict["overrides"][f"description_{filter_mode}_eval"] = "ALL"
+        filter_subscription_dict["overrides"][f"description_{filter_mode}_keywords"] = [
+            "descr",
+            "iption",
+        ]
+        subscription = Subscription.from_dict(
+            config=config,
+            preset_name=subscription_name,
+            preset_dict=filter_subscription_dict,
+        )
+
+        with mock_download_collection_entries(
+            is_youtube_channel=False, num_urls=1, is_dry_run=True
+        ):
+            transaction_log = subscription.download(dry_run=True)
+
+        assert_transaction_log_matches(
+            output_directory=output_directory,
+            transaction_log=transaction_log,
+            transaction_log_summary_file_name=f"integration/prebuilt_presets/description_filter_keywords_{filter_mode}.txt",
+        )
+
     @pytest.mark.parametrize(
         "keyword_variable",
         [
@@ -169,3 +231,35 @@ class TestFilterKeywords:
             pytest.raises(UserThrownRuntimeError, match="filter keywords must be strings"),
         ):
             _ = subscription.download(dry_run=True)
+
+    @pytest.mark.parametrize(
+        "keyword_variable",
+        [
+            "title_include",
+            "title_exclude",
+            "description_include",
+            "description_exclude",
+        ],
+    )
+    def test_error_not_correct_eval(
+        self,
+        config,
+        filter_subscription_dict,
+        output_directory,
+        subscription_name,
+        mock_download_collection_entries,
+        keyword_variable,
+    ):
+        filter_subscription_dict["overrides"][f"{keyword_variable}_keywords"] = ["hmm"]
+        filter_subscription_dict["overrides"][f"{keyword_variable}_eval"] = "LOL"
+        subscription = Subscription.from_dict(
+            config=config,
+            preset_name=subscription_name,
+            preset_dict=filter_subscription_dict,
+        )
+
+        with (
+            mock_download_collection_entries(is_youtube_channel=False, num_urls=1, is_dry_run=True),
+            pytest.raises(UserThrownRuntimeError, match="Keyword eval must be either ANY or ALL"),
+        ):
+            _ = subscription.download(dry_run=True)

tests/unit/script/functions/test_string_functions.py

@@ -126,6 +126,14 @@ class TestNumericFunctions:
         output = single_variable_output(f"{{%contains_any('a brown dog', {value})}}")
         assert output == expected_output
 
+    @pytest.mark.parametrize(
+        "value, expected_output",
+        [("['a', 'b', 'c']", False), ("['nope', [], {}]", False), ("['a', 'dog']", True)],
+    )
+    def test_contains_all(self, value, expected_output):
+        output = single_variable_output(f"{{%contains_all('a brown dog', {value})}}")
+        assert output == expected_output
+
     @pytest.mark.parametrize(
         "input_string, split, max_split, expected_output",
         [

tests/unit/validators/test_file_path_validators.py

@@ -54,19 +54,17 @@ class TestStringFormatterFilePathValidator:
         if "thumb" not in ext:  # do not put . in front of -thumb
             ext = f".{ext}"  # pytest args with . in the beginning act weird
 
-        base_file_name = "s2023.e031701 - 𝗪𝗔𝗥𝗡𝗜𝗡𝗚： LG Secretly Overhaul This OLED Feature on C３ & G３… Should You Buy C２ Instead？"
+        base_file_name = "s2023.e031701 - 𝗪𝗔𝗥𝗡𝗜𝗡𝗚： {LG} Secretly Overhaul This OLED Feature on C３ & G３… Should You Buy C２ Instead？"
 
         with tempfile.TemporaryDirectory() as temp_dir:
             file_path = str(Path(temp_dir) / f"{base_file_name}{ext}")
 
-            formatter = StringFormatterFileNameValidator(name="test", value=str(file_path))
+            formatter = StringFormatterFileNameValidator(name="test", value="")
-            truncated_file_path = formatter.post_process(
+            truncated_file_path = formatter.post_process(file_path)
-                Script({"file_name": formatter.format_string}).resolve().get_str("file_name")
-            )
 
             assert truncated_file_path == str(
                 Path(temp_dir)
-                / f"s2023.e031701 - 𝗪𝗔𝗥𝗡𝗜𝗡𝗚： LG Secretly Overhaul This OLED Feature on C３ & G３… Should You Buy C２ Instead？{ext}"
+                / f"s2023.e031701 - 𝗪𝗔𝗥𝗡𝗜𝗡𝗚： {{LG}} Secretly Overhaul This OLED Feature on C３ & G３… Should You Buy C２ Instead？{ext}"
             )
 
             # Ensure it can actually open the file