[FEATURE] Override variable support in ytdl_options (#1087)

Adds the ability to use override variables in the [ytdl_options](https://ytdl-sub.readthedocs.io/en/latest/config_reference/plugins.html#ytdl-options) section of the config.

docs/source/prebuilt_presets/helpers.rst

@@ -69,19 +69,34 @@ Supports the following override variables:
               - "To Catch a Smuggler"
 
 
-Chunk Initial Download
+Chunk Downloads
-----------------------
+---------------
 
-If you are archiving a large channel, ``ytdl-sub`` will try pulling each video's metadata from newest to oldest before starting any downloads. It is a long process and not ideal. A better method is to chunk the process by using the following preset:
+If you are archiving a large channel, ``ytdl-sub`` will try pulling each video's metadata from newest to oldest before
+starting any downloads. It is a long process and not ideal. A better method is to chunk the process by using the
+following preset:
 
-``chunk_initial_download``
+``Chunk Downloads``
 
-It will download videos starting from the oldest one, and only download 20 at a time. You can
+It will download videos starting from the oldest one, and only download 20 at a time by default. You can
-change this number by setting:
+change this number by setting the override variable ``chunk_max_downloads``.
 
 .. code-block:: yaml
 
-  ytdl_options:
+   __preset__:
-    max_downloads: 30  # Desired number to download per invocation
+     overrides:
+       chunk_max_downloads: 20
 
-Once the entire channel is downloaded, remove this preset. Then it will pull metadata from newest to oldest again, and stop pulling additional metadata once it reaches a video that has already been downloaded.
+   Plex TV Show by Date:
+
+     # Chunk these ones
+     = Documentaries | Chunk Downloads:
+       "NOVA PBS": "https://www.youtube.com/@novapbs"
+       "National Geographic": "https://www.youtube.com/@NatGeo"
+
+     # But not these ones
+     = Documentaries:
+       "Cosmos - What If": "https://www.youtube.com/playlist?list=PLZdXRHYAVxTJno6oFF9nLGuwXNGYHmE8U"
+
+Once the entire channel is downloaded, remove the usage of this preset. It will then pull metadata from newest to
+oldest again, and stop once it reaches a video that has already been downloaded.

src/ytdl_sub/config/overrides.py

@@ -11,12 +11,14 @@ from ytdl_sub.entries.variables.override_variables import REQUIRED_OVERRIDE_VARI
 from ytdl_sub.entries.variables.override_variables import OverrideHelpers
 from ytdl_sub.script.parser import parse
 from ytdl_sub.script.script import Script
+from ytdl_sub.script.types.resolvable import Resolvable
 from ytdl_sub.script.utils.exceptions import ScriptVariableNotResolved
 from ytdl_sub.utils.exceptions import InvalidVariableNameException
 from ytdl_sub.utils.exceptions import StringFormattingException
 from ytdl_sub.utils.exceptions import ValidationException
 from ytdl_sub.utils.script import ScriptUtils
 from ytdl_sub.utils.scriptable import Scriptable
+from ytdl_sub.validators.string_formatter_validators import OverridesStringFormatterValidator
 from ytdl_sub.validators.string_formatter_validators import StringFormatterValidator
 from ytdl_sub.validators.string_formatter_validators import UnstructuredDictFormatterValidator
 
@@ -144,11 +146,36 @@ class Overrides(UnstructuredDictFormatterValidator, Scriptable):
         self.update_script()
         return self
 
+    def _apply_to_resolvable(
+        self,
+        formatter: StringFormatterValidator,
+        entry: Optional[Entry],
+        function_overrides: Optional[Dict[str, str]],
+    ) -> Resolvable:
+        script: Script = self.script
+        unresolvable: Set[str] = self.unresolvable
+        if entry:
+            script = entry.script
+            unresolvable = entry.unresolvable
+
+        try:
+            return script.resolve_once(
+                dict({"tmp_var": formatter.format_string}, **(function_overrides or {})),
+                unresolvable=unresolvable,
+            )["tmp_var"]
+        except ScriptVariableNotResolved as exc:
+            raise StringFormattingException(
+                "Tried to resolve the following script, but could not due to unresolved "
+                f"variables:\n {formatter.format_string}\n"
+                "This is most likely due to circular dependencies in variables. "
+                "If you think otherwise, please file a bug on GitHub and post your config. Thanks!"
+            ) from exc
+
     def apply_formatter(
         self,
         formatter: StringFormatterValidator,
         entry: Optional[Entry] = None,
-        function_overrides: Dict[str, str] = None,
+        function_overrides: Optional[Dict[str, str]] = None,
     ) -> str:
         """
         Parameters
@@ -169,25 +196,28 @@ class Overrides(UnstructuredDictFormatterValidator, Scriptable):
         StringFormattingException
             If the formatter that is trying to be resolved cannot
         """
-        script: Script = self.script
+        return formatter.post_process(
-        unresolvable: Set[str] = self.unresolvable
+            str(
-        if entry:
+                self._apply_to_resolvable(
-            script = entry.script
+                    formatter=formatter, entry=entry, function_overrides=function_overrides
-            unresolvable = entry.unresolvable
-
-        try:
-            return formatter.post_process(
-                str(
-                    script.resolve_once(
-                        dict({"tmp_var": formatter.format_string}, **(function_overrides or {})),
-                        unresolvable=unresolvable,
-                    )["tmp_var"]
                 )
             )
-        except ScriptVariableNotResolved as exc:
+        )
-            raise StringFormattingException(
+
-                "Tried to resolve the following script, but could not due to unresolved "
+    def apply_overrides_formatter_to_native(
-                f"variables:\n {formatter.format_string}\n"
+        self,
-                "This is most likely due to circular dependencies in variables. "
+        formatter: OverridesStringFormatterValidator,
-                "If you think otherwise, please file a bug on GitHub and post your config. Thanks!"
+    ) -> Any:
-            ) from exc
+        """
+        Parameters
+        ----------
+        formatter
+            Overrides formatter to apply
+
+        Returns
+        -------
+        The native python form of the resolved variable
+        """
+        return self._apply_to_resolvable(
+            formatter=formatter, entry=None, function_overrides=None
+        ).native

src/ytdl_sub/config/preset_options.py

@@ -9,11 +9,13 @@ from ytdl_sub.validators.string_datetime import StringDatetimeValidator
 from ytdl_sub.validators.string_formatter_validators import OverridesIntegerFormatterValidator
 from ytdl_sub.validators.string_formatter_validators import OverridesStringFormatterValidator
 from ytdl_sub.validators.string_formatter_validators import StringFormatterValidator
+from ytdl_sub.validators.string_formatter_validators import (
+    UnstructuredOverridesDictFormatterValidator,
+)
 from ytdl_sub.validators.validators import BoolValidator
-from ytdl_sub.validators.validators import LiteralDictValidator
 
 
-class YTDLOptions(LiteralDictValidator):
+class YTDLOptions(UnstructuredOverridesDictFormatterValidator):
     """
     Allows you to add any ytdl argument to ytdl-sub's downloader.
     The argument names can differ slightly from the command-line argument names. See

src/ytdl_sub/prebuilt_presets/helpers/download_deletion_options.yaml

@@ -37,10 +37,12 @@ presets:
 
   chunk_initial_download:  # legacy preset name
     ytdl_options:
-      max_downloads: 20
+      max_downloads: "{chunk_max_downloads}"
       playlistreverse: True
       break_on_existing: False
+    overrides:
+      chunk_max_downloads: 20
 
-  "Download in Chunks":
+  "Chunk Downloads":
     preset:
       - chunk_initial_download

src/ytdl_sub/subscriptions/subscription_ytdl_options.py

@@ -110,7 +110,11 @@ class SubscriptionYTDLOptions:
 
     @property
     def _user_ytdl_options(self) -> Dict:
-        return self._preset.ytdl_options.dict
+        native_ytdl_options = {
+            key: self._overrides.apply_overrides_formatter_to_native(val)
+            for key, val in self._preset.ytdl_options.dict.items()
+        }
+        return native_ytdl_options
 
     @property
     def _plugin_match_filters(self) -> Dict:

tests/e2e/bandcamp/test_bandcamp.py

@@ -12,7 +12,10 @@ def subscription_dict(output_directory):
     return {
         "preset": "Bandcamp",
         "ytdl_options": {
-            "max_downloads": 15,
+            # Test that ytdl-options can handle overrides
+            # TODO: Move this to a local test
+            "max_downloads": "{max_downloads}",
+            "extractor_args": {"youtube": {"lang": ["en"]}},
         },
         "audio_extract": {"codec": "mp3", "quality": 320},
         "date_range": {"after": "20210110"},
@@ -20,6 +23,7 @@ def subscription_dict(output_directory):
             "subscription_value": "https://sithuayemusic.bandcamp.com/",
             "subscription_indent_1": "Progressive Metal",
             "music_directory": output_directory,
+            "max_downloads": 15,
         },
     }