Merge branch 'master' into j/staged-helper

docs/source/guides/getting_started/advanced_configuration.rst

@@ -30,14 +30,14 @@ You can modularize your presets via preset inheritance. For example,
   presets:
 
     TV Show:
-      presets:
+      preset:
         - "Jellyfin TV Show by Date"
 
       overrides:
         tv_show_directory: "/ytdl_sub_tv_shows"
 
     TV Show Only Recent:
-      presets:
+      preset:
         - "TV Show"
         - "Only Recent"
 

docs/source/guides/getting_started/automating_downloads.rst

@@ -21,7 +21,7 @@ Docker and Unraid
   
   .. tab-item:: GUI Image
     
-    The script that will execute automatically is located at ``/config/ytdl-sub-configs/run-cron``. 
+    The script that will execute automatically is located at ``/config/ytdl-sub-configs/run_cron``.
 
     Access your container at http://localhost:8443/, then in the GUI terminal run these commands:
 
@@ -81,7 +81,7 @@ Docker and Unraid
 
       docker compose restart
 
-    The script that will execute automatically is located at ``/config/run-cron``. 
+    The script that will execute automatically is located at ``/config/run_cron``.
 
     Access your container from the terminal by running:
 
@@ -93,13 +93,13 @@ Docker and Unraid
 
     .. code-block:: shell
 
-      echo '#!/bin/bash' > /config/ytdl-sub-configs/run_cron
+      echo '#!/bin/bash' > /config/run_cron
-      echo "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" >> /config/ytdl-sub-configs/run_cron
+      echo "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" >> /config/run_cron
-      echo "echo 'Cron started, running ytdl-sub...'" >> /config/ytdl-sub-configs/run_cron
+      echo "echo 'Cron started, running ytdl-sub...'" >> /config/run_cron
-      echo "cd /config/ytdl-sub-configs" >> /config/ytdl-sub-configs/run_cron
+      echo "cd /config" >> /config/run_cron
-      echo "ytdl-sub --config=config.yaml sub subscriptions.yaml" >> /config/ytdl-sub-configs/run_cron
+      echo "ytdl-sub --config=config.yaml sub subscriptions.yaml" >> /config/run_cron
-      chmod +x /config/ytdl-sub-configs/run_cron
+      chmod +x /config/run_cron
-      chown abc:abc /config/ytdl-sub-configs/run_cron
+      chown abc:abc /config/run_cron
 
     You can test the newly created script by running: 
 
@@ -137,4 +137,4 @@ To be tested (please contact code owner or join the discord server if you can te
 
 .. code-block:: powershell
 
-  ytdl-sub.exe --config \path\to\config\config.yaml sub \path\to\config\subscriptions.yaml
+  ytdl-sub.exe --config \path\to\config\config.yaml sub \path\to\config\subscriptions.yaml

docs/source/guides/install/docker.rst

@@ -49,22 +49,22 @@ Docker Compose is an easy "set it and forget it" install method. Follow the inst
       :caption: compose.yaml
 
       services:
-      ytdl-sub:
+        ytdl-sub:
-        image: ghcr.io/jmbannon/ytdl-sub-gui:latest
+          image: ghcr.io/jmbannon/ytdl-sub-gui:latest
-        container_name: ytdl-sub
+          container_name: ytdl-sub
-        environment:
+          environment:
-          - PUID=1000
+            - PUID=1000
-          - PGID=1000
+            - PGID=1000
-          - TZ=America/Los_Angeles
+            - TZ=America/Los_Angeles
-        volumes:
+          volumes:
-          - <path/to/ytdl-sub/config>:/config
+            - <path/to/ytdl-sub/config>:/config
-          - <path/to/tv_shows>:/tv_shows  # optional
+            - <path/to/tv_shows>:/tv_shows  # optional
-          - <path/to/movies>:/movies  # optional
+            - <path/to/movies>:/movies  # optional
-          - <path/to/music_videos>:/music_videos  # optional
+            - <path/to/music_videos>:/music_videos  # optional
-          - <path/to/music>:/music  # optional
+            - <path/to/music>:/music  # optional
-        ports:
+          ports:
-          - 8443:8443
+            - 8443:8443
-        restart: unless-stopped
+          restart: unless-stopped
 
   .. tab-item:: Headless Image
 
@@ -72,21 +72,21 @@ Docker Compose is an easy "set it and forget it" install method. Follow the inst
       :caption: compose.yaml
 
       services:
-      ytdl-sub:
+        ytdl-sub:
-        image: ghcr.io/jmbannon/ytdl-sub:latest
+          image: ghcr.io/jmbannon/ytdl-sub:latest
-        container_name: ytdl-sub
+          container_name: ytdl-sub
-        environment:
+          environment:
-          - PUID=1000
+            - PUID=1000
-          - PGID=1000
+            - PGID=1000
-          - TZ=America/Los_Angeles
+            - TZ=America/Los_Angeles
-          - DOCKER_MODS=linuxserver/mods:universal-cron
+            - DOCKER_MODS=linuxserver/mods:universal-cron
-        volumes:
+          volumes:
-          - <path/to/ytdl-sub/config>:/config
+            - <path/to/ytdl-sub/config>:/config
-          - <path/to/tv_shows>:/tv_shows  # optional
+            - <path/to/tv_shows>:/tv_shows  # optional
-          - <path/to/movies>:/movies  # optional
+            - <path/to/movies>:/movies  # optional
-          - <path/to/music_videos>:/music_videos  # optional
+            - <path/to/music_videos>:/music_videos  # optional
-          - <path/to/music>:/music  # optional
+            - <path/to/music>:/music  # optional
-        restart: unless-stopped
+          restart: unless-stopped
 
 Device Passthrough
 ~~~~~~~~~~~~~~~~~~~
@@ -120,19 +120,19 @@ GPU Passthrough
   :emphasize-lines: 5-13
 
   services:
-  ytdl-sub:
+    ytdl-sub:
-    image: ghcr.io/jmbannon/ytdl-sub-gui:latest
+      image: ghcr.io/jmbannon/ytdl-sub-gui:latest
-    container_name: ytdl-sub
+      container_name: ytdl-sub
-    environment:
+      environment:
-    - ..
+      - ..
-    - NVIDIA_DRIVER_CAPABILITIES=all  # Nvidia ENV args
+      - NVIDIA_DRIVER_CAPABILITIES=all  # Nvidia ENV args
-    - NVIDIA_VISIBLE_DEVICES=all
+      - NVIDIA_VISIBLE_DEVICES=all
-    deploy:
+      deploy:
-      resources:
+        resources:
-        reservations:
+          reservations:
-          devices:
+            devices:
-            - capabilities: ["gpu"]  # GPU passthrough
+              - capabilities: ["gpu"]  # GPU passthrough
-    restart: unless-stopped
+      restart: unless-stopped
 
 Docker CLI
 ----------

src/ytdl_sub/config/preset_options.py

@@ -1,7 +1,9 @@
 from typing import Any
+from typing import Dict
 from typing import Optional
 
 from ytdl_sub.config.defaults import DEFAULT_DOWNLOAD_ARCHIVE_NAME
+from ytdl_sub.config.overrides import Overrides
 from ytdl_sub.validators.file_path_validators import OverridesStringFormatterFilePathValidator
 from ytdl_sub.validators.file_path_validators import StringFormatterFileNameValidator
 from ytdl_sub.validators.strict_dict_validator import StrictDictValidator
@@ -48,6 +50,16 @@ class YTDLOptions(UnstructuredOverridesDictFormatterValidator):
     where each key is a ytdl argument. Include in the example are some popular ytdl_options.
     """
 
+    def to_native_dict(self, overrides: Overrides) -> Dict:
+        """
+        Materializes the entire ytdl-options dict from OverrideStringFormatters into
+        native python
+        """
+        return {
+            key: overrides.apply_overrides_formatter_to_native(val)
+            for key, val in self.dict.items()
+        }
+
 
 # Disable for proper docstring formatting
 # pylint: disable=line-too-long

src/ytdl_sub/downloaders/url/downloader.py

@@ -457,7 +457,7 @@ class MultiUrlDownloader(SourcePlugin[MultiUrlValidator]):
 
     def _download_metadata(self, url: str, validator: UrlValidator) -> Iterable[Entry]:
         metadata_ytdl_options = self.metadata_ytdl_options(
-            ytdl_option_overrides=validator.ytdl_options.dict
+            ytdl_option_overrides=validator.ytdl_options.to_native_dict(self.overrides)
         )
         download_reversed = ScriptUtils.bool_formatter_output(
             self.overrides.apply_formatter(validator.download_reverse)

src/ytdl_sub/script/parser.py

@@ -301,8 +301,16 @@ class _Parser:
                 self._pos += len(str_open_token)
                 return String(value=string_value)
 
-            self._pos += 1
+            # Read literal "\n" as newlines
-            string_value += ch
+            if self._read(increment_pos=False, length=2) == "\\n":
+                string_value += "\n"
+                self._pos += 2
+            elif self._read(increment_pos=False, length=2) == "\\t":
+                string_value += "\t"
+                self._pos += 2
+            else:
+                self._pos += 1
+                string_value += ch
 
         raise STRINGS_NOT_CLOSED
 

src/ytdl_sub/script/script.py

@@ -490,15 +490,18 @@ class Script:
             name: definition for name, definition in variables.items() if not _is_function(name)
         }
 
+        custom_function_names = set(self._functions.keys()) | functions_to_add.keys()
+        variable_names = (
+            set(self._variables.keys()) | variables_to_add.keys() | (unresolvable or set())
+        )
+
         for definitions in [functions_to_add, variables_to_add]:
             for name, definition in definitions.items():
                 parsed = parse(
                     text=definition,
                     name=name,
-                    custom_function_names=set(self._functions.keys()),
+                    custom_function_names=custom_function_names,
-                    variable_names=set(self._variables.keys())
+                    variable_names=variable_names,
-                    .union(variables.keys())
-                    .union(unresolvable or set()),
                 )
 
                 if parsed.maybe_resolvable is None:

src/ytdl_sub/subscriptions/subscription_ytdl_options.py

@@ -110,11 +110,7 @@ class SubscriptionYTDLOptions:
 
     @property
     def _user_ytdl_options(self) -> Dict:
-        native_ytdl_options = {
+        return self._preset.ytdl_options.to_native_dict(self._overrides)
-            key: self._overrides.apply_overrides_formatter_to_native(val)
-            for key, val in self._preset.ytdl_options.dict.items()
-        }
-        return native_ytdl_options
 
     @property
     def _plugin_match_filters(self) -> Dict:

tests/unit/script/functions/test_string_functions.py

@@ -132,6 +132,7 @@ class TestNumericFunctions:
             ("no splits", " | ", None, ["no splits"]),
             ("one | split", " | ", None, ["one", "split"]),
             ("max | split | one", " | ", 1, ["max", "split | one"]),
+            ("multiline\ndescription", "\\n", None, ["multiline", "description"]),
         ],
     )
     def test_split(

tests/unit/script/types/test_custom_function.py

@@ -22,6 +22,24 @@ class TestCustomFunction:
             }
         ).resolve() == ScriptOutput({"output": Integer(9)})
 
+    def test_custom_functions_any_order_via_add(self):
+        assert Script({}).add(
+            {
+                "%custom_cubed": "{%mul(%custom_square($0),$0)}",
+                "%custom_square": "{%mul($0, $0)}",
+                "output": "{%custom_cubed(3)}",
+            }
+        ).resolve() == ScriptOutput({"output": Integer(27)})
+
+    def test_custom_functions_any_order_via_init(self):
+        assert Script(
+            {
+                "%custom_cubed": "{%mul(%custom_square($0),$0)}",
+                "%custom_square": "{%mul($0, $0)}",
+                "output": "{%custom_cubed(3)}",
+            }
+        ).resolve() == ScriptOutput({"output": Integer(27)})
+
     def test_custom_function_cycle(self):
         with pytest.raises(
             CycleDetected, match=re.escape("The custom function %cycle_func cannot call itself.")

tests/unit/script/types/test_string.py

@@ -1,6 +1,7 @@
 import re
 
 import pytest
+from unit.script.conftest import single_variable_output
 
 from ytdl_sub.script.parser import STRINGS_NOT_CLOSED
 from ytdl_sub.script.parser import STRINGS_ONLY_ARGS
@@ -45,10 +46,13 @@ class TestString:
             ("{%string('backslash \\\\')}", "backslash \\\\"),
             ("{%string('''triple quote with \" ' \\''')}", "triple quote with \" ' \\"),
             ('{%string("""triple quote with " \' \\""")}', "triple quote with \" ' \\"),
+            ("{%string('literal \\n newlines')}", "literal \n newlines"),
+            ("{%string('supports \t tabs')}", "supports \t tabs"),
+            ("{%string('literal \\t tabs')}", "literal \t tabs"),
         ],
     )
     def test_string(self, string: str, expected_string: str):
-        assert Script({"out": string}).resolve() == ScriptOutput({"out": String(expected_string)})
+        assert single_variable_output(string) == expected_string
 
     def test_null_is_empty_string(self):
         assert Script({"out": "{%string(null)}"}).resolve() == ScriptOutput({"out": String("")})