From 88110a816d5d6d2682084d5c9c31a5ff06c08cfd Mon Sep 17 00:00:00 2001 From: Jesse Bannon Date: Fri, 25 Oct 2024 20:47:18 -0700 Subject: [PATCH 1/6] [BUGFIX] Fix playlists not downloading correctly Due to url ytdl-options not materializing --- src/ytdl_sub/config/preset_options.py | 12 ++++++++++++ src/ytdl_sub/downloaders/url/downloader.py | 2 +- .../subscriptions/subscription_ytdl_options.py | 6 +----- 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/src/ytdl_sub/config/preset_options.py b/src/ytdl_sub/config/preset_options.py index 2d907b29..f1ba8534 100644 --- a/src/ytdl_sub/config/preset_options.py +++ b/src/ytdl_sub/config/preset_options.py @@ -1,7 +1,9 @@ from typing import Any +from typing import Dict from typing import Optional from ytdl_sub.config.defaults import DEFAULT_DOWNLOAD_ARCHIVE_NAME +from ytdl_sub.config.overrides import Overrides from ytdl_sub.validators.file_path_validators import OverridesStringFormatterFilePathValidator from ytdl_sub.validators.file_path_validators import StringFormatterFileNameValidator from ytdl_sub.validators.strict_dict_validator import StrictDictValidator @@ -48,6 +50,16 @@ class YTDLOptions(UnstructuredOverridesDictFormatterValidator): where each key is a ytdl argument. Include in the example are some popular ytdl_options. """ + def to_native_dict(self, overrides: Overrides) -> Dict: + """ + Materializes the entire ytdl-options dict from OverrideStringFormatters into + native python + """ + return { + key: overrides.apply_overrides_formatter_to_native(val) + for key, val in self.dict.items() + } + # Disable for proper docstring formatting # pylint: disable=line-too-long diff --git a/src/ytdl_sub/downloaders/url/downloader.py b/src/ytdl_sub/downloaders/url/downloader.py index 81f310a7..f47cedee 100644 --- a/src/ytdl_sub/downloaders/url/downloader.py +++ b/src/ytdl_sub/downloaders/url/downloader.py @@ -457,7 +457,7 @@ class MultiUrlDownloader(SourcePlugin[MultiUrlValidator]): def _download_metadata(self, url: str, validator: UrlValidator) -> Iterable[Entry]: metadata_ytdl_options = self.metadata_ytdl_options( - ytdl_option_overrides=validator.ytdl_options.dict + ytdl_option_overrides=validator.ytdl_options.to_native_dict(self.overrides) ) download_reversed = ScriptUtils.bool_formatter_output( self.overrides.apply_formatter(validator.download_reverse) diff --git a/src/ytdl_sub/subscriptions/subscription_ytdl_options.py b/src/ytdl_sub/subscriptions/subscription_ytdl_options.py index d1f288ee..12efb951 100644 --- a/src/ytdl_sub/subscriptions/subscription_ytdl_options.py +++ b/src/ytdl_sub/subscriptions/subscription_ytdl_options.py @@ -110,11 +110,7 @@ class SubscriptionYTDLOptions: @property def _user_ytdl_options(self) -> Dict: - native_ytdl_options = { - key: self._overrides.apply_overrides_formatter_to_native(val) - for key, val in self._preset.ytdl_options.dict.items() - } - return native_ytdl_options + return self._preset.ytdl_options.to_native_dict(self._overrides) @property def _plugin_match_filters(self) -> Dict: From 102e2a90420c53bffd59714e034a979b57a636a7 Mon Sep 17 00:00:00 2001 From: Dan Hand Date: Sun, 27 Oct 2024 03:39:15 +0000 Subject: [PATCH 2/6] [DOCS] Fix documentation for headless cron path (#1100) --- .../getting_started/automating_downloads.rst | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/docs/source/guides/getting_started/automating_downloads.rst b/docs/source/guides/getting_started/automating_downloads.rst index 53791dde..5cea7cc8 100644 --- a/docs/source/guides/getting_started/automating_downloads.rst +++ b/docs/source/guides/getting_started/automating_downloads.rst @@ -21,7 +21,7 @@ Docker and Unraid .. tab-item:: GUI Image - The script that will execute automatically is located at ``/config/ytdl-sub-configs/run-cron``. + The script that will execute automatically is located at ``/config/ytdl-sub-configs/run_cron``. Access your container at http://localhost:8443/, then in the GUI terminal run these commands: @@ -81,7 +81,7 @@ Docker and Unraid docker compose restart - The script that will execute automatically is located at ``/config/run-cron``. + The script that will execute automatically is located at ``/config/run_cron``. Access your container from the terminal by running: @@ -93,13 +93,13 @@ Docker and Unraid .. code-block:: shell - echo '#!/bin/bash' > /config/ytdl-sub-configs/run_cron - echo "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" >> /config/ytdl-sub-configs/run_cron - echo "echo 'Cron started, running ytdl-sub...'" >> /config/ytdl-sub-configs/run_cron - echo "cd /config/ytdl-sub-configs" >> /config/ytdl-sub-configs/run_cron - echo "ytdl-sub --config=config.yaml sub subscriptions.yaml" >> /config/ytdl-sub-configs/run_cron - chmod +x /config/ytdl-sub-configs/run_cron - chown abc:abc /config/ytdl-sub-configs/run_cron + echo '#!/bin/bash' > /config/run_cron + echo "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" >> /config/run_cron + echo "echo 'Cron started, running ytdl-sub...'" >> /config/run_cron + echo "cd /config" >> /config/run_cron + echo "ytdl-sub --config=config.yaml sub subscriptions.yaml" >> /config/run_cron + chmod +x /config/run_cron + chown abc:abc /config/run_cron You can test the newly created script by running: @@ -137,4 +137,4 @@ To be tested (please contact code owner or join the discord server if you can te .. code-block:: powershell - ytdl-sub.exe --config \path\to\config\config.yaml sub \path\to\config\subscriptions.yaml \ No newline at end of file + ytdl-sub.exe --config \path\to\config\config.yaml sub \path\to\config\subscriptions.yaml From 43c10c19e0461656929c79ff23aed341073e368c Mon Sep 17 00:00:00 2001 From: Jesse Bannon Date: Sat, 26 Oct 2024 20:54:57 -0700 Subject: [PATCH 3/6] [DOCS] Fix docker indents (#1103) --- docs/source/guides/install/docker.rst | 88 +++++++++++++-------------- 1 file changed, 44 insertions(+), 44 deletions(-) diff --git a/docs/source/guides/install/docker.rst b/docs/source/guides/install/docker.rst index 04b5e456..bf9b04c4 100644 --- a/docs/source/guides/install/docker.rst +++ b/docs/source/guides/install/docker.rst @@ -49,22 +49,22 @@ Docker Compose is an easy "set it and forget it" install method. Follow the inst :caption: compose.yaml services: - ytdl-sub: - image: ghcr.io/jmbannon/ytdl-sub-gui:latest - container_name: ytdl-sub - environment: - - PUID=1000 - - PGID=1000 - - TZ=America/Los_Angeles - volumes: - - :/config - - :/tv_shows # optional - - :/movies # optional - - :/music_videos # optional - - :/music # optional - ports: - - 8443:8443 - restart: unless-stopped + ytdl-sub: + image: ghcr.io/jmbannon/ytdl-sub-gui:latest + container_name: ytdl-sub + environment: + - PUID=1000 + - PGID=1000 + - TZ=America/Los_Angeles + volumes: + - :/config + - :/tv_shows # optional + - :/movies # optional + - :/music_videos # optional + - :/music # optional + ports: + - 8443:8443 + restart: unless-stopped .. tab-item:: Headless Image @@ -72,21 +72,21 @@ Docker Compose is an easy "set it and forget it" install method. Follow the inst :caption: compose.yaml services: - ytdl-sub: - image: ghcr.io/jmbannon/ytdl-sub:latest - container_name: ytdl-sub - environment: - - PUID=1000 - - PGID=1000 - - TZ=America/Los_Angeles - - DOCKER_MODS=linuxserver/mods:universal-cron - volumes: - - :/config - - :/tv_shows # optional - - :/movies # optional - - :/music_videos # optional - - :/music # optional - restart: unless-stopped + ytdl-sub: + image: ghcr.io/jmbannon/ytdl-sub:latest + container_name: ytdl-sub + environment: + - PUID=1000 + - PGID=1000 + - TZ=America/Los_Angeles + - DOCKER_MODS=linuxserver/mods:universal-cron + volumes: + - :/config + - :/tv_shows # optional + - :/movies # optional + - :/music_videos # optional + - :/music # optional + restart: unless-stopped Device Passthrough ~~~~~~~~~~~~~~~~~~~ @@ -120,19 +120,19 @@ GPU Passthrough :emphasize-lines: 5-13 services: - ytdl-sub: - image: ghcr.io/jmbannon/ytdl-sub-gui:latest - container_name: ytdl-sub - environment: - - .. - - NVIDIA_DRIVER_CAPABILITIES=all # Nvidia ENV args - - NVIDIA_VISIBLE_DEVICES=all - deploy: - resources: - reservations: - devices: - - capabilities: ["gpu"] # GPU passthrough - restart: unless-stopped + ytdl-sub: + image: ghcr.io/jmbannon/ytdl-sub-gui:latest + container_name: ytdl-sub + environment: + - .. + - NVIDIA_DRIVER_CAPABILITIES=all # Nvidia ENV args + - NVIDIA_VISIBLE_DEVICES=all + deploy: + resources: + reservations: + devices: + - capabilities: ["gpu"] # GPU passthrough + restart: unless-stopped Docker CLI ---------- From 1bdc65f2e14a233993d83d762197c45deb843943 Mon Sep 17 00:00:00 2001 From: Jesse Bannon Date: Sat, 26 Oct 2024 21:16:17 -0700 Subject: [PATCH 4/6] [BUGFIX] Custom function ordering (#1104) Fixes a bug where custom functions would throw an error if they were used out-of-order from their definition --- src/ytdl_sub/script/script.py | 11 +++++++---- .../unit/script/types/test_custom_function.py | 18 ++++++++++++++++++ 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/src/ytdl_sub/script/script.py b/src/ytdl_sub/script/script.py index f7c20d88..ab506318 100644 --- a/src/ytdl_sub/script/script.py +++ b/src/ytdl_sub/script/script.py @@ -490,15 +490,18 @@ class Script: name: definition for name, definition in variables.items() if not _is_function(name) } + custom_function_names = set(self._functions.keys()) | functions_to_add.keys() + variable_names = ( + set(self._variables.keys()) | variables_to_add.keys() | (unresolvable or set()) + ) + for definitions in [functions_to_add, variables_to_add]: for name, definition in definitions.items(): parsed = parse( text=definition, name=name, - custom_function_names=set(self._functions.keys()), - variable_names=set(self._variables.keys()) - .union(variables.keys()) - .union(unresolvable or set()), + custom_function_names=custom_function_names, + variable_names=variable_names, ) if parsed.maybe_resolvable is None: diff --git a/tests/unit/script/types/test_custom_function.py b/tests/unit/script/types/test_custom_function.py index e7b76e80..61096cc3 100644 --- a/tests/unit/script/types/test_custom_function.py +++ b/tests/unit/script/types/test_custom_function.py @@ -22,6 +22,24 @@ class TestCustomFunction: } ).resolve() == ScriptOutput({"output": Integer(9)}) + def test_custom_functions_any_order_via_add(self): + assert Script({}).add( + { + "%custom_cubed": "{%mul(%custom_square($0),$0)}", + "%custom_square": "{%mul($0, $0)}", + "output": "{%custom_cubed(3)}", + } + ).resolve() == ScriptOutput({"output": Integer(27)}) + + def test_custom_functions_any_order_via_init(self): + assert Script( + { + "%custom_cubed": "{%mul(%custom_square($0),$0)}", + "%custom_square": "{%mul($0, $0)}", + "output": "{%custom_cubed(3)}", + } + ).resolve() == ScriptOutput({"output": Integer(27)}) + def test_custom_function_cycle(self): with pytest.raises( CycleDetected, match=re.escape("The custom function %cycle_func cannot call itself.") From 80054aa77b84318b63396ea06453c2edefad6f13 Mon Sep 17 00:00:00 2001 From: Jesse Bannon Date: Sun, 27 Oct 2024 08:09:05 -0700 Subject: [PATCH 5/6] [FEATURE] Support newlines and tabs in scripting strings (#1105) Allows the usage of `\n` and `\t` in scripting strings --- src/ytdl_sub/script/parser.py | 12 ++++++++++-- tests/unit/script/functions/test_string_functions.py | 1 + tests/unit/script/types/test_string.py | 6 +++++- 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/src/ytdl_sub/script/parser.py b/src/ytdl_sub/script/parser.py index 737d6ffa..0f844e34 100644 --- a/src/ytdl_sub/script/parser.py +++ b/src/ytdl_sub/script/parser.py @@ -301,8 +301,16 @@ class _Parser: self._pos += len(str_open_token) return String(value=string_value) - self._pos += 1 - string_value += ch + # Read literal "\n" as newlines + if self._read(increment_pos=False, length=2) == "\\n": + string_value += "\n" + self._pos += 2 + elif self._read(increment_pos=False, length=2) == "\\t": + string_value += "\t" + self._pos += 2 + else: + self._pos += 1 + string_value += ch raise STRINGS_NOT_CLOSED diff --git a/tests/unit/script/functions/test_string_functions.py b/tests/unit/script/functions/test_string_functions.py index e5d898f8..ea3d441d 100644 --- a/tests/unit/script/functions/test_string_functions.py +++ b/tests/unit/script/functions/test_string_functions.py @@ -132,6 +132,7 @@ class TestNumericFunctions: ("no splits", " | ", None, ["no splits"]), ("one | split", " | ", None, ["one", "split"]), ("max | split | one", " | ", 1, ["max", "split | one"]), + ("multiline\ndescription", "\\n", None, ["multiline", "description"]), ], ) def test_split( diff --git a/tests/unit/script/types/test_string.py b/tests/unit/script/types/test_string.py index c3749845..08700ace 100644 --- a/tests/unit/script/types/test_string.py +++ b/tests/unit/script/types/test_string.py @@ -1,6 +1,7 @@ import re import pytest +from unit.script.conftest import single_variable_output from ytdl_sub.script.parser import STRINGS_NOT_CLOSED from ytdl_sub.script.parser import STRINGS_ONLY_ARGS @@ -45,10 +46,13 @@ class TestString: ("{%string('backslash \\\\')}", "backslash \\\\"), ("{%string('''triple quote with \" ' \\''')}", "triple quote with \" ' \\"), ('{%string("""triple quote with " \' \\""")}', "triple quote with \" ' \\"), + ("{%string('literal \\n newlines')}", "literal \n newlines"), + ("{%string('supports \t tabs')}", "supports \t tabs"), + ("{%string('literal \\t tabs')}", "literal \t tabs"), ], ) def test_string(self, string: str, expected_string: str): - assert Script({"out": string}).resolve() == ScriptOutput({"out": String(expected_string)}) + assert single_variable_output(string) == expected_string def test_null_is_empty_string(self): assert Script({"out": "{%string(null)}"}).resolve() == ScriptOutput({"out": String("")}) From bf64be8a59185159c6fdd04beba15c81297c402a Mon Sep 17 00:00:00 2001 From: Jesse Bannon Date: Sun, 27 Oct 2024 08:25:26 -0700 Subject: [PATCH 6/6] [DOCS] Fix preset key (#1106) --- docs/source/guides/getting_started/advanced_configuration.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/guides/getting_started/advanced_configuration.rst b/docs/source/guides/getting_started/advanced_configuration.rst index e234b77a..7ab05e31 100644 --- a/docs/source/guides/getting_started/advanced_configuration.rst +++ b/docs/source/guides/getting_started/advanced_configuration.rst @@ -30,14 +30,14 @@ You can modularize your presets via preset inheritance. For example, presets: TV Show: - presets: + preset: - "Jellyfin TV Show by Date" overrides: tv_show_directory: "/ytdl_sub_tv_shows" TV Show Only Recent: - presets: + preset: - "TV Show" - "Only Recent"