[BACKEND] Array support for URLs (#1406)

Allows for the `download` plugin to take in a script-based array of URLs. Prebuilt presets are now greatly reduced in size by constructing urls as an array versus 100+ separate URL variables.

Eventually, we can completely remove the limit of the number of URLs in a subscription.

src/ytdl_sub/config/overrides.py

@@ -238,20 +238,25 @@ class Overrides(UnstructuredDictFormatterValidator, Scriptable):
     def apply_overrides_formatter_to_native(
         self,
         formatter: OverridesStringFormatterValidator,
+        function_overrides: Optional[Dict[str, str]] = None,
     ) -> Any:
         """
         Parameters
         ----------
         formatter
             Overrides formatter to apply
+        function_overrides
+            Optional. Explicit values to override the overrides themselves and source variables
 
         Returns
         -------
         The native python form of the resolved variable
         """
-        return self._apply_to_resolvable(
+        return formatter.post_process_native(
-            formatter=formatter, entry=None, function_overrides=None
+            self._apply_to_resolvable(
-        ).native
+                formatter=formatter, entry=None, function_overrides=function_overrides
+            ).native
+        )
 
     def evaluate_boolean(
         self, formatter: StringFormatterValidator, entry: Optional[Entry] = None

src/ytdl_sub/config/validators/variable_validation.py

@@ -84,6 +84,9 @@ class VariableValidation:
         )
         resolved_subscription["download"] = []
         for url_output in raw_download_output["download"]:
+            if isinstance(url_output["url"], list):
+                url_output["url"] = [url for url in url_output["url"] if bool(url)]
+
             if url_output["url"]:
                 resolved_subscription["download"].append(url_output)
 

src/ytdl_sub/downloaders/url/downloader.py

@@ -52,12 +52,12 @@ class UrlDownloaderBasePluginExtension(SourcePluginExtension[MultiUrlValidator])
 
         if 0 <= input_url_idx < len(self.plugin_options.urls.list):
             validator = self.plugin_options.urls.list[input_url_idx]
-            if self.overrides.apply_formatter(validator.url) == entry_input_url:
+            if entry_input_url in self.overrides.apply_overrides_formatter_to_native(validator.url):
                 return validator
 
         # Match the first validator based on the URL, if one exists
         for validator in self.plugin_options.urls.list:
-            if self.overrides.apply_formatter(validator.url) == entry_input_url:
+            if entry_input_url in self.overrides.apply_overrides_formatter_to_native(validator.url):
                 return validator
 
         # Return the first validator if none exist
@@ -487,19 +487,27 @@ class MultiUrlDownloader(SourcePlugin[MultiUrlValidator]):
         # download the bottom-most urls first since they are top-priority
         for idx, url_validator in reversed(list(enumerate(self.collection.urls.list))):
             # URLs can be empty. If they are, then skip
-            if not (url := self.overrides.apply_formatter(url_validator.url)):
+            if not (urls := self.overrides.apply_overrides_formatter_to_native(url_validator.url)):
                 continue
 
-            for entry in self._download_metadata(url=url, validator=url_validator):
+            assert isinstance(urls, list)
-                entry.initialize_script(self.overrides).add(
-                    {
-                        v.ytdl_sub_input_url: url,
-                        v.ytdl_sub_input_url_index: idx,
-                        v.ytdl_sub_input_url_count: len(self.collection.urls.list),
-                    }
-                )
 
-                yield entry
+            for url in reversed(urls):
+                assert isinstance(url, str)
+
+                if not url:
+                    continue
+
+                for entry in self._download_metadata(url=url, validator=url_validator):
+                    entry.initialize_script(self.overrides).add(
+                        {
+                            v.ytdl_sub_input_url: url,
+                            v.ytdl_sub_input_url_index: idx,
+                            v.ytdl_sub_input_url_count: len(self.collection.urls.list),
+                        }
+                    )
+
+                    yield entry
 
     def download(self, entry: Entry) -> Optional[Entry]:
         """

src/ytdl_sub/downloaders/url/validators.py

@@ -43,6 +43,19 @@ class UrlThumbnailListValidator(ListValidator[UrlThumbnailValidator]):
     _inner_list_type = UrlThumbnailValidator
 
 
+class OverridesOneOrManyUrlValidator(OverridesStringFormatterValidator):
+    def post_process_native(self, resolved: Any) -> Any:
+        if isinstance(resolved, str):
+            return [resolved]
+        if isinstance(resolved, list):
+            for value in resolved:
+                if not isinstance(value, str):
+                    raise self._validation_exception("Must be a string or an array of strings.")
+            return resolved
+
+        raise self._validation_exception("Must be a string or an array of strings.")
+
+
 class UrlValidator(StrictDictValidator):
     _required_keys = {"url"}
     _optional_keys = {
@@ -68,7 +81,7 @@ class UrlValidator(StrictDictValidator):
         super().__init__(name, value)
 
         # TODO: url validate using yt-dlp IE
-        self._url = self._validate_key(key="url", validator=OverridesStringFormatterValidator)
+        self._url = self._validate_key(key="url", validator=OverridesOneOrManyUrlValidator)
         self._variables = self._validate_key_if_present(
             key="variables", validator=DictFormatterValidator, default={}
         )

src/ytdl_sub/prebuilt_presets/helpers/url_categorized.yaml

@@ -108,103 +108,4 @@ presets:
         }
 
       subscription_map: "{ {} }"
-      url: "{ %get_url_i(1) }"
+      urls: "{ %array_apply(%range(100, 1), %get_url_i) }"
-      url2: "{ %get_url_i(2) }"
-      url3: "{ %get_url_i(3) }"
-      url4: "{ %get_url_i(4) }"
-      url5: "{ %get_url_i(5) }"
-      url6: "{ %get_url_i(6) }"
-      url7: "{ %get_url_i(7) }"
-      url8: "{ %get_url_i(8) }"
-      url9: "{ %get_url_i(9) }"
-      url10: "{ %get_url_i(10) }"
-      url11: "{ %get_url_i(11) }"
-      url12: "{ %get_url_i(12) }"
-      url13: "{ %get_url_i(13) }"
-      url14: "{ %get_url_i(14) }"
-      url15: "{ %get_url_i(15) }"
-      url16: "{ %get_url_i(16) }"
-      url17: "{ %get_url_i(17) }"
-      url18: "{ %get_url_i(18) }"
-      url19: "{ %get_url_i(19) }"
-      url20: "{ %get_url_i(20) }"
-      url21: "{ %get_url_i(21) }"
-      url22: "{ %get_url_i(22) }"
-      url23: "{ %get_url_i(23) }"
-      url24: "{ %get_url_i(24) }"
-      url25: "{ %get_url_i(25) }"
-      url26: "{ %get_url_i(26) }"
-      url27: "{ %get_url_i(27) }"
-      url28: "{ %get_url_i(28) }"
-      url29: "{ %get_url_i(29) }"
-      url30: "{ %get_url_i(30) }"
-      url31: "{ %get_url_i(31) }"
-      url32: "{ %get_url_i(32) }"
-      url33: "{ %get_url_i(33) }"
-      url34: "{ %get_url_i(34) }"
-      url35: "{ %get_url_i(35) }"
-      url36: "{ %get_url_i(36) }"
-      url37: "{ %get_url_i(37) }"
-      url38: "{ %get_url_i(38) }"
-      url39: "{ %get_url_i(39) }"
-      url40: "{ %get_url_i(40) }"
-      url41: "{ %get_url_i(41) }"
-      url42: "{ %get_url_i(42) }"
-      url43: "{ %get_url_i(43) }"
-      url44: "{ %get_url_i(44) }"
-      url45: "{ %get_url_i(45) }"
-      url46: "{ %get_url_i(46) }"
-      url47: "{ %get_url_i(47) }"
-      url48: "{ %get_url_i(48) }"
-      url49: "{ %get_url_i(49) }"
-      url50: "{ %get_url_i(50) }"
-      url51: "{ %get_url_i(51) }"
-      url52: "{ %get_url_i(52) }"
-      url53: "{ %get_url_i(53) }"
-      url54: "{ %get_url_i(54) }"
-      url55: "{ %get_url_i(55) }"
-      url56: "{ %get_url_i(56) }"
-      url57: "{ %get_url_i(57) }"
-      url58: "{ %get_url_i(58) }"
-      url59: "{ %get_url_i(59) }"
-      url60: "{ %get_url_i(60) }"
-      url61: "{ %get_url_i(61) }"
-      url62: "{ %get_url_i(62) }"
-      url63: "{ %get_url_i(63) }"
-      url64: "{ %get_url_i(64) }"
-      url65: "{ %get_url_i(65) }"
-      url66: "{ %get_url_i(66) }"
-      url67: "{ %get_url_i(67) }"
-      url68: "{ %get_url_i(68) }"
-      url69: "{ %get_url_i(69) }"
-      url70: "{ %get_url_i(70) }"
-      url71: "{ %get_url_i(71) }"
-      url72: "{ %get_url_i(72) }"
-      url73: "{ %get_url_i(73) }"
-      url74: "{ %get_url_i(74) }"
-      url75: "{ %get_url_i(75) }"
-      url76: "{ %get_url_i(76) }"
-      url77: "{ %get_url_i(77) }"
-      url78: "{ %get_url_i(78) }"
-      url79: "{ %get_url_i(79) }"
-      url80: "{ %get_url_i(80) }"
-      url81: "{ %get_url_i(81) }"
-      url82: "{ %get_url_i(82) }"
-      url83: "{ %get_url_i(83) }"
-      url84: "{ %get_url_i(84) }"
-      url85: "{ %get_url_i(85) }"
-      url86: "{ %get_url_i(86) }"
-      url87: "{ %get_url_i(87) }"
-      url88: "{ %get_url_i(88) }"
-      url89: "{ %get_url_i(89) }"
-      url90: "{ %get_url_i(90) }"
-      url91: "{ %get_url_i(91) }"
-      url92: "{ %get_url_i(92) }"
-      url93: "{ %get_url_i(93) }"
-      url94: "{ %get_url_i(94) }"
-      url95: "{ %get_url_i(95) }"
-      url96: "{ %get_url_i(96) }"
-      url97: "{ %get_url_i(97) }"
-      url98: "{ %get_url_i(98) }"
-      url99: "{ %get_url_i(99) }"
-      url100: "{ %get_url_i(100) }"

src/ytdl_sub/validators/string_formatter_validators.py

@@ -1,4 +1,5 @@
 from datetime import datetime
+from typing import Any
 from typing import Dict
 from typing import Set
 from typing import Union
@@ -89,6 +90,14 @@ class StringFormatterValidator(StringValidator):
         """
         return resolved
 
+    def post_process_native(self, resolved: Any) -> Any:
+        """
+        Returns
+        -------
+        Apply any post processing to the resolved native value.
+        """
+        return resolved
+
 
 class FloatFormatterValidator(StringFormatterValidator):
     _expected_value_type_name = "float"

tests/unit/config/test_subscription.py

@@ -1,4 +1,3 @@
-import json
 import re
 from contextlib import contextmanager
 from pathlib import Path
@@ -473,23 +472,19 @@ def test_advanced_tv_show_subscriptions(
     overrides = subs[5].overrides
 
     assert overrides.script.get("subscription_name").native == "Gardening with Ciscoe"
-    assert (
-        overrides.apply_formatter(overrides.dict["url"])
-        == "https://www.youtube.com/@gardeningwithciscoe4430"
-    )
-    assert (
-        overrides.apply_formatter(overrides.dict["url2"])
-        == "https://www.youtube.com/playlist?list=PLi8V8UemxeG6lo5if5H5g5EbsteELcb0_"
-    )
 
-    assert overrides.apply_formatter(overrides.dict["subscription_array"]) == json.dumps(
+    assert overrides.apply_overrides_formatter_to_native(overrides.dict["subscription_array"]) == [
-        [
+        "https://www.youtube.com/@gardeningwithciscoe4430",
-            "https://www.youtube.com/@gardeningwithciscoe4430",
+        "https://www.youtube.com/playlist?list=PLi8V8UemxeG6lo5if5H5g5EbsteELcb0_",
-            "https://www.youtube.com/playlist?list=PLi8V8UemxeG6lo5if5H5g5EbsteELcb0_",
+        "https://www.youtube.com/playlist?list=PLsJlQSR-KjmaQqqJ9jq18cF6XXXAR4kyn",
-            "https://www.youtube.com/playlist?list=PLsJlQSR-KjmaQqqJ9jq18cF6XXXAR4kyn",
+        "https://www.youtube.com/watch?v=2vq-vPubS5I",
-            "https://www.youtube.com/watch?v=2vq-vPubS5I",
+    ]
-        ]
+    assert overrides.apply_overrides_formatter_to_native(overrides.dict["urls"]) == [
-    )
+        "https://www.youtube.com/@gardeningwithciscoe4430",
+        "https://www.youtube.com/playlist?list=PLi8V8UemxeG6lo5if5H5g5EbsteELcb0_",
+        "https://www.youtube.com/playlist?list=PLsJlQSR-KjmaQqqJ9jq18cF6XXXAR4kyn",
+        "https://www.youtube.com/watch?v=2vq-vPubS5I",
+    ]
 
 
 def test_music_subscriptions(default_config: ConfigFile, music_subscriptions_path: Path):
@@ -525,7 +520,7 @@ def test_music_video_subscriptions(default_config: ConfigFile, music_video_subsc
     )
     assert jackson.get("subscription_indent_1").native == "Pop"
     assert (
-        jackson.get("url").native
+        jackson.get("urls").native[0]
         == "https://www.youtube.com/playlist?list=OLAK5uy_mnY03zP6abNWH929q2XhGzWD_2uKJ_n8E"
     )
 
@@ -533,12 +528,10 @@ def test_music_video_subscriptions(default_config: ConfigFile, music_video_subsc
     gnr = subs[3].overrides.script
 
     assert gnr.get("subscription_name").native == "Guns N' Roses"
-    assert (
+    gnr_urls = gnr.get("urls").native
-        gnr.get("url").native
+    assert gnr_urls[0] == "https://www.youtube.com/playlist?list=PLOTK54q5K4INNXaHKtmXYr6J7CajWjqeJ"
-        == "https://www.youtube.com/playlist?list=PLOTK54q5K4INNXaHKtmXYr6J7CajWjqeJ"
-    )
     assert gnr.get("subscription_indent_1").native == "Rock"
-    assert gnr.get("url2").native == "https://www.youtube.com/watch?v=OldpIhHPsbs"
+    assert gnr_urls[1] == "https://www.youtube.com/watch?v=OldpIhHPsbs"
 
 
 def test_default_docker_config_and_subscriptions(

tests/unit/prebuilt_presets/test_tv_show_by_date.py

@@ -24,3 +24,58 @@ class TestTvShowByDatePreset:
                     "overrides": {"tv_show_directory": "abc", "s01_url": "test"},
                 },
             )
+
+    def test_backward_compatibility_single(self, default_config):
+        a = Subscription.from_dict(
+            config=default_config,
+            preset_name="a",
+            preset_dict={
+                "preset": "Jellyfin TV Show by Date",
+                "overrides": {"tv_show_directory": "abc", "url": "test_1"},
+            },
+        )
+
+        b = Subscription.from_dict(
+            config=default_config,
+            preset_name="a",
+            preset_dict={
+                "preset": "Jellyfin TV Show by Date",
+                "overrides": {"tv_show_directory": "abc", "subscription_value": "test_1"},
+            },
+        )
+
+        assert a.resolved_yaml() == b.resolved_yaml()
+
+    def test_backward_compatibility_multi(self, default_config):
+        a = Subscription.from_dict(
+            config=default_config,
+            preset_name="a",
+            preset_dict={
+                "preset": "Jellyfin TV Show by Date",
+                "overrides": {"tv_show_directory": "abc", "url": "test_1", "url2": "test_2"},
+            },
+        )
+
+        b = Subscription.from_dict(
+            config=default_config,
+            preset_name="a",
+            preset_dict={
+                "preset": "Jellyfin TV Show by Date",
+                "overrides": {
+                    "tv_show_directory": "abc",
+                    "subscription_array": ["test_1", "test_2"],
+                },
+            },
+        )
+
+        c = Subscription.from_dict(
+            config=default_config,
+            preset_name="a",
+            preset_dict={
+                "preset": "Jellyfin TV Show by Date",
+                "overrides": {"tv_show_directory": "abc", "urls": ["test_1", "test_2"]},
+            },
+        )
+
+        assert a.resolved_yaml() == b.resolved_yaml()
+        assert a.resolved_yaml() == c.resolved_yaml()

tests/unit/prebuilt_presets/test_tv_show_collection.py

@@ -44,7 +44,7 @@ class TestTvShowCollectionPreset:
             preset_dict={"preset": "Jellyfin TV Show Collection", "overrides": overrides},
         )
 
-        assert len(sub.downloader_options.urls.list) == (num_seasons + 1) * num_urls_per_season * 2
+        assert len(sub.downloader_options.urls.list) == (num_seasons + 1) * 3
         url_list = sub.downloader_options.urls.list
         itr = 0
 
@@ -55,16 +55,20 @@ class TestTvShowCollectionPreset:
                 if season_num == num_seasons + 1:
                     season_num = 0
 
-                for i in range(num_urls_per_season):
+                # is_bilateral
-                    url = sub.overrides.apply_formatter(
+                if i == 0:
+                    url = sub.overrides.apply_overrides_formatter_to_native(
                         url_list[itr].url,
                         function_overrides={
                             # mock so bilateral url gets enabled
                             "subscription_has_download_archive": "True"
                         },
                     )
+                    assert url == [
+                        f"youtube.com/playlist?url_{season_num}_{i}"
+                        for i in range(num_urls_per_season)
+                    ]
                     variables = url_list[itr].variables.dict
-                    assert url == f"youtube.com/playlist?url_{season_num}_{i}"
                     assert (
                         sub.overrides.apply_formatter(variables["collection_season_number"])
                         == f"{season_num}"
@@ -73,5 +77,35 @@ class TestTvShowCollectionPreset:
                         sub.overrides.apply_formatter(variables["collection_season_name"])
                         == f"The Season {season_num}"
                     )
-
                     itr += 1
+                # not bilateral
+                else:
+                    for j in range(2):
+                        url = sub.overrides.apply_overrides_formatter_to_native(
+                            url_list[itr + j].url,
+                            function_overrides={
+                                # mock so bilateral url gets enabled
+                                "subscription_has_download_archive": "True"
+                            },
+                        )
+
+                        # First instance is the first url to get thumbnails
+                        if j == 0:
+                            assert url == [f"youtube.com/playlist?url_{season_num}_0"]
+                        # Next one contains remaining urls
+                        else:
+                            assert url == [
+                                f"youtube.com/playlist?url_{season_num}_{i}"
+                                for i in range(1, num_urls_per_season)
+                            ]
+
+                        variables = url_list[itr].variables.dict
+                        assert (
+                            sub.overrides.apply_formatter(variables["collection_season_number"])
+                            == f"{season_num}"
+                        )
+                        assert (
+                            sub.overrides.apply_formatter(variables["collection_season_name"])
+                            == f"The Season {season_num}"
+                        )
+                    itr += 2