[FEATURE] Multiple URLs supported for tv_show_by_date preset (#568)

* [BACKEND] Support empty urls for multi_url, add overrides for extra urls in tv_show_by_date preset

* need to check for missing urls in downloader

* use override in test

* docs about multi url

* wording

docs/presets.rst

@@ -58,6 +58,17 @@ and overriding the following variables:
        # episode_title: "{upload_date_standardized} - {title}"
        # episode_description: "{webpage_url}"
 
+In addition, you can add additional URLs to create a single TV by using the override variables
+``url2``, ``url3``, ..., ``url20``:
+
+.. code-block:: yaml
+
+     overrides:
+       tv_show_name: "Rick A"
+       tv_show_directory: "/path/to/youtube_shows"
+       url: "https://www.youtube.com/channel/UCuAXFkgsw1L7xaCfnd5JJOw"
+       url2: "https://www.youtube.com/@just.rick_6"
+
 
 TV Show Collection
 ^^^^^^^^^^^^^^^^^^

src/ytdl_sub/downloaders/url/downloader.py

@@ -443,14 +443,10 @@ class BaseUrlDownloader(BaseDownloader[BaseDownloaderOptionsT], ABC):
             ):
                 yield entry_child
 
-    def _download_url_metadata(
+    def _download_url_metadata(self, url: str) -> Tuple[List[EntryParent], List[Entry]]:
-        self, collection_url: UrlValidator
-    ) -> Tuple[List[EntryParent], List[Entry]]:
         """
         Downloads only info.json files and forms EntryParent trees
         """
-        url = self.overrides.apply_formatter(collection_url.url)
-
         with self._separate_download_archives():
             entry_dicts = YTDLP.extract_info_via_info_json(
                 working_directory=self.working_directory,
@@ -494,7 +490,11 @@ class BaseUrlDownloader(BaseDownloader[BaseDownloaderOptionsT], ABC):
         """The function to perform the download of all media entries"""
         # download the bottom-most urls first since they are top-priority
         for collection_url in reversed(self.collection.urls.list):
-            parents, orphan_entries = self._download_url_metadata(collection_url=collection_url)
+            # URLs can be empty. If they are, then skip
+            if not (url := self.overrides.apply_formatter(collection_url.url)):
+                continue
+
+            parents, orphan_entries = self._download_url_metadata(url=url)
 
             # TODO: Encapsulate this logic into its own class
             self._url_state = URLDownloadState(

src/ytdl_sub/downloaders/url/multi_url.py

@@ -1,3 +1,6 @@
+from typing import Dict
+from typing import List
+
 from ytdl_sub.downloaders.url.downloader import BaseUrlDownloader
 from ytdl_sub.downloaders.url.downloader import DownloaderValidator
 from ytdl_sub.downloaders.url.validators import MultiUrlValidator
@@ -43,6 +46,23 @@ class MultiUrlDownloadOptions(MultiUrlValidator, DownloaderValidator):
         """Returns itself!"""
         return self
 
+    def validate_with_variables(
+        self, source_variables: List[str], override_variables: Dict[str, str]
+    ) -> None:
+        """
+        Validates any source variables added by the collection
+        """
+        super().validate_with_variables(
+            source_variables=source_variables, override_variables=override_variables
+        )
+
+        has_non_empty_url = False
+        for url_validator in self.urls.list:
+            has_non_empty_url |= bool(url_validator.url.apply_formatter(override_variables))
+
+        if not has_non_empty_url:
+            raise self._validation_exception("Must contain at least one url that is non-empty")
+
 
 class MultiUrlDownloader(BaseUrlDownloader[MultiUrlDownloadOptions]):
     downloader_options_type = MultiUrlDownloadOptions

src/ytdl_sub/downloaders/url/validators.py

@@ -156,8 +156,8 @@ class UrlListValidator(ListValidator[UrlValidator]):
 
         added_variables: Dict[str, str] = self.list[0].variables.dict_with_format_strings
 
-        for idx, collection_url_validator in enumerate(self.list[1:]):
+        for idx, url_validator in enumerate(self.list[1:]):
-            collection_variables = collection_url_validator.variables.dict_with_format_strings
+            collection_variables = url_validator.variables.dict_with_format_strings
 
             # see if this collection contains new added vars (it should not)
             for var in collection_variables.keys():

src/ytdl_sub/prebuilt_presets/tv_show/tv_show.yaml

@@ -43,6 +43,45 @@ presets:
               uid: "avatar_uncropped"
             - name: "{tv_show_fanart_file_name}"
               uid: "banner_uncropped"
+        - url: "{url2}"
+        - url: "{url3}"
+        - url: "{url4}"
+        - url: "{url5}"
+        - url: "{url6}"
+        - url: "{url7}"
+        - url: "{url8}"
+        - url: "{url9}"
+        - url: "{url10}"
+        - url: "{url11}"
+        - url: "{url12}"
+        - url: "{url13}"
+        - url: "{url14}"
+        - url: "{url15}"
+        - url: "{url16}"
+        - url: "{url17}"
+        - url: "{url18}"
+        - url: "{url19}"
+        - url: "{url20}"
+    overrides:
+      url2: ""
+      url3: ""
+      url4: ""
+      url5: ""
+      url6: ""
+      url7: ""
+      url8: ""
+      url9: ""
+      url10: ""
+      url11: ""
+      url12: ""
+      url13: ""
+      url14: ""
+      url15: ""
+      url16: ""
+      url17: ""
+      url18: ""
+      url19: ""
+      url20: ""
 
 ####################################################################################################
 

tests/unit/config/test_preset.py

@@ -1,6 +1,9 @@
+import re
+
 import pytest
 
 from ytdl_sub.config.preset import Preset
+from ytdl_sub.downloaders.url.multi_url import MultiUrlDownloadOptions
 from ytdl_sub.plugins.nfo_tags import NfoTagsOptions
 from ytdl_sub.utils.exceptions import StringFormattingVariableNotFoundException
 from ytdl_sub.utils.exceptions import ValidationException
@@ -192,7 +195,7 @@ class TestPreset:
                 name="test",
                 value={
                     "download": youtube_video,
-                    "output_options": {"output_directory": "dir", "file_name": "file"},
+                    "output_options": output_options,
                     "output_directory_nfo_tags": {
                         "nfo_name": "the nfo name",
                         "nfo_root": "the root",
@@ -200,3 +203,39 @@ class TestPreset:
                     },
                 },
             )
+
+    def test_preset_with_multi_url__contains_empty_url(self, config_file, output_options):
+        _ = Preset(
+            config=config_file,
+            name="test",
+            value={
+                "download": {
+                    "download_strategy": "multi_url",
+                    "urls": [{"url": "non-empty url"}, {"url": ""}],  # empty url
+                },
+                "output_options": output_options,
+            },
+        )
+
+    def test_preset_with_multi_url__contains_all_empty_urls_errors(
+        self, config_file, output_options
+    ):
+        with pytest.raises(
+            ValidationException,
+            match=re.escape(
+                "Validation error in test.download: Must contain at least one "
+                "url that is non-empty"
+            ),
+        ):
+            _ = Preset(
+                config=config_file,
+                name="test",
+                value={
+                    "download": {
+                        "download_strategy": "multi_url",
+                        "urls": [{"url": "{url}"}, {"url": "{url2}"}],
+                    },
+                    "output_options": output_options,
+                    "overrides": {"url": "", "url2": ""},
+                },
+            )

tests/unit/prebuilt_presets/test_prebuilt_presets.py

@@ -115,9 +115,7 @@ class TestPrebuiltTVShowPresets:
             },
         }
         if is_many_urls:
-            preset_dict = dict(
+            preset_dict["overrides"]["url2"] = "https://url.number.2.here"
-                preset_dict, **{"download": {"urls": [{"url": "https://url.number.2.here"}]}}
-            )
 
         subscription = Subscription.from_dict(
             config=config,