From 1965bc0a40ae4a1ccff505c9b37a27133a303f90 Mon Sep 17 00:00:00 2001 From: Jesse Bannon Date: Thu, 30 Mar 2023 09:28:32 -0700 Subject: [PATCH] [FEATURE] Multiple URLs supported for `tv_show_by_date` preset (#568) * [BACKEND] Support empty urls for multi_url, add overrides for extra urls in tv_show_by_date preset * need to check for missing urls in downloader * use override in test * docs about multi url * wording --- docs/presets.rst | 11 +++++ src/ytdl_sub/downloaders/url/downloader.py | 12 +++--- src/ytdl_sub/downloaders/url/multi_url.py | 20 +++++++++ src/ytdl_sub/downloaders/url/validators.py | 4 +- .../prebuilt_presets/tv_show/tv_show.yaml | 39 ++++++++++++++++++ tests/unit/config/test_preset.py | 41 ++++++++++++++++++- .../prebuilt_presets/test_prebuilt_presets.py | 4 +- 7 files changed, 119 insertions(+), 12 deletions(-) diff --git a/docs/presets.rst b/docs/presets.rst index 28b46408..9ef718ec 100644 --- a/docs/presets.rst +++ b/docs/presets.rst @@ -58,6 +58,17 @@ and overriding the following variables: # episode_title: "{upload_date_standardized} - {title}" # episode_description: "{webpage_url}" +In addition, you can add additional URLs to create a single TV by using the override variables +``url2``, ``url3``, ..., ``url20``: + +.. code-block:: yaml + + overrides: + tv_show_name: "Rick A" + tv_show_directory: "/path/to/youtube_shows" + url: "https://www.youtube.com/channel/UCuAXFkgsw1L7xaCfnd5JJOw" + url2: "https://www.youtube.com/@just.rick_6" + TV Show Collection ^^^^^^^^^^^^^^^^^^ diff --git a/src/ytdl_sub/downloaders/url/downloader.py b/src/ytdl_sub/downloaders/url/downloader.py index dfebcf7c..1f9aeef3 100644 --- a/src/ytdl_sub/downloaders/url/downloader.py +++ b/src/ytdl_sub/downloaders/url/downloader.py @@ -443,14 +443,10 @@ class BaseUrlDownloader(BaseDownloader[BaseDownloaderOptionsT], ABC): ): yield entry_child - def _download_url_metadata( - self, collection_url: UrlValidator - ) -> Tuple[List[EntryParent], List[Entry]]: + def _download_url_metadata(self, url: str) -> Tuple[List[EntryParent], List[Entry]]: """ Downloads only info.json files and forms EntryParent trees """ - url = self.overrides.apply_formatter(collection_url.url) - with self._separate_download_archives(): entry_dicts = YTDLP.extract_info_via_info_json( working_directory=self.working_directory, @@ -494,7 +490,11 @@ class BaseUrlDownloader(BaseDownloader[BaseDownloaderOptionsT], ABC): """The function to perform the download of all media entries""" # download the bottom-most urls first since they are top-priority for collection_url in reversed(self.collection.urls.list): - parents, orphan_entries = self._download_url_metadata(collection_url=collection_url) + # URLs can be empty. If they are, then skip + if not (url := self.overrides.apply_formatter(collection_url.url)): + continue + + parents, orphan_entries = self._download_url_metadata(url=url) # TODO: Encapsulate this logic into its own class self._url_state = URLDownloadState( diff --git a/src/ytdl_sub/downloaders/url/multi_url.py b/src/ytdl_sub/downloaders/url/multi_url.py index 43599781..a4ada5e7 100644 --- a/src/ytdl_sub/downloaders/url/multi_url.py +++ b/src/ytdl_sub/downloaders/url/multi_url.py @@ -1,3 +1,6 @@ +from typing import Dict +from typing import List + from ytdl_sub.downloaders.url.downloader import BaseUrlDownloader from ytdl_sub.downloaders.url.downloader import DownloaderValidator from ytdl_sub.downloaders.url.validators import MultiUrlValidator @@ -43,6 +46,23 @@ class MultiUrlDownloadOptions(MultiUrlValidator, DownloaderValidator): """Returns itself!""" return self + def validate_with_variables( + self, source_variables: List[str], override_variables: Dict[str, str] + ) -> None: + """ + Validates any source variables added by the collection + """ + super().validate_with_variables( + source_variables=source_variables, override_variables=override_variables + ) + + has_non_empty_url = False + for url_validator in self.urls.list: + has_non_empty_url |= bool(url_validator.url.apply_formatter(override_variables)) + + if not has_non_empty_url: + raise self._validation_exception("Must contain at least one url that is non-empty") + class MultiUrlDownloader(BaseUrlDownloader[MultiUrlDownloadOptions]): downloader_options_type = MultiUrlDownloadOptions diff --git a/src/ytdl_sub/downloaders/url/validators.py b/src/ytdl_sub/downloaders/url/validators.py index 5694a7ba..36e1fc36 100644 --- a/src/ytdl_sub/downloaders/url/validators.py +++ b/src/ytdl_sub/downloaders/url/validators.py @@ -156,8 +156,8 @@ class UrlListValidator(ListValidator[UrlValidator]): added_variables: Dict[str, str] = self.list[0].variables.dict_with_format_strings - for idx, collection_url_validator in enumerate(self.list[1:]): - collection_variables = collection_url_validator.variables.dict_with_format_strings + for idx, url_validator in enumerate(self.list[1:]): + collection_variables = url_validator.variables.dict_with_format_strings # see if this collection contains new added vars (it should not) for var in collection_variables.keys(): diff --git a/src/ytdl_sub/prebuilt_presets/tv_show/tv_show.yaml b/src/ytdl_sub/prebuilt_presets/tv_show/tv_show.yaml index 9456b4cc..996d5fa9 100644 --- a/src/ytdl_sub/prebuilt_presets/tv_show/tv_show.yaml +++ b/src/ytdl_sub/prebuilt_presets/tv_show/tv_show.yaml @@ -43,6 +43,45 @@ presets: uid: "avatar_uncropped" - name: "{tv_show_fanart_file_name}" uid: "banner_uncropped" + - url: "{url2}" + - url: "{url3}" + - url: "{url4}" + - url: "{url5}" + - url: "{url6}" + - url: "{url7}" + - url: "{url8}" + - url: "{url9}" + - url: "{url10}" + - url: "{url11}" + - url: "{url12}" + - url: "{url13}" + - url: "{url14}" + - url: "{url15}" + - url: "{url16}" + - url: "{url17}" + - url: "{url18}" + - url: "{url19}" + - url: "{url20}" + overrides: + url2: "" + url3: "" + url4: "" + url5: "" + url6: "" + url7: "" + url8: "" + url9: "" + url10: "" + url11: "" + url12: "" + url13: "" + url14: "" + url15: "" + url16: "" + url17: "" + url18: "" + url19: "" + url20: "" #################################################################################################### diff --git a/tests/unit/config/test_preset.py b/tests/unit/config/test_preset.py index 4fc87fe0..e6d65fe8 100644 --- a/tests/unit/config/test_preset.py +++ b/tests/unit/config/test_preset.py @@ -1,6 +1,9 @@ +import re + import pytest from ytdl_sub.config.preset import Preset +from ytdl_sub.downloaders.url.multi_url import MultiUrlDownloadOptions from ytdl_sub.plugins.nfo_tags import NfoTagsOptions from ytdl_sub.utils.exceptions import StringFormattingVariableNotFoundException from ytdl_sub.utils.exceptions import ValidationException @@ -192,7 +195,7 @@ class TestPreset: name="test", value={ "download": youtube_video, - "output_options": {"output_directory": "dir", "file_name": "file"}, + "output_options": output_options, "output_directory_nfo_tags": { "nfo_name": "the nfo name", "nfo_root": "the root", @@ -200,3 +203,39 @@ class TestPreset: }, }, ) + + def test_preset_with_multi_url__contains_empty_url(self, config_file, output_options): + _ = Preset( + config=config_file, + name="test", + value={ + "download": { + "download_strategy": "multi_url", + "urls": [{"url": "non-empty url"}, {"url": ""}], # empty url + }, + "output_options": output_options, + }, + ) + + def test_preset_with_multi_url__contains_all_empty_urls_errors( + self, config_file, output_options + ): + with pytest.raises( + ValidationException, + match=re.escape( + "Validation error in test.download: Must contain at least one " + "url that is non-empty" + ), + ): + _ = Preset( + config=config_file, + name="test", + value={ + "download": { + "download_strategy": "multi_url", + "urls": [{"url": "{url}"}, {"url": "{url2}"}], + }, + "output_options": output_options, + "overrides": {"url": "", "url2": ""}, + }, + ) diff --git a/tests/unit/prebuilt_presets/test_prebuilt_presets.py b/tests/unit/prebuilt_presets/test_prebuilt_presets.py index a8a02da7..2cede012 100644 --- a/tests/unit/prebuilt_presets/test_prebuilt_presets.py +++ b/tests/unit/prebuilt_presets/test_prebuilt_presets.py @@ -115,9 +115,7 @@ class TestPrebuiltTVShowPresets: }, } if is_many_urls: - preset_dict = dict( - preset_dict, **{"download": {"urls": [{"url": "https://url.number.2.here"}]}} - ) + preset_dict["overrides"]["url2"] = "https://url.number.2.here" subscription = Subscription.from_dict( config=config,