From 06dffdb763745608a56ba3d43715db3773fabc51 Mon Sep 17 00:00:00 2001 From: Jesse Bannon Date: Wed, 28 Sep 2022 15:03:02 -0700 Subject: [PATCH] [BACKEND] Have generic url be overrides string formatter (#254) --- src/ytdl_sub/config/preset.py | 4 ++-- src/ytdl_sub/downloaders/downloader.py | 6 ++++-- src/ytdl_sub/downloaders/generic/collection_validator.py | 7 +++---- src/ytdl_sub/downloaders/youtube/merge_playlist.py | 4 +++- .../plugins/test_audio_extract_single.json | 4 ++-- .../plugins/test_subtitles_embedded.json | 4 ++-- .../plugins/test_subtitles_embedded_and_file.json | 4 ++-- 7 files changed, 18 insertions(+), 15 deletions(-) diff --git a/src/ytdl_sub/config/preset.py b/src/ytdl_sub/config/preset.py index 9e97e877..89c5b74e 100644 --- a/src/ytdl_sub/config/preset.py +++ b/src/ytdl_sub/config/preset.py @@ -315,8 +315,8 @@ class Preset(StrictDictValidator): # Append this preset (the subscription) last presets_to_merge.append(copy.deepcopy(self._value)) - # Merge all of the presets - self._value = mergedeep.merge({}, *presets_to_merge, strategy=mergedeep.Strategy.REPLACE) + # Merge all presets + self._value = mergedeep.merge({}, *presets_to_merge, strategy=mergedeep.Strategy.ADDITIVE) def __init__(self, config: ConfigFile, name: str, value: Any): super().__init__(name=name, value=value) diff --git a/src/ytdl_sub/downloaders/downloader.py b/src/ytdl_sub/downloaders/downloader.py index 93b76614..a2509ee7 100644 --- a/src/ytdl_sub/downloaders/downloader.py +++ b/src/ytdl_sub/downloaders/downloader.py @@ -428,15 +428,17 @@ class Downloader(DownloadArchiver, Generic[DownloaderOptionsT], ABC): """ Downloads only info.json files and forms EntryParent trees """ + url = self.overrides.apply_formatter(collection_url.url) + with self._separate_download_archives(): entry_dicts = self.extract_info_via_info_json( only_info_json=True, - url=collection_url.url, + url=url, log_prefix_on_info_json_dl="Downloading metadata for", ) self.parents = EntryParent.from_entry_dicts( - url=collection_url.url, + url=url, entry_dicts=entry_dicts, working_directory=self.working_directory, ) diff --git a/src/ytdl_sub/downloaders/generic/collection_validator.py b/src/ytdl_sub/downloaders/generic/collection_validator.py index 2772c6fc..5af9ebaf 100644 --- a/src/ytdl_sub/downloaders/generic/collection_validator.py +++ b/src/ytdl_sub/downloaders/generic/collection_validator.py @@ -8,7 +8,6 @@ from ytdl_sub.validators.string_formatter_validators import DictFormatterValidat from ytdl_sub.validators.string_formatter_validators import OverridesStringFormatterValidator from ytdl_sub.validators.string_formatter_validators import StringFormatterValidator from ytdl_sub.validators.validators import ListValidator -from ytdl_sub.validators.validators import StringValidator class CollectionThumbnailValidator(StrictDictValidator): @@ -47,7 +46,7 @@ class CollectionUrlValidator(StrictDictValidator): super().__init__(name, value) # TODO: url validate using yt-dlp IE - self._url = self._validate_key(key="url", validator=StringValidator) + self._url = self._validate_key(key="url", validator=OverridesStringFormatterValidator) self._variables = self._validate_key_if_present( key="variables", validator=DictFormatterValidator, default={} ) @@ -60,13 +59,13 @@ class CollectionUrlValidator(StrictDictValidator): ) @property - def url(self) -> str: + def url(self) -> OverridesStringFormatterValidator: """ URL to download from, listed in priority from lowest (top) to highest (bottom). If a download exists in more than one URL, it will resolve to the bottom-most one and inherit those variables. """ - return self._url.value + return self._url @property def variables(self) -> DictFormatterValidator: diff --git a/src/ytdl_sub/downloaders/youtube/merge_playlist.py b/src/ytdl_sub/downloaders/youtube/merge_playlist.py index f9117e6f..94712427 100644 --- a/src/ytdl_sub/downloaders/youtube/merge_playlist.py +++ b/src/ytdl_sub/downloaders/youtube/merge_playlist.py @@ -144,7 +144,9 @@ class YoutubeMergePlaylistDownloader(Downloader[YoutubeMergePlaylistDownloaderOp def download(self) -> List[Tuple[Entry, FileMetadata]]: """Download a single Youtube video, then split it into multiple videos""" - entry_dict = self.extract_info(url=self.collection.collection_urls.list[0].url) + url = self.overrides.apply_formatter(self.collection.collection_urls.list[0].url) + + entry_dict = self.extract_info(url=url) merged_video = self._to_merged_video(entry_dict=entry_dict) merged_video_metadata = self._get_chapters( diff --git a/tests/e2e/resources/expected_downloads_summaries/plugins/test_audio_extract_single.json b/tests/e2e/resources/expected_downloads_summaries/plugins/test_audio_extract_single.json index 3909511c..e8e24892 100644 --- a/tests/e2e/resources/expected_downloads_summaries/plugins/test_audio_extract_single.json +++ b/tests/e2e/resources/expected_downloads_summaries/plugins/test_audio_extract_single.json @@ -1,4 +1,4 @@ { - "YouTube Rewind 2019： For the Record ｜ #YouTubeRewind.info.json": "6ef6fd3f6b1b0f4f7736d85e618777ac", - "YouTube Rewind 2019： For the Record ｜ #YouTubeRewind.mp3": "3a156b122bd79c956cce5079d3530cc3" + "YouTube Rewind 2019： For the Record ｜ #YouTubeRewind.info.json": "8ea117f67fd7fcc9bd1d7ea1f9de3414", + "YouTube Rewind 2019： For the Record ｜ #YouTubeRewind.mp3": "d3687bf6c13a2a5f3a8a05fbce9be28c" } \ No newline at end of file diff --git a/tests/e2e/resources/expected_downloads_summaries/plugins/test_subtitles_embedded.json b/tests/e2e/resources/expected_downloads_summaries/plugins/test_subtitles_embedded.json index 2d0ae8c5..fbdb56d4 100644 --- a/tests/e2e/resources/expected_downloads_summaries/plugins/test_subtitles_embedded.json +++ b/tests/e2e/resources/expected_downloads_summaries/plugins/test_subtitles_embedded.json @@ -1,6 +1,6 @@ { "JMC - YouTube Rewind 2019： For the Record ｜ #YouTubeRewind-thumb.jpg": "50ee47c80f679029f5d3503bb91b045a", - "JMC - YouTube Rewind 2019： For the Record ｜ #YouTubeRewind.info.json": "c476aae219a3d3fa7a78b4afb145e8b5", - "JMC - YouTube Rewind 2019： For the Record ｜ #YouTubeRewind.mp4": "8562853314b75c1e47abd4f5ba97315c", + "JMC - YouTube Rewind 2019： For the Record ｜ #YouTubeRewind.info.json": "fdf364c33b30fccccd1f6af954d07e42", + "JMC - YouTube Rewind 2019： For the Record ｜ #YouTubeRewind.mp4": "1f1f91f85b162c20596a2413e704b809", "JMC - YouTube Rewind 2019： For the Record ｜ #YouTubeRewind.nfo": "c64964fab07574080e5da3242e3bfd48" } \ No newline at end of file diff --git a/tests/e2e/resources/expected_downloads_summaries/plugins/test_subtitles_embedded_and_file.json b/tests/e2e/resources/expected_downloads_summaries/plugins/test_subtitles_embedded_and_file.json index 783dd4fa..ae96c36e 100644 --- a/tests/e2e/resources/expected_downloads_summaries/plugins/test_subtitles_embedded_and_file.json +++ b/tests/e2e/resources/expected_downloads_summaries/plugins/test_subtitles_embedded_and_file.json @@ -2,7 +2,7 @@ "JMC - YouTube Rewind 2019： For the Record ｜ #YouTubeRewind-thumb.jpg": "50ee47c80f679029f5d3503bb91b045a", "JMC - YouTube Rewind 2019： For the Record ｜ #YouTubeRewind.de.srt": "b343c3bb9257b7ee7ba38f570a115b37", "JMC - YouTube Rewind 2019： For the Record ｜ #YouTubeRewind.en.srt": "fe8c6ee92cae6e059fd80fd61691adbe", - "JMC - YouTube Rewind 2019： For the Record ｜ #YouTubeRewind.info.json": "083591b51f393cd2d426fa9828d2e6fa", - "JMC - YouTube Rewind 2019： For the Record ｜ #YouTubeRewind.mp4": "8562853314b75c1e47abd4f5ba97315c", + "JMC - YouTube Rewind 2019： For the Record ｜ #YouTubeRewind.info.json": "a95841e816e23be1d54dde127a819934", + "JMC - YouTube Rewind 2019： For the Record ｜ #YouTubeRewind.mp4": "1f1f91f85b162c20596a2413e704b809", "JMC - YouTube Rewind 2019： For the Record ｜ #YouTubeRewind.nfo": "c64964fab07574080e5da3242e3bfd48" } \ No newline at end of file