From ef2ca5e6cc98ba59788eb5d391995751c7538423 Mon Sep 17 00:00:00 2001 From: Jesse Bannon Date: Fri, 10 Oct 2025 21:50:47 -0700 Subject: [PATCH] [FEATURE] Ability to modify webpage_url --- src/ytdl_sub/downloaders/url/downloader.py | 43 ++-- src/ytdl_sub/downloaders/url/validators.py | 17 ++ .../prebuilt_presets/helpers/url.yaml | 201 ++++++++++++++++++ .../integration/prebuilt_presets/test_url.py | 81 +++++++ 4 files changed, 321 insertions(+), 21 deletions(-) create mode 100644 tests/integration/prebuilt_presets/test_url.py diff --git a/src/ytdl_sub/downloaders/url/downloader.py b/src/ytdl_sub/downloaders/url/downloader.py index 1f5eb56d..ef5fbd5a 100644 --- a/src/ytdl_sub/downloaders/url/downloader.py +++ b/src/ytdl_sub/downloaders/url/downloader.py @@ -257,6 +257,16 @@ class MultiUrlDownloader(SourcePlugin[MultiUrlValidator]): .to_dict() ) + def webpage_url(self, entry: Entry) -> str: + """ + Returns + ------- + The webpage_url to use for the actual download + """ + url_idx = entry.get(v.ytdl_sub_input_url_index, int) + webpage_url_formatter = self.plugin_options.urls.list[url_idx].webpage_url + return self.overrides.apply_formatter(webpage_url_formatter, entry=entry) + def metadata_ytdl_options(self, ytdl_option_overrides: Dict) -> Dict: """ Returns @@ -358,7 +368,7 @@ class MultiUrlDownloader(SourcePlugin[MultiUrlValidator]): if (self.is_dry_run or not self.is_entry_thumbnails_enabled) else entry.is_thumbnail_downloaded_via_ytdlp ), - url=entry.webpage_url, + url=self.webpage_url(entry=entry), ) return Entry( download_entry_dict, @@ -366,13 +376,13 @@ class MultiUrlDownloader(SourcePlugin[MultiUrlValidator]): ) def _iterate_child_entries( - self, entries: List[Entry], download_reversed: bool + self, entries: List[Entry], validator: UrlValidator ) -> Iterator[Entry]: # Iterate a list of entries, and delete the entries after yielding entries_to_iter: List[Optional[Entry]] = entries indices = list(range(len(entries_to_iter))) - if download_reversed: + if self.overrides.evaluate_boolean(validator.download_reverse): indices = reversed(indices) for idx in indices: @@ -394,17 +404,13 @@ class MultiUrlDownloader(SourcePlugin[MultiUrlValidator]): entries_to_iter[idx] = None def _iterate_parent_entry( - self, parent: EntryParent, download_reversed: bool + self, parent: EntryParent, validator: UrlValidator ) -> Iterator[Entry]: - yield from self._iterate_child_entries( - entries=parent.entry_children(), download_reversed=download_reversed - ) + yield from self._iterate_child_entries(entries=parent.entry_children(), validator=validator) # Recursion the parent's parent entries for parent_child in reversed(parent.parent_children()): - yield from self._iterate_parent_entry( - parent=parent_child, download_reversed=download_reversed - ) + yield from self._iterate_parent_entry(parent=parent_child, validator=validator) def _download_url_metadata( self, url: str, include_sibling_metadata: bool, ytdl_options_overrides: Dict @@ -438,7 +444,7 @@ class MultiUrlDownloader(SourcePlugin[MultiUrlValidator]): self, parents: List[EntryParent], orphans: List[Entry], - download_reversed: bool, + validator: UrlValidator, ) -> Iterator[Entry]: """ Downloads the leaf entries from EntryParent trees @@ -446,19 +452,15 @@ class MultiUrlDownloader(SourcePlugin[MultiUrlValidator]): # Delete info json files afterwards so other collection URLs do not use them with self._separate_download_archives(clear_info_json_files=True): for parent in parents: - yield from self._iterate_parent_entry( - parent=parent, download_reversed=download_reversed - ) + yield from self._iterate_parent_entry(parent=parent, validator=validator) - yield from self._iterate_child_entries( - entries=orphans, download_reversed=download_reversed - ) + yield from self._iterate_child_entries(entries=orphans, validator=validator) def _download_metadata(self, url: str, validator: UrlValidator) -> Iterable[Entry]: metadata_ytdl_options = self.metadata_ytdl_options( ytdl_option_overrides=validator.ytdl_options.to_native_dict(self.overrides) ) - download_reversed = self.overrides.evaluate_boolean(validator.download_reverse) + include_sibling_metadata = self.overrides.evaluate_boolean( validator.include_sibling_metadata ) @@ -469,16 +471,15 @@ class MultiUrlDownloader(SourcePlugin[MultiUrlValidator]): ytdl_options_overrides=metadata_ytdl_options, ) - # TODO: Encapsulate this logic into its own class self._url_state = URLDownloadState( - entries_total=sum(parent.num_children() for parent in parents) + len(orphan_entries) + entries_total=sum(parent.num_children() for parent in parents) + len(orphan_entries), ) download_logger.info("Beginning downloads for %s", url) yield from self._iterate_entries( parents=parents, orphans=orphan_entries, - download_reversed=download_reversed, + validator=validator, ) def download_metadata(self) -> Iterable[Entry]: diff --git a/src/ytdl_sub/downloaders/url/validators.py b/src/ytdl_sub/downloaders/url/validators.py index 1bd9a79f..4504eccf 100644 --- a/src/ytdl_sub/downloaders/url/validators.py +++ b/src/ytdl_sub/downloaders/url/validators.py @@ -52,6 +52,7 @@ class UrlValidator(StrictDictValidator): "download_reverse", "ytdl_options", "include_sibling_metadata", + "webpage_url", } @classmethod @@ -89,6 +90,9 @@ class UrlValidator(StrictDictValidator): validator=OverridesBooleanFormatterValidator, default="False", ) + self._webpage_url = self._validate_key( + key="webpage_url", validator=StringFormatterValidator, default="{webpage_url}" + ) @property def url(self) -> OverridesStringFormatterValidator: @@ -180,6 +184,19 @@ class UrlValidator(StrictDictValidator): """ return self._include_sibling_metadata + @property + def webpage_url(self) -> StringFormatterValidator: + """ + Optional. After ytdl-sub performs the metadata download, it will inspect each + entry's .info.json file and perform the actual download from yt-dlp using + `webpage_url `. This + can be overwritten by supplying parameter with a modification to ``webpage_url`` in the + form of an override variable. + + Defaults to ``{webpage_url}``. + """ + return self._webpage_url + class UrlStringOrDictValidator(UrlValidator): """ diff --git a/src/ytdl_sub/prebuilt_presets/helpers/url.yaml b/src/ytdl_sub/prebuilt_presets/helpers/url.yaml index 2f7d8f37..5c4439e7 100644 --- a/src/ytdl_sub/prebuilt_presets/helpers/url.yaml +++ b/src/ytdl_sub/prebuilt_presets/helpers/url.yaml @@ -26,208 +26,309 @@ presets: - name: "{banner_uncropped_thumbnail_file_name}" uid: "banner_uncropped" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url2}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url3}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url4}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url5}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url6}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url7}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url8}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url9}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url10}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url11}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url12}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url13}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url14}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url15}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url16}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url17}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url18}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url19}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url20}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url21}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url22}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url23}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url24}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url25}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url26}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url27}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url28}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url29}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url30}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url31}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url32}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url33}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url34}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url35}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url36}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url37}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url38}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url39}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url40}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url41}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url42}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url43}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url44}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url45}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url46}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url47}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url48}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url49}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url50}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url51}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url52}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url53}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url54}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url55}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url56}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url57}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url58}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url59}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url60}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url61}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url62}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url63}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url64}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url65}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url66}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url67}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url68}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url69}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url70}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url71}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url72}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url73}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url74}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url75}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url76}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url77}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url78}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url79}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url80}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url81}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url82}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url83}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url84}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url85}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url86}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url87}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url88}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url89}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url90}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url91}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url92}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url93}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url94}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url95}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url96}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url97}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url98}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url99}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url100}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" overrides: avatar_uncropped_thumbnail_file_name: "" banner_uncropped_thumbnail_file_name: "" include_sibling_metadata: False + modified_webpage_url: "{webpage_url}" subscription_array: "{ [] }" subscription_value: "" @@ -445,402 +546,502 @@ presets: download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url2) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url3) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url4) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url5) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url6) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url7) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url8) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url9) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url10) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url11) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url12) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url13) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url14) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url15) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url16) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url17) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url18) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url19) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url20) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url21) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url22) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url23) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url24) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url25) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url26) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url27) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url28) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url29) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url30) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url31) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url32) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url33) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url34) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url35) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url36) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url37) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url38) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url39) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url40) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url41) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url42) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url43) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url44) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url45) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url46) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url47) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url48) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url49) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url50) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url51) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url52) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url53) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url54) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url55) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url56) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url57) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url58) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url59) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url60) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url61) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url62) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url63) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url64) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url65) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url66) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url67) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url68) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url69) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url70) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url71) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url72) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url73) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url74) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url75) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url76) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url77) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url78) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url79) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url80) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url81) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url82) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url83) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url84) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url85) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url86) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url87) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url88) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url89) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url90) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url91) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url92) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url93) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url94) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url95) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url96) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url97) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url98) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url99) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{%bilateral_url(url100) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" _multi_url_bilateral: preset: diff --git a/tests/integration/prebuilt_presets/test_url.py b/tests/integration/prebuilt_presets/test_url.py new file mode 100644 index 00000000..e93621bc --- /dev/null +++ b/tests/integration/prebuilt_presets/test_url.py @@ -0,0 +1,81 @@ +import pytest +from conftest import assert_logs + +from ytdl_sub.downloaders.url.downloader import MultiUrlDownloader +from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder +from ytdl_sub.downloaders.ytdlp import YTDLP +from ytdl_sub.entries.entry import Entry +from ytdl_sub.entries.script.variable_definitions import VARIABLES +from ytdl_sub.entries.script.variable_definitions import VariableDefinitions +from ytdl_sub.subscriptions.subscription import Subscription + +v: VariableDefinitions = VARIABLES + + +@pytest.fixture +def subscription_dict(output_directory): + return { + "preset": [ + "Plex TV Show by Date", + "Filter Keywords", + ], + "overrides": { + "url": "https://your.name.here", + "tv_show_directory": output_directory, + "title_include_keywords": ["MOCK ENTRY 20-3"], + "modified_webpage_url": "{webpage_url}_append_test", + }, + } + + +class TestUrl: + + def test_modified_url_when_downloading( + self, + config, + subscription_dict, + subscription_name, + working_directory, + ): + subscription = Subscription.from_dict( + config=config, + preset_name=subscription_name, + preset_dict=subscription_dict, + ) + + assert set( + url.webpage_url.format_string for url in subscription.downloader_options.urls.list + ) == {"{modified_webpage_url}"} + downloader = MultiUrlDownloader( + options=subscription.downloader_options, + enhanced_download_archive=subscription.download_archive, + download_ytdl_options=YTDLOptionsBuilder(), + metadata_ytdl_options=YTDLOptionsBuilder(), + overrides=subscription.overrides, + ) + + entry = Entry( + entry_dict={ + v.uid.metadata_key: "0", + v.extractor.metadata_key: "test", + v.extractor_key.metadata_key: "test", + v.epoch.metadata_key: 123, + v.webpage_url.metadata_key: "youtube.com/test_url", + v.ext.metadata_key: "mp4", + }, + working_directory=working_directory, + ) + + entry.initialize_base_script() + + entry.add_injected_variables( + download_entry=entry, + download_idx=0, + upload_date_idx=0, + ).initialize_script(subscription.overrides) + + entry.add({ + v.ytdl_sub_input_url_index.variable_name: 0 + }) + + assert downloader.webpage_url(entry) == "youtube.com/test_url_append_test"