diff --git a/docs/source/prebuilt_presets/helpers.rst b/docs/source/prebuilt_presets/helpers.rst index 86752ed4..4e952009 100644 --- a/docs/source/prebuilt_presets/helpers.rst +++ b/docs/source/prebuilt_presets/helpers.rst @@ -1,3 +1,6 @@ + + + ============== Helper Presets ============== @@ -194,3 +197,41 @@ to your subscription, like so: url: "https://youtube.com/@channel" resolution_assert_ignore_titles: - "This 360p Video Title" + +_url +---- + +All prebuilt presets share the same internal ``_multi_url`` preset which comes equipped with +a few available customizations. + +Sibling Metadata +~~~~~~~~~~~~~~~~ + +*Sibling* refers to any entry within the same *playlist*. For channel downloads, this would +imply **every** video that gets downloaded since yt-dlp treats the channel as the *playlist*. + +Setting the variable ``include_sibling_metadata`` will include all sibling metadata within +each individual entry's metadata. This is used specifically for music presets. When downloading +a playlist as an album for example, it will take the max year amongst all the other sibling's metadata +to have a consistent album year that can be used in file or directory naming. + +Webpage URL +~~~~~~~~~~~ + +``ytdl-sub`` performs downloads in two stages. + +1. Metadata scrape from the original URL +2. Individual entry downloads + +For step 2, ``ytdl-sub`` will use the ``webpage_url`` variable by default for the input URL to yt-dlp. +This can be modified in case it's not working as expected by using the variable ``modified_webpage_url``. + +Example: + +.. code-block:: yaml + :caption: + Removes yt-dlp smuggle data from the URL + + overrides: + modified_webpage_url: >- + { %regex_sub("#__youtubedl_smuggle=.*", "", webpage_url) } \ No newline at end of file diff --git a/src/ytdl_sub/downloaders/url/downloader.py b/src/ytdl_sub/downloaders/url/downloader.py index 1f5eb56d..ef5fbd5a 100644 --- a/src/ytdl_sub/downloaders/url/downloader.py +++ b/src/ytdl_sub/downloaders/url/downloader.py @@ -257,6 +257,16 @@ class MultiUrlDownloader(SourcePlugin[MultiUrlValidator]): .to_dict() ) + def webpage_url(self, entry: Entry) -> str: + """ + Returns + ------- + The webpage_url to use for the actual download + """ + url_idx = entry.get(v.ytdl_sub_input_url_index, int) + webpage_url_formatter = self.plugin_options.urls.list[url_idx].webpage_url + return self.overrides.apply_formatter(webpage_url_formatter, entry=entry) + def metadata_ytdl_options(self, ytdl_option_overrides: Dict) -> Dict: """ Returns @@ -358,7 +368,7 @@ class MultiUrlDownloader(SourcePlugin[MultiUrlValidator]): if (self.is_dry_run or not self.is_entry_thumbnails_enabled) else entry.is_thumbnail_downloaded_via_ytdlp ), - url=entry.webpage_url, + url=self.webpage_url(entry=entry), ) return Entry( download_entry_dict, @@ -366,13 +376,13 @@ class MultiUrlDownloader(SourcePlugin[MultiUrlValidator]): ) def _iterate_child_entries( - self, entries: List[Entry], download_reversed: bool + self, entries: List[Entry], validator: UrlValidator ) -> Iterator[Entry]: # Iterate a list of entries, and delete the entries after yielding entries_to_iter: List[Optional[Entry]] = entries indices = list(range(len(entries_to_iter))) - if download_reversed: + if self.overrides.evaluate_boolean(validator.download_reverse): indices = reversed(indices) for idx in indices: @@ -394,17 +404,13 @@ class MultiUrlDownloader(SourcePlugin[MultiUrlValidator]): entries_to_iter[idx] = None def _iterate_parent_entry( - self, parent: EntryParent, download_reversed: bool + self, parent: EntryParent, validator: UrlValidator ) -> Iterator[Entry]: - yield from self._iterate_child_entries( - entries=parent.entry_children(), download_reversed=download_reversed - ) + yield from self._iterate_child_entries(entries=parent.entry_children(), validator=validator) # Recursion the parent's parent entries for parent_child in reversed(parent.parent_children()): - yield from self._iterate_parent_entry( - parent=parent_child, download_reversed=download_reversed - ) + yield from self._iterate_parent_entry(parent=parent_child, validator=validator) def _download_url_metadata( self, url: str, include_sibling_metadata: bool, ytdl_options_overrides: Dict @@ -438,7 +444,7 @@ class MultiUrlDownloader(SourcePlugin[MultiUrlValidator]): self, parents: List[EntryParent], orphans: List[Entry], - download_reversed: bool, + validator: UrlValidator, ) -> Iterator[Entry]: """ Downloads the leaf entries from EntryParent trees @@ -446,19 +452,15 @@ class MultiUrlDownloader(SourcePlugin[MultiUrlValidator]): # Delete info json files afterwards so other collection URLs do not use them with self._separate_download_archives(clear_info_json_files=True): for parent in parents: - yield from self._iterate_parent_entry( - parent=parent, download_reversed=download_reversed - ) + yield from self._iterate_parent_entry(parent=parent, validator=validator) - yield from self._iterate_child_entries( - entries=orphans, download_reversed=download_reversed - ) + yield from self._iterate_child_entries(entries=orphans, validator=validator) def _download_metadata(self, url: str, validator: UrlValidator) -> Iterable[Entry]: metadata_ytdl_options = self.metadata_ytdl_options( ytdl_option_overrides=validator.ytdl_options.to_native_dict(self.overrides) ) - download_reversed = self.overrides.evaluate_boolean(validator.download_reverse) + include_sibling_metadata = self.overrides.evaluate_boolean( validator.include_sibling_metadata ) @@ -469,16 +471,15 @@ class MultiUrlDownloader(SourcePlugin[MultiUrlValidator]): ytdl_options_overrides=metadata_ytdl_options, ) - # TODO: Encapsulate this logic into its own class self._url_state = URLDownloadState( - entries_total=sum(parent.num_children() for parent in parents) + len(orphan_entries) + entries_total=sum(parent.num_children() for parent in parents) + len(orphan_entries), ) download_logger.info("Beginning downloads for %s", url) yield from self._iterate_entries( parents=parents, orphans=orphan_entries, - download_reversed=download_reversed, + validator=validator, ) def download_metadata(self) -> Iterable[Entry]: diff --git a/src/ytdl_sub/downloaders/url/validators.py b/src/ytdl_sub/downloaders/url/validators.py index 1bd9a79f..4504eccf 100644 --- a/src/ytdl_sub/downloaders/url/validators.py +++ b/src/ytdl_sub/downloaders/url/validators.py @@ -52,6 +52,7 @@ class UrlValidator(StrictDictValidator): "download_reverse", "ytdl_options", "include_sibling_metadata", + "webpage_url", } @classmethod @@ -89,6 +90,9 @@ class UrlValidator(StrictDictValidator): validator=OverridesBooleanFormatterValidator, default="False", ) + self._webpage_url = self._validate_key( + key="webpage_url", validator=StringFormatterValidator, default="{webpage_url}" + ) @property def url(self) -> OverridesStringFormatterValidator: @@ -180,6 +184,19 @@ class UrlValidator(StrictDictValidator): """ return self._include_sibling_metadata + @property + def webpage_url(self) -> StringFormatterValidator: + """ + Optional. After ytdl-sub performs the metadata download, it will inspect each + entry's .info.json file and perform the actual download from yt-dlp using + `webpage_url `. This + can be overwritten by supplying parameter with a modification to ``webpage_url`` in the + form of an override variable. + + Defaults to ``{webpage_url}``. + """ + return self._webpage_url + class UrlStringOrDictValidator(UrlValidator): """ diff --git a/src/ytdl_sub/entries/script/variable_definitions.py b/src/ytdl_sub/entries/script/variable_definitions.py index 75496a96..67505473 100644 --- a/src/ytdl_sub/entries/script/variable_definitions.py +++ b/src/ytdl_sub/entries/script/variable_definitions.py @@ -1162,6 +1162,7 @@ class VariableDefinitions( """ return { self.uid, + self.extractor, self.extractor_key, self.epoch, self.webpage_url, diff --git a/src/ytdl_sub/prebuilt_presets/helpers/url.yaml b/src/ytdl_sub/prebuilt_presets/helpers/url.yaml index 2f7d8f37..5c4439e7 100644 --- a/src/ytdl_sub/prebuilt_presets/helpers/url.yaml +++ b/src/ytdl_sub/prebuilt_presets/helpers/url.yaml @@ -26,208 +26,309 @@ presets: - name: "{banner_uncropped_thumbnail_file_name}" uid: "banner_uncropped" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url2}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url3}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url4}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url5}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url6}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url7}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url8}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url9}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url10}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url11}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url12}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url13}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url14}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url15}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url16}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url17}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url18}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url19}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url20}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url21}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url22}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url23}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url24}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url25}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url26}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url27}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url28}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url29}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url30}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url31}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url32}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url33}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url34}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url35}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url36}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url37}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url38}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url39}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url40}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url41}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url42}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url43}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url44}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url45}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url46}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url47}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url48}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url49}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url50}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url51}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url52}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url53}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url54}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url55}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url56}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url57}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url58}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url59}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url60}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url61}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url62}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url63}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url64}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url65}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url66}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url67}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url68}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url69}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url70}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url71}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url72}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url73}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url74}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url75}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url76}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url77}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url78}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url79}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url80}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url81}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url82}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url83}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url84}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url85}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url86}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url87}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url88}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url89}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url90}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url91}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url92}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url93}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url94}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url95}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url96}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url97}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url98}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url99}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" - url: "{url100}" include_sibling_metadata: "{include_sibling_metadata}" + webpage_url: "{modified_webpage_url}" overrides: avatar_uncropped_thumbnail_file_name: "" banner_uncropped_thumbnail_file_name: "" include_sibling_metadata: False + modified_webpage_url: "{webpage_url}" subscription_array: "{ [] }" subscription_value: "" @@ -445,402 +546,502 @@ presets: download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url2) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url3) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url4) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url5) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url6) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url7) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url8) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url9) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url10) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url11) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url12) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url13) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url14) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url15) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url16) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url17) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url18) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url19) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url20) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url21) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url22) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url23) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url24) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url25) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url26) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url27) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url28) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url29) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url30) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url31) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url32) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url33) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url34) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url35) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url36) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url37) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url38) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url39) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url40) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url41) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url42) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url43) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url44) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url45) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url46) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url47) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url48) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url49) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url50) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url51) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url52) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url53) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url54) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url55) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url56) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url57) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url58) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url59) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url60) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url61) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url62) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url63) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url64) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url65) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url66) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url67) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url68) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url69) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url70) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url71) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url72) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url73) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url74) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url75) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url76) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url77) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url78) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url79) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url80) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url81) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url82) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url83) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url84) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url85) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url86) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url87) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url88) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url89) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url90) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url91) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url92) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url93) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url94) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url95) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url96) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url97) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url98) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{ %bilateral_url(url99) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" - url: "{%bilateral_url(url100) }" download_reverse: False ytdl_options: playlist_items: "-1:0:-1" + webpage_url: "{modified_webpage_url}" _multi_url_bilateral: preset: diff --git a/tests/integration/prebuilt_presets/test_url.py b/tests/integration/prebuilt_presets/test_url.py new file mode 100644 index 00000000..0da8d4b5 --- /dev/null +++ b/tests/integration/prebuilt_presets/test_url.py @@ -0,0 +1,79 @@ +import pytest +from conftest import assert_logs + +from ytdl_sub.downloaders.url.downloader import MultiUrlDownloader +from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder +from ytdl_sub.downloaders.ytdlp import YTDLP +from ytdl_sub.entries.entry import Entry +from ytdl_sub.entries.script.variable_definitions import VARIABLES +from ytdl_sub.entries.script.variable_definitions import VariableDefinitions +from ytdl_sub.subscriptions.subscription import Subscription + +v: VariableDefinitions = VARIABLES + + +@pytest.fixture +def subscription_dict(output_directory): + return { + "preset": [ + "Plex TV Show by Date", + "Filter Keywords", + ], + "overrides": { + "url": "https://your.name.here", + "tv_show_directory": output_directory, + "title_include_keywords": ["MOCK ENTRY 20-3"], + "modified_webpage_url": "{webpage_url}_append_test", + }, + } + + +class TestUrl: + + def test_modified_url_when_downloading( + self, + config, + subscription_dict, + subscription_name, + working_directory, + ): + subscription = Subscription.from_dict( + config=config, + preset_name=subscription_name, + preset_dict=subscription_dict, + ) + + assert set( + url.webpage_url.format_string for url in subscription.downloader_options.urls.list + ) == {"{modified_webpage_url}"} + downloader = MultiUrlDownloader( + options=subscription.downloader_options, + enhanced_download_archive=subscription.download_archive, + download_ytdl_options=YTDLOptionsBuilder(), + metadata_ytdl_options=YTDLOptionsBuilder(), + overrides=subscription.overrides, + ) + + entry = Entry( + entry_dict={ + v.uid.metadata_key: "0", + v.extractor.metadata_key: "test", + v.extractor_key.metadata_key: "test", + v.epoch.metadata_key: 123, + v.webpage_url.metadata_key: "youtube.com/test_url", + v.ext.metadata_key: "mp4", + }, + working_directory=working_directory, + ) + + entry.initialize_base_script() + + entry.add_injected_variables( + download_entry=entry, + download_idx=0, + upload_date_idx=0, + ).initialize_script(subscription.overrides) + + entry.add({v.ytdl_sub_input_url_index.variable_name: 0}) + + assert downloader.webpage_url(entry) == "youtube.com/test_url_append_test" diff --git a/tests/unit/config/test_config_file.py b/tests/unit/config/test_config_file.py index 1bed7504..ae1d2231 100644 --- a/tests/unit/config/test_config_file.py +++ b/tests/unit/config/test_config_file.py @@ -77,7 +77,7 @@ class TestConfigFilePartiallyValidatesPresets: expected_error_message="Validation error in partial_preset.download.1: " "'partial_preset.download.1' contains the field 'bad_key' which is not allowed. " "Allowed fields: download_reverse, include_sibling_metadata, playlist_thumbnails, " - "source_thumbnails, url, variables, ytdl_options", + "source_thumbnails, url, variables, webpage_url, ytdl_options", ) @pytest.mark.parametrize( diff --git a/tests/unit/entries/test_entry.py b/tests/unit/entries/test_entry.py index 4797092d..07b7a5a3 100644 --- a/tests/unit/entries/test_entry.py +++ b/tests/unit/entries/test_entry.py @@ -58,3 +58,25 @@ class TestEntry(object): assert entry.get(v.upload_day_of_year_reversed, int) == day_year_rev assert entry.get(v.upload_day_of_year_padded, str) == day_year_pad assert entry.get(v.upload_day_of_year_reversed_padded, str) == day_year_rev_pad + + def test_entry_required_variables(self): + entry_dict = { + v.uid.metadata_key: "0", + v.extractor.metadata_key: "test", + v.extractor_key.metadata_key: "test", + v.epoch.metadata_key: 123, + v.webpage_url.metadata_key: "youtube.com/test_url", + v.ext.metadata_key: "mp4", + } + + assert set(entry_dict.keys()) == {_.metadata_key for _ in v.required_entry_variables()} + + # Ensure adding variables works + entry = Entry( + entry_dict=entry_dict, + working_directory=".", + ).initialize_script() + + entry.add({v.channel: "can add"}) + + assert entry.get(v.channel, str) == "can add"