diff --git a/docs/config.rst b/docs/config.rst index 3b0be2ee..79bb5266 100644 --- a/docs/config.rst +++ b/docs/config.rst @@ -63,7 +63,7 @@ download strategy has its own set of parameters. url ''' .. autoclass:: ytdl_sub.downloaders.generic.url.UrlDownloadOptions() - :members: url, playlist_thumbnails, source_thumbnails + :members: url, playlist_thumbnails, source_thumbnails, download_reverse :member-order: bysource multi_url diff --git a/src/ytdl_sub/downloaders/downloader.py b/src/ytdl_sub/downloaders/downloader.py index c20b6987..862d4927 100644 --- a/src/ytdl_sub/downloaders/downloader.py +++ b/src/ytdl_sub/downloaders/downloader.py @@ -476,10 +476,14 @@ class Downloader(DownloadArchiver, Generic[DownloaderOptionsT], ABC): return entry - def _download_entries(self, entries: List[Entry]) -> Generator[Entry, None, None]: - # Download entries in reverse order since they are scraped in the opposite direction. - # Helps deal with break_on_existing - for entry in reversed(entries): + def _download_entries( + self, url_validator: UrlValidator, entries: List[Entry] + ) -> Generator[Entry, None, None]: + entries_to_iterate = entries + if url_validator.download_reverse: + entries_to_iterate = reversed(entries) + + for entry in entries_to_iterate: self._url_state.entries_downloaded += 1 if self._is_downloaded(entry): @@ -500,13 +504,19 @@ class Downloader(DownloadArchiver, Generic[DownloaderOptionsT], ABC): yield self._download_entry(entry) self._mark_downloaded(entry) - def _download_parent_entry(self, parent: EntryParent) -> Generator[Entry, None, None]: - for entry_child in self._download_entries(parent.entry_children()): + def _download_parent_entry( + self, url_validator: UrlValidator, parent: EntryParent + ) -> Generator[Entry, None, None]: + for entry_child in self._download_entries( + url_validator=url_validator, entries=parent.entry_children() + ): yield entry_child # Recursion the parent's parent entries for parent_child in reversed(parent.parent_children()): - for entry_child in self._download_parent_entry(parent=parent_child): + for entry_child in self._download_parent_entry( + url_validator=url_validator, parent=parent_child + ): yield entry_child def _set_collection_variables(self, collection_url: UrlValidator, entry: Entry | EntryParent): @@ -554,6 +564,7 @@ class Downloader(DownloadArchiver, Generic[DownloaderOptionsT], ABC): def _download( self, + url_validator: UrlValidator, parents: List[EntryParent], orphans: List[Entry], ) -> Generator[Entry, None, None]: @@ -563,10 +574,12 @@ class Downloader(DownloadArchiver, Generic[DownloaderOptionsT], ABC): # Delete info json files afterwards so other collection URLs do not use them with self._separate_download_archives(clear_info_json_files=True): for parent in parents: - for entry_child in self._download_parent_entry(parent=parent): + for entry_child in self._download_parent_entry( + url_validator=url_validator, parent=parent + ): yield entry_child - for orphan in self._download_entries(orphans): + for orphan in self._download_entries(url_validator=url_validator, entries=orphans): yield orphan def download( @@ -585,7 +598,9 @@ class Downloader(DownloadArchiver, Generic[DownloaderOptionsT], ABC): download_logger.info( "Beginning downloads for %s", self.overrides.apply_formatter(collection_url.url) ) - for entry in self._download(parents=parents, orphans=orphan_entries): + for entry in self._download( + url_validator=collection_url, parents=parents, orphans=orphan_entries + ): # Update thumbnails in case of last_entry self._download_url_thumbnails(collection_url=collection_url, entry=entry) yield entry diff --git a/src/ytdl_sub/downloaders/generic/url.py b/src/ytdl_sub/downloaders/generic/url.py index 9a3b1735..bf29ef59 100644 --- a/src/ytdl_sub/downloaders/generic/url.py +++ b/src/ytdl_sub/downloaders/generic/url.py @@ -24,6 +24,7 @@ class UrlDownloadOptions(UrlValidator, DownloaderValidator): uid: "avatar_uncropped" - name: "fanart.jpg" uid: "banner_uncropped" + download_reverse: True """ @property diff --git a/src/ytdl_sub/downloaders/generic/validators.py b/src/ytdl_sub/downloaders/generic/validators.py index 905c2f2a..5f94975c 100644 --- a/src/ytdl_sub/downloaders/generic/validators.py +++ b/src/ytdl_sub/downloaders/generic/validators.py @@ -8,6 +8,7 @@ from ytdl_sub.validators.strict_dict_validator import StrictDictValidator from ytdl_sub.validators.string_formatter_validators import DictFormatterValidator from ytdl_sub.validators.string_formatter_validators import OverridesStringFormatterValidator from ytdl_sub.validators.string_formatter_validators import StringFormatterValidator +from ytdl_sub.validators.validators import BoolValidator from ytdl_sub.validators.validators import ListValidator @@ -41,7 +42,7 @@ class UrlThumbnailListValidator(ListValidator[UrlThumbnailValidator]): class UrlValidator(StrictDictValidator): _required_keys = {"url"} - _optional_keys = {"variables", "source_thumbnails", "playlist_thumbnails"} + _optional_keys = {"variables", "source_thumbnails", "playlist_thumbnails", "download_reverse"} @classmethod def partial_validate(cls, name: str, value: Any) -> None: @@ -67,6 +68,9 @@ class UrlValidator(StrictDictValidator): self._playlist_thumbnails = self._validate_key_if_present( key="playlist_thumbnails", validator=UrlThumbnailListValidator, default=[] ) + self._download_reverse = self._validate_key( + key="download_reverse", validator=BoolValidator, default=True + ) @property def url(self) -> OverridesStringFormatterValidator: @@ -134,6 +138,14 @@ class UrlValidator(StrictDictValidator): """ return self._source_thumbnails + @property + def download_reverse(self) -> bool: + """ + Optional. Whether to download entries in the reverse order of the metadata downloaded. + Defaults to True. + """ + return self._download_reverse.value + class UrlListValidator(ListValidator[UrlValidator]): _inner_list_type = UrlValidator