[BACKEND] Download descending, process ascending for YouTube channels and playlists

src/ytdl_sub/downloaders/downloader.py

@@ -58,7 +58,7 @@ class Downloader(DownloadArchiver, Generic[DownloaderOptionsT, DownloaderEntryT]
     supports_download_archive: bool = True
 
     _extract_entry_num_retries: int = 5
-    _extract_entry_retry_wait_sec: int = 1
+    _extract_entry_retry_wait_sec: int = 3
 
     @classmethod
     def ytdl_option_overrides(cls) -> Dict:
@@ -212,6 +212,29 @@ class Downloader(DownloadArchiver, Generic[DownloaderOptionsT, DownloaderEntryT]
             f"yt-dlp failed to download an entry with these arguments: {error_dict}"
         )
 
+    def _filter_entry_dicts(
+        self, entry_dicts: List[Dict], extractor: Optional[str] = None
+    ) -> List[Dict]:
+        """
+        Parameters
+        ----------
+        entry_dicts
+            entry dicts to filter
+        extractor
+            Optional. Extractor that the entry dicts must have. If None, defaults to the
+            entry type's extractor
+
+        Returns
+        -------
+        filtered entry dicts
+        """
+        if extractor is None:
+            extractor = self.downloader_entry_type.entry_extractor
+
+        return [
+            entry_dict for entry_dict in entry_dicts if entry_dict.get("extractor") == extractor
+        ]
+
     def _get_entry_dicts_from_info_json_files(self) -> List[Dict]:
         """
         Returns

src/ytdl_sub/downloaders/soundcloud/albums_and_singles.py

@@ -84,16 +84,15 @@ class SoundcloudAlbumsAndSinglesDownloader(
         albums: Dict[str, SoundcloudAlbum] = {}
 
         # First, get the albums themselves
-        for entry_dict in entry_dicts:
+        for entry_dict in self._filter_entry_dicts(entry_dicts, extractor="soundcloud:set"):
-            if entry_dict.get("extractor") == "soundcloud:set":
+            albums[entry_dict["id"]] = SoundcloudAlbum(
-                albums[entry_dict["id"]] = SoundcloudAlbum(
+                entry_dict=entry_dict, working_directory=self.working_directory
-                    entry_dict=entry_dict, working_directory=self.working_directory
+            )
-                )
 
         # Then, get all tracks that belong to the album
-        for entry_dict in entry_dicts:
+        for entry_dict in self._filter_entry_dicts(entry_dicts):
             album_id = entry_dict.get("playlist_id")
-            if entry_dict.get("extractor") == "soundcloud" and album_id in albums:
+            if album_id in albums:
                 albums[album_id].tracks.append(
                     SoundcloudTrack(entry_dict=entry_dict, working_directory=self.working_directory)
                 )
@@ -106,10 +105,8 @@ class SoundcloudAlbumsAndSinglesDownloader(
         tracks: List[SoundcloudTrack] = []
 
         # Get all tracks that are not part of an album
-        for entry_dict in entry_dicts:
+        for entry_dict in self._filter_entry_dicts(entry_dicts):
-            if entry_dict.get("extractor") == "soundcloud" and not any(
+            if not any(entry_dict in album for album in albums):
-                entry_dict in album for album in albums
-            ):
                 tracks.append(
                     SoundcloudTrack(entry_dict=entry_dict, working_directory=self.working_directory)
                 )

src/ytdl_sub/downloaders/youtube/channel.py

@@ -124,12 +124,10 @@ class YoutubeChannelDownloader(YoutubeDownloader[YoutubeChannelDownloaderOptions
         )
         self.channel: Optional[YoutubeChannel] = None
 
-    def _get_channel_from_entry_dicts(self, entry_dicts: List[Dict]) -> YoutubeChannel:
+    def _get_channel(self, entry_dicts: List[Dict]) -> YoutubeChannel:
-        channel_entry_dict = [
-            entry_dict for entry_dict in entry_dicts if entry_dict.get("extractor") == "youtube:tab"
-        ][0]
         return YoutubeChannel(
-            entry_dict=channel_entry_dict, working_directory=self.working_directory
+            entry_dict=self._filter_entry_dicts(entry_dicts, extractor="youtube:tab")[0],
+            working_directory=self.working_directory,
         )
 
     def download(self) -> Generator[YoutubeVideo, None, None]:
@@ -151,26 +149,27 @@ class YoutubeChannelDownloader(YoutubeDownloader[YoutubeChannelDownloaderOptions
             log_prefix_on_info_json_dl="Downloading metadata for",
             url=self.download_options.channel_url,
         )
-        self.channel = self._get_channel_from_entry_dicts(entry_dicts=entry_dicts)
+        self.channel = self._get_channel(entry_dicts=entry_dicts)
+        channel_videos = self._filter_entry_dicts(entry_dicts=entry_dicts)
 
-        for idx, entry_dict in enumerate(entry_dicts, start=1):
+        # Iterate in reverse to process older videos first. In case an error occurs and a
-            if entry_dict.get("extractor") == self.downloader_entry_type.entry_extractor:
+        # the channel must be redownloaded, it will fetch most recent metadata first, and break
-                video = YoutubeVideo(
+        # on the older video that's been processed and is in the download archive.
-                    entry_dict=entry_dict, working_directory=self.working_directory
+        for idx, entry_dict in enumerate(reversed(channel_videos), start=1):
+            video = YoutubeVideo(entry_dict=entry_dict, working_directory=self.working_directory)
+            download_logger.info("Downloading %d/%d %s", idx, len(entry_dicts), video.title)
+
+            # Only do the individual download if it is not dry-run
+            if not self.is_dry_run:
+                _ = self.extract_info_with_retry(
+                    is_downloaded_fn=video.is_downloaded,
+                    ytdl_options_overrides={
+                        "playlist_items": str(video.kwargs("playlist_index")),
+                        "writeinfojson": False,
+                    },
+                    url=self.download_options.channel_url,
                 )
-                download_logger.info("Downloading %d/%d %s", idx, len(entry_dicts), video.title)
+            yield video
-
-                # Only do the individual download if it is not dry-run
-                if not self.is_dry_run:
-                    _ = self.extract_info_with_retry(
-                        is_downloaded_fn=video.is_downloaded,
-                        ytdl_options_overrides={
-                            "playlist_items": str(video.kwargs("playlist_index")),
-                            "writeinfojson": False,
-                        },
-                        url=self.download_options.channel_url,
-                    )
-                yield video
 
     def _download_thumbnail(
         self,

src/ytdl_sub/downloaders/youtube/playlist.py

@@ -78,23 +78,26 @@ class YoutubePlaylistDownloader(
             log_prefix_on_info_json_dl="Downloading metadata for",
             url=self.download_options.playlist_url,
         )
+        playlist_videos = self._filter_entry_dicts(entry_dicts=entry_dicts)
 
-        for idx, entry_dict in enumerate(entry_dicts, start=1):
+        # Iterate in reverse to process older videos first. In case an error occurs and a
-            if entry_dict.get("extractor") == self.downloader_entry_type.entry_extractor:
+        # the playlist must be redownloaded, it will fetch most recent metadata first, and break
-                video = YoutubePlaylistVideo(
+        # on the older video that's been processed and is in the download archive.
-                    entry_dict=entry_dict, working_directory=self.working_directory
+        for idx, entry_dict in enumerate(reversed(playlist_videos), start=1):
+            video = YoutubePlaylistVideo(
+                entry_dict=entry_dict, working_directory=self.working_directory
+            )
+            download_logger.info("Downloading %d/%d %s", idx, len(entry_dicts), video.title)
+
+            # Only do the individual download if it is not dry-run and downloading individually
+            if not self.is_dry_run:
+                _ = self.extract_info_with_retry(
+                    is_downloaded_fn=video.is_downloaded,
+                    ytdl_options_overrides={
+                        "playlist_items": str(entry_dict.get("playlist_index")),
+                        "writeinfojson": False,
+                    },
+                    url=self.download_options.playlist_url,
                 )
-                download_logger.info("Downloading %d/%d %s", idx, len(entry_dicts), video.title)
 
-                # Only do the individual download if it is not dry-run and downloading individually
+            yield video
-                if not self.is_dry_run:
-                    _ = self.extract_info_with_retry(
-                        is_downloaded_fn=video.is_downloaded,
-                        ytdl_options_overrides={
-                            "playlist_items": str(entry_dict.get("playlist_index")),
-                            "writeinfojson": False,
-                        },
-                        url=self.download_options.playlist_url,
-                    )
-
-                yield video