From 8033b1a44c820d0b08364066c3c7c13c703395f5 Mon Sep 17 00:00:00 2001 From: Jesse Bannon Date: Sat, 6 Aug 2022 22:02:18 -0700 Subject: [PATCH] [BACKEND] Download descending, process ascending for YouTube channels and playlists (#159) --- src/ytdl_sub/downloaders/downloader.py | 25 ++++++++++- .../soundcloud/albums_and_singles.py | 19 ++++---- src/ytdl_sub/downloaders/youtube/channel.py | 45 +++++++++---------- src/ytdl_sub/downloaders/youtube/playlist.py | 37 ++++++++------- 4 files changed, 74 insertions(+), 52 deletions(-) diff --git a/src/ytdl_sub/downloaders/downloader.py b/src/ytdl_sub/downloaders/downloader.py index eb73b43c..539ab514 100644 --- a/src/ytdl_sub/downloaders/downloader.py +++ b/src/ytdl_sub/downloaders/downloader.py @@ -58,7 +58,7 @@ class Downloader(DownloadArchiver, Generic[DownloaderOptionsT, DownloaderEntryT] supports_download_archive: bool = True _extract_entry_num_retries: int = 5 - _extract_entry_retry_wait_sec: int = 1 + _extract_entry_retry_wait_sec: int = 3 @classmethod def ytdl_option_overrides(cls) -> Dict: @@ -212,6 +212,29 @@ class Downloader(DownloadArchiver, Generic[DownloaderOptionsT, DownloaderEntryT] f"yt-dlp failed to download an entry with these arguments: {error_dict}" ) + def _filter_entry_dicts( + self, entry_dicts: List[Dict], extractor: Optional[str] = None + ) -> List[Dict]: + """ + Parameters + ---------- + entry_dicts + entry dicts to filter + extractor + Optional. Extractor that the entry dicts must have. If None, defaults to the + entry type's extractor + + Returns + ------- + filtered entry dicts + """ + if extractor is None: + extractor = self.downloader_entry_type.entry_extractor + + return [ + entry_dict for entry_dict in entry_dicts if entry_dict.get("extractor") == extractor + ] + def _get_entry_dicts_from_info_json_files(self) -> List[Dict]: """ Returns diff --git a/src/ytdl_sub/downloaders/soundcloud/albums_and_singles.py b/src/ytdl_sub/downloaders/soundcloud/albums_and_singles.py index 16bd31e9..e5e9f760 100644 --- a/src/ytdl_sub/downloaders/soundcloud/albums_and_singles.py +++ b/src/ytdl_sub/downloaders/soundcloud/albums_and_singles.py @@ -84,16 +84,15 @@ class SoundcloudAlbumsAndSinglesDownloader( albums: Dict[str, SoundcloudAlbum] = {} # First, get the albums themselves - for entry_dict in entry_dicts: - if entry_dict.get("extractor") == "soundcloud:set": - albums[entry_dict["id"]] = SoundcloudAlbum( - entry_dict=entry_dict, working_directory=self.working_directory - ) + for entry_dict in self._filter_entry_dicts(entry_dicts, extractor="soundcloud:set"): + albums[entry_dict["id"]] = SoundcloudAlbum( + entry_dict=entry_dict, working_directory=self.working_directory + ) # Then, get all tracks that belong to the album - for entry_dict in entry_dicts: + for entry_dict in self._filter_entry_dicts(entry_dicts): album_id = entry_dict.get("playlist_id") - if entry_dict.get("extractor") == "soundcloud" and album_id in albums: + if album_id in albums: albums[album_id].tracks.append( SoundcloudTrack(entry_dict=entry_dict, working_directory=self.working_directory) ) @@ -106,10 +105,8 @@ class SoundcloudAlbumsAndSinglesDownloader( tracks: List[SoundcloudTrack] = [] # Get all tracks that are not part of an album - for entry_dict in entry_dicts: - if entry_dict.get("extractor") == "soundcloud" and not any( - entry_dict in album for album in albums - ): + for entry_dict in self._filter_entry_dicts(entry_dicts): + if not any(entry_dict in album for album in albums): tracks.append( SoundcloudTrack(entry_dict=entry_dict, working_directory=self.working_directory) ) diff --git a/src/ytdl_sub/downloaders/youtube/channel.py b/src/ytdl_sub/downloaders/youtube/channel.py index 33252f02..34d4ca6a 100644 --- a/src/ytdl_sub/downloaders/youtube/channel.py +++ b/src/ytdl_sub/downloaders/youtube/channel.py @@ -124,12 +124,10 @@ class YoutubeChannelDownloader(YoutubeDownloader[YoutubeChannelDownloaderOptions ) self.channel: Optional[YoutubeChannel] = None - def _get_channel_from_entry_dicts(self, entry_dicts: List[Dict]) -> YoutubeChannel: - channel_entry_dict = [ - entry_dict for entry_dict in entry_dicts if entry_dict.get("extractor") == "youtube:tab" - ][0] + def _get_channel(self, entry_dicts: List[Dict]) -> YoutubeChannel: return YoutubeChannel( - entry_dict=channel_entry_dict, working_directory=self.working_directory + entry_dict=self._filter_entry_dicts(entry_dicts, extractor="youtube:tab")[0], + working_directory=self.working_directory, ) def download(self) -> Generator[YoutubeVideo, None, None]: @@ -151,26 +149,27 @@ class YoutubeChannelDownloader(YoutubeDownloader[YoutubeChannelDownloaderOptions log_prefix_on_info_json_dl="Downloading metadata for", url=self.download_options.channel_url, ) - self.channel = self._get_channel_from_entry_dicts(entry_dicts=entry_dicts) + self.channel = self._get_channel(entry_dicts=entry_dicts) + channel_videos = self._filter_entry_dicts(entry_dicts=entry_dicts) - for idx, entry_dict in enumerate(entry_dicts, start=1): - if entry_dict.get("extractor") == self.downloader_entry_type.entry_extractor: - video = YoutubeVideo( - entry_dict=entry_dict, working_directory=self.working_directory + # Iterate in reverse to process older videos first. In case an error occurs and a + # the channel must be redownloaded, it will fetch most recent metadata first, and break + # on the older video that's been processed and is in the download archive. + for idx, entry_dict in enumerate(reversed(channel_videos), start=1): + video = YoutubeVideo(entry_dict=entry_dict, working_directory=self.working_directory) + download_logger.info("Downloading %d/%d %s", idx, len(entry_dicts), video.title) + + # Only do the individual download if it is not dry-run + if not self.is_dry_run: + _ = self.extract_info_with_retry( + is_downloaded_fn=video.is_downloaded, + ytdl_options_overrides={ + "playlist_items": str(video.kwargs("playlist_index")), + "writeinfojson": False, + }, + url=self.download_options.channel_url, ) - download_logger.info("Downloading %d/%d %s", idx, len(entry_dicts), video.title) - - # Only do the individual download if it is not dry-run - if not self.is_dry_run: - _ = self.extract_info_with_retry( - is_downloaded_fn=video.is_downloaded, - ytdl_options_overrides={ - "playlist_items": str(video.kwargs("playlist_index")), - "writeinfojson": False, - }, - url=self.download_options.channel_url, - ) - yield video + yield video def _download_thumbnail( self, diff --git a/src/ytdl_sub/downloaders/youtube/playlist.py b/src/ytdl_sub/downloaders/youtube/playlist.py index 050a3f16..295463d0 100644 --- a/src/ytdl_sub/downloaders/youtube/playlist.py +++ b/src/ytdl_sub/downloaders/youtube/playlist.py @@ -78,23 +78,26 @@ class YoutubePlaylistDownloader( log_prefix_on_info_json_dl="Downloading metadata for", url=self.download_options.playlist_url, ) + playlist_videos = self._filter_entry_dicts(entry_dicts=entry_dicts) - for idx, entry_dict in enumerate(entry_dicts, start=1): - if entry_dict.get("extractor") == self.downloader_entry_type.entry_extractor: - video = YoutubePlaylistVideo( - entry_dict=entry_dict, working_directory=self.working_directory + # Iterate in reverse to process older videos first. In case an error occurs and a + # the playlist must be redownloaded, it will fetch most recent metadata first, and break + # on the older video that's been processed and is in the download archive. + for idx, entry_dict in enumerate(reversed(playlist_videos), start=1): + video = YoutubePlaylistVideo( + entry_dict=entry_dict, working_directory=self.working_directory + ) + download_logger.info("Downloading %d/%d %s", idx, len(entry_dicts), video.title) + + # Only do the individual download if it is not dry-run and downloading individually + if not self.is_dry_run: + _ = self.extract_info_with_retry( + is_downloaded_fn=video.is_downloaded, + ytdl_options_overrides={ + "playlist_items": str(entry_dict.get("playlist_index")), + "writeinfojson": False, + }, + url=self.download_options.playlist_url, ) - download_logger.info("Downloading %d/%d %s", idx, len(entry_dicts), video.title) - # Only do the individual download if it is not dry-run and downloading individually - if not self.is_dry_run: - _ = self.extract_info_with_retry( - is_downloaded_fn=video.is_downloaded, - ytdl_options_overrides={ - "playlist_items": str(entry_dict.get("playlist_index")), - "writeinfojson": False, - }, - url=self.download_options.playlist_url, - ) - - yield video + yield video