danvics/ytdl-sub
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

[BACKEND] Download descending, process ascending for YouTube channels and playlists (#159)

Browse source
This commit is contained in:
Jesse Bannon 2022-08-06 22:02:18 -07:00 committed by GitHub
parent 034855fbc4
commit 8033b1a44c
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
4 changed files with 74 additions and 52 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

25
src/ytdl_sub/downloaders/downloader.py
View file

@ -58,7 +58,7 @@ class Downloader(DownloadArchiver, Generic[DownloaderOptionsT, DownloaderEntryT]
supports_download_archive: bool = True
_extract_entry_num_retries: int = 5
_extract_entry_retry_wait_sec: int = 1
_extract_entry_retry_wait_sec: int = 3
@classmethod
def ytdl_option_overrides(cls) -> Dict:
@ -212,6 +212,29 @@ class Downloader(DownloadArchiver, Generic[DownloaderOptionsT, DownloaderEntryT]
f"yt-dlp failed to download an entry with these arguments: {error_dict}"
)
def _filter_entry_dicts(
self, entry_dicts: List[Dict], extractor: Optional[str] = None
) -> List[Dict]:
"""
Parameters
----------
entry_dicts
entry dicts to filter
extractor
Optional. Extractor that the entry dicts must have. If None, defaults to the
entry type's extractor
Returns
-------
filtered entry dicts
"""
if extractor is None:
extractor = self.downloader_entry_type.entry_extractor
return [
entry_dict for entry_dict in entry_dicts if entry_dict.get("extractor") == extractor
]
def _get_entry_dicts_from_info_json_files(self) -> List[Dict]:
"""
Returns

19
src/ytdl_sub/downloaders/soundcloud/albums_and_singles.py
View file

@ -84,16 +84,15 @@ class SoundcloudAlbumsAndSinglesDownloader(
albums: Dict[str, SoundcloudAlbum] = {}
# First, get the albums themselves
for entry_dict in entry_dicts:
if entry_dict.get("extractor") == "soundcloud:set":
albums[entry_dict["id"]] = SoundcloudAlbum(
entry_dict=entry_dict, working_directory=self.working_directory
)
for entry_dict in self._filter_entry_dicts(entry_dicts, extractor="soundcloud:set"):
albums[entry_dict["id"]] = SoundcloudAlbum(
entry_dict=entry_dict, working_directory=self.working_directory
)
# Then, get all tracks that belong to the album
for entry_dict in entry_dicts:
for entry_dict in self._filter_entry_dicts(entry_dicts):
album_id = entry_dict.get("playlist_id")
if entry_dict.get("extractor") == "soundcloud" and album_id in albums:
if album_id in albums:
albums[album_id].tracks.append(
SoundcloudTrack(entry_dict=entry_dict, working_directory=self.working_directory)
)
@ -106,10 +105,8 @@ class SoundcloudAlbumsAndSinglesDownloader(
tracks: List[SoundcloudTrack] = []
# Get all tracks that are not part of an album
for entry_dict in entry_dicts:
if entry_dict.get("extractor") == "soundcloud" and not any(
entry_dict in album for album in albums
):
for entry_dict in self._filter_entry_dicts(entry_dicts):
if not any(entry_dict in album for album in albums):
tracks.append(
SoundcloudTrack(entry_dict=entry_dict, working_directory=self.working_directory)
)

45
src/ytdl_sub/downloaders/youtube/channel.py
View file

@ -124,12 +124,10 @@ class YoutubeChannelDownloader(YoutubeDownloader[YoutubeChannelDownloaderOptions
)
self.channel: Optional[YoutubeChannel] = None
def _get_channel_from_entry_dicts(self, entry_dicts: List[Dict]) -> YoutubeChannel:
channel_entry_dict = [
entry_dict for entry_dict in entry_dicts if entry_dict.get("extractor") == "youtube:tab"
][0]
def _get_channel(self, entry_dicts: List[Dict]) -> YoutubeChannel:
return YoutubeChannel(
entry_dict=channel_entry_dict, working_directory=self.working_directory
entry_dict=self._filter_entry_dicts(entry_dicts, extractor="youtube:tab")[0],
working_directory=self.working_directory,
)
def download(self) -> Generator[YoutubeVideo, None, None]:
@ -151,26 +149,27 @@ class YoutubeChannelDownloader(YoutubeDownloader[YoutubeChannelDownloaderOptions
log_prefix_on_info_json_dl="Downloading metadata for",
url=self.download_options.channel_url,
)
self.channel = self._get_channel_from_entry_dicts(entry_dicts=entry_dicts)
self.channel = self._get_channel(entry_dicts=entry_dicts)
channel_videos = self._filter_entry_dicts(entry_dicts=entry_dicts)
for idx, entry_dict in enumerate(entry_dicts, start=1):
if entry_dict.get("extractor") == self.downloader_entry_type.entry_extractor:
video = YoutubeVideo(
entry_dict=entry_dict, working_directory=self.working_directory
# Iterate in reverse to process older videos first. In case an error occurs and a
# the channel must be redownloaded, it will fetch most recent metadata first, and break
# on the older video that's been processed and is in the download archive.
for idx, entry_dict in enumerate(reversed(channel_videos), start=1):
video = YoutubeVideo(entry_dict=entry_dict, working_directory=self.working_directory)
download_logger.info("Downloading %d/%d %s", idx, len(entry_dicts), video.title)
# Only do the individual download if it is not dry-run
if not self.is_dry_run:
_ = self.extract_info_with_retry(
is_downloaded_fn=video.is_downloaded,
ytdl_options_overrides={
"playlist_items": str(video.kwargs("playlist_index")),
"writeinfojson": False,
},
url=self.download_options.channel_url,
)
download_logger.info("Downloading %d/%d %s", idx, len(entry_dicts), video.title)
# Only do the individual download if it is not dry-run
if not self.is_dry_run:
_ = self.extract_info_with_retry(
is_downloaded_fn=video.is_downloaded,
ytdl_options_overrides={
"playlist_items": str(video.kwargs("playlist_index")),
"writeinfojson": False,
},
url=self.download_options.channel_url,
)
yield video
yield video
def _download_thumbnail(
self,

37
src/ytdl_sub/downloaders/youtube/playlist.py
View file

@ -78,23 +78,26 @@ class YoutubePlaylistDownloader(
log_prefix_on_info_json_dl="Downloading metadata for",
url=self.download_options.playlist_url,
)
playlist_videos = self._filter_entry_dicts(entry_dicts=entry_dicts)
for idx, entry_dict in enumerate(entry_dicts, start=1):
if entry_dict.get("extractor") == self.downloader_entry_type.entry_extractor:
video = YoutubePlaylistVideo(
entry_dict=entry_dict, working_directory=self.working_directory
# Iterate in reverse to process older videos first. In case an error occurs and a
# the playlist must be redownloaded, it will fetch most recent metadata first, and break
# on the older video that's been processed and is in the download archive.
for idx, entry_dict in enumerate(reversed(playlist_videos), start=1):
video = YoutubePlaylistVideo(
entry_dict=entry_dict, working_directory=self.working_directory
)
download_logger.info("Downloading %d/%d %s", idx, len(entry_dicts), video.title)
# Only do the individual download if it is not dry-run and downloading individually
if not self.is_dry_run:
_ = self.extract_info_with_retry(
is_downloaded_fn=video.is_downloaded,
ytdl_options_overrides={
"playlist_items": str(entry_dict.get("playlist_index")),
"writeinfojson": False,
},
url=self.download_options.playlist_url,
)
download_logger.info("Downloading %d/%d %s", idx, len(entry_dicts), video.title)
# Only do the individual download if it is not dry-run and downloading individually
if not self.is_dry_run:
_ = self.extract_info_with_retry(
is_downloaded_fn=video.is_downloaded,
ytdl_options_overrides={
"playlist_items": str(entry_dict.get("playlist_index")),
"writeinfojson": False,
},
url=self.download_options.playlist_url,
)
yield video
yield video