[BACKEND] Use generic collection download for every playlist strategy

This commit is contained in:
Jesse Bannon 2022-09-14 22:07:36 -07:00
parent c08ea64b5a
commit dd190332ad
5 changed files with 68 additions and 11 deletions

View file

@ -235,6 +235,10 @@ class CollectionDownloader(Downloader[CollectionDownloadOptions, Entry]):
def _download_parent_entry(self, parent: EntryParent) -> Generator[Entry, None, None]:
"""Download in reverse order, that way we download older entries ones first"""
if parent.is_entry():
yield parent.to_type(Entry)
return
for entry_child in reversed(parent.entry_children()):
if _entry_key(entry_child) in self.downloaded_entries:
continue

View file

@ -59,7 +59,7 @@ class YoutubeChannelDownloaderOptions(YoutubeDownloaderOptions):
self.collection_validator = CollectionDownloadOptions(
name=self._name,
value={"urls": [{"url": self._channel_url}]},
value={"urls": [{"url": self.channel_url}]},
)
@property

View file

@ -3,6 +3,7 @@ from typing import List
from typing import Optional
from typing import Tuple
from ytdl_sub.downloaders.generic.collection import CollectionDownloader
from ytdl_sub.downloaders.youtube.abc import YoutubeDownloader
from ytdl_sub.downloaders.youtube.playlist import YoutubePlaylistDownloaderOptions
from ytdl_sub.entries.youtube import YoutubeVideo
@ -147,9 +148,25 @@ class YoutubeMergePlaylistDownloader(
def download(self) -> List[Tuple[YoutubeVideo, FileMetadata]]:
"""Download a single Youtube video, then split it into multiple videos"""
merged_video = self._to_merged_video(
entry_dict=self.extract_info(url=self.download_options.playlist_url)
downloader = CollectionDownloader(
download_options=self.download_options.collection_validator,
enhanced_download_archive=self._enhanced_download_archive,
ytdl_options_builder=self._ytdl_options_builder,
overrides=self.overrides,
)
collection_url = self.download_options.collection_validator.collection_urls.list[0]
parents = downloader.download_url_metadata(collection_url=collection_url)
assert len(parents) == 1, "Playlist should be the only entry parent"
playlist = parents[0]
# perform the download of all entries in the playlist
_ = list(downloader.download_url(collection_url=collection_url, parents=parents))
# pylint: disable=protected-access
merged_video = self._to_merged_video(entry_dict=playlist._kwargs)
# pylint: enable=protected-access
merged_video_metadata = self._get_chapters(
merged_video=merged_video, add_chapters=self.download_options.add_chapters
)

View file

@ -1,13 +1,19 @@
from typing import Dict
from typing import Generator
from typing import List
from typing import Optional
from ytdl_sub.config.preset_options import Overrides
from ytdl_sub.downloaders.downloader import DownloaderOptionsT
from ytdl_sub.downloaders.generic.collection import CollectionDownloader
from ytdl_sub.downloaders.generic.collection import CollectionDownloadOptions
from ytdl_sub.downloaders.youtube.abc import YoutubeDownloader
from ytdl_sub.downloaders.youtube.abc import YoutubeDownloaderOptions
from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder
from ytdl_sub.entries.entry_parent import EntryParent
from ytdl_sub.entries.youtube import YoutubePlaylistVideo
from ytdl_sub.validators.url_validator import YoutubePlaylistUrlValidator
from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive
class YoutubePlaylistDownloaderOptions(YoutubeDownloaderOptions):
@ -36,7 +42,7 @@ class YoutubePlaylistDownloaderOptions(YoutubeDownloaderOptions):
self.collection_validator = CollectionDownloadOptions(
name=self._name,
value={"urls": [{"url": self._playlist_url}]},
value={"urls": [{"url": self.playlist_url}]},
)
@property
@ -87,6 +93,22 @@ class YoutubePlaylistDownloader(
# pylint: enable=line-too-long
def __init__(
self,
download_options: DownloaderOptionsT,
enhanced_download_archive: EnhancedDownloadArchive,
ytdl_options_builder: YTDLOptionsBuilder,
overrides: Overrides,
):
super().__init__(
download_options=download_options,
enhanced_download_archive=enhanced_download_archive,
ytdl_options_builder=ytdl_options_builder,
overrides=overrides,
)
self.playlist: Optional[EntryParent] = None
def download(self) -> Generator[YoutubePlaylistVideo, None, None]:
"""
Downloads all videos in a Youtube playlist.
@ -101,14 +123,14 @@ class YoutubePlaylistDownloader(
parents = downloader.download_url_metadata(collection_url=collection_url)
assert len(parents) == 1, "Playlist should be the only entry parent"
playlist = parents[0]
self.playlist = parents[0]
# TODO: Handle this better
self.overrides.add_override_variables(
variables_to_add={
"source_title": playlist.title,
"source_uploader": playlist.kwargs_get("uploader", "__failed_to_scrape__"),
"source_description": playlist.kwargs_get("description", ""),
"source_title": self.playlist.title,
"source_uploader": self.playlist.kwargs_get("uploader", "__failed_to_scrape__"),
"source_description": self.playlist.kwargs_get("description", ""),
}
)

View file

@ -1,6 +1,8 @@
from typing import Dict
from typing import List
from ytdl_sub.downloaders.generic.collection import CollectionDownloader
from ytdl_sub.downloaders.generic.collection import CollectionDownloadOptions
from ytdl_sub.downloaders.youtube.abc import YoutubeDownloader
from ytdl_sub.downloaders.youtube.abc import YoutubeDownloaderOptions
from ytdl_sub.entries.youtube import YoutubeVideo
@ -36,6 +38,11 @@ class YoutubeVideoDownloaderOptions(YoutubeDownloaderOptions):
super().__init__(name, value)
self._video_url = self._validate_key("video_url", YoutubeVideoUrlValidator).video_url
self.collection_validator = CollectionDownloadOptions(
name=self._name,
value={"urls": [{"url": self.video_url}]},
)
@property
def video_url(self) -> str:
"""
@ -65,7 +72,14 @@ class YoutubeVideoDownloader(YoutubeDownloader[YoutubeVideoDownloaderOptions, Yo
def download(self) -> List[YoutubeVideo]:
"""Download a single Youtube video"""
entry_dict = self.extract_info(url=self.download_options.video_url)
video = YoutubeVideo(entry_dict=entry_dict, working_directory=self.working_directory)
downloader = CollectionDownloader(
download_options=self.download_options.collection_validator,
enhanced_download_archive=self._enhanced_download_archive,
ytdl_options_builder=self._ytdl_options_builder,
overrides=self.overrides,
)
return [video]
for entry in downloader.download():
# pylint: disable=protected-access
yield YoutubeVideo(entry_dict=entry._kwargs, working_directory=self.working_directory)
# pylint: enable=protected-access