From 72442ab846676d9f68f466bcab758e58b16aec1d Mon Sep 17 00:00:00 2001 From: Jesse Bannon Date: Fri, 22 Jul 2022 00:09:08 -0700 Subject: [PATCH] [BACKEND] Download and process Soundcloud tracks individually (#117) --- .../soundcloud/albums_and_singles.py | 82 +++++++++++++++---- src/ytdl_sub/downloaders/youtube/channel.py | 10 +-- src/ytdl_sub/downloaders/youtube/playlist.py | 11 +-- tests/e2e/conftest.py | 5 ++ .../soundcloud/test_soundcloud_discography.py | 43 +++------- 5 files changed, 89 insertions(+), 62 deletions(-) diff --git a/src/ytdl_sub/downloaders/soundcloud/albums_and_singles.py b/src/ytdl_sub/downloaders/soundcloud/albums_and_singles.py index 85cb28f6..f6a06d9e 100644 --- a/src/ytdl_sub/downloaders/soundcloud/albums_and_singles.py +++ b/src/ytdl_sub/downloaders/soundcloud/albums_and_singles.py @@ -1,9 +1,11 @@ from typing import Dict +from typing import Generator from typing import List from ytdl_sub.downloaders.soundcloud.abc import SoundcloudDownloader from ytdl_sub.downloaders.soundcloud.abc import SoundcloudDownloaderOptions from ytdl_sub.entries.soundcloud import SoundcloudAlbum +from ytdl_sub.entries.soundcloud import SoundcloudAlbumTrack from ytdl_sub.entries.soundcloud import SoundcloudTrack from ytdl_sub.validators.url_validator import SoundcloudUsernameUrlValidator @@ -67,7 +69,7 @@ class SoundcloudAlbumsAndSinglesDownloader( }, ) - def _get_albums(self, entry_dicts: List[Dict]) -> List[SoundcloudAlbum]: + def _get_albums_from_entry_dicts(self, entry_dicts: List[Dict]) -> List[SoundcloudAlbum]: """ Parameters ---------- @@ -113,28 +115,72 @@ class SoundcloudAlbumsAndSinglesDownloader( return tracks - def download(self) -> List[SoundcloudTrack]: - """ - Soundcloud subscription to download albums and tracks as singles. - """ + def _get_albums(self) -> List[SoundcloudAlbum]: + # Dry-run to get the info json files artist_albums_url = self.artist_albums_url(artist_url=self.download_options.url) + album_entry_dicts = self.extract_info_via_info_json( + url=artist_albums_url, + ytdl_options_overrides={ + "skip_download": True, + "writethumbnail": False, + }, + ) + + albums = self._get_albums_from_entry_dicts(entry_dicts=album_entry_dicts) + return albums + + def _get_album_tracks( + self, albums: List[SoundcloudAlbum] + ) -> Generator[SoundcloudAlbumTrack, None, None]: + for album in albums: + for track in album.album_tracks(): + if self.download_options.skip_premiere_tracks and track.is_premiere(): + continue + + if not self.is_dry_run: + _ = self.extract_info( + url=album.kwargs("webpage_url"), + ytdl_options_overrides={ + "playlist_items": str(track.kwargs("playlist_index")), + "writeinfojson": False, + }, + ) + + yield track + + def _get_single_tracks( + self, albums: List[SoundcloudAlbum] + ) -> Generator[SoundcloudTrack, None, None]: artist_tracks_url = self.artist_tracks_url(artist_url=self.download_options.url) - - album_entry_dicts = self.extract_info_via_info_json(url=artist_albums_url) - tracks_entry_dicts = self.extract_info_via_info_json(url=artist_tracks_url) - - # Get all of the artist's albums - albums = self._get_albums(entry_dicts=album_entry_dicts) + tracks_entry_dicts = self.extract_info_via_info_json( + url=artist_tracks_url, + ytdl_options_overrides={ + "skip_download": True, + "writethumbnail": False, + }, + ) # Then, get all singles tracks = self._get_singles(entry_dicts=tracks_entry_dicts, albums=albums) + for track in tracks: + # Filter any premiere tracks if specified + if self.download_options.skip_premiere_tracks and track.is_premiere(): + continue - # Append all album tracks as SoundcloudAlbumTrack classes to the singles - for album in albums: - tracks += album.album_tracks() + if not self.is_dry_run: + _ = self.extract_info( + url=track.kwargs("webpage_url"), ytdl_options_overrides={"writeinfojson": False} + ) - # Filter any premiere tracks if specified - if self.download_options.skip_premiere_tracks: - tracks = [track for track in tracks if not track.is_premiere()] + yield track - return tracks + def download(self) -> Generator[SoundcloudTrack, None, None]: + """ + Soundcloud subscription to download albums and tracks as singles. + """ + albums = self._get_albums() + for album_track in self._get_album_tracks(albums=albums): + yield album_track + + for single_track in self._get_single_tracks(albums=albums): + yield single_track diff --git a/src/ytdl_sub/downloaders/youtube/channel.py b/src/ytdl_sub/downloaders/youtube/channel.py index 7525c8d1..bfa3ac2a 100644 --- a/src/ytdl_sub/downloaders/youtube/channel.py +++ b/src/ytdl_sub/downloaders/youtube/channel.py @@ -171,18 +171,16 @@ class YoutubeChannelDownloader(YoutubeDownloader[YoutubeChannelDownloaderOptions ) self.channel = self._get_channel_from_entry_dicts(entry_dicts=entry_dicts) - # If downloading individually, remove the skip_download to actually download the video - if self.download_options.download_individually: - del ytdl_options_overrides["skip_download"] - del ytdl_options_overrides["writethumbnail"] - for entry_dict in entry_dicts: if entry_dict.get("extractor") == "youtube": # Only do the individual download if it is not dry-run and downloading individually if not self.is_dry_run and self.download_options.download_individually: ytdl_options_overrides["playlist_items"] = str(entry_dict.get("playlist_index")) _ = self.extract_info( - ytdl_options_overrides=ytdl_options_overrides, + ytdl_options_overrides={ + "playlist_items": str(entry_dict.get("playlist_index")), + "writeinfojson": False, + }, url=self.download_options.channel_url, ) yield YoutubeVideo(entry_dict=entry_dict, working_directory=self.working_directory) diff --git a/src/ytdl_sub/downloaders/youtube/playlist.py b/src/ytdl_sub/downloaders/youtube/playlist.py index e36f5869..1a476dba 100644 --- a/src/ytdl_sub/downloaders/youtube/playlist.py +++ b/src/ytdl_sub/downloaders/youtube/playlist.py @@ -98,19 +98,16 @@ class YoutubePlaylistDownloader( ytdl_options_overrides=ytdl_options_overrides, url=self.download_options.playlist_url ) - # If downloading individually, remove the skip_download to actually download the video - if self.download_options.download_individually: - del ytdl_options_overrides["skip_download"] - del ytdl_options_overrides["writethumbnail"] - for entry_dict in entry_dicts: if entry_dict.get("extractor") == "youtube": # Only do the individual download if it is not dry-run and downloading individually if not self.is_dry_run and self.download_options.download_individually: - ytdl_options_overrides["playlist_items"] = str(entry_dict.get("playlist_index")) _ = self.extract_info( - ytdl_options_overrides=ytdl_options_overrides, + ytdl_options_overrides={ + "playlist_items": str(entry_dict.get("playlist_index")), + "writeinfojson": False, + }, url=self.download_options.playlist_url, ) diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py index c80ba719..951602c7 100644 --- a/tests/e2e/conftest.py +++ b/tests/e2e/conftest.py @@ -19,3 +19,8 @@ def music_video_config(): @pytest.fixture() def channel_as_tv_show_config(): return ConfigFile.from_file_path(config_path="examples/kodi_tv_shows_config.yaml") + + +@pytest.fixture +def soundcloud_discography_config(): + return ConfigFile.from_file_path(config_path="examples/soundcloud_discography_config.yaml") diff --git a/tests/e2e/soundcloud/test_soundcloud_discography.py b/tests/e2e/soundcloud/test_soundcloud_discography.py index eac92645..d1674e8d 100644 --- a/tests/e2e/soundcloud/test_soundcloud_discography.py +++ b/tests/e2e/soundcloud/test_soundcloud_discography.py @@ -11,22 +11,7 @@ from ytdl_sub.subscriptions.subscription import Subscription @pytest.fixture -def config_path(): - return "examples/soundcloud_discography_config.yaml" - - -@pytest.fixture -def subscription_name(): - return "jb" - - -@pytest.fixture -def config(config_path): - return ConfigFile.from_file_path(config_path=config_path) - - -@pytest.fixture -def subscription_dict(output_directory, subscription_name): +def subscription_dict(output_directory): return { "preset": "sc_discography", "soundcloud": {"url": "https://soundcloud.com/jessebannon"}, @@ -40,20 +25,6 @@ def subscription_dict(output_directory, subscription_name): } -@pytest.fixture -def discography_subscription(config, subscription_name, subscription_dict): - discography_preset = Preset.from_dict( - config=config, - preset_name=subscription_name, - preset_dict=subscription_dict, - ) - - return Subscription.from_preset( - preset=discography_preset, - config=config, - ) - - @pytest.fixture def expected_discography_download(): # turn off black formatter here for readability @@ -96,8 +67,18 @@ class TestSoundcloudDiscography: @pytest.mark.parametrize("dry_run", [True, False]) def test_discography_download( - self, discography_subscription, expected_discography_download, output_directory, dry_run + self, + subscription_dict, + soundcloud_discography_config, + expected_discography_download, + output_directory, + dry_run, ): + discography_subscription = Subscription.from_dict( + preset_dict=subscription_dict, + preset_name="jb", + config=soundcloud_discography_config, + ) transaction_log = discography_subscription.download(dry_run=dry_run) assert_transaction_log_matches( output_directory=output_directory,