From a6cf9c3a1112231e440c6a0b8ac7303c24d71fa6 Mon Sep 17 00:00:00 2001 From: Jesse Bannon Date: Mon, 30 May 2022 00:25:41 -0700 Subject: [PATCH] Add soundcloud discography example w/test (#49) * example sc yaml * discography test * json sort_keys --- examples/soundcloud_discography_config.yaml | 75 +++++++++++++ .../downloaders/soundcloud_downloader.py | 47 +++++--- src/ytdl_sub/entries/soundcloud.py | 10 ++ .../enhanced_download_archive.py | 1 + tests/e2e/soundcloud/__init__.py | 0 .../soundcloud/test_soundcloud_discography.py | 101 ++++++++++++++++++ .../youtube/test_channel_as_kodi_tv_show.py | 6 +- .../test_playlist_as_kodi_music_videos.py | 2 +- 8 files changed, 222 insertions(+), 20 deletions(-) create mode 100644 examples/soundcloud_discography_config.yaml create mode 100644 tests/e2e/soundcloud/__init__.py create mode 100644 tests/e2e/soundcloud/test_soundcloud_discography.py diff --git a/examples/soundcloud_discography_config.yaml b/examples/soundcloud_discography_config.yaml new file mode 100644 index 00000000..dfdb6704 --- /dev/null +++ b/examples/soundcloud_discography_config.yaml @@ -0,0 +1,75 @@ +# This example shows how to download and format a Soundcloud artist's +# discography with tags and album ar. We will configure this to make +# the output directory formatted as: +# +# /path/to/music +# /The Be Sharps +# /[2020] My first upload +# 01 - My first upload.mp3 +# folder.jpg +# /[2021] My first album +# 01 - Track one.mp3 +# 02 - Track two.mp3 +# folder.jpg +# /Another artist +# ... +# +configuration: + working_directory: '.ytdl-sub-downloads' + +presets: + sc_discography: + # A Soundcloud artist's albums and singles will be our source/download + # strategy. This will group together album tracks, and treat any track + # not part of an album as a single. + soundcloud: + download_strategy: "albums_and_singles" + skip_premiere_tracks: True + + # For advanced YTDL users only. + # This sets the download format to grab the best mp3 available. + # Other formats should work but have not been tested. + ytdl_options: + format: 'bestaudio[ext=mp3]' + + # For each song downloaded, set the file and thumbnail name here. + # The output directory stores all artists in a shared {music_directory}. + # We store the audio file in its respective {album_directory_name} folder + # with the track number and title. The (thumbnail) album art is stored + # in the album folder. + # + # Another field worth mentioning is maintain_download_archive=True. This + # is generally a good thing to enable with artists because it will + # store previously downloaded song IDs to tell YTDL not to re-download + # them on a successive invocation. + output_options: + output_directory: "/{music_directory}/{artist_sanitized}" + file_name: "{album_directory_name}/{track_number_padded} - {title_sanitized}.{ext}" + thumbnail_name: "{album_directory_name}/folder.jpg" + maintain_download_archive: True + + # Always convert the video thumbnail to jpg + # TODO: always convert all thumbnails to jpg in code + convert_thumbnail: + to: "jpg" + + # For each song downloaded, populate the audio file with music tags. + # Tagging should work with most audio file formats. See + # https://ytdl-sub.readthedocs.io/en/latest/config.html#music-tags + # for more details. + music_tags: + tags: + artist: "{artist}" + albumartist: "{artist}" + title: "{title}" + album: "{album}" + track: "{track_number}" + year: "{album_year}" + genre: "Unset" + + # Overrides is a section where we can define our own variables, and use them in + # any other section. We define our music directory and album directory names here, + # which gets reused above for the audio file name and album art path. + overrides: + album_directory_name: "[{album_year}] {album_sanitized}" + music_directory: "/path/to/music" \ No newline at end of file diff --git a/src/ytdl_sub/downloaders/soundcloud_downloader.py b/src/ytdl_sub/downloaders/soundcloud_downloader.py index 89bea5c2..961b5708 100644 --- a/src/ytdl_sub/downloaders/soundcloud_downloader.py +++ b/src/ytdl_sub/downloaders/soundcloud_downloader.py @@ -65,6 +65,24 @@ class SoundcloudDownloader( """Returns full artist url""" return f"https://soundcloud.com/{artist_name}" + @classmethod + def artist_albums_url(cls, artist_name: str) -> str: + """ + Returns + ------- + Full artist album url + """ + return cls.artist_url(artist_name) + "/albums" + + @classmethod + def artist_tracks_url(cls, artist_name: str) -> str: + """ + Returns + ------- + Full artist tracks url + """ + return cls.artist_url(artist_name) + "/tracks" + ############################################################################### # Soundcloud albums and singles downloader + options @@ -139,21 +157,15 @@ class SoundcloudAlbumsAndSinglesDownloader( return list(albums.values()) - def _get_singles(self, entry_dicts: List[Dict]) -> List[SoundcloudTrack]: - artist_id = None + def _get_singles( + self, entry_dicts: List[Dict], albums: List[SoundcloudAlbum] + ) -> List[SoundcloudTrack]: tracks: List[SoundcloudTrack] = [] - # First, get the artist entry. All single tracks that do not belong to an album will belong - # to the 'artist' playlist + # Get all tracks that are not part of an album for entry_dict in entry_dicts: - if entry_dict.get("extractor") == "soundcloud:user": - artist_id = entry_dict["id"] - - # Then, get all singles that belong to the 'artist' playlist - for entry_dict in entry_dicts: - if ( - entry_dict.get("extractor") == "soundcloud" - and entry_dict.get("playlist_id") == artist_id + if entry_dict.get("extractor") == "soundcloud" and not any( + entry_dict in album for album in albums ): tracks.append( SoundcloudTrack(entry_dict=entry_dict, working_directory=self.working_directory) @@ -165,14 +177,17 @@ class SoundcloudAlbumsAndSinglesDownloader( """ Soundcloud subscription to download albums and tracks as singles. """ - artist_url = self.artist_url(artist_name=self.download_options.username) - entry_dicts = self.extract_info_via_info_json(url=artist_url) + artist_albums_url = self.artist_albums_url(artist_name=self.download_options.username) + artist_tracks_url = self.artist_tracks_url(artist_name=self.download_options.username) + + album_entry_dicts = self.extract_info_via_info_json(url=artist_albums_url) + tracks_entry_dicts = self.extract_info_via_info_json(url=artist_tracks_url) # Get all of the artist's albums - albums = self._get_albums(entry_dicts=entry_dicts) + albums = self._get_albums(entry_dicts=album_entry_dicts) # Then, get all singles - tracks = self._get_singles(entry_dicts=entry_dicts) + tracks = self._get_singles(entry_dicts=tracks_entry_dicts, albums=albums) # Append all album tracks as SoundcloudAlbumTrack classes to the singles for album in albums: diff --git a/src/ytdl_sub/entries/soundcloud.py b/src/ytdl_sub/entries/soundcloud.py index e6c2353e..cec40761 100644 --- a/src/ytdl_sub/entries/soundcloud.py +++ b/src/ytdl_sub/entries/soundcloud.py @@ -134,3 +134,13 @@ class SoundcloudAlbum(Entry): Number of tracks in the album (technically a playlist) """ return self.kwargs("playlist_count") + + def __contains__(self, item): + """ + Returns + ------- + True if the the item (entry_dict) has the same id as one of the tracks. False otherwise. + """ + if isinstance(item, dict): + return any(item.get("id") == track.uid for track in self.tracks) + return False diff --git a/src/ytdl_sub/ytdl_additions/enhanced_download_archive.py b/src/ytdl_sub/ytdl_additions/enhanced_download_archive.py index fe86c95e..8e343294 100644 --- a/src/ytdl_sub/ytdl_additions/enhanced_download_archive.py +++ b/src/ytdl_sub/ytdl_additions/enhanced_download_archive.py @@ -278,6 +278,7 @@ class DownloadMappings: ) }, indent=2, + sort_keys=True, ) with open(output_json_file, "w", encoding="utf8") as file: diff --git a/tests/e2e/soundcloud/__init__.py b/tests/e2e/soundcloud/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/e2e/soundcloud/test_soundcloud_discography.py b/tests/e2e/soundcloud/test_soundcloud_discography.py new file mode 100644 index 00000000..f840175e --- /dev/null +++ b/tests/e2e/soundcloud/test_soundcloud_discography.py @@ -0,0 +1,101 @@ +from pathlib import Path + +import pytest +from e2e.expected_download import ExpectedDownload + +from ytdl_sub.config.config_file import ConfigFile +from ytdl_sub.config.preset import Preset +from ytdl_sub.subscriptions.subscription import Subscription + + +@pytest.fixture +def config_path(): + return "examples/soundcloud_discography_config.yaml" + + +@pytest.fixture +def subscription_name(): + return "jb" + + +@pytest.fixture +def config(config_path): + return ConfigFile.from_file_path(config_path=config_path) + + +@pytest.fixture +def subscription_dict(output_directory, subscription_name): + return { + "preset": "sc_discography", + "soundcloud": {"username": "jessebannon"}, + # override the output directory with our fixture-generated dir + "output_options": {"output_directory": output_directory + "/{artist_sanitized}"}, + # download the worst format so it is fast + "ytdl_options": { + "format": "worst[ext=mp3]", + }, + "overrides": {"artist": "j_b"}, + } + + +@pytest.fixture +def discography_subscription(config, subscription_name, subscription_dict): + discography_preset = Preset.from_dict( + config=config, + preset_name=subscription_name, + preset_dict=subscription_dict, + ) + + return Subscription.from_preset( + preset=discography_preset, + config=config, + ) + + +@pytest.fixture +def expected_discography_download(): + # turn off black formatter here for readability + # fmt: off + return ExpectedDownload( + expected_md5_file_hashes={ + # Download mapping + Path("j_b/.ytdl-sub-jb-download-archive.json"): "ae55de93b71267b5712c9a3d06c07c26", + + # Entry files (singles) + Path("j_b/[2021] Baby Santana's Dorian Groove/01 - Baby Santana's Dorian Groove.mp3"): "bffbd558e12c6a9e029dc136a88342c4", + Path("j_b/[2021] Baby Santana's Dorian Groove/folder.jpg"): "511c43d7e939c70953cf2cd3cd437072", + + Path("j_b/[2021] Purple Clouds/01 - Purple Clouds.mp3"): "038db58aebe2ba875b733932b42a94d6", + Path("j_b/[2021] Purple Clouds/folder.jpg"): "511c43d7e939c70953cf2cd3cd437072", + + # Entry files (albums) + Path("j_b/[2022] Acoustic Treats/01 - 20160426 184214.mp3"): "e145f0a2f6012768280c38655ca58065", + Path("j_b/[2022] Acoustic Treats/02 - 20160502 123150.mp3"): "60c8b8817a197a13e4bb90903af612c5", + Path("j_b/[2022] Acoustic Treats/03 - 20160504 143832.mp3"): "8265b7e4f79878af877bc6ecd9757efe", + Path("j_b/[2022] Acoustic Treats/04 - 20160601 221234.mp3"): "accf46b76891d2954b893d0f91d82816", + Path("j_b/[2022] Acoustic Treats/05 - 20160601 222440.mp3"): "e1f584f523336160d5c1104a61de77f3", + Path("j_b/[2022] Acoustic Treats/06 - 20170604 190236.mp3"): "f6885b25901177f0357649afe97328cc", + Path("j_b/[2022] Acoustic Treats/07 - 20170612 193646.mp3"): "fa057f221cbe4cf2442cd2fdb960743e", + Path("j_b/[2022] Acoustic Treats/08 - 20170628 215206.mp3"): "7794ae812c64580e2ac8fc457d5cc85f", + Path("j_b/[2022] Acoustic Treats/09 - Finding Home.mp3"): "adbf02eddb2090c008eb497d13ff84b9", + Path("j_b/[2022] Acoustic Treats/10 - Shallow Water WIP.mp3"): "65bb10c84366c71498161734f953e93d", + Path("j_b/[2022] Acoustic Treats/11 - Untold History.mp3"): "6904b2918e5dc38d9a9f72d967eb74bf", + Path("j_b/[2022] Acoustic Treats/folder.jpg"): "511c43d7e939c70953cf2cd3cd437072", + + + } + ) + # fmt: on + + +class TestSoundcloudDiscography: + """ + Downloads my (bad) SC recordings I made. Ensure the above files exist and have the + expected md5 file hashes. + """ + + def test_discography_download( + self, discography_subscription, expected_discography_download, output_directory + ): + discography_subscription.download() + expected_discography_download.assert_files_exist(relative_directory=output_directory) diff --git a/tests/e2e/youtube/test_channel_as_kodi_tv_show.py b/tests/e2e/youtube/test_channel_as_kodi_tv_show.py index e7a6ae28..f64576e8 100644 --- a/tests/e2e/youtube/test_channel_as_kodi_tv_show.py +++ b/tests/e2e/youtube/test_channel_as_kodi_tv_show.py @@ -65,7 +65,7 @@ def expected_full_channel_download(): return ExpectedDownload( expected_md5_file_hashes={ # Download mapping - Path("pz/.ytdl-sub-pz-download-archive.json"): "add71021318bf87a3facb965fd38bd7f", + Path("pz/.ytdl-sub-pz-download-archive.json"): "b7e7c19d2cf0277e4e42453a64fbaa90", # Output directory files Path("pz/fanart.jpg"): "e6e323373c8902568e96e374817179cf", @@ -159,7 +159,7 @@ def expected_recent_channel_download(): return ExpectedDownload( expected_md5_file_hashes={ # Download mapping - Path("pz/.ytdl-sub-pz-download-archive.json"): "a133d9ea8a63e239cd41b799b9031fd5", + Path("pz/.ytdl-sub-pz-download-archive.json"): "b1675ca4d9f0d4b9c2102b6749e4cdfd", # Output directory files Path("pz/fanart.jpg"): "e6e323373c8902568e96e374817179cf", @@ -215,7 +215,7 @@ def expected_rolling_recent_channel_download(): return ExpectedDownload( expected_md5_file_hashes={ # Download mapping - Path("pz/.ytdl-sub-pz-download-archive.json"): "8013b4d2ba6921c9347c014ac915e3f6", + Path("pz/.ytdl-sub-pz-download-archive.json"): "9ae3463bd2dc39830003aba68a276df4", # Output directory files Path("pz/fanart.jpg"): "e6e323373c8902568e96e374817179cf", diff --git a/tests/e2e/youtube/test_playlist_as_kodi_music_videos.py b/tests/e2e/youtube/test_playlist_as_kodi_music_videos.py index 5d559adb..411e9bca 100644 --- a/tests/e2e/youtube/test_playlist_as_kodi_music_videos.py +++ b/tests/e2e/youtube/test_playlist_as_kodi_music_videos.py @@ -64,7 +64,7 @@ def expected_playlist_download(): return ExpectedDownload( expected_md5_file_hashes={ # Download mapping - Path(".ytdl-sub-jmc-download-archive.json"): "4c368c5a12dc3ddb4c9d68fd9a782f24", + Path(".ytdl-sub-jmc-download-archive.json"): "7541aa75606b86bff5ff276895520cf0", # Entry files Path("JMC - Jesse's Minecraft Server [Trailer - Feb.1].jpg"): "048a19cf0f674437351872c3f312ebf1",