Add soundcloud discography example w/test (#49)

* example sc yaml

* discography test

* json sort_keys

examples/soundcloud_discography_config.yaml

@@ -0,0 +1,75 @@
+# This example shows how to download and format a Soundcloud artist's
+# discography with tags and album ar. We will configure this to make
+# the output directory formatted as:
+#
+#   /path/to/music
+#       /The Be Sharps
+#           /[2020] My first upload
+#               01 - My first upload.mp3
+#               folder.jpg
+#           /[2021] My first album
+#               01 - Track one.mp3
+#               02 - Track two.mp3
+#               folder.jpg
+#       /Another artist
+#           ...
+#
+configuration:
+  working_directory: '.ytdl-sub-downloads'
+
+presets:
+  sc_discography:
+    # A Soundcloud artist's albums and singles will be our source/download
+    # strategy. This will group together album tracks, and treat any track
+    # not part of an album as a single.
+    soundcloud:
+      download_strategy: "albums_and_singles"
+      skip_premiere_tracks: True
+
+    # For advanced YTDL users only.
+    # This sets the download format to grab the best mp3 available.
+    # Other formats should work but have not been tested.
+    ytdl_options:
+      format: 'bestaudio[ext=mp3]'
+
+    # For each song downloaded, set the file and thumbnail name here.
+    # The output directory stores all artists in a shared {music_directory}.
+    # We store the audio file in its respective {album_directory_name} folder
+    # with the track number and title. The (thumbnail) album art is stored
+    # in the album folder.
+    #
+    # Another field worth mentioning is maintain_download_archive=True. This
+    # is generally a good thing to enable with artists because it will
+    # store previously downloaded song IDs to tell YTDL not to re-download
+    # them on a successive invocation.
+    output_options:
+      output_directory: "/{music_directory}/{artist_sanitized}"
+      file_name: "{album_directory_name}/{track_number_padded} - {title_sanitized}.{ext}"
+      thumbnail_name: "{album_directory_name}/folder.jpg"
+      maintain_download_archive: True
+
+    # Always convert the video thumbnail to jpg
+    # TODO: always convert all thumbnails to jpg in code
+    convert_thumbnail:
+      to: "jpg"
+
+    # For each song downloaded, populate the audio file with music tags.
+    # Tagging should work with most audio file formats. See
+    # https://ytdl-sub.readthedocs.io/en/latest/config.html#music-tags
+    # for more details.
+    music_tags:
+      tags:
+        artist: "{artist}"
+        albumartist: "{artist}"
+        title: "{title}"
+        album: "{album}"
+        track: "{track_number}"
+        year: "{album_year}"
+        genre: "Unset"
+
+    # Overrides is a section where we can define our own variables, and use them in
+    # any other section. We define our music directory and album directory names here,
+    # which gets reused above for the audio file name and album art path.
+    overrides:
+      album_directory_name: "[{album_year}] {album_sanitized}"
+      music_directory: "/path/to/music"

src/ytdl_sub/downloaders/soundcloud_downloader.py

@@ -65,6 +65,24 @@ class SoundcloudDownloader(
         """Returns full artist url"""
         return f"https://soundcloud.com/{artist_name}"
 
+    @classmethod
+    def artist_albums_url(cls, artist_name: str) -> str:
+        """
+        Returns
+        -------
+        Full artist album url
+        """
+        return cls.artist_url(artist_name) + "/albums"
+
+    @classmethod
+    def artist_tracks_url(cls, artist_name: str) -> str:
+        """
+        Returns
+        -------
+        Full artist tracks url
+        """
+        return cls.artist_url(artist_name) + "/tracks"
+
 
 ###############################################################################
 # Soundcloud albums and singles downloader + options
@@ -139,21 +157,15 @@ class SoundcloudAlbumsAndSinglesDownloader(
 
         return list(albums.values())
 
-    def _get_singles(self, entry_dicts: List[Dict]) -> List[SoundcloudTrack]:
-        artist_id = None
+    def _get_singles(
+        self, entry_dicts: List[Dict], albums: List[SoundcloudAlbum]
+    ) -> List[SoundcloudTrack]:
         tracks: List[SoundcloudTrack] = []
 
-        # First, get the artist entry. All single tracks that do not belong to an album will belong
-        # to the 'artist' playlist
+        # Get all tracks that are not part of an album
         for entry_dict in entry_dicts:
-            if entry_dict.get("extractor") == "soundcloud:user":
-                artist_id = entry_dict["id"]
-
-        # Then, get all singles that belong to the 'artist' playlist
-        for entry_dict in entry_dicts:
-            if (
-                entry_dict.get("extractor") == "soundcloud"
-                and entry_dict.get("playlist_id") == artist_id
+            if entry_dict.get("extractor") == "soundcloud" and not any(
+                entry_dict in album for album in albums
             ):
                 tracks.append(
                     SoundcloudTrack(entry_dict=entry_dict, working_directory=self.working_directory)
@@ -165,14 +177,17 @@ class SoundcloudAlbumsAndSinglesDownloader(
         """
         Soundcloud subscription to download albums and tracks as singles.
         """
-        artist_url = self.artist_url(artist_name=self.download_options.username)
-        entry_dicts = self.extract_info_via_info_json(url=artist_url)
+        artist_albums_url = self.artist_albums_url(artist_name=self.download_options.username)
+        artist_tracks_url = self.artist_tracks_url(artist_name=self.download_options.username)
+
+        album_entry_dicts = self.extract_info_via_info_json(url=artist_albums_url)
+        tracks_entry_dicts = self.extract_info_via_info_json(url=artist_tracks_url)
 
         # Get all of the artist's albums
-        albums = self._get_albums(entry_dicts=entry_dicts)
+        albums = self._get_albums(entry_dicts=album_entry_dicts)
 
         # Then, get all singles
-        tracks = self._get_singles(entry_dicts=entry_dicts)
+        tracks = self._get_singles(entry_dicts=tracks_entry_dicts, albums=albums)
 
         # Append all album tracks as SoundcloudAlbumTrack classes to the singles
         for album in albums:

src/ytdl_sub/entries/soundcloud.py

@@ -134,3 +134,13 @@ class SoundcloudAlbum(Entry):
         Number of tracks in the album (technically a playlist)
         """
         return self.kwargs("playlist_count")
+
+    def __contains__(self, item):
+        """
+        Returns
+        -------
+        True if the the item (entry_dict) has the same id as one of the tracks. False otherwise.
+        """
+        if isinstance(item, dict):
+            return any(item.get("id") == track.uid for track in self.tracks)
+        return False

src/ytdl_sub/ytdl_additions/enhanced_download_archive.py

@@ -278,6 +278,7 @@ class DownloadMappings:
                 )
             },
             indent=2,
+            sort_keys=True,
         )
 
         with open(output_json_file, "w", encoding="utf8") as file:

tests/e2e/soundcloud/test_soundcloud_discography.py

@@ -0,0 +1,101 @@
+from pathlib import Path
+
+import pytest
+from e2e.expected_download import ExpectedDownload
+
+from ytdl_sub.config.config_file import ConfigFile
+from ytdl_sub.config.preset import Preset
+from ytdl_sub.subscriptions.subscription import Subscription
+
+
+@pytest.fixture
+def config_path():
+    return "examples/soundcloud_discography_config.yaml"
+
+
+@pytest.fixture
+def subscription_name():
+    return "jb"
+
+
+@pytest.fixture
+def config(config_path):
+    return ConfigFile.from_file_path(config_path=config_path)
+
+
+@pytest.fixture
+def subscription_dict(output_directory, subscription_name):
+    return {
+        "preset": "sc_discography",
+        "soundcloud": {"username": "jessebannon"},
+        # override the output directory with our fixture-generated dir
+        "output_options": {"output_directory": output_directory + "/{artist_sanitized}"},
+        # download the worst format so it is fast
+        "ytdl_options": {
+            "format": "worst[ext=mp3]",
+        },
+        "overrides": {"artist": "j_b"},
+    }
+
+
+@pytest.fixture
+def discography_subscription(config, subscription_name, subscription_dict):
+    discography_preset = Preset.from_dict(
+        config=config,
+        preset_name=subscription_name,
+        preset_dict=subscription_dict,
+    )
+
+    return Subscription.from_preset(
+        preset=discography_preset,
+        config=config,
+    )
+
+
+@pytest.fixture
+def expected_discography_download():
+    # turn off black formatter here for readability
+    # fmt: off
+    return ExpectedDownload(
+        expected_md5_file_hashes={
+            # Download mapping
+            Path("j_b/.ytdl-sub-jb-download-archive.json"): "ae55de93b71267b5712c9a3d06c07c26",
+
+            # Entry files (singles)
+            Path("j_b/[2021] Baby Santana's Dorian Groove/01 - Baby Santana's Dorian Groove.mp3"): "bffbd558e12c6a9e029dc136a88342c4",
+            Path("j_b/[2021] Baby Santana's Dorian Groove/folder.jpg"): "511c43d7e939c70953cf2cd3cd437072",
+
+            Path("j_b/[2021] Purple Clouds/01 - Purple Clouds.mp3"): "038db58aebe2ba875b733932b42a94d6",
+            Path("j_b/[2021] Purple Clouds/folder.jpg"): "511c43d7e939c70953cf2cd3cd437072",
+
+            # Entry files (albums)
+            Path("j_b/[2022] Acoustic Treats/01 - 20160426 184214.mp3"): "e145f0a2f6012768280c38655ca58065",
+            Path("j_b/[2022] Acoustic Treats/02 - 20160502 123150.mp3"): "60c8b8817a197a13e4bb90903af612c5",
+            Path("j_b/[2022] Acoustic Treats/03 - 20160504 143832.mp3"): "8265b7e4f79878af877bc6ecd9757efe",
+            Path("j_b/[2022] Acoustic Treats/04 - 20160601 221234.mp3"): "accf46b76891d2954b893d0f91d82816",
+            Path("j_b/[2022] Acoustic Treats/05 - 20160601 222440.mp3"): "e1f584f523336160d5c1104a61de77f3",
+            Path("j_b/[2022] Acoustic Treats/06 - 20170604 190236.mp3"): "f6885b25901177f0357649afe97328cc",
+            Path("j_b/[2022] Acoustic Treats/07 - 20170612 193646.mp3"): "fa057f221cbe4cf2442cd2fdb960743e",
+            Path("j_b/[2022] Acoustic Treats/08 - 20170628 215206.mp3"): "7794ae812c64580e2ac8fc457d5cc85f",
+            Path("j_b/[2022] Acoustic Treats/09 - Finding Home.mp3"): "adbf02eddb2090c008eb497d13ff84b9",
+            Path("j_b/[2022] Acoustic Treats/10 - Shallow Water WIP.mp3"): "65bb10c84366c71498161734f953e93d",
+            Path("j_b/[2022] Acoustic Treats/11 - Untold History.mp3"): "6904b2918e5dc38d9a9f72d967eb74bf",
+            Path("j_b/[2022] Acoustic Treats/folder.jpg"): "511c43d7e939c70953cf2cd3cd437072",
+
+
+        }
+    )
+    # fmt: on
+
+
+class TestSoundcloudDiscography:
+    """
+    Downloads my (bad) SC recordings I made. Ensure the above files exist and have the
+    expected md5 file hashes.
+    """
+
+    def test_discography_download(
+        self, discography_subscription, expected_discography_download, output_directory
+    ):
+        discography_subscription.download()
+        expected_discography_download.assert_files_exist(relative_directory=output_directory)

tests/e2e/youtube/test_channel_as_kodi_tv_show.py

@@ -65,7 +65,7 @@ def expected_full_channel_download():
     return ExpectedDownload(
         expected_md5_file_hashes={
             # Download mapping
-            Path("pz/.ytdl-sub-pz-download-archive.json"): "add71021318bf87a3facb965fd38bd7f",
+            Path("pz/.ytdl-sub-pz-download-archive.json"): "b7e7c19d2cf0277e4e42453a64fbaa90",
 
             # Output directory files
             Path("pz/fanart.jpg"): "e6e323373c8902568e96e374817179cf",
@@ -159,7 +159,7 @@ def expected_recent_channel_download():
     return ExpectedDownload(
         expected_md5_file_hashes={
             # Download mapping
-            Path("pz/.ytdl-sub-pz-download-archive.json"): "a133d9ea8a63e239cd41b799b9031fd5",
+            Path("pz/.ytdl-sub-pz-download-archive.json"): "b1675ca4d9f0d4b9c2102b6749e4cdfd",
 
             # Output directory files
             Path("pz/fanart.jpg"): "e6e323373c8902568e96e374817179cf",
@@ -215,7 +215,7 @@ def expected_rolling_recent_channel_download():
     return ExpectedDownload(
         expected_md5_file_hashes={
             # Download mapping
-            Path("pz/.ytdl-sub-pz-download-archive.json"): "8013b4d2ba6921c9347c014ac915e3f6",
+            Path("pz/.ytdl-sub-pz-download-archive.json"): "9ae3463bd2dc39830003aba68a276df4",
 
             # Output directory files
             Path("pz/fanart.jpg"): "e6e323373c8902568e96e374817179cf",

tests/e2e/youtube/test_playlist_as_kodi_music_videos.py

@@ -64,7 +64,7 @@ def expected_playlist_download():
     return ExpectedDownload(
         expected_md5_file_hashes={
             # Download mapping
-            Path(".ytdl-sub-jmc-download-archive.json"): "4c368c5a12dc3ddb4c9d68fd9a782f24",
+            Path(".ytdl-sub-jmc-download-archive.json"): "7541aa75606b86bff5ff276895520cf0",
 
             # Entry files
             Path("JMC - Jesse's Minecraft Server [Trailer - Feb.1].jpg"): "048a19cf0f674437351872c3f312ebf1",