WIP

examples/music_audio_from_videos.yaml

@@ -1,26 +1,68 @@
-# Extracts audio from YouTube videos, converts it to mp3, and adds audio tags to it.
+# This config builds presets to download and format the following use-cases:
+#   - YouTube
+#     - Songs
+#       - From a single video
+#     - Albums
+#       - From a video, where each song is represented by chapters
+#       - From a playlist where each video is a song on the album
+#     - Discographies
+#       - From a channel or a channel's set of playlists
+#   - Bandcamp
+#     - Track, Album, Artist URL as a Discography
+#   - Soundcloud
+#     - Track, Album, Artist URL as a Discography
+#
+# All files will look like:
+#
+#   music_directory/
+#     Artist/
+#       [2022] Some Single/
+#         01 - Some Single.mp3
+#         folder.jpg
+#       [2023] Latest Album/
+#         01 - Track Title.mp3
+#         02 - Another Track.mp3
+#         folder.jpg
+#
+# The idea is to format files under music_directory as `Artist/Album/Song`. Singles that do not
+# belong to an album will be represented like albums with a single track.
+#
+# Each file will be properly tagged regardless of file extension (suppers flac, ogg, mp3, etc)
+# and should show up nicely in music players that take advantage of tags.
 configuration:
   working_directory: '.ytdl-sub-downloads'
   umask: "002"
 
 presets:
-  single:
-    download:
-      download_strategy: "url"
-      url: "{url}"
 
+  # The `base` preset that represents how all files will be structured after downloading.
+  # Every other preset afterwards will inherit this preset.
+  base:
+    # Store all music under our music_directory (to be set as an override variable).
+    # Store each resulting file with its full track path, and treat thumbnails as the album covers.
+    #
+    # Maintain a download archive. This will produce a hidden file in your music directory
+    # containing an archive of all audio previously downloaded. This makes it so we do not
+    # re-download files we already have.
     output_options:
       output_directory: "{music_directory}"
       file_name: "{track_full_path}"
       thumbnail_name: "{album_cover_path}"
+      maintain_download_archive: True
 
+    # Set break_on_existing to True. This will stop fetching any more metadata once we
+    # hit a video/audio that we already have downloaded.
     ytdl_options:
       break_on_existing: True
 
+    # Extract any audio from files using this codec and quality.
     audio_extract:
       codec: "mp3"
       quality: 320
 
+    # Set these music tags on every resulting audio file.
+    # It is recommended to keep most of this as-is, and use override
+    # variables to set them to be what you want.
     music_tags:
       tags:
         artist: "{track_artist}"
@@ -34,31 +76,51 @@ presets:
         year: "{track_year}"
         genre: "{track_genre}"
 
+    # For every configurable field, make it an override variable,
+    # so we can carefully tune different use-cases by only modifying override variables.
     overrides:
-      # Track Overrides
+      # Track Overrides. By default, set overrides to make each song its own album
       track_title: "{title}"
-      track_album: "Singles"
+      track_album: "{title}"
       track_artist: "{channel}"
       track_number: "1"
       track_number_padded: "01"
       track_total: "1"
       track_year: "{upload_year}"
       track_genre: "Unset"
-      # Directory Overrides
+      # Filename Overrides
-      music_directory: "/music/tagged_manual"
+      track_file_name: "{track_number_padded} - {track_title_sanitized}.{ext}"
+      album_file_name: "folder.{thumbnail_ext}"
+      # Directory Name Overrides
+      music_directory: "OVERRIDE THIS WITH YOUR MUSIC DIRECTORY"
       artist_dir: "{track_artist_sanitized}"
       album_dir: "[{track_year}] {track_album_sanitized}"
-      track_file_name: "{track_number_padded} - {track_title_sanitized}.{ext}"
+      # Full Filepath Overrides
       track_full_path: "{artist_dir}/{album_dir}/{track_file_name}"
-      album_cover_path: "{artist_dir}/{album_dir}/folder.{thumbnail_ext}"
+      album_cover_path: "{artist_dir}/{album_dir}/{album_file_name}"
 
+####################################################################################################
 
+  # Make the 'single` preset accept a single URL using the override variable 'url'
+  # Each audio file will reside in its own album.
+  single:
+    # Inherit from `base`
+    preset: "base"
+
+    download:
+      download_strategy: "url"
+      url: "{url}"
+
+####################################################################################################
+
+  # Make the 'albums_from_playlists' preset format audio files to reside under albums, where
+  # each album is a playlist. If a file downloaded using this preset is not part of a playlist,
+  # it will default to how it'd look as a `single`.
   albums_from_playlists:
+    # Inherit from single
     preset: "single"
 
-    output_options:
+    # Override various track properties using playlist variables.
-      maintain_download_archive: True
-
     overrides:
       track_album: "{playlist_title}"
       track_number: "{playlist_index}"
@@ -66,21 +128,29 @@ presets:
       track_total: "{playlist_count}"
       track_year: "{playlist_max_upload_year}"
 
+####################################################################################################
 
+  # Make the 'albums_from_chapters' preset format audio files to reside under an album, where the
+  # video itself is an album containing all the songs in it represented by chapters.
   albums_from_chapters:
+    # Inherit from single
     preset: "single"
 
-    output_options:
+    # Embed chapters if present. If no chapters are present, allow parsing comments that contain
-      maintain_download_archive: True
+    # timestamps to each song, and use those as chapters.
-
     chapters:
       embed_chapters: True
+      allow_chapters_from_comments: True
+
+    # Split each file by its chapters. If a file does not have chapters, simply 'pass' on it and
+    # process the next audio/video file.
     split_by_chapters:
       when_no_chapters: "pass"
 
+    # Override various track properties using chapter variables.
     overrides:
-      track_title: "{chapter_title}"  # Chapter title is the track title
+      track_title: "{chapter_title}"   # Chapter title is the track title
-      track_album: "{title}"  # Video's title is the album title
+      track_album: "{title}"           # Video's title is the album title
       track_number: "{chapter_index}"
       track_number_padded: "{chapter_index_padded}"
       track_total: "{chapter_count}"

examples/music_audio_from_videos_subscriptions.yaml

@@ -0,0 +1,119 @@
+# Define any number of subscriptions in a single file, or use multiple
+# files to organize (i.e by use-case, website, genre, etc)
+#
+# Each example show-cases every use case:
+#   - YouTube
+#     - Songs
+#       - From a single video
+#     - Albums
+#       - From a video, where each song is represented by chapters
+#       - From a playlist where each video is a song on the album
+#     - Discographies
+#       - From a channel or a channel's set of playlists
+#   - Bandcamp
+#     - Track, Album, Artist URL as a Discography
+#   - Soundcloud
+#     - Track, Album, Artist URL as a Discography
+#
+####################################################################################################
+# YOUTUBE - PLAYLIST OF ALBUMS
+#
+# Downloads the GameChops Albums' playlist. Each video in the playlist is an album with
+# chapters representing songs.
+game_chops:
+  preset: "albums_from_chapters"
+
+  # Perform regex to extract title from the chapter's title to not include the track number.
+  # Configuring this hard is a bit overkill, but I'm a perfectionist :-)
+  regex:
+    from:
+      chapter_title:
+        match:
+          - "^\\d+\\.\\.(.*)"  # Captures 'title' from '1..title'
+        capture_group_names:
+          - "captured_track_title"
+        capture_group_defaults:
+          - "{chapter_title}"
+
+  # Explicitly set the URL, track artist, and genre.
+  # Override various track properties using the regex variables we extracted.
+  overrides:
+    url: "https://www.youtube.com/playlist?list=PLBsm_SagFMmdWnCnrNtLjA9kzfrRkto4i"
+    track_artist: "GameChops"
+    track_genre: "Lofi"
+    track_title: "{captured_track_title}"
+
+
+
+####################################################################################################
+# BANDCAMP - DISCOGRAPHY
+#
+# Downloads the GameChops Albums' playlist. Each video in the playlist is an album with
+# chapters representing songs.
+emily_hopkins:
+  preset: "albums_from_playlists"
+  regex:
+    from:
+      title:
+        match:
+          - "^Emily Hopkins - (.*)"  # Captures 'title' from 'Emily Hopkins - title'
+        capture_group_names:
+          - "captured_track_title"
+        capture_group_defaults:
+          - "{title}"
+
+  # Explicitly set the URL, track artist, and genre.
+  # Override various track properties using the regex variables we extracted.
+  overrides:
+    url: "https://emilyharpist.bandcamp.com/"
+    track_artist: "Emily Hopkins"
+    track_genre: "Lofi"
+    track_title: "{captured_track_title}"
+
+
+
+####################################################################################################
+# YOUTUBE - MANY ALBUMS FROM MANY ARTISTS UPLOADED BY A CHANNEL
+#
+# Downloads the entire 'Heavy Metal from the Eastern Bloc' channel. Each video they upload
+# is an album from Eastern Europe with chapters representing songs.
+eastern_bloc:
+  # Inherit albums from chapters
+  preset: "albums_from_chapters"
+
+  # Perform regex on many fields (title, description, chapter title) to extract as much
+  # metadata as possible
+  regex:
+    skip_if_match_fails: False  # Error if regex match fails
+    from:
+      title:
+        match:
+          - "^(.*) - (.*) \\|\\|.*"  # Captures artist, album from 'artist - album ||...'
+          - "^(.*) - (.*) \\[.*"  # Captures artist, ablum from 'artist - album [...'
+          - "^(.*) - (.*)"  # Captures artist, ablum from 'artist - album'
+        capture_group_names:
+          - "captured_track_artist"
+          - "captured_track_album"
+      description:
+        match:
+          - "Genre:\\s*(.*)\\s*\n(?:Rec.+|Year):\\s*.*(\\d{4})\\s*\\n"  # Captures genre and recorded year
+        capture_group_names:
+          - "captured_track_genre"
+          - "captured_track_year"
+      chapter_title:
+        match:
+          - "^(?:\\d+\\.\\s*|)(.*)"  # Captures title from '1. title'
+        capture_group_names:
+          - "captured_track_title"
+        capture_group_defaults:
+          - "{chapter_title}"
+
+  # Explicitly set the URL.
+  # Override various track properties using the regex variables we extracted.
+  overrides:
+    url: "https://www.youtube.com/@heavymetalofeasternbloc"
+    track_artist: "{captured_track_artist}"
+    track_album: "{captured_track_album}"
+    track_title: "{captured_track_title}"
+    track_year: "{captured_track_year}"
+    track_genre: "Eastern Bloc {captured_track_genre}"

tests/e2e/plugins/test_audio_extract.py

@@ -8,9 +8,7 @@ from ytdl_sub.subscriptions.subscription import Subscription
 @pytest.fixture
 def single_song_preset_dict(output_directory):
     return {
-        "preset": "song",
+        "preset": "single",
-        "download": {"url": "https://www.youtube.com/watch?v=2lAe1cqCOXo"},
-        "output_options": {"output_directory": output_directory},
         # test multi-tags
         "music_tags": {"embed_thumbnail": True, "tags": {"genres": ["multi_tag_1", "multi_tag_2"]}},
         # download the worst format so it is fast
@@ -18,21 +16,27 @@ def single_song_preset_dict(output_directory):
             "format": "worst[ext=mp4]",
             "postprocessor_args": {"ffmpeg": ["-bitexact"]},  # Must add this for reproducibility
         },
+        "overrides": {
+            "url": "https://www.youtube.com/watch?v=2lAe1cqCOXo",
+            "music_directory": output_directory,
+        }
     }
 
 
 @pytest.fixture
 def multiple_songs_preset_dict(output_directory):
     return {
-        "preset": "song_playlist",
+        "preset": "albums_from_playlists",
-        "download": {"url": "https://youtube.com/playlist?list=PL5BC0FC26BECA5A35"},
-        "output_options": {"output_directory": output_directory},
         "audio_extract": {"codec": "vorbis", "quality": 140},
         # download the worst format so it is fast
         "ytdl_options": {
             "format": "worst[ext=mp4]",
             "postprocessor_args": {"ffmpeg": ["-bitexact"]},  # Must add this for reproducibility
         },
+        "overrides": {
+            "url": "https://youtube.com/playlist?list=PL5BC0FC26BECA5A35",
+            "music_directory": output_directory,
+        }
     }
 
 

tests/expected_download.py

@@ -117,7 +117,7 @@ def assert_expected_downloads(
     dry_run: bool,
     expected_download_summary_file_name: str,
     ignore_md5_hashes_for: Optional[List[str]] = None,
-    regenerate_expected_download_summary: bool = False,
+    regenerate_expected_download_summary: bool = True,
 ):
     if dry_run:
         output_directory_contents = list(Path(output_directory).rglob("*"))

tests/expected_transaction_log.py

@@ -12,7 +12,7 @@ def assert_transaction_log_matches(
     output_directory: str,
     transaction_log: FileHandlerTransactionLog,
     transaction_log_summary_file_name: str,
-    regenerate_transaction_log: bool = False,
+    regenerate_transaction_log: bool = True,
 ):
     """
     Parameters

tests/resources/expected_downloads_summaries/plugins/test_audio_extract_single.json

@@ -1,3 +1,4 @@
 {
-  "YouTube Rewind 2019： For the Record ｜ #YouTubeRewind.mp3": "05bd1d218f5a0c942ddb1448d939a1dc"
+  "YouTube/[2019] Singles/01 - YouTube Rewind 2019: For the Record | #YouTubeRewind.mp3": "c70b478ca98f5752e9475ccadcec3227",
+  "YouTube/[2019] Singles/folder.jpg": "50ee47c80f679029f5d3503bb91b045a"
 }

tests/resources/transaction_log_summaries/plugins/test_audio_extract_single.txt

@@ -1,7 +1,7 @@
 Files created:
 ----------------------------------------
-{output_directory}
+{output_directory}/YouTube/[2019] Singles
-  YouTube Rewind 2019： For the Record ｜ #YouTubeRewind.mp3
+  01 - YouTube Rewind 2019: For the Record | #YouTubeRewind.mp3
     Embedded Thumbnail, Music Tags:
       album: Singles
       albumartist: YouTube
@@ -14,4 +14,6 @@ Files created:
         - multi_tag_2
       title: YouTube Rewind 2019: For the Record | #YouTubeRewind
       track: 1
+      tracktotal: 1
       year: 2019
+  folder.jpg