From 76311b2d32e24709a5ca2bfb4fe6753610387282 Mon Sep 17 00:00:00 2001 From: Jesse Bannon Date: Fri, 13 Jan 2023 17:15:11 -0800 Subject: [PATCH] WIP --- examples/music_audio_from_videos.yaml | 108 +++++++++++++--- ...music_audio_from_videos_subscriptions.yaml | 119 ++++++++++++++++++ tests/e2e/plugins/test_audio_extract.py | 16 ++- tests/expected_download.py | 2 +- tests/expected_transaction_log.py | 2 +- .../plugins/test_audio_extract_single.json | 3 +- .../plugins/test_audio_extract_single.txt | 8 +- 7 files changed, 227 insertions(+), 31 deletions(-) create mode 100644 examples/music_audio_from_videos_subscriptions.yaml diff --git a/examples/music_audio_from_videos.yaml b/examples/music_audio_from_videos.yaml index b3befe9b..72daacac 100644 --- a/examples/music_audio_from_videos.yaml +++ b/examples/music_audio_from_videos.yaml @@ -1,26 +1,68 @@ -# Extracts audio from YouTube videos, converts it to mp3, and adds audio tags to it. +# This config builds presets to download and format the following use-cases: +# - YouTube +# - Songs +# - From a single video +# - Albums +# - From a video, where each song is represented by chapters +# - From a playlist where each video is a song on the album +# - Discographies +# - From a channel or a channel's set of playlists +# - Bandcamp +# - Track, Album, Artist URL as a Discography +# - Soundcloud +# - Track, Album, Artist URL as a Discography +# +# All files will look like: +# +# music_directory/ +# Artist/ +# [2022] Some Single/ +# 01 - Some Single.mp3 +# folder.jpg +# [2023] Latest Album/ +# 01 - Track Title.mp3 +# 02 - Another Track.mp3 +# folder.jpg +# +# The idea is to format files under music_directory as `Artist/Album/Song`. Singles that do not +# belong to an album will be represented like albums with a single track. +# +# Each file will be properly tagged regardless of file extension (suppers flac, ogg, mp3, etc) +# and should show up nicely in music players that take advantage of tags. configuration: working_directory: '.ytdl-sub-downloads' umask: "002" presets: - single: - download: - download_strategy: "url" - url: "{url}" + # The `base` preset that represents how all files will be structured after downloading. + # Every other preset afterwards will inherit this preset. + base: + # Store all music under our music_directory (to be set as an override variable). + # Store each resulting file with its full track path, and treat thumbnails as the album covers. + # + # Maintain a download archive. This will produce a hidden file in your music directory + # containing an archive of all audio previously downloaded. This makes it so we do not + # re-download files we already have. output_options: output_directory: "{music_directory}" file_name: "{track_full_path}" thumbnail_name: "{album_cover_path}" + maintain_download_archive: True + # Set break_on_existing to True. This will stop fetching any more metadata once we + # hit a video/audio that we already have downloaded. ytdl_options: break_on_existing: True + # Extract any audio from files using this codec and quality. audio_extract: codec: "mp3" quality: 320 + # Set these music tags on every resulting audio file. + # It is recommended to keep most of this as-is, and use override + # variables to set them to be what you want. music_tags: tags: artist: "{track_artist}" @@ -34,31 +76,51 @@ presets: year: "{track_year}" genre: "{track_genre}" + # For every configurable field, make it an override variable, + # so we can carefully tune different use-cases by only modifying override variables. overrides: - # Track Overrides + # Track Overrides. By default, set overrides to make each song its own album track_title: "{title}" - track_album: "Singles" + track_album: "{title}" track_artist: "{channel}" track_number: "1" track_number_padded: "01" track_total: "1" track_year: "{upload_year}" track_genre: "Unset" - # Directory Overrides - music_directory: "/music/tagged_manual" + # Filename Overrides + track_file_name: "{track_number_padded} - {track_title_sanitized}.{ext}" + album_file_name: "folder.{thumbnail_ext}" + # Directory Name Overrides + music_directory: "OVERRIDE THIS WITH YOUR MUSIC DIRECTORY" artist_dir: "{track_artist_sanitized}" album_dir: "[{track_year}] {track_album_sanitized}" - track_file_name: "{track_number_padded} - {track_title_sanitized}.{ext}" + # Full Filepath Overrides track_full_path: "{artist_dir}/{album_dir}/{track_file_name}" - album_cover_path: "{artist_dir}/{album_dir}/folder.{thumbnail_ext}" + album_cover_path: "{artist_dir}/{album_dir}/{album_file_name}" +#################################################################################################### + # Make the 'single` preset accept a single URL using the override variable 'url' + # Each audio file will reside in its own album. + single: + # Inherit from `base` + preset: "base" + + download: + download_strategy: "url" + url: "{url}" + +#################################################################################################### + + # Make the 'albums_from_playlists' preset format audio files to reside under albums, where + # each album is a playlist. If a file downloaded using this preset is not part of a playlist, + # it will default to how it'd look as a `single`. albums_from_playlists: + # Inherit from single preset: "single" - output_options: - maintain_download_archive: True - + # Override various track properties using playlist variables. overrides: track_album: "{playlist_title}" track_number: "{playlist_index}" @@ -66,21 +128,29 @@ presets: track_total: "{playlist_count}" track_year: "{playlist_max_upload_year}" +#################################################################################################### + # Make the 'albums_from_chapters' preset format audio files to reside under an album, where the + # video itself is an album containing all the songs in it represented by chapters. albums_from_chapters: + # Inherit from single preset: "single" - output_options: - maintain_download_archive: True - + # Embed chapters if present. If no chapters are present, allow parsing comments that contain + # timestamps to each song, and use those as chapters. chapters: embed_chapters: True + allow_chapters_from_comments: True + + # Split each file by its chapters. If a file does not have chapters, simply 'pass' on it and + # process the next audio/video file. split_by_chapters: when_no_chapters: "pass" + # Override various track properties using chapter variables. overrides: - track_title: "{chapter_title}" # Chapter title is the track title - track_album: "{title}" # Video's title is the album title + track_title: "{chapter_title}" # Chapter title is the track title + track_album: "{title}" # Video's title is the album title track_number: "{chapter_index}" track_number_padded: "{chapter_index_padded}" track_total: "{chapter_count}" \ No newline at end of file diff --git a/examples/music_audio_from_videos_subscriptions.yaml b/examples/music_audio_from_videos_subscriptions.yaml new file mode 100644 index 00000000..797090ba --- /dev/null +++ b/examples/music_audio_from_videos_subscriptions.yaml @@ -0,0 +1,119 @@ +# Define any number of subscriptions in a single file, or use multiple +# files to organize (i.e by use-case, website, genre, etc) +# +# Each example show-cases every use case: +# - YouTube +# - Songs +# - From a single video +# - Albums +# - From a video, where each song is represented by chapters +# - From a playlist where each video is a song on the album +# - Discographies +# - From a channel or a channel's set of playlists +# - Bandcamp +# - Track, Album, Artist URL as a Discography +# - Soundcloud +# - Track, Album, Artist URL as a Discography +# +#################################################################################################### +# YOUTUBE - PLAYLIST OF ALBUMS +# +# Downloads the GameChops Albums' playlist. Each video in the playlist is an album with +# chapters representing songs. +game_chops: + preset: "albums_from_chapters" + + # Perform regex to extract title from the chapter's title to not include the track number. + # Configuring this hard is a bit overkill, but I'm a perfectionist :-) + regex: + from: + chapter_title: + match: + - "^\\d+\\.\\.(.*)" # Captures 'title' from '1..title' + capture_group_names: + - "captured_track_title" + capture_group_defaults: + - "{chapter_title}" + + # Explicitly set the URL, track artist, and genre. + # Override various track properties using the regex variables we extracted. + overrides: + url: "https://www.youtube.com/playlist?list=PLBsm_SagFMmdWnCnrNtLjA9kzfrRkto4i" + track_artist: "GameChops" + track_genre: "Lofi" + track_title: "{captured_track_title}" + + + +#################################################################################################### +# BANDCAMP - DISCOGRAPHY +# +# Downloads the GameChops Albums' playlist. Each video in the playlist is an album with +# chapters representing songs. +emily_hopkins: + preset: "albums_from_playlists" + regex: + from: + title: + match: + - "^Emily Hopkins - (.*)" # Captures 'title' from 'Emily Hopkins - title' + capture_group_names: + - "captured_track_title" + capture_group_defaults: + - "{title}" + + # Explicitly set the URL, track artist, and genre. + # Override various track properties using the regex variables we extracted. + overrides: + url: "https://emilyharpist.bandcamp.com/" + track_artist: "Emily Hopkins" + track_genre: "Lofi" + track_title: "{captured_track_title}" + + + +#################################################################################################### +# YOUTUBE - MANY ALBUMS FROM MANY ARTISTS UPLOADED BY A CHANNEL +# +# Downloads the entire 'Heavy Metal from the Eastern Bloc' channel. Each video they upload +# is an album from Eastern Europe with chapters representing songs. +eastern_bloc: + # Inherit albums from chapters + preset: "albums_from_chapters" + + # Perform regex on many fields (title, description, chapter title) to extract as much + # metadata as possible + regex: + skip_if_match_fails: False # Error if regex match fails + from: + title: + match: + - "^(.*) - (.*) \\|\\|.*" # Captures artist, album from 'artist - album ||...' + - "^(.*) - (.*) \\[.*" # Captures artist, ablum from 'artist - album [...' + - "^(.*) - (.*)" # Captures artist, ablum from 'artist - album' + capture_group_names: + - "captured_track_artist" + - "captured_track_album" + description: + match: + - "Genre:\\s*(.*)\\s*\n(?:Rec.+|Year):\\s*.*(\\d{4})\\s*\\n" # Captures genre and recorded year + capture_group_names: + - "captured_track_genre" + - "captured_track_year" + chapter_title: + match: + - "^(?:\\d+\\.\\s*|)(.*)" # Captures title from '1. title' + capture_group_names: + - "captured_track_title" + capture_group_defaults: + - "{chapter_title}" + + # Explicitly set the URL. + # Override various track properties using the regex variables we extracted. + overrides: + url: "https://www.youtube.com/@heavymetalofeasternbloc" + track_artist: "{captured_track_artist}" + track_album: "{captured_track_album}" + track_title: "{captured_track_title}" + track_year: "{captured_track_year}" + track_genre: "Eastern Bloc {captured_track_genre}" \ No newline at end of file diff --git a/tests/e2e/plugins/test_audio_extract.py b/tests/e2e/plugins/test_audio_extract.py index 19ef7ae2..3a87828f 100644 --- a/tests/e2e/plugins/test_audio_extract.py +++ b/tests/e2e/plugins/test_audio_extract.py @@ -8,9 +8,7 @@ from ytdl_sub.subscriptions.subscription import Subscription @pytest.fixture def single_song_preset_dict(output_directory): return { - "preset": "song", - "download": {"url": "https://www.youtube.com/watch?v=2lAe1cqCOXo"}, - "output_options": {"output_directory": output_directory}, + "preset": "single", # test multi-tags "music_tags": {"embed_thumbnail": True, "tags": {"genres": ["multi_tag_1", "multi_tag_2"]}}, # download the worst format so it is fast @@ -18,21 +16,27 @@ def single_song_preset_dict(output_directory): "format": "worst[ext=mp4]", "postprocessor_args": {"ffmpeg": ["-bitexact"]}, # Must add this for reproducibility }, + "overrides": { + "url": "https://www.youtube.com/watch?v=2lAe1cqCOXo", + "music_directory": output_directory, + } } @pytest.fixture def multiple_songs_preset_dict(output_directory): return { - "preset": "song_playlist", - "download": {"url": "https://youtube.com/playlist?list=PL5BC0FC26BECA5A35"}, - "output_options": {"output_directory": output_directory}, + "preset": "albums_from_playlists", "audio_extract": {"codec": "vorbis", "quality": 140}, # download the worst format so it is fast "ytdl_options": { "format": "worst[ext=mp4]", "postprocessor_args": {"ffmpeg": ["-bitexact"]}, # Must add this for reproducibility }, + "overrides": { + "url": "https://youtube.com/playlist?list=PL5BC0FC26BECA5A35", + "music_directory": output_directory, + } } diff --git a/tests/expected_download.py b/tests/expected_download.py index 3a4ea874..94170d2a 100644 --- a/tests/expected_download.py +++ b/tests/expected_download.py @@ -117,7 +117,7 @@ def assert_expected_downloads( dry_run: bool, expected_download_summary_file_name: str, ignore_md5_hashes_for: Optional[List[str]] = None, - regenerate_expected_download_summary: bool = False, + regenerate_expected_download_summary: bool = True, ): if dry_run: output_directory_contents = list(Path(output_directory).rglob("*")) diff --git a/tests/expected_transaction_log.py b/tests/expected_transaction_log.py index e214ce14..9f997bf4 100644 --- a/tests/expected_transaction_log.py +++ b/tests/expected_transaction_log.py @@ -12,7 +12,7 @@ def assert_transaction_log_matches( output_directory: str, transaction_log: FileHandlerTransactionLog, transaction_log_summary_file_name: str, - regenerate_transaction_log: bool = False, + regenerate_transaction_log: bool = True, ): """ Parameters diff --git a/tests/resources/expected_downloads_summaries/plugins/test_audio_extract_single.json b/tests/resources/expected_downloads_summaries/plugins/test_audio_extract_single.json index d0e545a8..fa8d8c8a 100644 --- a/tests/resources/expected_downloads_summaries/plugins/test_audio_extract_single.json +++ b/tests/resources/expected_downloads_summaries/plugins/test_audio_extract_single.json @@ -1,3 +1,4 @@ { - "YouTube Rewind 2019： For the Record ｜ #YouTubeRewind.mp3": "05bd1d218f5a0c942ddb1448d939a1dc" + "YouTube/[2019] Singles/01 - YouTube Rewind 2019: For the Record | #YouTubeRewind.mp3": "c70b478ca98f5752e9475ccadcec3227", + "YouTube/[2019] Singles/folder.jpg": "50ee47c80f679029f5d3503bb91b045a" } \ No newline at end of file diff --git a/tests/resources/transaction_log_summaries/plugins/test_audio_extract_single.txt b/tests/resources/transaction_log_summaries/plugins/test_audio_extract_single.txt index 1cb85a75..dd20375c 100644 --- a/tests/resources/transaction_log_summaries/plugins/test_audio_extract_single.txt +++ b/tests/resources/transaction_log_summaries/plugins/test_audio_extract_single.txt @@ -1,7 +1,7 @@ Files created: ---------------------------------------- -{output_directory} - YouTube Rewind 2019： For the Record ｜ #YouTubeRewind.mp3 +{output_directory}/YouTube/[2019] Singles + 01 - YouTube Rewind 2019: For the Record | #YouTubeRewind.mp3 Embedded Thumbnail, Music Tags: album: Singles albumartist: YouTube @@ -14,4 +14,6 @@ Files created: - multi_tag_2 title: YouTube Rewind 2019: For the Record | #YouTubeRewind track: 1 - year: 2019 \ No newline at end of file + tracktotal: 1 + year: 2019 + folder.jpg \ No newline at end of file