danvics/ytdl-sub
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

WIP

Browse source
This commit is contained in:
Jesse Bannon 2023-01-13 17:15:11 -08:00
parent 6c06359c1e
commit 76311b2d32
7 changed files with 227 additions and 31 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

108
examples/music_audio_from_videos.yaml
View file

@ -1,26 +1,68 @@
# Extracts audio from YouTube videos, converts it to mp3, and adds audio tags to it.
# This config builds presets to download and format the following use-cases:
# - YouTube
# - Songs
# - From a single video
# - Albums
# - From a video, where each song is represented by chapters
# - From a playlist where each video is a song on the album
# - Discographies
# - From a channel or a channel's set of playlists
# - Bandcamp
# - Track, Album, Artist URL as a Discography
# - Soundcloud
# - Track, Album, Artist URL as a Discography
#
# All files will look like:
#
# music_directory/
# Artist/
# [2022] Some Single/
# 01 - Some Single.mp3
# folder.jpg
# [2023] Latest Album/
# 01 - Track Title.mp3
# 02 - Another Track.mp3
# folder.jpg
#
# The idea is to format files under music_directory as `Artist/Album/Song`. Singles that do not
# belong to an album will be represented like albums with a single track.
#
# Each file will be properly tagged regardless of file extension (suppers flac, ogg, mp3, etc)
# and should show up nicely in music players that take advantage of tags.
configuration:
working_directory: '.ytdl-sub-downloads'
umask: "002"
presets:
single:
download:
download_strategy: "url"
url: "{url}"
# The `base` preset that represents how all files will be structured after downloading.
# Every other preset afterwards will inherit this preset.
base:
# Store all music under our music_directory (to be set as an override variable).
# Store each resulting file with its full track path, and treat thumbnails as the album covers.
#
# Maintain a download archive. This will produce a hidden file in your music directory
# containing an archive of all audio previously downloaded. This makes it so we do not
# re-download files we already have.
output_options:
output_directory: "{music_directory}"
file_name: "{track_full_path}"
thumbnail_name: "{album_cover_path}"
maintain_download_archive: True
# Set break_on_existing to True. This will stop fetching any more metadata once we
# hit a video/audio that we already have downloaded.
ytdl_options:
break_on_existing: True
# Extract any audio from files using this codec and quality.
audio_extract:
codec: "mp3"
quality: 320
# Set these music tags on every resulting audio file.
# It is recommended to keep most of this as-is, and use override
# variables to set them to be what you want.
music_tags:
tags:
artist: "{track_artist}"
@ -34,31 +76,51 @@ presets:
year: "{track_year}"
genre: "{track_genre}"
# For every configurable field, make it an override variable,
# so we can carefully tune different use-cases by only modifying override variables.
overrides:
# Track Overrides
# Track Overrides. By default, set overrides to make each song its own album
track_title: "{title}"
track_album: "Singles"
track_album: "{title}"
track_artist: "{channel}"
track_number: "1"
track_number_padded: "01"
track_total: "1"
track_year: "{upload_year}"
track_genre: "Unset"
# Directory Overrides
music_directory: "/music/tagged_manual"
# Filename Overrides
track_file_name: "{track_number_padded} - {track_title_sanitized}.{ext}"
album_file_name: "folder.{thumbnail_ext}"
# Directory Name Overrides
music_directory: "OVERRIDE THIS WITH YOUR MUSIC DIRECTORY"
artist_dir: "{track_artist_sanitized}"
album_dir: "[{track_year}] {track_album_sanitized}"
track_file_name: "{track_number_padded} - {track_title_sanitized}.{ext}"
# Full Filepath Overrides
track_full_path: "{artist_dir}/{album_dir}/{track_file_name}"
album_cover_path: "{artist_dir}/{album_dir}/folder.{thumbnail_ext}"
album_cover_path: "{artist_dir}/{album_dir}/{album_file_name}"
####################################################################################################
# Make the 'single` preset accept a single URL using the override variable 'url'
# Each audio file will reside in its own album.
single:
# Inherit from `base`
preset: "base"
download:
download_strategy: "url"
url: "{url}"
####################################################################################################
# Make the 'albums_from_playlists' preset format audio files to reside under albums, where
# each album is a playlist. If a file downloaded using this preset is not part of a playlist,
# it will default to how it'd look as a `single`.
albums_from_playlists:
# Inherit from single
preset: "single"
output_options:
maintain_download_archive: True
# Override various track properties using playlist variables.
overrides:
track_album: "{playlist_title}"
track_number: "{playlist_index}"
@ -66,21 +128,29 @@ presets:
track_total: "{playlist_count}"
track_year: "{playlist_max_upload_year}"
####################################################################################################
# Make the 'albums_from_chapters' preset format audio files to reside under an album, where the
# video itself is an album containing all the songs in it represented by chapters.
albums_from_chapters:
# Inherit from single
preset: "single"
output_options:
maintain_download_archive: True
# Embed chapters if present. If no chapters are present, allow parsing comments that contain
# timestamps to each song, and use those as chapters.
chapters:
embed_chapters: True
allow_chapters_from_comments: True
# Split each file by its chapters. If a file does not have chapters, simply 'pass' on it and
# process the next audio/video file.
split_by_chapters:
when_no_chapters: "pass"
# Override various track properties using chapter variables.
overrides:
track_title: "{chapter_title}" # Chapter title is the track title
track_album: "{title}" # Video's title is the album title
track_title: "{chapter_title}" # Chapter title is the track title
track_album: "{title}" # Video's title is the album title
track_number: "{chapter_index}"
track_number_padded: "{chapter_index_padded}"
track_total: "{chapter_count}"

119
examples/music_audio_from_videos_subscriptions.yaml Normal file
View file

@ -0,0 +1,119 @@
# Define any number of subscriptions in a single file, or use multiple
# files to organize (i.e by use-case, website, genre, etc)
#
# Each example show-cases every use case:
# - YouTube
# - Songs
# - From a single video
# - Albums
# - From a video, where each song is represented by chapters
# - From a playlist where each video is a song on the album
# - Discographies
# - From a channel or a channel's set of playlists
# - Bandcamp
# - Track, Album, Artist URL as a Discography
# - Soundcloud
# - Track, Album, Artist URL as a Discography
#
####################################################################################################
# YOUTUBE - PLAYLIST OF ALBUMS
#
# Downloads the GameChops Albums' playlist. Each video in the playlist is an album with
# chapters representing songs.
game_chops:
preset: "albums_from_chapters"
# Perform regex to extract title from the chapter's title to not include the track number.
# Configuring this hard is a bit overkill, but I'm a perfectionist :-)
regex:
from:
chapter_title:
match:
- "^\\d+\\.\\.(.*)" # Captures 'title' from '1..title'
capture_group_names:
- "captured_track_title"
capture_group_defaults:
- "{chapter_title}"
# Explicitly set the URL, track artist, and genre.
# Override various track properties using the regex variables we extracted.
overrides:
url: "https://www.youtube.com/playlist?list=PLBsm_SagFMmdWnCnrNtLjA9kzfrRkto4i"
track_artist: "GameChops"
track_genre: "Lofi"
track_title: "{captured_track_title}"
####################################################################################################
# BANDCAMP - DISCOGRAPHY
#
# Downloads the GameChops Albums' playlist. Each video in the playlist is an album with
# chapters representing songs.
emily_hopkins:
preset: "albums_from_playlists"
regex:
from:
title:
match:
- "^Emily Hopkins - (.*)" # Captures 'title' from 'Emily Hopkins - title'
capture_group_names:
- "captured_track_title"
capture_group_defaults:
- "{title}"
# Explicitly set the URL, track artist, and genre.
# Override various track properties using the regex variables we extracted.
overrides:
url: "https://emilyharpist.bandcamp.com/"
track_artist: "Emily Hopkins"
track_genre: "Lofi"
track_title: "{captured_track_title}"
####################################################################################################
# YOUTUBE - MANY ALBUMS FROM MANY ARTISTS UPLOADED BY A CHANNEL
#
# Downloads the entire 'Heavy Metal from the Eastern Bloc' channel. Each video they upload
# is an album from Eastern Europe with chapters representing songs.
eastern_bloc:
# Inherit albums from chapters
preset: "albums_from_chapters"
# Perform regex on many fields (title, description, chapter title) to extract as much
# metadata as possible
regex:
skip_if_match_fails: False # Error if regex match fails
from:
title:
match:
- "^(.*) - (.*) \\|\\|.*" # Captures artist, album from 'artist - album ||...'
- "^(.*) - (.*) \\[.*" # Captures artist, ablum from 'artist - album [...'
- "^(.*) - (.*)" # Captures artist, ablum from 'artist - album'
capture_group_names:
- "captured_track_artist"
- "captured_track_album"
description:
match:
- "Genre:\\s*(.*)\\s*\n(?:Rec.+|Year):\\s*.*(\\d{4})\\s*\\n" # Captures genre and recorded year
capture_group_names:
- "captured_track_genre"
- "captured_track_year"
chapter_title:
match:
- "^(?:\\d+\\.\\s*|)(.*)" # Captures title from '1. title'
capture_group_names:
- "captured_track_title"
capture_group_defaults:
- "{chapter_title}"
# Explicitly set the URL.
# Override various track properties using the regex variables we extracted.
overrides:
url: "https://www.youtube.com/@heavymetalofeasternbloc"
track_artist: "{captured_track_artist}"
track_album: "{captured_track_album}"
track_title: "{captured_track_title}"
track_year: "{captured_track_year}"
track_genre: "Eastern Bloc {captured_track_genre}"

16
tests/e2e/plugins/test_audio_extract.py
View file

@ -8,9 +8,7 @@ from ytdl_sub.subscriptions.subscription import Subscription
@pytest.fixture
def single_song_preset_dict(output_directory):
return {
"preset": "song",
"download": {"url": "https://www.youtube.com/watch?v=2lAe1cqCOXo"},
"output_options": {"output_directory": output_directory},
"preset": "single",
# test multi-tags
"music_tags": {"embed_thumbnail": True, "tags": {"genres": ["multi_tag_1", "multi_tag_2"]}},
# download the worst format so it is fast
@ -18,21 +16,27 @@ def single_song_preset_dict(output_directory):
"format": "worst[ext=mp4]",
"postprocessor_args": {"ffmpeg": ["-bitexact"]}, # Must add this for reproducibility
},
"overrides": {
"url": "https://www.youtube.com/watch?v=2lAe1cqCOXo",
"music_directory": output_directory,
}
}
@pytest.fixture
def multiple_songs_preset_dict(output_directory):
return {
"preset": "song_playlist",
"download": {"url": "https://youtube.com/playlist?list=PL5BC0FC26BECA5A35"},
"output_options": {"output_directory": output_directory},
"preset": "albums_from_playlists",
"audio_extract": {"codec": "vorbis", "quality": 140},
# download the worst format so it is fast
"ytdl_options": {
"format": "worst[ext=mp4]",
"postprocessor_args": {"ffmpeg": ["-bitexact"]}, # Must add this for reproducibility
},
"overrides": {
"url": "https://youtube.com/playlist?list=PL5BC0FC26BECA5A35",
"music_directory": output_directory,
}
}

2
tests/expected_download.py
View file

@ -117,7 +117,7 @@ def assert_expected_downloads(
dry_run: bool,
expected_download_summary_file_name: str,
ignore_md5_hashes_for: Optional[List[str]] = None,
regenerate_expected_download_summary: bool = False,
regenerate_expected_download_summary: bool = True,
):
if dry_run:
output_directory_contents = list(Path(output_directory).rglob("*"))

2
tests/expected_transaction_log.py
View file

@ -12,7 +12,7 @@ def assert_transaction_log_matches(
output_directory: str,
transaction_log: FileHandlerTransactionLog,
transaction_log_summary_file_name: str,
regenerate_transaction_log: bool = False,
regenerate_transaction_log: bool = True,
):
"""
Parameters

3
tests/resources/expected_downloads_summaries/plugins/test_audio_extract_single.json
View file

@ -1,3 +1,4 @@
{
"YouTube Rewind 2019： For the Record ｜ #YouTubeRewind.mp3": "05bd1d218f5a0c942ddb1448d939a1dc"
"YouTube/[2019] Singles/01 - YouTube Rewind 2019: For the Record | #YouTubeRewind.mp3": "c70b478ca98f5752e9475ccadcec3227",
"YouTube/[2019] Singles/folder.jpg": "50ee47c80f679029f5d3503bb91b045a"
}

8
tests/resources/transaction_log_summaries/plugins/test_audio_extract_single.txt
View file

@ -1,7 +1,7 @@
Files created:
----------------------------------------
{output_directory}
YouTube Rewind 2019： For the Record ｜ #YouTubeRewind.mp3
{output_directory}/YouTube/[2019] Singles
01 - YouTube Rewind 2019: For the Record | #YouTubeRewind.mp3
Embedded Thumbnail, Music Tags:
album: Singles
albumartist: YouTube
@ -14,4 +14,6 @@ Files created:
- multi_tag_2
title: YouTube Rewind 2019: For the Record | #YouTubeRewind
track: 1
year: 2019
tracktotal: 1
year: 2019
folder.jpg