From c4e1e5bfcfe6d441caaf1f5574743848d0fc21a8 Mon Sep 17 00:00:00 2001 From: Jesse Bannon Date: Sun, 6 Nov 2022 17:31:02 -0800 Subject: [PATCH] [REFACTOR] Use `download` type in examples --- examples/kodi_music_videos_subscriptions.yaml | 59 ------------ ...udio.yaml => music_audio_from_videos.yaml} | 17 ++-- ...s_config.yaml => music_videos_config.yaml} | 42 ++++---- examples/music_videos_subscriptions.yaml | 24 +++++ src/ytdl_sub/config/config_validator.py | 8 +- tests/e2e/conftest.py | 4 +- tests/e2e/plugins/test_audio_extract.py | 9 +- tests/e2e/plugins/test_chapters.py | 4 +- tests/e2e/plugins/test_file_convert.py | 4 +- tests/e2e/plugins/test_nfo_tags.py | 4 +- tests/e2e/plugins/test_regex.py | 4 +- tests/e2e/plugins/test_subtitles.py | 4 +- tests/e2e/youtube/test_merge_playlist.py | 95 ------------------- tests/e2e/youtube/test_playlist.py | 6 +- tests/e2e/youtube/test_video.py | 4 +- tests/unit/cli/test_download_args_parser.py | 4 +- 16 files changed, 76 insertions(+), 216 deletions(-) delete mode 100644 examples/kodi_music_videos_subscriptions.yaml rename examples/{youtube_extract_and_tag_audio.yaml => music_audio_from_videos.yaml} (88%) rename examples/{kodi_music_videos_config.yaml => music_videos_config.yaml} (62%) create mode 100644 examples/music_videos_subscriptions.yaml delete mode 100644 tests/e2e/youtube/test_merge_playlist.py diff --git a/examples/kodi_music_videos_subscriptions.yaml b/examples/kodi_music_videos_subscriptions.yaml deleted file mode 100644 index 8682226c..00000000 --- a/examples/kodi_music_videos_subscriptions.yaml +++ /dev/null @@ -1,59 +0,0 @@ -# This example shows how we can use the `kodi_music_videos_config.yaml` preset -# to download music videos in a few different ways. We will use made-up artists -# in each example - -############################################################################### -# LEVEL 1 - DOWNLOAD MUSIC VIDEO PLAYLIST - -# Subscription names are defined by you. We will call this one john_smith -# for simplicity, and it will download every single video in john_smith's music -# video playlist. Many artists maintain a playlist of all their music videos, -# which makes this an easy way to grab all of them. -john_smith: - # We must define a preset to use from our config. We named the one in the - # config example "yt_music_video", so set that here. - preset: "yt_music_video_playlist" - - # Since our preset download strategy is set to 'playlist', set the playlist url - youtube: - playlist_url: "https://youtube.com/playlist?list=UCsvn_Po0SmunchJYtttWpOxMg" - - # Overrides can be defined per-subscription. If you noticed, we used {artist} - # and {artist_sanitized} in our "yt_music_video" preset. We intended to reserve - # that variable to be defined for each individual subscription. Each override - # defined here will create a '_sanitized' version that is safe for file systems. - # - # A note for Kodi music videos, it is important to make sure your artist name - # exactly matches how it is formatted in Kodi itself, otherwise it will be - # read in as a new artist - overrides: - artist: "John Smith and the Instrument Players" - -############################################################################### -# LEVEL 2 - DOWNLOAD SINGLE MUSIC VIDEO - -# It is not always ideal to download all of an artist's music videos. -# Maybe you only like one song of theirs. We can reuse our preset -# to download a single video instead. -# -# The only difference between this example and the one above is -# - preset -# Use the music video preset, not the playlist preset -# - youtube.video_url -# The video url to download -# -# Of course, defining yaml configuration to download a single video once -# and never again seems weird. Instead, we can perform this download via -# command: -# -# ytdl-sub dl \ -# --preset "yt_music_video" \ -# --youtube.video_url "https://youtube.com/watch?v=QhY6r6oAErg" \ -# --overrides.artist "John Smith and the Instrument Players" -# -john_smith_one_hit_wonder: - preset: "yt_music_video" - youtube: - video_url: "https://youtube.com/watch?v=QhY6r6oAErg" - overrides: - artist: "John Smith and the Instrument Players" diff --git a/examples/youtube_extract_and_tag_audio.yaml b/examples/music_audio_from_videos.yaml similarity index 88% rename from examples/youtube_extract_and_tag_audio.yaml rename to examples/music_audio_from_videos.yaml index 7d11240c..921aec21 100644 --- a/examples/youtube_extract_and_tag_audio.yaml +++ b/examples/music_audio_from_videos.yaml @@ -3,9 +3,10 @@ configuration: working_directory: '.ytdl-sub-downloads' presets: - yt_song: - youtube: - download_strategy: "video" + song: + download: + download_strategy: "url" + url: "{url}" output_options: output_directory: "{music_directory}" @@ -34,15 +35,13 @@ presets: custom_track_number: "1" # TODO: make a playlist of individual songs into an album. Need playlist_title - yt_song_playlist: - preset: yt_song - youtube: - download_strategy: "playlist" + song_playlist: + preset: "song" overrides: custom_track_number: "{playlist_index}" - yt_album_as_chapters: - preset: "yt_song" + album_from_chapters: + preset: "song" output_options: file_name: "{custom_album_name_sanitized}/{chapter_index_padded} - {custom_track_name_sanitized}.{ext}" thumbnail_name: "{custom_album_name_sanitized}/folder.{thumbnail_ext}" diff --git a/examples/kodi_music_videos_config.yaml b/examples/music_videos_config.yaml similarity index 62% rename from examples/kodi_music_videos_config.yaml rename to examples/music_videos_config.yaml index 94fc64d7..477d286a 100644 --- a/examples/kodi_music_videos_config.yaml +++ b/examples/music_videos_config.yaml @@ -1,12 +1,13 @@ -# This example shows how to download and format a Youtube video OR playlist -# to display in Kodi as a music video. Kodi requires music videos to be in -# a shared directory, so we will configure this to make the output directory -# formatted as: +# This example shows how to download and format a music video OR playlist +# of music videos to display in Kodi as a music video. The format will +# look like: # # /path/to/Music Videos +# Elton John/ # Elton John - Rocketman-thumb.jpg # Elton John - Rocketman.mp4 # Elton John - Rocketman.nfo +# System of a Down/ # System of a Down - Chop Suey-thumb.jpg # System of a Down - Chop Suey.mp4 # System of a Down - Chop Suey.nfo @@ -16,12 +17,13 @@ configuration: working_directory: '.ytdl-sub-downloads' presets: - yt_music_video: - # A single YouTube video is our source/download strategy. However, this - # can be overwritten to download music videos from a "playlist", as we - # will see in a preset below - youtube: - download_strategy: "video" + music_video: + # Set the download details + download: + # We will only use a single URL to download music video(s). + # Make {url} an override variable to set later. + download_strategy: "url" + url: "{url}" # For advanced YTDL users only; any YTDL parameter can be set here. # To download age-restricted videos, you will need to set your cookie @@ -39,6 +41,7 @@ presets: file_name: "{music_video_name}.{ext}" thumbnail_name: "{music_video_name}-thumb.jpg" info_json_name: "{music_video_name}.{info_json_ext}" + maintain_download_archive: True # For each video downloaded, add a music video NFO file for it. Populate it # with tags that Kodi will read and use to display it in the music or music @@ -55,23 +58,10 @@ presets: # Overrides is a section where we can define our own variables, and use them in # any other section. We define our music video directory and episode file name # here, which gets reused above for the video, thumbnail, and NFO file. - # Recommended to override the artist variable. + # Recommended to override the artist variable since {channel} is not always + # the artist's exact name. overrides: music_video_directory: "path/to/Music Videos" - music_video_name: "{artist_sanitized} - {title_sanitized}" + music_video_name: "{artist_sanitized}/{artist_sanitized} - {title_sanitized}" artist: "{channel}" - # It is not always ideal to download all of an artist's music videos. - # Maybe you only like one song of theirs. We can reuse our preset above - # to download a single video instead. - yt_music_video_playlist: - preset: "yt_music_video" - youtube: - download_strategy: "playlist" - - # Setting maintain_download_archive=True is generally a good thing to enable - # with playlists and channels because it will store previously downloaded - # video ids to tell YTDL not to re-download them on a successive invocation. - output_options: - maintain_download_archive: True - diff --git a/examples/music_videos_subscriptions.yaml b/examples/music_videos_subscriptions.yaml new file mode 100644 index 00000000..5da045e4 --- /dev/null +++ b/examples/music_videos_subscriptions.yaml @@ -0,0 +1,24 @@ +############################################################################### +# DOWNLOAD MUSIC VIDEO PLAYLIST +john_smith: + preset: "music_video" + overrides: + url: "https://youtube.com/playlist?list=UCsvn_Po0SmunchJYtttWpOxMg" + artist: "John Smith and the Instrument Players" + +############################################################################### +# DOWNLOAD SINGLE MUSIC VIDEO VIA CLI + +# It is not always ideal to download all of an artist's music videos. +# Maybe you only like one song of theirs. We can reuse our preset +# to download a single video instead. +# +# Of course, defining yaml configuration to download a single video once +# and never again seems weird. Instead, we can perform this download via +# command-line: +# +# ytdl-sub dl \ +# --preset "music_video" \ +# --override.url "https://youtube.com/watch?v=QhY6r6oAErg" \ +# --overrides.artist "John Smith and the Instrument Players" +# diff --git a/src/ytdl_sub/config/config_validator.py b/src/ytdl_sub/config/config_validator.py index 31a15b04..c12fb860 100644 --- a/src/ytdl_sub/config/config_validator.py +++ b/src/ytdl_sub/config/config_validator.py @@ -51,20 +51,20 @@ class ConfigOptions(StrictDictValidator): configuration: dl_aliases: - mv: "--preset yt_music_video" - v: "--youtube.video_url" + mv: "--preset music_video" + u: "--download.url" Simplifies .. code-block:: bash - ytdl-sub dl --preset "yt_music_video" --youtube.video_url "youtube.com/watch?v=a1b2c3" + ytdl-sub dl --preset "music_video" --download.url "youtube.com/watch?v=a1b2c3" to .. code-block:: bash - ytdl-sub dl --mv --v "youtube.com/watch?v=a1b2c3" + ytdl-sub dl --mv --u "youtube.com/watch?v=a1b2c3" """ if self._dl_aliases: return self._dl_aliases.dict diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py index 3783d842..b12160e8 100644 --- a/tests/e2e/conftest.py +++ b/tests/e2e/conftest.py @@ -14,7 +14,7 @@ from ytdl_sub.utils.file_handler import FileHandlerTransactionLog @pytest.fixture() def music_video_config_path(): - return "examples/kodi_music_videos_config.yaml" + return "examples/music_videos_config.yaml" @pytest.fixture() @@ -34,7 +34,7 @@ def soundcloud_discography_config(): @pytest.fixture() def youtube_audio_config_path(): - return "examples/youtube_extract_and_tag_audio.yaml" + return "examples/music_audio_from_videos.yaml" @pytest.fixture() diff --git a/tests/e2e/plugins/test_audio_extract.py b/tests/e2e/plugins/test_audio_extract.py index e3ddb2e7..4832bc0a 100644 --- a/tests/e2e/plugins/test_audio_extract.py +++ b/tests/e2e/plugins/test_audio_extract.py @@ -8,8 +8,8 @@ from ytdl_sub.subscriptions.subscription import Subscription @pytest.fixture def single_song_preset_dict(output_directory): return { - "preset": "yt_song", - "youtube": {"video_url": "https://www.youtube.com/watch?v=2lAe1cqCOXo"}, + "preset": "song", + "download": {"url": "https://www.youtube.com/watch?v=2lAe1cqCOXo"}, "output_options": {"output_directory": output_directory}, # download the worst format so it is fast "ytdl_options": { @@ -22,8 +22,8 @@ def single_song_preset_dict(output_directory): @pytest.fixture def multiple_songs_preset_dict(output_directory): return { - "preset": "yt_song_playlist", - "youtube": {"playlist_url": "https://youtube.com/playlist?list=PL5BC0FC26BECA5A35"}, + "preset": "song_playlist", + "download": {"url": "https://youtube.com/playlist?list=PL5BC0FC26BECA5A35"}, "output_options": {"output_directory": output_directory}, "audio_extract": {"codec": "vorbis", "quality": 140}, # download the worst format so it is fast @@ -34,6 +34,7 @@ def multiple_songs_preset_dict(output_directory): } +# TODO: Test album from chapters class TestAudioExtract: @pytest.mark.parametrize("dry_run", [True, False]) def test_audio_extract_single_song( diff --git a/tests/e2e/plugins/test_chapters.py b/tests/e2e/plugins/test_chapters.py index 1333851f..144ea060 100644 --- a/tests/e2e/plugins/test_chapters.py +++ b/tests/e2e/plugins/test_chapters.py @@ -8,8 +8,8 @@ from ytdl_sub.subscriptions.subscription import Subscription @pytest.fixture def single_video_sponsorblock_and_embedded_subs_preset_dict(output_directory): return { - "preset": "yt_music_video", - "youtube": {"video_url": "https://www.youtube.com/watch?v=-wJOUAuKZm8"}, + "preset": "music_video", + "download": {"url": "https://www.youtube.com/watch?v=-wJOUAuKZm8"}, # override the output directory with our fixture-generated dir "output_options": {"output_directory": output_directory}, "subtitles": { diff --git a/tests/e2e/plugins/test_file_convert.py b/tests/e2e/plugins/test_file_convert.py index 894cbc00..fdb24dfe 100644 --- a/tests/e2e/plugins/test_file_convert.py +++ b/tests/e2e/plugins/test_file_convert.py @@ -8,8 +8,8 @@ from ytdl_sub.subscriptions.subscription import Subscription @pytest.fixture def preset_dict(output_directory): return { - "preset": "yt_music_video", - "youtube": {"video_url": "https://www.youtube.com/watch?v=2zYF9JLHDmA"}, + "preset": "music_video", + "download": {"url": "https://www.youtube.com/watch?v=2zYF9JLHDmA"}, "output_options": {"output_directory": output_directory}, # download the worst format so it is fast "ytdl_options": { diff --git a/tests/e2e/plugins/test_nfo_tags.py b/tests/e2e/plugins/test_nfo_tags.py index 67eb5d59..72856cc3 100644 --- a/tests/e2e/plugins/test_nfo_tags.py +++ b/tests/e2e/plugins/test_nfo_tags.py @@ -7,8 +7,8 @@ from ytdl_sub.subscriptions.subscription import Subscription @pytest.fixture def subscription_dict(output_directory): return { - "preset": "yt_music_video", - "youtube": {"video_url": "https://www.youtube.com/shorts/ucYmEqmlhFw"}, + "preset": "music_video", + "download": {"url": "https://www.youtube.com/shorts/ucYmEqmlhFw"}, # override the output directory with our fixture-generated dir "output_options": {"output_directory": output_directory}, # download the worst format so it is fast diff --git a/tests/e2e/plugins/test_regex.py b/tests/e2e/plugins/test_regex.py index 82f9bff8..fa32d5ab 100644 --- a/tests/e2e/plugins/test_regex.py +++ b/tests/e2e/plugins/test_regex.py @@ -13,8 +13,8 @@ from ytdl_sub.utils.exceptions import ValidationException @pytest.fixture def regex_subscription_dict(output_directory): return { - "preset": "yt_music_video_playlist", - "youtube": {"playlist_url": "https://youtube.com/playlist?list=PL5BC0FC26BECA5A35"}, + "preset": "music_video", + "download": {"url": "https://youtube.com/playlist?list=PL5BC0FC26BECA5A35"}, # override the output directory with our fixture-generated dir "output_options": {"output_directory": output_directory}, # download the worst format so it is fast diff --git a/tests/e2e/plugins/test_subtitles.py b/tests/e2e/plugins/test_subtitles.py index d62974ff..a77b9886 100644 --- a/tests/e2e/plugins/test_subtitles.py +++ b/tests/e2e/plugins/test_subtitles.py @@ -8,8 +8,8 @@ from ytdl_sub.subscriptions.subscription import Subscription @pytest.fixture def single_video_subs_embed_preset_dict(output_directory): return { - "preset": "yt_music_video", - "youtube": {"video_url": "https://www.youtube.com/watch?v=2lAe1cqCOXo"}, + "preset": "music_video", + "download": {"url": "https://www.youtube.com/watch?v=2lAe1cqCOXo"}, # override the output directory with our fixture-generated dir "output_options": {"output_directory": output_directory}, "subtitles": { diff --git a/tests/e2e/youtube/test_merge_playlist.py b/tests/e2e/youtube/test_merge_playlist.py deleted file mode 100644 index 437075a5..00000000 --- a/tests/e2e/youtube/test_merge_playlist.py +++ /dev/null @@ -1,95 +0,0 @@ -import pytest -from expected_download import assert_expected_downloads -from expected_transaction_log import assert_transaction_log_matches - -from ytdl_sub.config.config_file import ConfigFile -from ytdl_sub.config.preset import Preset -from ytdl_sub.subscriptions.subscription import Subscription - - -@pytest.fixture -def config_path(): - return "examples/kodi_music_videos_config.yaml" - - -@pytest.fixture -def subscription_name(): - return "jmc" - - -@pytest.fixture -def config(config_path): - return ConfigFile.from_file_path(config_path=config_path) - - -@pytest.fixture -def subscription_dict(output_directory, subscription_name): - return { - "preset": "yt_music_video_playlist", - "youtube": { - "download_strategy": "merge_playlist", - "playlist_url": "https://youtube.com/playlist?list=PL5BC0FC26BECA5A35", - "add_chapters": True, - }, - # override the output directory with our fixture-generated dir - "output_options": {"output_directory": output_directory}, - # download the worst format so it is fast - "ytdl_options": { - "format": "best[height<=480]", - "postprocessor_args": {"ffmpeg": ["-bitexact"]}, # Must add this for reproducibility - }, - "overrides": {"artist": "JMC"}, - } - - -#################################################################################################### -# PLAYLIST FIXTURES - - -@pytest.fixture -def playlist_subscription(config, subscription_name, subscription_dict): - playlist_preset = Preset.from_dict( - config=config, - preset_name=subscription_name, - preset_dict=subscription_dict, - ) - - return Subscription.from_preset( - preset=playlist_preset, - config=config, - ) - - -# @pytest.fixture -# def expected_playlist_download(): -# # fmt: off -# return ExpectedDownloads( -# expected_downloads=[ -# ExpectedDownloadFile(path=Path("JMC - Jesse's Minecraft Server-thumb.jpg"), md5="a3f1910f9c51f6442f845a528e190829"), -# ExpectedDownloadFile(path=Path("JMC - Jesse's Minecraft Server.mkv")), # not bitexact TODO: check size -# ExpectedDownloadFile(path=Path("JMC - Jesse's Minecraft Server.nfo"), md5="10df5dcdb65ab18ecf21b3503c77e48b"), -# ] -# ) -# # fmt: on - - -class TestYoutubeMergePlaylist: - """ - Downloads my old minecraft youtube channel, pretends they are music videos. Ensure the above - files exist and have the expected md5 file hashes. - """ - - @pytest.mark.parametrize("dry_run", [True, False]) - def test_merge_playlist_download(self, playlist_subscription, output_directory, dry_run): - transaction_log = playlist_subscription.download(dry_run=dry_run) - assert_transaction_log_matches( - output_directory=output_directory, - transaction_log=transaction_log, - transaction_log_summary_file_name="youtube/test_merge_playlist.txt", - ) - assert_expected_downloads( - output_directory=output_directory, - dry_run=dry_run, - ignore_md5_hashes_for=["JMC - Jesse's Minecraft Server.mkv"], # TODO, better test here - expected_download_summary_file_name="youtube/test_merge_playlist.json", - ) diff --git a/tests/e2e/youtube/test_playlist.py b/tests/e2e/youtube/test_playlist.py index dd9a8638..f05f159b 100644 --- a/tests/e2e/youtube/test_playlist.py +++ b/tests/e2e/youtube/test_playlist.py @@ -11,10 +11,10 @@ from ytdl_sub.subscriptions.subscription import Subscription @pytest.fixture def playlist_preset_dict(output_directory): return { - "preset": "yt_music_video_playlist", + "preset": "music_video", "youtube": { - "playlist_url": "https://youtube.com/playlist?list=PL5BC0FC26BECA5A35", - "playlist_thumbnail_name": "poster.jpg", + "url": "https://youtube.com/playlist?list=PL5BC0FC26BECA5A35", + "playlist_thumbnails": {"name": "poster.jpg", "uid": "latest_entry"}, }, # override the output directory with our fixture-generated dir "output_options": {"output_directory": output_directory}, diff --git a/tests/e2e/youtube/test_video.py b/tests/e2e/youtube/test_video.py index 314679c6..af40a95f 100644 --- a/tests/e2e/youtube/test_video.py +++ b/tests/e2e/youtube/test_video.py @@ -10,8 +10,8 @@ from ytdl_sub.subscriptions.subscription import Subscription @pytest.fixture def single_video_preset_dict(output_directory): return { - "preset": "yt_music_video", - "youtube": {"video_url": "https://youtube.com/watch?v=HKTNxEqsN3Q"}, + "preset": "music_video", + "download": {"url": "https://youtube.com/watch?v=HKTNxEqsN3Q"}, # override the output directory with our fixture-generated dir "output_options": {"output_directory": output_directory}, # download the worst format so it is fast diff --git a/tests/unit/cli/test_download_args_parser.py b/tests/unit/cli/test_download_args_parser.py index d528c6c1..bc73384f 100644 --- a/tests/unit/cli/test_download_args_parser.py +++ b/tests/unit/cli/test_download_args_parser.py @@ -44,9 +44,9 @@ class TestDownloadArgsParser: "aliases, cmd, expected_sub_dict", [ ( - {"mv": "--preset yt_music_video", "v": "--youtube.video_url"}, + {"mv": "--preset music_video", "v": "--download.url"}, "dl --mv --v 123abc", - {"preset": "yt_music_video", "youtube": {"video_url": "123abc"}}, + {"preset": "music_video", "download": {"url": "123abc"}}, ), ( {