From 0cbd2af4d32536c02d7df0528644dc9c7ffb42eb Mon Sep 17 00:00:00 2001 From: Jesse Bannon Date: Mon, 24 Jul 2023 12:07:36 -0700 Subject: [PATCH] [FEATURE] `embed_thumbnail` plugin (#659) * [REFACTOR] OptionsValidator + OptionsDictValidator * more * [FEATURE] Dedicated embed_thumbnail plugin * tests * fix test * docs * deprecation notices * priority to run after file convert * naming * fixtures --- docs/config.rst | 11 +- docs/deprecation_notices.rst | 25 +++++ docs/index.rst | 1 + src/ytdl_sub/config/preset_class_mappings.py | 2 + src/ytdl_sub/plugins/embed_thumbnail.py | 102 ++++++++++++++++++ src/ytdl_sub/plugins/music_tags.py | 7 +- src/ytdl_sub/plugins/plugin.py | 3 + tests/e2e/plugins/test_audio_extract.py | 2 + tests/e2e/youtube/test_video.py | 2 + .../youtube/test_video.json | 4 +- .../youtube/test_video_cli.json | 4 +- .../plugins/test_audio_extract_single.txt | 1 + .../youtube/test_video.txt | 1 + .../youtube/test_video_cli.txt | 1 + tests/unit/config/test_config_file.py | 6 +- 15 files changed, 164 insertions(+), 8 deletions(-) create mode 100644 docs/deprecation_notices.rst create mode 100644 src/ytdl_sub/plugins/embed_thumbnail.py diff --git a/docs/config.rst b/docs/config.rst index 99ce10cc..adb05693 100644 --- a/docs/config.rst +++ b/docs/config.rst @@ -165,6 +165,15 @@ date_range :member-order: bysource :exclude-members: partial_validate +------------------------------------------------------------------------------- + +embed_thumbnail +'''''''''''''''' + +.. autoclass:: ytdl_sub.plugins.embed_thumbnail.EmbedThumbnailOptions() + +------------------------------------------------------------------------------- + file_convert '''''''''''' .. autoclass:: ytdl_sub.plugins.file_convert.FileConvertOptions() @@ -186,7 +195,7 @@ music_tags '''''''''' .. autoclass:: ytdl_sub.plugins.music_tags.MusicTagsOptions() :members: - :exclude-members: partial_validate + :exclude-members: partial_validate, embed_thumbnail ------------------------------------------------------------------------------- diff --git a/docs/deprecation_notices.rst b/docs/deprecation_notices.rst new file mode 100644 index 00000000..2a05181c --- /dev/null +++ b/docs/deprecation_notices.rst @@ -0,0 +1,25 @@ +Deprecation Notices +=================== + +July 2023 +--------- + +embed_thumbnail +^^^^^^^^^^^^^^^ + +Embedding thumbnails has its own dedicated plugin now, which supports both audio and video files. +It will be removed from ``music_tags`` in October 2023. Convert from: + +.. code-block:: yaml + + my_example_preset: + music_tags: + embed_thumbnail: True + +To the following: + +.. code-block:: yaml + + my_example_preset: + embed_thumbnail: True + diff --git a/docs/index.rst b/docs/index.rst index 7974add8..1ef6df9f 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -26,3 +26,4 @@ Contents getting_started presets config + deprecation_notices diff --git a/src/ytdl_sub/config/preset_class_mappings.py b/src/ytdl_sub/config/preset_class_mappings.py index dac9e996..a77e4988 100644 --- a/src/ytdl_sub/config/preset_class_mappings.py +++ b/src/ytdl_sub/config/preset_class_mappings.py @@ -8,6 +8,7 @@ from ytdl_sub.downloaders.url.url import UrlDownloader from ytdl_sub.plugins.audio_extract import AudioExtractPlugin from ytdl_sub.plugins.chapters import ChaptersPlugin from ytdl_sub.plugins.date_range import DateRangePlugin +from ytdl_sub.plugins.embed_thumbnail import EmbedThumbnailPlugin from ytdl_sub.plugins.file_convert import FileConvertPlugin from ytdl_sub.plugins.internal.view import ViewPlugin from ytdl_sub.plugins.match_filters import MatchFiltersPlugin @@ -107,6 +108,7 @@ class PluginMapping: "_view": ViewPlugin, "audio_extract": AudioExtractPlugin, "date_range": DateRangePlugin, + "embed_thumbnail": EmbedThumbnailPlugin, "file_convert": FileConvertPlugin, "match_filters": MatchFiltersPlugin, "music_tags": MusicTagsPlugin, diff --git a/src/ytdl_sub/plugins/embed_thumbnail.py b/src/ytdl_sub/plugins/embed_thumbnail.py new file mode 100644 index 00000000..fac7e050 --- /dev/null +++ b/src/ytdl_sub/plugins/embed_thumbnail.py @@ -0,0 +1,102 @@ +from typing import List +from typing import Optional + +import mediafile + +from ytdl_sub.config.preset_options import OptionsValidator +from ytdl_sub.entries.entry import Entry +from ytdl_sub.plugins.plugin import Plugin +from ytdl_sub.plugins.plugin import PluginPriority +from ytdl_sub.utils.ffmpeg import FFMPEG +from ytdl_sub.utils.file_handler import FileHandler +from ytdl_sub.utils.file_handler import FileMetadata +from ytdl_sub.utils.logger import Logger +from ytdl_sub.utils.thumbnail import convert_download_thumbnail +from ytdl_sub.validators.audo_codec_validator import AUDIO_CODEC_EXTS +from ytdl_sub.validators.validators import BoolValidator + +logger = Logger.get("embed_thumbnail") + + +class EmbedThumbnailOptions(BoolValidator, OptionsValidator): + """ + Whether to embed thumbnails to the audio/video file or not. + + Usage: + + .. code-block:: yaml + + presets: + my_example_preset: + embed_thumbnail: True + """ + + +class EmbedThumbnailPlugin(Plugin[EmbedThumbnailOptions]): + plugin_options_type = EmbedThumbnailOptions + priority = PluginPriority(post_process=PluginPriority.POST_PROCESS_AFTER_FILE_CONVERT) + + @property + def _embed_thumbnail(self) -> bool: + return self.plugin_options.value + + @classmethod + def _embed_video_thumbnail(cls, entry: Entry) -> None: + file_path = entry.get_download_file_path() + thumbnail_path = entry.get_download_thumbnail_path() + tmp_file_path = FFMPEG.tmp_file_path(file_path) + try: + ffmpeg_args: List[str] = [ + "-i", + file_path, + "-i", + thumbnail_path, + "-map", + "1", + "-map", + "0", + "-dn", # ignore data streams + "-c", + "copy", + "-bitexact", # for reproducibility + "-disposition:0", + "attached_pic", + tmp_file_path, + ] + FFMPEG.run(ffmpeg_args) + FileHandler.move(tmp_file_path, file_path) + finally: + FileHandler.delete(tmp_file_path) + + @classmethod + def _embed_audio_file(cls, entry: Entry) -> None: + audio_file = mediafile.MediaFile(entry.get_download_file_path()) + with open(entry.get_download_thumbnail_path(), "rb") as thumb: + mediafile_img = mediafile.Image( + data=thumb.read(), desc="cover", type=mediafile.ImageType.front + ) + + audio_file.images = [mediafile_img] + audio_file.save() + + def post_process_entry(self, entry: Entry) -> Optional[FileMetadata]: + """ + Maybe embed the thumbnail + """ + if not self._embed_thumbnail: + return None + + if entry.ext == "webm": + logger.warning("webm does not support embedded thumbnails, skipping") + return None + + if not self.is_dry_run: + # convert the entry thumbnail so it is embedded as jpg + convert_download_thumbnail(entry=entry) + + if entry.ext in AUDIO_CODEC_EXTS: + self._embed_audio_file(entry) + else: + self._embed_video_thumbnail(entry) + + return FileMetadata("Embedded thumbnail") diff --git a/src/ytdl_sub/plugins/music_tags.py b/src/ytdl_sub/plugins/music_tags.py index 97280bd6..4f671d26 100644 --- a/src/ytdl_sub/plugins/music_tags.py +++ b/src/ytdl_sub/plugins/music_tags.py @@ -70,8 +70,6 @@ class MusicTagsOptions(OptionsDictValidator): albumartists: - "{artist}" - "ytdl-sub" - # Optional - embed_thumbnail: False """ _required_keys = {"tags"} @@ -149,6 +147,11 @@ class MusicTagsPlugin(Plugin[MusicTagsOptions]): setattr(audio_file, tag_name, tag_value[0]) if self.plugin_options.embed_thumbnail: + logger.warning( + "music_tags.embed_thumbnail is now deprecated. Use the dedicated " + "embed_thumbnail plugin instead. This will be removed in October of 2023." + ) + # convert the entry thumbnail so it is embedded as jpg convert_download_thumbnail(entry=entry) diff --git a/src/ytdl_sub/plugins/plugin.py b/src/ytdl_sub/plugins/plugin.py index 5f5ec03c..630f2270 100644 --- a/src/ytdl_sub/plugins/plugin.py +++ b/src/ytdl_sub/plugins/plugin.py @@ -24,6 +24,9 @@ class PluginPriority: # If modify_entry priority is >= to this value, run after split MODIFY_ENTRY_AFTER_SPLIT = 10 + # if post_process is >= to this value, run after file_convert + POST_PROCESS_AFTER_FILE_CONVERT = 10 + MODIFY_ENTRY_FIRST = 0 def __init__(self, modify_entry: int = 5, post_process: int = 5): diff --git a/tests/e2e/plugins/test_audio_extract.py b/tests/e2e/plugins/test_audio_extract.py index 1a3563c7..48aeb9d0 100644 --- a/tests/e2e/plugins/test_audio_extract.py +++ b/tests/e2e/plugins/test_audio_extract.py @@ -11,6 +11,8 @@ def single_song_preset_dict(output_directory): "preset": "single", # test multi-tags "music_tags": {"embed_thumbnail": True, "tags": {"genres": ["multi_tag_1", "multi_tag_2"]}}, + # test the new embed_thumbnail plugin + "embed_thumbnail": True, # download the worst format so it is fast "ytdl_options": { "format": "worst[ext=mp4]", diff --git a/tests/e2e/youtube/test_video.py b/tests/e2e/youtube/test_video.py index e7a1f2f6..3219692d 100644 --- a/tests/e2e/youtube/test_video.py +++ b/tests/e2e/youtube/test_video.py @@ -17,6 +17,8 @@ def single_video_preset_dict(output_directory): "output_directory": output_directory, "maintain_download_archive": False, }, + # embed thumb into the video + "embed_thumbnail": True, # download the worst format so it is fast "ytdl_options": { "format": "worst[ext=mp4]", diff --git a/tests/resources/expected_downloads_summaries/youtube/test_video.json b/tests/resources/expected_downloads_summaries/youtube/test_video.json index f7d75ddc..bdf3a223 100644 --- a/tests/resources/expected_downloads_summaries/youtube/test_video.json +++ b/tests/resources/expected_downloads_summaries/youtube/test_video.json @@ -1,6 +1,6 @@ { "JMC/JMC - Oblivion Mod ＂Falcor＂ p.1-thumb.jpg": "fb95b510681676e81c321171fc23143e", - "JMC/JMC - Oblivion Mod ＂Falcor＂ p.1.info.json": "73dd7eb872fe150364581de40a66777d", - "JMC/JMC - Oblivion Mod ＂Falcor＂ p.1.mp4": "3744c49f2e447bd7712a5aad5ed36be2", + "JMC/JMC - Oblivion Mod ＂Falcor＂ p.1.info.json": "08b0f7d93488d625bd7311ab925cec77", + "JMC/JMC - Oblivion Mod ＂Falcor＂ p.1.mp4": "797b44f3207be01651780d6d86cb70bb", "JMC/JMC - Oblivion Mod ＂Falcor＂ p.1.nfo": "24cc4e17d2bebc89b2759ce5471d403e" } \ No newline at end of file diff --git a/tests/resources/expected_downloads_summaries/youtube/test_video_cli.json b/tests/resources/expected_downloads_summaries/youtube/test_video_cli.json index 27e58215..ba733bde 100644 --- a/tests/resources/expected_downloads_summaries/youtube/test_video_cli.json +++ b/tests/resources/expected_downloads_summaries/youtube/test_video_cli.json @@ -1,6 +1,6 @@ { "JMC/JMC - Oblivion Mod ＂Falcor＂ p.1-thumb.jpg": "fb95b510681676e81c321171fc23143e", - "JMC/JMC - Oblivion Mod ＂Falcor＂ p.1.info.json": "7ffe51eae87c1d6fd10151a366c9d207", - "JMC/JMC - Oblivion Mod ＂Falcor＂ p.1.mp4": "3744c49f2e447bd7712a5aad5ed36be2", + "JMC/JMC - Oblivion Mod ＂Falcor＂ p.1.info.json": "ebb744c59a37dff5df6e33f926c8eebb", + "JMC/JMC - Oblivion Mod ＂Falcor＂ p.1.mp4": "797b44f3207be01651780d6d86cb70bb", "JMC/JMC - Oblivion Mod ＂Falcor＂ p.1.nfo": "24cc4e17d2bebc89b2759ce5471d403e" } \ No newline at end of file diff --git a/tests/resources/transaction_log_summaries/plugins/test_audio_extract_single.txt b/tests/resources/transaction_log_summaries/plugins/test_audio_extract_single.txt index 6327e106..5c9b995a 100644 --- a/tests/resources/transaction_log_summaries/plugins/test_audio_extract_single.txt +++ b/tests/resources/transaction_log_summaries/plugins/test_audio_extract_single.txt @@ -18,4 +18,5 @@ Files created: track: 1 tracktotal: 1 year: 2019 + Embedded thumbnail folder.jpg \ No newline at end of file diff --git a/tests/resources/transaction_log_summaries/youtube/test_video.txt b/tests/resources/transaction_log_summaries/youtube/test_video.txt index 8162ad7f..43da748f 100644 --- a/tests/resources/transaction_log_summaries/youtube/test_video.txt +++ b/tests/resources/transaction_log_summaries/youtube/test_video.txt @@ -6,6 +6,7 @@ Files created: JMC - Oblivion Mod ＂Falcor＂ p.1.mp4 Video Tags: title: Oblivion Mod "Falcor" p.1 + Embedded thumbnail JMC - Oblivion Mod ＂Falcor＂ p.1.nfo NFO tags: musicvideo: diff --git a/tests/resources/transaction_log_summaries/youtube/test_video_cli.txt b/tests/resources/transaction_log_summaries/youtube/test_video_cli.txt index 8162ad7f..43da748f 100644 --- a/tests/resources/transaction_log_summaries/youtube/test_video_cli.txt +++ b/tests/resources/transaction_log_summaries/youtube/test_video_cli.txt @@ -6,6 +6,7 @@ Files created: JMC - Oblivion Mod ＂Falcor＂ p.1.mp4 Video Tags: title: Oblivion Mod "Falcor" p.1 + Embedded thumbnail JMC - Oblivion Mod ＂Falcor＂ p.1.nfo NFO tags: musicvideo: diff --git a/tests/unit/config/test_config_file.py b/tests/unit/config/test_config_file.py index 2bda9621..24de3269 100644 --- a/tests/unit/config/test_config_file.py +++ b/tests/unit/config/test_config_file.py @@ -51,7 +51,11 @@ class TestConfigFilePartiallyValidatesPresets: @pytest.mark.parametrize("plugin", PluginMapping.plugins()) def test_success__empty_plugins(self, plugin: str): - self._partial_validate({plugin: {}}) + excluded_plugins = [ + "embed_thumbnail", # value is bool, not dict + ] + if plugin not in excluded_plugins: + self._partial_validate({plugin: {}}) @pytest.mark.parametrize("source", DownloadStrategyMapping.sources()) def test_success__empty_sources(self, source: str):