[FEATURE] embed_thumbnail plugin (#659)

* [REFACTOR] OptionsValidator + OptionsDictValidator

* more

* [FEATURE] Dedicated embed_thumbnail plugin

* tests

* fix test

* docs

* deprecation notices

* priority to run after file convert

* naming

* fixtures

docs/config.rst

@@ -165,6 +165,15 @@ date_range
   :member-order: bysource
   :exclude-members: partial_validate
 
+-------------------------------------------------------------------------------
+
+embed_thumbnail
+''''''''''''''''
+
+.. autoclass:: ytdl_sub.plugins.embed_thumbnail.EmbedThumbnailOptions()
+
+-------------------------------------------------------------------------------
+
 file_convert
 ''''''''''''
 .. autoclass:: ytdl_sub.plugins.file_convert.FileConvertOptions()
@@ -186,7 +195,7 @@ music_tags
 ''''''''''
 .. autoclass:: ytdl_sub.plugins.music_tags.MusicTagsOptions()
   :members:
-  :exclude-members: partial_validate
+  :exclude-members: partial_validate, embed_thumbnail
 
 -------------------------------------------------------------------------------
 

docs/deprecation_notices.rst

@@ -0,0 +1,25 @@
+Deprecation Notices
+===================
+
+July 2023
+---------
+
+embed_thumbnail
+^^^^^^^^^^^^^^^
+
+Embedding thumbnails has its own dedicated plugin now, which supports both audio and video files.
+It will be removed from ``music_tags`` in October 2023. Convert from:
+
+.. code-block:: yaml
+
+   my_example_preset:
+     music_tags:
+       embed_thumbnail: True
+
+To the following:
+
+.. code-block:: yaml
+
+   my_example_preset:
+     embed_thumbnail: True
+

docs/index.rst

@@ -26,3 +26,4 @@ Contents
    getting_started
    presets
    config
+   deprecation_notices

src/ytdl_sub/config/preset_class_mappings.py

@@ -8,6 +8,7 @@ from ytdl_sub.downloaders.url.url import UrlDownloader
 from ytdl_sub.plugins.audio_extract import AudioExtractPlugin
 from ytdl_sub.plugins.chapters import ChaptersPlugin
 from ytdl_sub.plugins.date_range import DateRangePlugin
+from ytdl_sub.plugins.embed_thumbnail import EmbedThumbnailPlugin
 from ytdl_sub.plugins.file_convert import FileConvertPlugin
 from ytdl_sub.plugins.internal.view import ViewPlugin
 from ytdl_sub.plugins.match_filters import MatchFiltersPlugin
@@ -107,6 +108,7 @@ class PluginMapping:
         "_view": ViewPlugin,
         "audio_extract": AudioExtractPlugin,
         "date_range": DateRangePlugin,
+        "embed_thumbnail": EmbedThumbnailPlugin,
         "file_convert": FileConvertPlugin,
         "match_filters": MatchFiltersPlugin,
         "music_tags": MusicTagsPlugin,

src/ytdl_sub/plugins/embed_thumbnail.py

@@ -0,0 +1,102 @@
+from typing import List
+from typing import Optional
+
+import mediafile
+
+from ytdl_sub.config.preset_options import OptionsValidator
+from ytdl_sub.entries.entry import Entry
+from ytdl_sub.plugins.plugin import Plugin
+from ytdl_sub.plugins.plugin import PluginPriority
+from ytdl_sub.utils.ffmpeg import FFMPEG
+from ytdl_sub.utils.file_handler import FileHandler
+from ytdl_sub.utils.file_handler import FileMetadata
+from ytdl_sub.utils.logger import Logger
+from ytdl_sub.utils.thumbnail import convert_download_thumbnail
+from ytdl_sub.validators.audo_codec_validator import AUDIO_CODEC_EXTS
+from ytdl_sub.validators.validators import BoolValidator
+
+logger = Logger.get("embed_thumbnail")
+
+
+class EmbedThumbnailOptions(BoolValidator, OptionsValidator):
+    """
+    Whether to embed thumbnails to the audio/video file or not.
+
+    Usage:
+
+    .. code-block:: yaml
+
+       presets:
+         my_example_preset:
+           embed_thumbnail: True
+    """
+
+
+class EmbedThumbnailPlugin(Plugin[EmbedThumbnailOptions]):
+    plugin_options_type = EmbedThumbnailOptions
+    priority = PluginPriority(post_process=PluginPriority.POST_PROCESS_AFTER_FILE_CONVERT)
+
+    @property
+    def _embed_thumbnail(self) -> bool:
+        return self.plugin_options.value
+
+    @classmethod
+    def _embed_video_thumbnail(cls, entry: Entry) -> None:
+        file_path = entry.get_download_file_path()
+        thumbnail_path = entry.get_download_thumbnail_path()
+        tmp_file_path = FFMPEG.tmp_file_path(file_path)
+        try:
+            ffmpeg_args: List[str] = [
+                "-i",
+                file_path,
+                "-i",
+                thumbnail_path,
+                "-map",
+                "1",
+                "-map",
+                "0",
+                "-dn",  # ignore data streams
+                "-c",
+                "copy",
+                "-bitexact",  # for reproducibility
+                "-disposition:0",
+                "attached_pic",
+                tmp_file_path,
+            ]
+            FFMPEG.run(ffmpeg_args)
+            FileHandler.move(tmp_file_path, file_path)
+        finally:
+            FileHandler.delete(tmp_file_path)
+
+    @classmethod
+    def _embed_audio_file(cls, entry: Entry) -> None:
+        audio_file = mediafile.MediaFile(entry.get_download_file_path())
+        with open(entry.get_download_thumbnail_path(), "rb") as thumb:
+            mediafile_img = mediafile.Image(
+                data=thumb.read(), desc="cover", type=mediafile.ImageType.front
+            )
+
+        audio_file.images = [mediafile_img]
+        audio_file.save()
+
+    def post_process_entry(self, entry: Entry) -> Optional[FileMetadata]:
+        """
+        Maybe embed the thumbnail
+        """
+        if not self._embed_thumbnail:
+            return None
+
+        if entry.ext == "webm":
+            logger.warning("webm does not support embedded thumbnails, skipping")
+            return None
+
+        if not self.is_dry_run:
+            # convert the entry thumbnail so it is embedded as jpg
+            convert_download_thumbnail(entry=entry)
+
+            if entry.ext in AUDIO_CODEC_EXTS:
+                self._embed_audio_file(entry)
+            else:
+                self._embed_video_thumbnail(entry)
+
+        return FileMetadata("Embedded thumbnail")

src/ytdl_sub/plugins/music_tags.py

@@ -70,8 +70,6 @@ class MusicTagsOptions(OptionsDictValidator):
                albumartists:
                  - "{artist}"
                  - "ytdl-sub"
-             # Optional
-             embed_thumbnail: False
     """
 
     _required_keys = {"tags"}
@@ -149,6 +147,11 @@ class MusicTagsPlugin(Plugin[MusicTagsOptions]):
                     setattr(audio_file, tag_name, tag_value[0])
 
             if self.plugin_options.embed_thumbnail:
+                logger.warning(
+                    "music_tags.embed_thumbnail is now deprecated. Use the dedicated "
+                    "embed_thumbnail plugin instead. This will be removed in October of 2023."
+                )
+
                 # convert the entry thumbnail so it is embedded as jpg
                 convert_download_thumbnail(entry=entry)
 

src/ytdl_sub/plugins/plugin.py

@@ -24,6 +24,9 @@ class PluginPriority:
     # If modify_entry priority is >= to this value, run after split
     MODIFY_ENTRY_AFTER_SPLIT = 10
 
+    # if post_process is >= to this value, run after file_convert
+    POST_PROCESS_AFTER_FILE_CONVERT = 10
+
     MODIFY_ENTRY_FIRST = 0
 
     def __init__(self, modify_entry: int = 5, post_process: int = 5):

tests/e2e/plugins/test_audio_extract.py

@@ -11,6 +11,8 @@ def single_song_preset_dict(output_directory):
         "preset": "single",
         # test multi-tags
         "music_tags": {"embed_thumbnail": True, "tags": {"genres": ["multi_tag_1", "multi_tag_2"]}},
+        # test the new embed_thumbnail plugin
+        "embed_thumbnail": True,
         # download the worst format so it is fast
         "ytdl_options": {
             "format": "worst[ext=mp4]",

tests/e2e/youtube/test_video.py

@@ -17,6 +17,8 @@ def single_video_preset_dict(output_directory):
             "output_directory": output_directory,
             "maintain_download_archive": False,
         },
+        # embed thumb into the video
+        "embed_thumbnail": True,
         # download the worst format so it is fast
         "ytdl_options": {
             "format": "worst[ext=mp4]",

tests/resources/expected_downloads_summaries/youtube/test_video.json

@@ -1,6 +1,6 @@
 {
   "JMC/JMC - Oblivion Mod ＂Falcor＂ p.1-thumb.jpg": "fb95b510681676e81c321171fc23143e",
-  "JMC/JMC - Oblivion Mod ＂Falcor＂ p.1.info.json": "73dd7eb872fe150364581de40a66777d",
+  "JMC/JMC - Oblivion Mod ＂Falcor＂ p.1.info.json": "08b0f7d93488d625bd7311ab925cec77",
-  "JMC/JMC - Oblivion Mod ＂Falcor＂ p.1.mp4": "3744c49f2e447bd7712a5aad5ed36be2",
+  "JMC/JMC - Oblivion Mod ＂Falcor＂ p.1.mp4": "797b44f3207be01651780d6d86cb70bb",
   "JMC/JMC - Oblivion Mod ＂Falcor＂ p.1.nfo": "24cc4e17d2bebc89b2759ce5471d403e"
 }

tests/resources/expected_downloads_summaries/youtube/test_video_cli.json

@@ -1,6 +1,6 @@
 {
   "JMC/JMC - Oblivion Mod ＂Falcor＂ p.1-thumb.jpg": "fb95b510681676e81c321171fc23143e",
-  "JMC/JMC - Oblivion Mod ＂Falcor＂ p.1.info.json": "7ffe51eae87c1d6fd10151a366c9d207",
+  "JMC/JMC - Oblivion Mod ＂Falcor＂ p.1.info.json": "ebb744c59a37dff5df6e33f926c8eebb",
-  "JMC/JMC - Oblivion Mod ＂Falcor＂ p.1.mp4": "3744c49f2e447bd7712a5aad5ed36be2",
+  "JMC/JMC - Oblivion Mod ＂Falcor＂ p.1.mp4": "797b44f3207be01651780d6d86cb70bb",
   "JMC/JMC - Oblivion Mod ＂Falcor＂ p.1.nfo": "24cc4e17d2bebc89b2759ce5471d403e"
 }

tests/resources/transaction_log_summaries/plugins/test_audio_extract_single.txt

@@ -18,4 +18,5 @@ Files created:
       track: 1
       tracktotal: 1
       year: 2019
+    Embedded thumbnail
   folder.jpg

tests/resources/transaction_log_summaries/youtube/test_video.txt

@@ -6,6 +6,7 @@ Files created:
   JMC - Oblivion Mod ＂Falcor＂ p.1.mp4
     Video Tags:
       title: Oblivion Mod "Falcor" p.1
+    Embedded thumbnail
   JMC - Oblivion Mod ＂Falcor＂ p.1.nfo
     NFO tags:
       musicvideo:

tests/resources/transaction_log_summaries/youtube/test_video_cli.txt

@@ -6,6 +6,7 @@ Files created:
   JMC - Oblivion Mod ＂Falcor＂ p.1.mp4
     Video Tags:
       title: Oblivion Mod "Falcor" p.1
+    Embedded thumbnail
   JMC - Oblivion Mod ＂Falcor＂ p.1.nfo
     NFO tags:
       musicvideo:

tests/unit/config/test_config_file.py

@@ -51,7 +51,11 @@ class TestConfigFilePartiallyValidatesPresets:
 
     @pytest.mark.parametrize("plugin", PluginMapping.plugins())
     def test_success__empty_plugins(self, plugin: str):
-        self._partial_validate({plugin: {}})
+        excluded_plugins = [
+            "embed_thumbnail",  # value is bool, not dict
+        ]
+        if plugin not in excluded_plugins:
+            self._partial_validate({plugin: {}})
 
     @pytest.mark.parametrize("source", DownloadStrategyMapping.sources())
     def test_success__empty_sources(self, source: str):