Merge branch 'master' into jesse/split_chapters

docs/config.rst

@@ -177,6 +177,14 @@ audio_extract
 
 -------------------------------------------------------------------------------
 
+chapters
+''''''''
+.. autoclass:: ytdl_sub.plugins.chapters.ChaptersOptions()
+  :members:
+  :member-order: bysource
+
+-------------------------------------------------------------------------------
+
 music_tags
 ''''''''''
 .. autoclass:: ytdl_sub.plugins.music_tags.MusicTagsOptions()

src/ytdl_sub/downloaders/downloader.py

@@ -57,6 +57,7 @@ class Downloader(DownloadArchiver, Generic[DownloaderOptionsT, DownloaderEntryT]
 
     supports_download_archive: bool = True
     supports_subtitles: bool = True
+    supports_chapters: bool = True
 
     _extract_entry_num_retries: int = 5
     _extract_entry_retry_wait_sec: int = 3

src/ytdl_sub/downloaders/soundcloud/albums_and_singles.py

@@ -52,6 +52,7 @@ class SoundcloudAlbumsAndSinglesDownloader(
 ):
     downloader_options_type = SoundcloudAlbumsAndSinglesDownloadOptions
     supports_subtitles = False
+    supports_chapters = False
 
     @classmethod
     def ytdl_option_defaults(cls) -> Dict:

src/ytdl_sub/downloaders/youtube/merge_playlist.py

@@ -65,6 +65,7 @@ class YoutubeMergePlaylistDownloader(
     downloader_entry_type = YoutubeVideo
     supports_download_archive = False
     supports_subtitles = False
+    supports_chapters = False
 
     @classmethod
     def ytdl_option_defaults(cls) -> Dict:

src/ytdl_sub/plugins/chapters.py

@@ -28,6 +28,18 @@ SPONSORBLOCK_CATEGORIES: Set[str] = SPONSORBLOCK_HIGHLIGHT_CATEGORIES | {
 }
 
 
+def _chapters(entry: Entry) -> List[Dict]:
+    if entry.kwargs_contains("chapters"):
+        return entry.kwargs("chapters")
+    return []
+
+
+def _sponsorblock_chapters(entry: Entry) -> List[Dict]:
+    if entry.kwargs_contains("sponsorblock_chapters"):
+        return entry.kwargs("sponsorblock_chapters")
+    return []
+
+
 class SponsorBlockCategoriesValidator(StringSelectValidator):
     _expected_value_type_name = "sponsorblock category"
     _select_values = {"all"} | SPONSORBLOCK_CATEGORIES
@@ -40,8 +52,8 @@ class SponsorBlockCategoryListValidator(ListValidator[SponsorBlockCategoriesVali
 
 class ChaptersOptions(PluginOptions):
     """
-    Add chapters to video files if they are present. Additional options to add SponsorBlock
+    Embeds chapters to video files if they are present. Additional options to add SponsorBlock
-    chapters and remove specific ones. Can also remove chapters using regex patterns.
+    chapters and remove specific ones. Can also remove chapters using regex.
 
     Note that at this time, chapter removal with regex will not work with chapters added via
     timestamp file.
@@ -53,6 +65,7 @@ class ChaptersOptions(PluginOptions):
        presets:
          my_example_preset:
            chapters:
+             embed_chapters: True
              sponsorblock_categories:
                - "outro"
                - "selfpromo"
@@ -69,6 +82,7 @@ class ChaptersOptions(PluginOptions):
     """
 
     _optional_keys = {
+        "embed_chapters",
         "sponsorblock_categories",
         "remove_sponsorblock_categories",
         "remove_chapters_regex",
@@ -77,6 +91,9 @@ class ChaptersOptions(PluginOptions):
 
     def __init__(self, name, value):
         super().__init__(name, value)
+        self._embed_chapters = self._validate_key_if_present(
+            key="embed_chapters", validator=BoolValidator, default=True
+        ).value
         self._sponsorblock_categories = self._validate_key_if_present(
             key="sponsorblock_categories", validator=SponsorBlockCategoryListValidator
         )
@@ -95,8 +112,20 @@ class ChaptersOptions(PluginOptions):
                 "Must specify sponsorblock_categories if you are going to remove any of them"
             )
 
+    @property
+    def embed_chapters(self) -> Optional[bool]:
+        """
+        Optional. Embed chapters into the file. Defaults to True.
+        """
+        return self._embed_chapters
+
     @property
     def sponsorblock_categories(self) -> Optional[List[str]]:
+        """
+        Optional. List of SponsorBlock categories to embed as chapters. Supports "sponsor",
+        "intro", "outro", "selfpromo", "preview", "filler", "interaction", "music_offtopic",
+        "poi_highlight", or "all" to include all categories.
+        """
         if self._sponsorblock_categories:
             category_list = [validator.value for validator in self._sponsorblock_categories.list]
             if "all" in category_list:
@@ -106,6 +135,11 @@ class ChaptersOptions(PluginOptions):
 
     @property
     def remove_sponsorblock_categories(self) -> Optional[List[str]]:
+        """
+        Optional. List of SponsorBlock categories to remove from the output file. Can only remove
+        categories that are specified in ``sponsorblock_categories`` or "all", which removes
+        everything specified in ``sponsorblock_categories``.
+        """
         if self._remove_sponsorblock_categories:
             category_list = [
                 validator.value for validator in self._remove_sponsorblock_categories.list
@@ -117,12 +151,21 @@ class ChaptersOptions(PluginOptions):
 
     @property
     def remove_chapters_regex(self) -> Optional[List[re.Pattern]]:
+        """
+        Optional. List of regex patterns to match chapter titles against and remove them from the
+        entry.
+        """
         if self._remove_chapters_regex:
             return [validator.compiled_regex for validator in self._remove_chapters_regex.list]
         return None
 
     @property
     def force_key_frames(self) -> bool:
+        """
+        Optional. Force keyframes at cuts when removing sections. This is slow due to needing a
+        re-encode, but the resulting video may have fewer artifacts around the cuts. Defaults to
+        False.
+        """
         return self._force_key_frames
 
 
@@ -137,6 +180,11 @@ class ChaptersPlugin(Plugin[ChaptersOptions]):
         )
 
     def ytdl_options(self) -> Optional[Dict]:
+        """
+        Returns
+        -------
+        YTDL options to embed chapters, add/remove SponsorBlock segments, remove chapters via regex
+        """
         builder = YTDLOptionsBuilder()
         if self.plugin_options.sponsorblock_categories:
             builder.add(
@@ -152,8 +200,14 @@ class ChaptersPlugin(Plugin[ChaptersOptions]):
                 }
             )
 
-        # Always add chapters
+        if self.plugin_options.embed_chapters:
-        builder.add({"postprocessors": [{"key": "FFmpegMetadata", "add_chapters": True}]})
+            builder.add(
+                {
+                    "postprocessors": [
+                        {"key": "FFmpegMetadata", "add_chapters": True, "add_metadata": False}
+                    ]
+                }
+            )
 
         if self._is_removing_chapters:
             remove_chapters_post_processor = {
@@ -169,31 +223,23 @@ class ChaptersPlugin(Plugin[ChaptersOptions]):
                     "remove_chapters_patterns"
                 ] = self.plugin_options.remove_chapters_regex
 
-            builder.add(
+            if self.plugin_options.embed_chapters:
-                {
+                builder.add(
-                    "postprocessors": [
+                    {
-                        remove_chapters_post_processor,
+                        # re-add chapters
-                        {"key": "FFmpegMetadata", "add_chapters": True},  # re-add chapters
+                        "postprocessors": [
-                    ]
+                            remove_chapters_post_processor,
-                }
+                            {"key": "FFmpegMetadata", "add_chapters": True, "add_metadata": False},
-            )
+                        ]
+                    }
+                )
 
         return builder.to_dict()
 
-    def _chapters(self, entry: Entry) -> List[Dict]:
-        if entry.kwargs_contains("chapters"):
-            return entry.kwargs("chapters")
-        return []
-
-    def _sponsorblock_chapters(self, entry: Entry) -> List[Dict]:
-        if entry.kwargs_contains("sponsorblock_chapters"):
-            return entry.kwargs("sponsorblock_chapters")
-        return []
-
     def _get_removed_chapters(self, entry: Entry) -> List[str]:
         removed_chapters: List[str] = []
         for pattern in self.plugin_options.remove_chapters_regex or []:
-            for chapter in self._chapters(entry):
+            for chapter in _chapters(entry):
                 if pattern.search(chapter["title"]):
                     removed_chapters.append(chapter["title"])
         return removed_chapters
@@ -201,7 +247,7 @@ class ChaptersPlugin(Plugin[ChaptersOptions]):
     def _get_removed_sponsorblock_category_counts(self, entry: Entry) -> Dict:
         removed_category_counts = collections.Counter()
         for category in self.plugin_options.remove_sponsorblock_categories or []:
-            for chapter in self._sponsorblock_chapters(entry):
+            for chapter in _sponsorblock_chapters(entry):
                 if chapter["category"] == category:
                     removed_category_counts.update({chapter["title"]: 1})
 
@@ -214,6 +260,16 @@ class ChaptersPlugin(Plugin[ChaptersOptions]):
         )
 
     def post_process_entry(self, entry: Entry) -> Optional[FileMetadata]:
+        """
+        Parameters
+        ----------
+        entry:
+            Entry with possibly removed chapters
+
+        Returns
+        -------
+        FileMetadata outlining which chapters/SponsorBlock segments got removed
+        """
         metadata_dict = {}
         removed_chapters = self._get_removed_chapters(entry)
         removed_sponsorblock = self._get_removed_sponsorblock_category_counts(entry)

src/ytdl_sub/subscriptions/subscription_ytdl_options.py

@@ -105,6 +105,10 @@ class SubscriptionYTDLOptions:
         if not (chapters_plugin := self._get_plugin(ChaptersPlugin)):
             return {}
 
+        if not self._downloader.supports_chapters:
+            # TODO: warn here
+            return {}
+
         return chapters_plugin.ytdl_options()
 
     @property

tests/e2e/plugins/test_audio_extract.py

@@ -7,16 +7,15 @@ from ytdl_sub.subscriptions.subscription import Subscription
 
 
 @pytest.fixture
-def single_song_preset_dict(output_directory, timestamps_file_path):
+def single_song_preset_dict(output_directory):
     return {
         "preset": "yt_song",
-        "youtube": {
+        "youtube": {"video_url": "https://www.youtube.com/watch?v=2lAe1cqCOXo"},
-            "video_url": "https://www.youtube.com/watch?v=gr0XWmEbiMQ",
-        },
         "output_options": {"output_directory": output_directory},
         # download the worst format so it is fast
         "ytdl_options": {
             "format": "worst[ext=mp4]",
+            "postprocessor_args": {"ffmpeg": ["-bitexact"]},  # Must add this for reproducibility
         },
     }
 
@@ -31,6 +30,7 @@ def multiple_songs_preset_dict(output_directory):
         # download the worst format so it is fast
         "ytdl_options": {
             "format": "worst[ext=mp4]",
+            "postprocessor_args": {"ffmpeg": ["-bitexact"]},  # Must add this for reproducibility
         },
     }
 
@@ -55,13 +55,11 @@ class TestAudioExtract:
             output_directory=output_directory,
             transaction_log=transaction_log,
             transaction_log_summary_file_name="plugins/test_audio_extract_single.txt",
-            regenerate_transaction_log=True,
         )
         assert_expected_downloads(
             output_directory=output_directory,
             dry_run=dry_run,
             expected_download_summary_file_name="plugins/test_audio_extract_single.json",
-            regenerate_expected_download_summary=True,
         )
 
     @pytest.mark.parametrize("dry_run", [True, False])
@@ -83,11 +81,9 @@ class TestAudioExtract:
             output_directory=output_directory,
             transaction_log=transaction_log,
             transaction_log_summary_file_name="plugins/test_audio_extract_playlist.txt",
-            regenerate_transaction_log=True,
         )
         assert_expected_downloads(
             output_directory=output_directory,
             dry_run=dry_run,
             expected_download_summary_file_name="plugins/test_audio_extract_playlist.json",
-            regenerate_expected_download_summary=True,
         )

tests/e2e/plugins/test_chapters.py

@@ -17,10 +17,6 @@ def single_video_sponsorblock_and_embedded_subs_preset_dict(output_directory):
             "languages": ["en", "de"],
             "allow_auto_generated_subtitles": True,
         },
-        "audio_extract": {
-            "codec": "mp3",
-            "quality": 128,
-        },
         "chapters": {
             "sponsorblock_categories": [
                 "outro",
@@ -40,6 +36,7 @@ def single_video_sponsorblock_and_embedded_subs_preset_dict(output_directory):
         # download the worst format so it is fast
         "ytdl_options": {
             "format": "worst[ext=mp4]",
+            "postprocessor_args": {"ffmpeg": ["-bitexact"]},  # Must add this for reproducibility
         },
         "overrides": {"artist": "JMC"},
     }
@@ -65,11 +62,12 @@ class TestChapters:
             output_directory=output_directory,
             transaction_log=transaction_log,
             transaction_log_summary_file_name="plugins/test_chapters_sb_and_embedded_subs.txt",
-            regenerate_transaction_log=True,
         )
         assert_expected_downloads(
             output_directory=output_directory,
             dry_run=dry_run,
             expected_download_summary_file_name="plugins/test_chapters_sb_and_embedded_subs.json",
-            regenerate_expected_download_summary=True,
+            ignore_md5_hashes_for=[
+                "JMC - This GPU SLIDES into this Case! - Silverstone SUGO 16 ITX Case.mp4"
+            ],
         )

tests/e2e/resources/expected_downloads_summaries/plugins/test_audio_extract_playlist.json

@@ -1,5 +1,5 @@
 {
-  "Jesse's Minecraft Server [Trailer - Feb.1].ogg": "67b19b495756fb6dc332f49bfa2957d3",
+  "Jesse's Minecraft Server [Trailer - Feb.1].ogg": "c37c289bc9b7c79464aa8bcc6df423e3",
-  "Jesse's Minecraft Server [Trailer - Feb.27].ogg": "273c96652e198171ee60345051addc7a",
+  "Jesse's Minecraft Server [Trailer - Feb.27].ogg": "37def0736bb5c0a7fba0c1685e90bf3c",
-  "Jesse's Minecraft Server [Trailer - Mar.21].ogg": "e91db58c153f0724aaed5ce53de2a736"
+  "Jesse's Minecraft Server [Trailer - Mar.21].ogg": "1fcc317ee5ce24675f6e4e6c2eae40d3"
 }

tests/e2e/resources/expected_downloads_summaries/plugins/test_audio_extract_single.json

@@ -1,3 +1,3 @@
 {
-  "YouTube Rewind 2019： For the Record ｜ #YouTubeRewind.mp3": "c4ce74e2ddc4e6a0ab823b5b622e2275"
+  "YouTube Rewind 2019： For the Record ｜ #YouTubeRewind.mp3": "3a156b122bd79c956cce5079d3530cc3"
 }

tests/e2e/resources/expected_downloads_summaries/plugins/test_chapters_sb_and_embedded_subs.json

@@ -1,5 +1,5 @@
 {
   "JMC - This GPU SLIDES into this Case! - Silverstone SUGO 16 ITX Case-thumb.jpg": "a81457393418b5abed785a82122f3352",
-  "JMC - This GPU SLIDES into this Case! - Silverstone SUGO 16 ITX Case.mp4": "ff829c47452173eac92e09507feb64a0",
+  "JMC - This GPU SLIDES into this Case! - Silverstone SUGO 16 ITX Case.mp4": "2c417b31c9f5eb8cbecf0bf1fc1f9e53",
   "JMC - This GPU SLIDES into this Case! - Silverstone SUGO 16 ITX Case.nfo": "0c06fe6874588209fccbd9276a446750"
 }

tests/e2e/resources/transaction_log_summaries/plugins/test_audio_extract_single.txt

@@ -1,11 +1,11 @@
 Files created in '{output_directory}'
 ----------------------------------------
-Gabor Szabo - Dreams (1968) [full album].mp3
+YouTube Rewind 2019： For the Record ｜ #YouTubeRewind.mp3
   Music Tags:
     album: Singles
-    albumartist: Cubensis Records
+    albumartist: YouTube
-    artist: Cubensis Records
+    artist: YouTube
     genre: Unset
-    title: Gabor Szabo - Dreams (1968) [full album]
+    title: YouTube Rewind 2019: For the Record | #YouTubeRewind
     track: 1
-    year: 2015
+    year: 2019

tests/e2e/resources/transaction_log_summaries/plugins/test_chapters_sb_and_embedded_subs.txt

@@ -1,9 +1,14 @@
 Files created in '{output_directory}'
 ----------------------------------------
 JMC - This GPU SLIDES into this Case! - Silverstone SUGO 16 ITX Case-thumb.jpg
-JMC - This GPU SLIDES into this Case! - Silverstone SUGO 16 ITX Case.mp3
+JMC - This GPU SLIDES into this Case! - Silverstone SUGO 16 ITX Case.mp4
   Embedded Chapters
     Removed Chapter(s): Intro, Outro
+    Removed SponsorBlock Category Count(s):
+      Sponsor: 2
+      Endcards/Credits: 1
+      Intermission/Intro Animation: 1
+      Unpaid/Self Promotion: 1
   Embedded subtitles with lang(s) en, de
 JMC - This GPU SLIDES into this Case! - Silverstone SUGO 16 ITX Case.nfo
   NFO tags: