diff --git a/docs/config.rst b/docs/config.rst index a646b080..b058dcf1 100644 --- a/docs/config.rst +++ b/docs/config.rst @@ -177,6 +177,14 @@ audio_extract ------------------------------------------------------------------------------- +chapters +'''''''' +.. autoclass:: ytdl_sub.plugins.chapters.ChaptersOptions() + :members: + :member-order: bysource + +------------------------------------------------------------------------------- + music_tags '''''''''' .. autoclass:: ytdl_sub.plugins.music_tags.MusicTagsOptions() diff --git a/src/ytdl_sub/downloaders/downloader.py b/src/ytdl_sub/downloaders/downloader.py index ae808f95..0ac150e8 100644 --- a/src/ytdl_sub/downloaders/downloader.py +++ b/src/ytdl_sub/downloaders/downloader.py @@ -57,6 +57,7 @@ class Downloader(DownloadArchiver, Generic[DownloaderOptionsT, DownloaderEntryT] supports_download_archive: bool = True supports_subtitles: bool = True + supports_chapters: bool = True _extract_entry_num_retries: int = 5 _extract_entry_retry_wait_sec: int = 3 diff --git a/src/ytdl_sub/downloaders/soundcloud/albums_and_singles.py b/src/ytdl_sub/downloaders/soundcloud/albums_and_singles.py index a2ad1720..3052576b 100644 --- a/src/ytdl_sub/downloaders/soundcloud/albums_and_singles.py +++ b/src/ytdl_sub/downloaders/soundcloud/albums_and_singles.py @@ -52,6 +52,7 @@ class SoundcloudAlbumsAndSinglesDownloader( ): downloader_options_type = SoundcloudAlbumsAndSinglesDownloadOptions supports_subtitles = False + supports_chapters = False @classmethod def ytdl_option_defaults(cls) -> Dict: diff --git a/src/ytdl_sub/downloaders/youtube/merge_playlist.py b/src/ytdl_sub/downloaders/youtube/merge_playlist.py index 7f94397d..4ef24b61 100644 --- a/src/ytdl_sub/downloaders/youtube/merge_playlist.py +++ b/src/ytdl_sub/downloaders/youtube/merge_playlist.py @@ -65,6 +65,7 @@ class YoutubeMergePlaylistDownloader( downloader_entry_type = YoutubeVideo supports_download_archive = False supports_subtitles = False + supports_chapters = False @classmethod def ytdl_option_defaults(cls) -> Dict: diff --git a/src/ytdl_sub/plugins/chapters.py b/src/ytdl_sub/plugins/chapters.py index 0c4e9bc0..dead5838 100644 --- a/src/ytdl_sub/plugins/chapters.py +++ b/src/ytdl_sub/plugins/chapters.py @@ -28,6 +28,18 @@ SPONSORBLOCK_CATEGORIES: Set[str] = SPONSORBLOCK_HIGHLIGHT_CATEGORIES | { } +def _chapters(entry: Entry) -> List[Dict]: + if entry.kwargs_contains("chapters"): + return entry.kwargs("chapters") + return [] + + +def _sponsorblock_chapters(entry: Entry) -> List[Dict]: + if entry.kwargs_contains("sponsorblock_chapters"): + return entry.kwargs("sponsorblock_chapters") + return [] + + class SponsorBlockCategoriesValidator(StringSelectValidator): _expected_value_type_name = "sponsorblock category" _select_values = {"all"} | SPONSORBLOCK_CATEGORIES @@ -40,8 +52,8 @@ class SponsorBlockCategoryListValidator(ListValidator[SponsorBlockCategoriesVali class ChaptersOptions(PluginOptions): """ - Add chapters to video files if they are present. Additional options to add SponsorBlock - chapters and remove specific ones. Can also remove chapters using regex patterns. + Embeds chapters to video files if they are present. Additional options to add SponsorBlock + chapters and remove specific ones. Can also remove chapters using regex. Note that at this time, chapter removal with regex will not work with chapters added via timestamp file. @@ -53,6 +65,7 @@ class ChaptersOptions(PluginOptions): presets: my_example_preset: chapters: + embed_chapters: True sponsorblock_categories: - "outro" - "selfpromo" @@ -69,6 +82,7 @@ class ChaptersOptions(PluginOptions): """ _optional_keys = { + "embed_chapters", "sponsorblock_categories", "remove_sponsorblock_categories", "remove_chapters_regex", @@ -77,6 +91,9 @@ class ChaptersOptions(PluginOptions): def __init__(self, name, value): super().__init__(name, value) + self._embed_chapters = self._validate_key_if_present( + key="embed_chapters", validator=BoolValidator, default=True + ).value self._sponsorblock_categories = self._validate_key_if_present( key="sponsorblock_categories", validator=SponsorBlockCategoryListValidator ) @@ -95,8 +112,20 @@ class ChaptersOptions(PluginOptions): "Must specify sponsorblock_categories if you are going to remove any of them" ) + @property + def embed_chapters(self) -> Optional[bool]: + """ + Optional. Embed chapters into the file. Defaults to True. + """ + return self._embed_chapters + @property def sponsorblock_categories(self) -> Optional[List[str]]: + """ + Optional. List of SponsorBlock categories to embed as chapters. Supports "sponsor", + "intro", "outro", "selfpromo", "preview", "filler", "interaction", "music_offtopic", + "poi_highlight", or "all" to include all categories. + """ if self._sponsorblock_categories: category_list = [validator.value for validator in self._sponsorblock_categories.list] if "all" in category_list: @@ -106,6 +135,11 @@ class ChaptersOptions(PluginOptions): @property def remove_sponsorblock_categories(self) -> Optional[List[str]]: + """ + Optional. List of SponsorBlock categories to remove from the output file. Can only remove + categories that are specified in ``sponsorblock_categories`` or "all", which removes + everything specified in ``sponsorblock_categories``. + """ if self._remove_sponsorblock_categories: category_list = [ validator.value for validator in self._remove_sponsorblock_categories.list @@ -117,12 +151,21 @@ class ChaptersOptions(PluginOptions): @property def remove_chapters_regex(self) -> Optional[List[re.Pattern]]: + """ + Optional. List of regex patterns to match chapter titles against and remove them from the + entry. + """ if self._remove_chapters_regex: return [validator.compiled_regex for validator in self._remove_chapters_regex.list] return None @property def force_key_frames(self) -> bool: + """ + Optional. Force keyframes at cuts when removing sections. This is slow due to needing a + re-encode, but the resulting video may have fewer artifacts around the cuts. Defaults to + False. + """ return self._force_key_frames @@ -137,6 +180,11 @@ class ChaptersPlugin(Plugin[ChaptersOptions]): ) def ytdl_options(self) -> Optional[Dict]: + """ + Returns + ------- + YTDL options to embed chapters, add/remove SponsorBlock segments, remove chapters via regex + """ builder = YTDLOptionsBuilder() if self.plugin_options.sponsorblock_categories: builder.add( @@ -152,8 +200,14 @@ class ChaptersPlugin(Plugin[ChaptersOptions]): } ) - # Always add chapters - builder.add({"postprocessors": [{"key": "FFmpegMetadata", "add_chapters": True}]}) + if self.plugin_options.embed_chapters: + builder.add( + { + "postprocessors": [ + {"key": "FFmpegMetadata", "add_chapters": True, "add_metadata": False} + ] + } + ) if self._is_removing_chapters: remove_chapters_post_processor = { @@ -169,31 +223,23 @@ class ChaptersPlugin(Plugin[ChaptersOptions]): "remove_chapters_patterns" ] = self.plugin_options.remove_chapters_regex - builder.add( - { - "postprocessors": [ - remove_chapters_post_processor, - {"key": "FFmpegMetadata", "add_chapters": True}, # re-add chapters - ] - } - ) + if self.plugin_options.embed_chapters: + builder.add( + { + # re-add chapters + "postprocessors": [ + remove_chapters_post_processor, + {"key": "FFmpegMetadata", "add_chapters": True, "add_metadata": False}, + ] + } + ) return builder.to_dict() - def _chapters(self, entry: Entry) -> List[Dict]: - if entry.kwargs_contains("chapters"): - return entry.kwargs("chapters") - return [] - - def _sponsorblock_chapters(self, entry: Entry) -> List[Dict]: - if entry.kwargs_contains("sponsorblock_chapters"): - return entry.kwargs("sponsorblock_chapters") - return [] - def _get_removed_chapters(self, entry: Entry) -> List[str]: removed_chapters: List[str] = [] for pattern in self.plugin_options.remove_chapters_regex or []: - for chapter in self._chapters(entry): + for chapter in _chapters(entry): if pattern.search(chapter["title"]): removed_chapters.append(chapter["title"]) return removed_chapters @@ -201,7 +247,7 @@ class ChaptersPlugin(Plugin[ChaptersOptions]): def _get_removed_sponsorblock_category_counts(self, entry: Entry) -> Dict: removed_category_counts = collections.Counter() for category in self.plugin_options.remove_sponsorblock_categories or []: - for chapter in self._sponsorblock_chapters(entry): + for chapter in _sponsorblock_chapters(entry): if chapter["category"] == category: removed_category_counts.update({chapter["title"]: 1}) @@ -214,6 +260,16 @@ class ChaptersPlugin(Plugin[ChaptersOptions]): ) def post_process_entry(self, entry: Entry) -> Optional[FileMetadata]: + """ + Parameters + ---------- + entry: + Entry with possibly removed chapters + + Returns + ------- + FileMetadata outlining which chapters/SponsorBlock segments got removed + """ metadata_dict = {} removed_chapters = self._get_removed_chapters(entry) removed_sponsorblock = self._get_removed_sponsorblock_category_counts(entry) diff --git a/src/ytdl_sub/subscriptions/subscription_ytdl_options.py b/src/ytdl_sub/subscriptions/subscription_ytdl_options.py index 4d8cca11..64833e22 100644 --- a/src/ytdl_sub/subscriptions/subscription_ytdl_options.py +++ b/src/ytdl_sub/subscriptions/subscription_ytdl_options.py @@ -105,6 +105,10 @@ class SubscriptionYTDLOptions: if not (chapters_plugin := self._get_plugin(ChaptersPlugin)): return {} + if not self._downloader.supports_chapters: + # TODO: warn here + return {} + return chapters_plugin.ytdl_options() @property diff --git a/tests/e2e/plugins/test_audio_extract.py b/tests/e2e/plugins/test_audio_extract.py index 2c10ce4f..c3492cd3 100644 --- a/tests/e2e/plugins/test_audio_extract.py +++ b/tests/e2e/plugins/test_audio_extract.py @@ -7,16 +7,15 @@ from ytdl_sub.subscriptions.subscription import Subscription @pytest.fixture -def single_song_preset_dict(output_directory, timestamps_file_path): +def single_song_preset_dict(output_directory): return { "preset": "yt_song", - "youtube": { - "video_url": "https://www.youtube.com/watch?v=gr0XWmEbiMQ", - }, + "youtube": {"video_url": "https://www.youtube.com/watch?v=2lAe1cqCOXo"}, "output_options": {"output_directory": output_directory}, # download the worst format so it is fast "ytdl_options": { "format": "worst[ext=mp4]", + "postprocessor_args": {"ffmpeg": ["-bitexact"]}, # Must add this for reproducibility }, } @@ -31,6 +30,7 @@ def multiple_songs_preset_dict(output_directory): # download the worst format so it is fast "ytdl_options": { "format": "worst[ext=mp4]", + "postprocessor_args": {"ffmpeg": ["-bitexact"]}, # Must add this for reproducibility }, } @@ -55,13 +55,11 @@ class TestAudioExtract: output_directory=output_directory, transaction_log=transaction_log, transaction_log_summary_file_name="plugins/test_audio_extract_single.txt", - regenerate_transaction_log=True, ) assert_expected_downloads( output_directory=output_directory, dry_run=dry_run, expected_download_summary_file_name="plugins/test_audio_extract_single.json", - regenerate_expected_download_summary=True, ) @pytest.mark.parametrize("dry_run", [True, False]) @@ -83,11 +81,9 @@ class TestAudioExtract: output_directory=output_directory, transaction_log=transaction_log, transaction_log_summary_file_name="plugins/test_audio_extract_playlist.txt", - regenerate_transaction_log=True, ) assert_expected_downloads( output_directory=output_directory, dry_run=dry_run, expected_download_summary_file_name="plugins/test_audio_extract_playlist.json", - regenerate_expected_download_summary=True, ) diff --git a/tests/e2e/plugins/test_chapters.py b/tests/e2e/plugins/test_chapters.py index ff444ab8..9e6ebb30 100644 --- a/tests/e2e/plugins/test_chapters.py +++ b/tests/e2e/plugins/test_chapters.py @@ -17,10 +17,6 @@ def single_video_sponsorblock_and_embedded_subs_preset_dict(output_directory): "languages": ["en", "de"], "allow_auto_generated_subtitles": True, }, - "audio_extract": { - "codec": "mp3", - "quality": 128, - }, "chapters": { "sponsorblock_categories": [ "outro", @@ -40,6 +36,7 @@ def single_video_sponsorblock_and_embedded_subs_preset_dict(output_directory): # download the worst format so it is fast "ytdl_options": { "format": "worst[ext=mp4]", + "postprocessor_args": {"ffmpeg": ["-bitexact"]}, # Must add this for reproducibility }, "overrides": {"artist": "JMC"}, } @@ -65,11 +62,12 @@ class TestChapters: output_directory=output_directory, transaction_log=transaction_log, transaction_log_summary_file_name="plugins/test_chapters_sb_and_embedded_subs.txt", - regenerate_transaction_log=True, ) assert_expected_downloads( output_directory=output_directory, dry_run=dry_run, expected_download_summary_file_name="plugins/test_chapters_sb_and_embedded_subs.json", - regenerate_expected_download_summary=True, + ignore_md5_hashes_for=[ + "JMC - This GPU SLIDES into this Case! - Silverstone SUGO 16 ITX Case.mp4" + ], ) diff --git a/tests/e2e/resources/expected_downloads_summaries/plugins/test_audio_extract_playlist.json b/tests/e2e/resources/expected_downloads_summaries/plugins/test_audio_extract_playlist.json index 35c71317..a849b2ae 100644 --- a/tests/e2e/resources/expected_downloads_summaries/plugins/test_audio_extract_playlist.json +++ b/tests/e2e/resources/expected_downloads_summaries/plugins/test_audio_extract_playlist.json @@ -1,5 +1,5 @@ { - "Jesse's Minecraft Server [Trailer - Feb.1].ogg": "67b19b495756fb6dc332f49bfa2957d3", - "Jesse's Minecraft Server [Trailer - Feb.27].ogg": "273c96652e198171ee60345051addc7a", - "Jesse's Minecraft Server [Trailer - Mar.21].ogg": "e91db58c153f0724aaed5ce53de2a736" + "Jesse's Minecraft Server [Trailer - Feb.1].ogg": "c37c289bc9b7c79464aa8bcc6df423e3", + "Jesse's Minecraft Server [Trailer - Feb.27].ogg": "37def0736bb5c0a7fba0c1685e90bf3c", + "Jesse's Minecraft Server [Trailer - Mar.21].ogg": "1fcc317ee5ce24675f6e4e6c2eae40d3" } \ No newline at end of file diff --git a/tests/e2e/resources/expected_downloads_summaries/plugins/test_audio_extract_single.json b/tests/e2e/resources/expected_downloads_summaries/plugins/test_audio_extract_single.json index 1a2dbb73..4ac1b8be 100644 --- a/tests/e2e/resources/expected_downloads_summaries/plugins/test_audio_extract_single.json +++ b/tests/e2e/resources/expected_downloads_summaries/plugins/test_audio_extract_single.json @@ -1,3 +1,3 @@ { - "YouTube Rewind 2019： For the Record ｜ #YouTubeRewind.mp3": "c4ce74e2ddc4e6a0ab823b5b622e2275" + "YouTube Rewind 2019： For the Record ｜ #YouTubeRewind.mp3": "3a156b122bd79c956cce5079d3530cc3" } \ No newline at end of file diff --git a/tests/e2e/resources/expected_downloads_summaries/plugins/test_chapters_sb_and_embedded_subs.json b/tests/e2e/resources/expected_downloads_summaries/plugins/test_chapters_sb_and_embedded_subs.json index 264aa7d5..42560d25 100644 --- a/tests/e2e/resources/expected_downloads_summaries/plugins/test_chapters_sb_and_embedded_subs.json +++ b/tests/e2e/resources/expected_downloads_summaries/plugins/test_chapters_sb_and_embedded_subs.json @@ -1,5 +1,5 @@ { "JMC - This GPU SLIDES into this Case! - Silverstone SUGO 16 ITX Case-thumb.jpg": "a81457393418b5abed785a82122f3352", - "JMC - This GPU SLIDES into this Case! - Silverstone SUGO 16 ITX Case.mp4": "ff829c47452173eac92e09507feb64a0", + "JMC - This GPU SLIDES into this Case! - Silverstone SUGO 16 ITX Case.mp4": "2c417b31c9f5eb8cbecf0bf1fc1f9e53", "JMC - This GPU SLIDES into this Case! - Silverstone SUGO 16 ITX Case.nfo": "0c06fe6874588209fccbd9276a446750" } \ No newline at end of file diff --git a/tests/e2e/resources/transaction_log_summaries/plugins/test_audio_extract_single.txt b/tests/e2e/resources/transaction_log_summaries/plugins/test_audio_extract_single.txt index 6ff96145..2ee2d4a4 100644 --- a/tests/e2e/resources/transaction_log_summaries/plugins/test_audio_extract_single.txt +++ b/tests/e2e/resources/transaction_log_summaries/plugins/test_audio_extract_single.txt @@ -1,11 +1,11 @@ Files created in '{output_directory}' ---------------------------------------- -Gabor Szabo - Dreams (1968) [full album].mp3 +YouTube Rewind 2019： For the Record ｜ #YouTubeRewind.mp3 Music Tags: album: Singles - albumartist: Cubensis Records - artist: Cubensis Records + albumartist: YouTube + artist: YouTube genre: Unset - title: Gabor Szabo - Dreams (1968) [full album] + title: YouTube Rewind 2019: For the Record | #YouTubeRewind track: 1 - year: 2015 \ No newline at end of file + year: 2019 \ No newline at end of file diff --git a/tests/e2e/resources/transaction_log_summaries/plugins/test_chapters_sb_and_embedded_subs.txt b/tests/e2e/resources/transaction_log_summaries/plugins/test_chapters_sb_and_embedded_subs.txt index ac4d145b..127e6543 100644 --- a/tests/e2e/resources/transaction_log_summaries/plugins/test_chapters_sb_and_embedded_subs.txt +++ b/tests/e2e/resources/transaction_log_summaries/plugins/test_chapters_sb_and_embedded_subs.txt @@ -1,9 +1,14 @@ Files created in '{output_directory}' ---------------------------------------- JMC - This GPU SLIDES into this Case! - Silverstone SUGO 16 ITX Case-thumb.jpg -JMC - This GPU SLIDES into this Case! - Silverstone SUGO 16 ITX Case.mp3 +JMC - This GPU SLIDES into this Case! - Silverstone SUGO 16 ITX Case.mp4 Embedded Chapters Removed Chapter(s): Intro, Outro + Removed SponsorBlock Category Count(s): + Sponsor: 2 + Endcards/Credits: 1 + Intermission/Intro Animation: 1 + Unpaid/Self Promotion: 1 Embedded subtitles with lang(s) en, de JMC - This GPU SLIDES into this Case! - Silverstone SUGO 16 ITX Case.nfo NFO tags: