Merge branch 'master' into jesse/split_chapters
This commit is contained in:
jbannon
commit
fe16083773
13 changed files with 119 additions and 49 deletions
|
|
@ -177,6 +177,14 @@ audio_extract
|
|||
|
||||
-------------------------------------------------------------------------------
|
||||
|
||||
chapters
|
||||
''''''''
|
||||
.. autoclass:: ytdl_sub.plugins.chapters.ChaptersOptions()
|
||||
:members:
|
||||
:member-order: bysource
|
||||
|
||||
-------------------------------------------------------------------------------
|
||||
|
||||
music_tags
|
||||
''''''''''
|
||||
.. autoclass:: ytdl_sub.plugins.music_tags.MusicTagsOptions()
|
||||
|
|
|
|||
|
|
@ -57,6 +57,7 @@ class Downloader(DownloadArchiver, Generic[DownloaderOptionsT, DownloaderEntryT]
|
|||
|
||||
supports_download_archive: bool = True
|
||||
supports_subtitles: bool = True
|
||||
supports_chapters: bool = True
|
||||
|
||||
_extract_entry_num_retries: int = 5
|
||||
_extract_entry_retry_wait_sec: int = 3
|
||||
|
|
|
|||
|
|
@ -52,6 +52,7 @@ class SoundcloudAlbumsAndSinglesDownloader(
|
|||
):
|
||||
downloader_options_type = SoundcloudAlbumsAndSinglesDownloadOptions
|
||||
supports_subtitles = False
|
||||
supports_chapters = False
|
||||
|
||||
@classmethod
|
||||
def ytdl_option_defaults(cls) -> Dict:
|
||||
|
|
|
|||
|
|
@ -65,6 +65,7 @@ class YoutubeMergePlaylistDownloader(
|
|||
downloader_entry_type = YoutubeVideo
|
||||
supports_download_archive = False
|
||||
supports_subtitles = False
|
||||
supports_chapters = False
|
||||
|
||||
@classmethod
|
||||
def ytdl_option_defaults(cls) -> Dict:
|
||||
|
|
|
|||
|
|
@ -28,6 +28,18 @@ SPONSORBLOCK_CATEGORIES: Set[str] = SPONSORBLOCK_HIGHLIGHT_CATEGORIES | {
|
|||
}
|
||||
|
||||
|
||||
def _chapters(entry: Entry) -> List[Dict]:
|
||||
if entry.kwargs_contains("chapters"):
|
||||
return entry.kwargs("chapters")
|
||||
return []
|
||||
|
||||
|
||||
def _sponsorblock_chapters(entry: Entry) -> List[Dict]:
|
||||
if entry.kwargs_contains("sponsorblock_chapters"):
|
||||
return entry.kwargs("sponsorblock_chapters")
|
||||
return []
|
||||
|
||||
|
||||
class SponsorBlockCategoriesValidator(StringSelectValidator):
|
||||
_expected_value_type_name = "sponsorblock category"
|
||||
_select_values = {"all"} | SPONSORBLOCK_CATEGORIES
|
||||
|
|
@ -40,8 +52,8 @@ class SponsorBlockCategoryListValidator(ListValidator[SponsorBlockCategoriesVali
|
|||
|
||||
class ChaptersOptions(PluginOptions):
|
||||
"""
|
||||
Add chapters to video files if they are present. Additional options to add SponsorBlock
|
||||
chapters and remove specific ones. Can also remove chapters using regex patterns.
|
||||
Embeds chapters to video files if they are present. Additional options to add SponsorBlock
|
||||
chapters and remove specific ones. Can also remove chapters using regex.
|
||||
|
||||
Note that at this time, chapter removal with regex will not work with chapters added via
|
||||
timestamp file.
|
||||
|
|
@ -53,6 +65,7 @@ class ChaptersOptions(PluginOptions):
|
|||
presets:
|
||||
my_example_preset:
|
||||
chapters:
|
||||
embed_chapters: True
|
||||
sponsorblock_categories:
|
||||
- "outro"
|
||||
- "selfpromo"
|
||||
|
|
@ -69,6 +82,7 @@ class ChaptersOptions(PluginOptions):
|
|||
"""
|
||||
|
||||
_optional_keys = {
|
||||
"embed_chapters",
|
||||
"sponsorblock_categories",
|
||||
"remove_sponsorblock_categories",
|
||||
"remove_chapters_regex",
|
||||
|
|
@ -77,6 +91,9 @@ class ChaptersOptions(PluginOptions):
|
|||
|
||||
def __init__(self, name, value):
|
||||
super().__init__(name, value)
|
||||
self._embed_chapters = self._validate_key_if_present(
|
||||
key="embed_chapters", validator=BoolValidator, default=True
|
||||
).value
|
||||
self._sponsorblock_categories = self._validate_key_if_present(
|
||||
key="sponsorblock_categories", validator=SponsorBlockCategoryListValidator
|
||||
)
|
||||
|
|
@ -95,8 +112,20 @@ class ChaptersOptions(PluginOptions):
|
|||
"Must specify sponsorblock_categories if you are going to remove any of them"
|
||||
)
|
||||
|
||||
@property
|
||||
def embed_chapters(self) -> Optional[bool]:
|
||||
"""
|
||||
Optional. Embed chapters into the file. Defaults to True.
|
||||
"""
|
||||
return self._embed_chapters
|
||||
|
||||
@property
|
||||
def sponsorblock_categories(self) -> Optional[List[str]]:
|
||||
"""
|
||||
Optional. List of SponsorBlock categories to embed as chapters. Supports "sponsor",
|
||||
"intro", "outro", "selfpromo", "preview", "filler", "interaction", "music_offtopic",
|
||||
"poi_highlight", or "all" to include all categories.
|
||||
"""
|
||||
if self._sponsorblock_categories:
|
||||
category_list = [validator.value for validator in self._sponsorblock_categories.list]
|
||||
if "all" in category_list:
|
||||
|
|
@ -106,6 +135,11 @@ class ChaptersOptions(PluginOptions):
|
|||
|
||||
@property
|
||||
def remove_sponsorblock_categories(self) -> Optional[List[str]]:
|
||||
"""
|
||||
Optional. List of SponsorBlock categories to remove from the output file. Can only remove
|
||||
categories that are specified in ``sponsorblock_categories`` or "all", which removes
|
||||
everything specified in ``sponsorblock_categories``.
|
||||
"""
|
||||
if self._remove_sponsorblock_categories:
|
||||
category_list = [
|
||||
validator.value for validator in self._remove_sponsorblock_categories.list
|
||||
|
|
@ -117,12 +151,21 @@ class ChaptersOptions(PluginOptions):
|
|||
|
||||
@property
|
||||
def remove_chapters_regex(self) -> Optional[List[re.Pattern]]:
|
||||
"""
|
||||
Optional. List of regex patterns to match chapter titles against and remove them from the
|
||||
entry.
|
||||
"""
|
||||
if self._remove_chapters_regex:
|
||||
return [validator.compiled_regex for validator in self._remove_chapters_regex.list]
|
||||
return None
|
||||
|
||||
@property
|
||||
def force_key_frames(self) -> bool:
|
||||
"""
|
||||
Optional. Force keyframes at cuts when removing sections. This is slow due to needing a
|
||||
re-encode, but the resulting video may have fewer artifacts around the cuts. Defaults to
|
||||
False.
|
||||
"""
|
||||
return self._force_key_frames
|
||||
|
||||
|
||||
|
|
@ -137,6 +180,11 @@ class ChaptersPlugin(Plugin[ChaptersOptions]):
|
|||
)
|
||||
|
||||
def ytdl_options(self) -> Optional[Dict]:
|
||||
"""
|
||||
Returns
|
||||
-------
|
||||
YTDL options to embed chapters, add/remove SponsorBlock segments, remove chapters via regex
|
||||
"""
|
||||
builder = YTDLOptionsBuilder()
|
||||
if self.plugin_options.sponsorblock_categories:
|
||||
builder.add(
|
||||
|
|
@ -152,8 +200,14 @@ class ChaptersPlugin(Plugin[ChaptersOptions]):
|
|||
}
|
||||
)
|
||||
|
||||
# Always add chapters
|
||||
builder.add({"postprocessors": [{"key": "FFmpegMetadata", "add_chapters": True}]})
|
||||
if self.plugin_options.embed_chapters:
|
||||
builder.add(
|
||||
{
|
||||
"postprocessors": [
|
||||
{"key": "FFmpegMetadata", "add_chapters": True, "add_metadata": False}
|
||||
]
|
||||
}
|
||||
)
|
||||
|
||||
if self._is_removing_chapters:
|
||||
remove_chapters_post_processor = {
|
||||
|
|
@ -169,31 +223,23 @@ class ChaptersPlugin(Plugin[ChaptersOptions]):
|
|||
"remove_chapters_patterns"
|
||||
] = self.plugin_options.remove_chapters_regex
|
||||
|
||||
builder.add(
|
||||
{
|
||||
"postprocessors": [
|
||||
remove_chapters_post_processor,
|
||||
{"key": "FFmpegMetadata", "add_chapters": True}, # re-add chapters
|
||||
]
|
||||
}
|
||||
)
|
||||
if self.plugin_options.embed_chapters:
|
||||
builder.add(
|
||||
{
|
||||
# re-add chapters
|
||||
"postprocessors": [
|
||||
remove_chapters_post_processor,
|
||||
{"key": "FFmpegMetadata", "add_chapters": True, "add_metadata": False},
|
||||
]
|
||||
}
|
||||
)
|
||||
|
||||
return builder.to_dict()
|
||||
|
||||
def _chapters(self, entry: Entry) -> List[Dict]:
|
||||
if entry.kwargs_contains("chapters"):
|
||||
return entry.kwargs("chapters")
|
||||
return []
|
||||
|
||||
def _sponsorblock_chapters(self, entry: Entry) -> List[Dict]:
|
||||
if entry.kwargs_contains("sponsorblock_chapters"):
|
||||
return entry.kwargs("sponsorblock_chapters")
|
||||
return []
|
||||
|
||||
def _get_removed_chapters(self, entry: Entry) -> List[str]:
|
||||
removed_chapters: List[str] = []
|
||||
for pattern in self.plugin_options.remove_chapters_regex or []:
|
||||
for chapter in self._chapters(entry):
|
||||
for chapter in _chapters(entry):
|
||||
if pattern.search(chapter["title"]):
|
||||
removed_chapters.append(chapter["title"])
|
||||
return removed_chapters
|
||||
|
|
@ -201,7 +247,7 @@ class ChaptersPlugin(Plugin[ChaptersOptions]):
|
|||
def _get_removed_sponsorblock_category_counts(self, entry: Entry) -> Dict:
|
||||
removed_category_counts = collections.Counter()
|
||||
for category in self.plugin_options.remove_sponsorblock_categories or []:
|
||||
for chapter in self._sponsorblock_chapters(entry):
|
||||
for chapter in _sponsorblock_chapters(entry):
|
||||
if chapter["category"] == category:
|
||||
removed_category_counts.update({chapter["title"]: 1})
|
||||
|
||||
|
|
@ -214,6 +260,16 @@ class ChaptersPlugin(Plugin[ChaptersOptions]):
|
|||
)
|
||||
|
||||
def post_process_entry(self, entry: Entry) -> Optional[FileMetadata]:
|
||||
"""
|
||||
Parameters
|
||||
----------
|
||||
entry:
|
||||
Entry with possibly removed chapters
|
||||
|
||||
Returns
|
||||
-------
|
||||
FileMetadata outlining which chapters/SponsorBlock segments got removed
|
||||
"""
|
||||
metadata_dict = {}
|
||||
removed_chapters = self._get_removed_chapters(entry)
|
||||
removed_sponsorblock = self._get_removed_sponsorblock_category_counts(entry)
|
||||
|
|
|
|||
|
|
@ -105,6 +105,10 @@ class SubscriptionYTDLOptions:
|
|||
if not (chapters_plugin := self._get_plugin(ChaptersPlugin)):
|
||||
return {}
|
||||
|
||||
if not self._downloader.supports_chapters:
|
||||
# TODO: warn here
|
||||
return {}
|
||||
|
||||
return chapters_plugin.ytdl_options()
|
||||
|
||||
@property
|
||||
|
|
|
|||
|
|
@ -7,16 +7,15 @@ from ytdl_sub.subscriptions.subscription import Subscription
|
|||
|
||||
|
||||
@pytest.fixture
|
||||
def single_song_preset_dict(output_directory, timestamps_file_path):
|
||||
def single_song_preset_dict(output_directory):
|
||||
return {
|
||||
"preset": "yt_song",
|
||||
"youtube": {
|
||||
"video_url": "https://www.youtube.com/watch?v=gr0XWmEbiMQ",
|
||||
},
|
||||
"youtube": {"video_url": "https://www.youtube.com/watch?v=2lAe1cqCOXo"},
|
||||
"output_options": {"output_directory": output_directory},
|
||||
# download the worst format so it is fast
|
||||
"ytdl_options": {
|
||||
"format": "worst[ext=mp4]",
|
||||
"postprocessor_args": {"ffmpeg": ["-bitexact"]}, # Must add this for reproducibility
|
||||
},
|
||||
}
|
||||
|
||||
|
|
@ -31,6 +30,7 @@ def multiple_songs_preset_dict(output_directory):
|
|||
# download the worst format so it is fast
|
||||
"ytdl_options": {
|
||||
"format": "worst[ext=mp4]",
|
||||
"postprocessor_args": {"ffmpeg": ["-bitexact"]}, # Must add this for reproducibility
|
||||
},
|
||||
}
|
||||
|
||||
|
|
@ -55,13 +55,11 @@ class TestAudioExtract:
|
|||
output_directory=output_directory,
|
||||
transaction_log=transaction_log,
|
||||
transaction_log_summary_file_name="plugins/test_audio_extract_single.txt",
|
||||
regenerate_transaction_log=True,
|
||||
)
|
||||
assert_expected_downloads(
|
||||
output_directory=output_directory,
|
||||
dry_run=dry_run,
|
||||
expected_download_summary_file_name="plugins/test_audio_extract_single.json",
|
||||
regenerate_expected_download_summary=True,
|
||||
)
|
||||
|
||||
@pytest.mark.parametrize("dry_run", [True, False])
|
||||
|
|
@ -83,11 +81,9 @@ class TestAudioExtract:
|
|||
output_directory=output_directory,
|
||||
transaction_log=transaction_log,
|
||||
transaction_log_summary_file_name="plugins/test_audio_extract_playlist.txt",
|
||||
regenerate_transaction_log=True,
|
||||
)
|
||||
assert_expected_downloads(
|
||||
output_directory=output_directory,
|
||||
dry_run=dry_run,
|
||||
expected_download_summary_file_name="plugins/test_audio_extract_playlist.json",
|
||||
regenerate_expected_download_summary=True,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -17,10 +17,6 @@ def single_video_sponsorblock_and_embedded_subs_preset_dict(output_directory):
|
|||
"languages": ["en", "de"],
|
||||
"allow_auto_generated_subtitles": True,
|
||||
},
|
||||
"audio_extract": {
|
||||
"codec": "mp3",
|
||||
"quality": 128,
|
||||
},
|
||||
"chapters": {
|
||||
"sponsorblock_categories": [
|
||||
"outro",
|
||||
|
|
@ -40,6 +36,7 @@ def single_video_sponsorblock_and_embedded_subs_preset_dict(output_directory):
|
|||
# download the worst format so it is fast
|
||||
"ytdl_options": {
|
||||
"format": "worst[ext=mp4]",
|
||||
"postprocessor_args": {"ffmpeg": ["-bitexact"]}, # Must add this for reproducibility
|
||||
},
|
||||
"overrides": {"artist": "JMC"},
|
||||
}
|
||||
|
|
@ -65,11 +62,12 @@ class TestChapters:
|
|||
output_directory=output_directory,
|
||||
transaction_log=transaction_log,
|
||||
transaction_log_summary_file_name="plugins/test_chapters_sb_and_embedded_subs.txt",
|
||||
regenerate_transaction_log=True,
|
||||
)
|
||||
assert_expected_downloads(
|
||||
output_directory=output_directory,
|
||||
dry_run=dry_run,
|
||||
expected_download_summary_file_name="plugins/test_chapters_sb_and_embedded_subs.json",
|
||||
regenerate_expected_download_summary=True,
|
||||
ignore_md5_hashes_for=[
|
||||
"JMC - This GPU SLIDES into this Case! - Silverstone SUGO 16 ITX Case.mp4"
|
||||
],
|
||||
)
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
{
|
||||
"Jesse's Minecraft Server [Trailer - Feb.1].ogg": "67b19b495756fb6dc332f49bfa2957d3",
|
||||
"Jesse's Minecraft Server [Trailer - Feb.27].ogg": "273c96652e198171ee60345051addc7a",
|
||||
"Jesse's Minecraft Server [Trailer - Mar.21].ogg": "e91db58c153f0724aaed5ce53de2a736"
|
||||
"Jesse's Minecraft Server [Trailer - Feb.1].ogg": "c37c289bc9b7c79464aa8bcc6df423e3",
|
||||
"Jesse's Minecraft Server [Trailer - Feb.27].ogg": "37def0736bb5c0a7fba0c1685e90bf3c",
|
||||
"Jesse's Minecraft Server [Trailer - Mar.21].ogg": "1fcc317ee5ce24675f6e4e6c2eae40d3"
|
||||
}
|
||||
|
|
@ -1,3 +1,3 @@
|
|||
{
|
||||
"YouTube Rewind 2019： For the Record ｜ #YouTubeRewind.mp3": "c4ce74e2ddc4e6a0ab823b5b622e2275"
|
||||
"YouTube Rewind 2019： For the Record ｜ #YouTubeRewind.mp3": "3a156b122bd79c956cce5079d3530cc3"
|
||||
}
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
{
|
||||
"JMC - This GPU SLIDES into this Case! - Silverstone SUGO 16 ITX Case-thumb.jpg": "a81457393418b5abed785a82122f3352",
|
||||
"JMC - This GPU SLIDES into this Case! - Silverstone SUGO 16 ITX Case.mp4": "ff829c47452173eac92e09507feb64a0",
|
||||
"JMC - This GPU SLIDES into this Case! - Silverstone SUGO 16 ITX Case.mp4": "2c417b31c9f5eb8cbecf0bf1fc1f9e53",
|
||||
"JMC - This GPU SLIDES into this Case! - Silverstone SUGO 16 ITX Case.nfo": "0c06fe6874588209fccbd9276a446750"
|
||||
}
|
||||
|
|
@ -1,11 +1,11 @@
|
|||
Files created in '{output_directory}'
|
||||
----------------------------------------
|
||||
Gabor Szabo - Dreams (1968) [full album].mp3
|
||||
YouTube Rewind 2019： For the Record ｜ #YouTubeRewind.mp3
|
||||
Music Tags:
|
||||
album: Singles
|
||||
albumartist: Cubensis Records
|
||||
artist: Cubensis Records
|
||||
albumartist: YouTube
|
||||
artist: YouTube
|
||||
genre: Unset
|
||||
title: Gabor Szabo - Dreams (1968) [full album]
|
||||
title: YouTube Rewind 2019: For the Record | #YouTubeRewind
|
||||
track: 1
|
||||
year: 2015
|
||||
year: 2019
|
||||
|
|
@ -1,9 +1,14 @@
|
|||
Files created in '{output_directory}'
|
||||
----------------------------------------
|
||||
JMC - This GPU SLIDES into this Case! - Silverstone SUGO 16 ITX Case-thumb.jpg
|
||||
JMC - This GPU SLIDES into this Case! - Silverstone SUGO 16 ITX Case.mp3
|
||||
JMC - This GPU SLIDES into this Case! - Silverstone SUGO 16 ITX Case.mp4
|
||||
Embedded Chapters
|
||||
Removed Chapter(s): Intro, Outro
|
||||
Removed SponsorBlock Category Count(s):
|
||||
Sponsor: 2
|
||||
Endcards/Credits: 1
|
||||
Intermission/Intro Animation: 1
|
||||
Unpaid/Self Promotion: 1
|
||||
Embedded subtitles with lang(s) en, de
|
||||
JMC - This GPU SLIDES into this Case! - Silverstone SUGO 16 ITX Case.nfo
|
||||
NFO tags:
|
||||
|
|
|
|||
Loading…
Reference in a new issue