From 103e5aaac2ebffd5f595d6cd7c71d98b668074c9 Mon Sep 17 00:00:00 2001 From: jbannon Date: Fri, 12 Aug 2022 06:45:58 +0000 Subject: [PATCH] subtitles seem to work, need to manually verify --- src/ytdl_sub/config/preset_options.py | 23 +++++- src/ytdl_sub/plugins/subtitles.py | 73 +++++++++++++------ .../subscription_ytdl_options.py | 47 ++++++++---- tests/e2e/plugins/test_subtitles.py | 56 ++++++++++++++ .../plugins/test_subtitles_video.json | 7 ++ .../plugins/test_subtitles_video.txt | 13 ++++ tests/e2e/youtube/test_video.py | 29 -------- 7 files changed, 180 insertions(+), 68 deletions(-) create mode 100644 tests/e2e/plugins/test_subtitles.py create mode 100644 tests/e2e/resources/expected_downloads_summaries/plugins/test_subtitles_video.json create mode 100644 tests/e2e/resources/transaction_log_summaries/plugins/test_subtitles_video.txt diff --git a/src/ytdl_sub/config/preset_options.py b/src/ytdl_sub/config/preset_options.py index c2df0ff9..b3bcc53e 100644 --- a/src/ytdl_sub/config/preset_options.py +++ b/src/ytdl_sub/config/preset_options.py @@ -1,3 +1,4 @@ +from typing import Dict from typing import Optional from yt_dlp.utils import DateRange @@ -74,15 +75,31 @@ class Overrides(DictFormatterValidator): ) def apply_formatter( - self, formatter: StringFormatterValidator, entry: Optional[Entry] = None + self, + formatter: StringFormatterValidator, + entry: Optional[Entry] = None, + function_overrides: Dict[str, str] = None, ) -> str: """ - Returns the format_string after .format has been called on it using entry (if provided) and - override values + Parameters + ---------- + formatter + Formatter to apply + entry + Optional. Entry to add source variables to the formatter + function_overrides + Optional. Explicit values to override the overrides themselves and source variables + + Returns + ------- + The format_string after .format has been called """ variable_dict = self.dict_with_format_strings if entry: variable_dict = dict(entry.to_dict(), **variable_dict) + if function_overrides: + variable_dict = dict(variable_dict, **function_overrides) + return formatter.apply_formatter(variable_dict) diff --git a/src/ytdl_sub/plugins/subtitles.py b/src/ytdl_sub/plugins/subtitles.py index 26d00589..d4f14fc4 100644 --- a/src/ytdl_sub/plugins/subtitles.py +++ b/src/ytdl_sub/plugins/subtitles.py @@ -1,3 +1,4 @@ +from pathlib import Path from typing import Dict from typing import List from typing import Optional @@ -6,7 +7,7 @@ from typing import Set from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder from ytdl_sub.entries.entry import Entry from ytdl_sub.plugins.plugin import Plugin -from ytdl_sub.validators.strict_dict_validator import StrictDictValidator +from ytdl_sub.plugins.plugin import PluginOptions from ytdl_sub.validators.string_formatter_validators import StringFormatterValidator from ytdl_sub.validators.string_select_validator import StringSelectValidator from ytdl_sub.validators.validators import BoolValidator @@ -15,12 +16,20 @@ from ytdl_sub.validators.validators import StringListValidator SUBTITLE_EXTENSIONS: Set[str] = {"srt", "vtt", "ass", "lrc"} +def _is_entry_subtitle_file(path: Path, entry: Entry) -> bool: + if path.is_file() and path.name.startswith(entry.uid): + for ext in SUBTITLE_EXTENSIONS: + if path.name.endswith(f".{ext}"): + return True + return False + + class SubtitlesTypeValidator(StringSelectValidator): _expected_value_type_name = "subtitles type" _select_values = SUBTITLE_EXTENSIONS -class SubtitleOptions(StrictDictValidator): +class SubtitleOptions(PluginOptions): """ Defines how to download and store subtitles. @@ -56,10 +65,10 @@ class SubtitleOptions(StrictDictValidator): ) self._subtitles_type = self._validate_key_if_present( key="subtitles_type", validator=SubtitlesTypeValidator, default="srt" - ) + ).value self._embed_subtitles = self._validate_key_if_present( key="embed_subtitles", validator=BoolValidator - ) + ).value self._languages = self._validate_key_if_present( key="languages", validator=StringListValidator, default=["en"] ).list @@ -86,9 +95,9 @@ class SubtitleOptions(StrictDictValidator): @property def embed_subtitles(self) -> Optional[bool]: """ - Optional. Whether to embed the subtitles into the video file. + Optional. Whether to embed the subtitles into the video file. Defaults to False. """ - return self._subtitles_type + return self._embed_subtitles @property def languages(self) -> Optional[List[str]]: @@ -104,6 +113,14 @@ class SubtitleOptions(StrictDictValidator): """ return self._allow_auto_generated_subtitles + def added_source_variables(self) -> List[str]: + """ + Returns + ------- + List of new source variables created by using the subtitles plugin + """ + return ["lang", "subtitle_ext"] + class SubtitlesPlugin(Plugin[SubtitleOptions]): plugin_options_type = SubtitleOptions @@ -144,6 +161,18 @@ class SubtitlesPlugin(Plugin[SubtitleOptions]): } ).to_dict() + def modify_entry(self, entry: Entry) -> Optional[Entry]: + requested_subtitles = entry.kwargs("requested_subtitles") + languages = sorted(requested_subtitles.keys()) + entry.add_variables( + variables_to_add={ + "subtitle_ext": self.plugin_options.subtitles_type, + "lang": ",".join(languages), + } + ) + + return entry + def post_process_entry(self, entry: Entry) -> None: """ Creates an entry's NFO file using values defined in the metadata options @@ -151,20 +180,22 @@ class SubtitlesPlugin(Plugin[SubtitleOptions]): Parameters ---------- entry: - Entry to create an NFO file for + Entry to create subtitles for """ + if not self.plugin_options.subtitles_name: + return - # def get_ytdlp_download_subtitle_paths(self) -> List[str]: - # possible_subtitle_exts = SUBTITLE_EXTENSIONS - # subtitle_paths: List[str] = [] - # - # for ext in possible_subtitle_exts: - # for path in Path(self.working_directory()).rglob("*"): - # if ( - # path.is_file() - # and path.name.startswith(self.uid) - # and path.name.endswith(f".{ext}") - # ): - # subtitle_paths.append(str(path)) - # - # return subtitle_paths + requested_subtitles = entry.kwargs("requested_subtitles") + for lang in requested_subtitles.keys(): + subtitle_file_name = f"{entry.uid}.{lang}.{self.plugin_options.subtitles_type}" + output_subtitle_file_name = self.overrides.apply_formatter( + formatter=self.plugin_options.subtitles_name, + entry=entry, + function_overrides={"lang": lang}, + ) + + self.save_file( + file_name=subtitle_file_name, + output_file_name=output_subtitle_file_name, + entry=entry, + ) diff --git a/src/ytdl_sub/subscriptions/subscription_ytdl_options.py b/src/ytdl_sub/subscriptions/subscription_ytdl_options.py index a4b9436d..a7040f11 100644 --- a/src/ytdl_sub/subscriptions/subscription_ytdl_options.py +++ b/src/ytdl_sub/subscriptions/subscription_ytdl_options.py @@ -67,8 +67,8 @@ class SubscriptionYTDLOptions: def _dry_run_options(self) -> Dict: return { "skip_download": True, + "simulate": True, "writethumbnail": False, - "writesubtitles": False, } @property @@ -90,30 +90,45 @@ class SubscriptionYTDLOptions: # TODO: warn here return {} - ytdl_options: Dict = {} + builder = YTDLOptionsBuilder() subtitle_options: SubtitleOptions = subtitle_plugin.plugin_options write_subtitle_file: bool = subtitle_options.subtitles_name is not None if write_subtitle_file: - ytdl_options["writesubtitles"] = True - ytdl_options["postprocessors"] = [ - {"key": "FFmpegSubtitlesConvertor", "format": subtitle_options.subtitles_type} - ] + builder.add( + { + "writesubtitles": True, + "postprocessors": [ + { + "key": "FFmpegSubtitlesConvertor", + "format": subtitle_options.subtitles_type, + } + ], + } + ) if subtitle_options.embed_subtitles: - ytdl_options["postprocessors"] = [ - # already_have_subtitle=True means keep the subtitle files. False means delete - {"key": "FFmpegEmbedSubtitle", "already_have_subtitle": write_subtitle_file} - ] + builder.add( + { + "postprocessors": [ + # already_have_subtitle=True means we downloaded the subtitle files. + {"key": "FFmpegEmbedSubtitle", "already_have_subtitle": write_subtitle_file} + ] + } + ) # If neither subtitles_name or embed_subtitles is set, do not set any other flags - if not ytdl_options: + if not builder.to_dict(): return {} - ytdl_options["writeautomaticsub"] = subtitle_options.allow_auto_generated_subtitles - ytdl_options["subtitleslangs"] = subtitle_options.languages + builder.add( + { + "writeautomaticsub": subtitle_options.allow_auto_generated_subtitles, + "subtitleslangs": subtitle_options.languages, + } + ) - return ytdl_options + return builder.to_dict() @property def _user_ytdl_options(self) -> Dict: @@ -128,7 +143,9 @@ class SubscriptionYTDLOptions: """ ytdl_options_builder = YTDLOptionsBuilder().add(self._global_options) if self._dry_run: - ytdl_options_builder.add(self._dry_run_options, self._user_ytdl_options) + ytdl_options_builder.add( + self._subtitle_options, self._user_ytdl_options, self._dry_run_options + ) else: ytdl_options_builder.add( self._output_options, self._subtitle_options, self._user_ytdl_options diff --git a/tests/e2e/plugins/test_subtitles.py b/tests/e2e/plugins/test_subtitles.py new file mode 100644 index 00000000..1467fe6d --- /dev/null +++ b/tests/e2e/plugins/test_subtitles.py @@ -0,0 +1,56 @@ +import pytest +from e2e.expected_download import assert_expected_downloads +from e2e.expected_transaction_log import assert_transaction_log_matches + +from ytdl_sub.subscriptions.subscription import Subscription + + +@pytest.fixture +def single_video_subs_preset_dict(output_directory): + return { + "preset": "yt_music_video", + "youtube": {"video_url": "https://www.youtube.com/watch?v=2lAe1cqCOXo"}, + # override the output directory with our fixture-generated dir + "output_options": {"output_directory": output_directory}, + "subtitles": { + "subtitles_name": "{music_video_name}.{lang}.{subtitles_ext}", + "embed_subtitles": True, + "languages": ["en", "de"], + "allow_auto_generated_subtitles": True, + }, + # download the worst format so it is fast + "ytdl_options": { + "format": "worst[ext=mp4]", + }, + "overrides": {"artist": "JMC"}, + } + + +class TestSubtitles: + @pytest.mark.parametrize("dry_run", [True, False]) + def test_single_video_download_with_subtitles( + self, + music_video_config, + single_video_subs_preset_dict, + output_directory, + dry_run, + ): + subscription = Subscription.from_dict( + config=music_video_config, + preset_name="subtitles_video_test", + preset_dict=single_video_subs_preset_dict, + ) + + transaction_log = subscription.download(dry_run=dry_run) + assert_transaction_log_matches( + output_directory=output_directory, + transaction_log=transaction_log, + transaction_log_summary_file_name="plugins/test_subtitles_video.txt", + regenerate_transaction_log=True, + ) + assert_expected_downloads( + output_directory=output_directory, + dry_run=dry_run, + expected_download_summary_file_name="plugins/test_subtitles_video.json", + regenerate_expected_download_summary=True, + ) diff --git a/tests/e2e/resources/expected_downloads_summaries/plugins/test_subtitles_video.json b/tests/e2e/resources/expected_downloads_summaries/plugins/test_subtitles_video.json new file mode 100644 index 00000000..8c38ba09 --- /dev/null +++ b/tests/e2e/resources/expected_downloads_summaries/plugins/test_subtitles_video.json @@ -0,0 +1,7 @@ +{ + "JMC - YouTube Rewind 2019： For the Record ｜ #YouTubeRewind-thumb.jpg": "704246dd78074e8a0ec001dd8d03fd60", + "JMC - YouTube Rewind 2019： For the Record ｜ #YouTubeRewind.de.srt": "b343c3bb9257b7ee7ba38f570a115b37", + "JMC - YouTube Rewind 2019： For the Record ｜ #YouTubeRewind.en.srt": "fe8c6ee92cae6e059fd80fd61691adbe", + "JMC - YouTube Rewind 2019： For the Record ｜ #YouTubeRewind.mp4": "383213d2809cdc2e86e3a2c4c8deb685", + "JMC - YouTube Rewind 2019： For the Record ｜ #YouTubeRewind.nfo": "ffa10f1cbc098ace7b1c7a8fbe3097a8" +} \ No newline at end of file diff --git a/tests/e2e/resources/transaction_log_summaries/plugins/test_subtitles_video.txt b/tests/e2e/resources/transaction_log_summaries/plugins/test_subtitles_video.txt new file mode 100644 index 00000000..4d6bda7c --- /dev/null +++ b/tests/e2e/resources/transaction_log_summaries/plugins/test_subtitles_video.txt @@ -0,0 +1,13 @@ +Files created in '{output_directory}' +---------------------------------------- +JMC - YouTube Rewind 2019： For the Record ｜ #YouTubeRewind-thumb.jpg +JMC - YouTube Rewind 2019： For the Record ｜ #YouTubeRewind.de.srt +JMC - YouTube Rewind 2019： For the Record ｜ #YouTubeRewind.en.srt +JMC - YouTube Rewind 2019： For the Record ｜ #YouTubeRewind.mp4 +JMC - YouTube Rewind 2019： For the Record ｜ #YouTubeRewind.nfo + NFO tags: + musicvideo: + album: Music Videos + artist: JMC + title: YouTube Rewind 2019: For the Record | #YouTubeRewind + year: 2019 \ No newline at end of file diff --git a/tests/e2e/youtube/test_video.py b/tests/e2e/youtube/test_video.py index adb07c5d..7c044c67 100644 --- a/tests/e2e/youtube/test_video.py +++ b/tests/e2e/youtube/test_video.py @@ -54,35 +54,6 @@ class TestYoutubeVideo: expected_download_summary_file_name="youtube/test_video.json", ) - # @pytest.mark.parametrize("dry_run", [True, False]) - # def test_single_video_download_with_subtitles( - # self, - # music_video_config, - # single_video_preset_dict, - # expected_single_video_download, - # output_directory, - # dry_run, - # ): - # single_video_preset_dict["youtube"][ - # "video_url" - # ] = "https://www.youtube.com/watch?v=dQw4w9WgXcQ" - # single_video_preset_dict["subtitles"] = {} - # single_video_preset_dict["subtitles"]["subtitles_name"] = "{music_video_name}.srt" - # single_video_subscription = Subscription.from_dict( - # config=music_video_config, - # preset_name="music_video_single_video_test", - # preset_dict=single_video_preset_dict, - # ) - # - # transaction_log = single_video_subscription.download(dry_run=dry_run) - # assert_transaction_log_matches( - # output_directory=output_directory, - # transaction_log=transaction_log, - # transaction_log_summary_file_name="youtube/test_video.txt", - # ) - # if not dry_run: - # expected_single_video_download.assert_files_exist(relative_directory=output_directory) - @pytest.mark.parametrize("dry_run", [True, False]) def test_single_video_download_from_cli_dl( self,