diff --git a/src/ytdl_sub/downloaders/downloader.py b/src/ytdl_sub/downloaders/downloader.py index 22e83dc7..e8a872a9 100644 --- a/src/ytdl_sub/downloaders/downloader.py +++ b/src/ytdl_sub/downloaders/downloader.py @@ -132,7 +132,7 @@ class Downloader(DownloadArchiver, Generic[DownloaderOptionsT, DownloaderEntryT] def extract_info_with_retry( self, - is_downloaded_fn: Callable[[], bool], + is_downloaded_fn: Optional[Callable[[], bool]], ytdl_options_overrides: Optional[Dict] = None, **kwargs, ) -> Dict: @@ -146,7 +146,7 @@ class Downloader(DownloadArchiver, Generic[DownloaderOptionsT, DownloaderEntryT] Parameters ---------- is_downloaded_fn - Function to check if the entry is downloaded + Optional. Function to check if the entry is downloaded ytdl_options_overrides Optional. Dict containing ytdl args to override other predefined ytdl args **kwargs @@ -162,7 +162,7 @@ class Downloader(DownloadArchiver, Generic[DownloaderOptionsT, DownloaderEntryT] while not entry_files_exist and num_tries < self._extract_entry_num_retries: entry_dict = self.extract_info(ytdl_options_overrides=ytdl_options_overrides, **kwargs) - if is_downloaded_fn(): + if is_downloaded_fn is None or is_downloaded_fn(): return entry_dict time.sleep(self._extract_entry_retry_wait_sec) @@ -298,7 +298,6 @@ class Downloader(DownloadArchiver, Generic[DownloaderOptionsT, DownloaderEntryT] { "skip_download": True, "writethumbnail": False, - "writesubtitles": False, } ) diff --git a/src/ytdl_sub/downloaders/youtube/channel.py b/src/ytdl_sub/downloaders/youtube/channel.py index e84056dd..f3f93e9e 100644 --- a/src/ytdl_sub/downloaders/youtube/channel.py +++ b/src/ytdl_sub/downloaders/youtube/channel.py @@ -160,16 +160,19 @@ class YoutubeChannelDownloader(YoutubeDownloader[YoutubeChannelDownloaderOptions video = YoutubeVideo(entry_dict=entry_dict, working_directory=self.working_directory) download_logger.info("Downloading %d/%d %s", idx, len(entry_dicts), video.title) - # Only do the individual download if it is not dry-run - if not self.is_dry_run: - _ = self.extract_info_with_retry( - is_downloaded_fn=video.is_downloaded, - ytdl_options_overrides={ - "playlist_items": str(video.kwargs("playlist_index")), - "writeinfojson": False, - }, - url=self.download_options.channel_url, - ) + # Re-download the contents even if it's a dry-run as a single video. At this time, + # channels do not download subtitles or subtitle metadata + as_single_video_dict = self.extract_info_with_retry( + is_downloaded_fn=None if self.is_dry_run else video.is_downloaded, + ytdl_options_overrides={"writeinfojson": False, "skip_download": self.is_dry_run}, + url=video.kwargs("webpage_url"), + ) + + # Workaround for the ytdlp issue + # pylint: disable=protected-access + video._kwargs["requested_subtitles"] = as_single_video_dict.get("requested_subtitles") + # pylint: enable=protected-access + yield video def _download_thumbnail( diff --git a/src/ytdl_sub/downloaders/youtube/playlist.py b/src/ytdl_sub/downloaders/youtube/playlist.py index 514d2ab2..b6edf69c 100644 --- a/src/ytdl_sub/downloaders/youtube/playlist.py +++ b/src/ytdl_sub/downloaders/youtube/playlist.py @@ -89,15 +89,17 @@ class YoutubePlaylistDownloader( ) download_logger.info("Downloading %d/%d %s", idx, len(entry_dicts), video.title) - # Only do the individual download if it is not dry-run and downloading individually - if not self.is_dry_run: - _ = self.extract_info_with_retry( - is_downloaded_fn=video.is_downloaded, - ytdl_options_overrides={ - "playlist_items": str(entry_dict.get("playlist_index")), - "writeinfojson": False, - }, - url=self.download_options.playlist_url, - ) + # Re-download the contents even if it's a dry-run as a single video. At this time, + # playlists do not download subtitles or subtitle metadata + as_single_video_dict = self.extract_info_with_retry( + is_downloaded_fn=None if self.is_dry_run else video.is_downloaded, + ytdl_options_overrides={"writeinfojson": False, "skip_download": self.is_dry_run}, + url=video.kwargs("webpage_url"), + ) + + # Workaround for the ytdlp issue + # pylint: disable=protected-access + video._kwargs["requested_subtitles"] = as_single_video_dict.get("requested_subtitles") + # pylint: enable=protected-access yield video diff --git a/src/ytdl_sub/plugins/subtitles.py b/src/ytdl_sub/plugins/subtitles.py index 662a1b39..1724deb6 100644 --- a/src/ytdl_sub/plugins/subtitles.py +++ b/src/ytdl_sub/plugins/subtitles.py @@ -9,6 +9,7 @@ from ytdl_sub.entries.entry import Entry from ytdl_sub.plugins.plugin import Plugin from ytdl_sub.plugins.plugin import PluginOptions from ytdl_sub.utils.file_handler import FileMetadata +from ytdl_sub.utils.logger import Logger from ytdl_sub.validators.string_formatter_validators import StringFormatterValidator from ytdl_sub.validators.string_select_validator import StringSelectValidator from ytdl_sub.validators.validators import BoolValidator @@ -17,6 +18,9 @@ from ytdl_sub.validators.validators import StringListValidator SUBTITLE_EXTENSIONS: Set[str] = {"srt", "vtt", "ass", "lrc"} +logger = Logger.get(name="subtitles") + + def _is_entry_subtitle_file(path: Path, entry: Entry) -> bool: if path.is_file() and path.name.startswith(entry.uid): for ext in SUBTITLE_EXTENSIONS: @@ -67,7 +71,9 @@ class SubtitleOptions(PluginOptions): key="subtitles_type", validator=SubtitlesTypeValidator, default="srt" ).value self._embed_subtitles = self._validate_key_if_present( - key="embed_subtitles", validator=BoolValidator + key="embed_subtitles", + validator=BoolValidator, + default=False, ).value self._languages = self._validate_key_if_present( key="languages", validator=StringListValidator, default=["en"] @@ -166,6 +172,9 @@ class SubtitlesPlugin(Plugin[SubtitleOptions]): def modify_entry(self, entry: Entry) -> Optional[Entry]: requested_subtitles = entry.kwargs("requested_subtitles") + if not requested_subtitles: + return entry + languages = sorted(requested_subtitles.keys()) entry.add_variables( variables_to_add={ @@ -186,6 +195,10 @@ class SubtitlesPlugin(Plugin[SubtitleOptions]): Entry to create subtitles for """ requested_subtitles = entry.kwargs("requested_subtitles") + if not requested_subtitles: + logger.info("subtitles not found for %s", entry.title) + return None + file_metadata: Optional[FileMetadata] = None langs = list(requested_subtitles.keys()) diff --git a/tests/e2e/resources/expected_downloads_summaries/youtube/test_channel_full.json b/tests/e2e/resources/expected_downloads_summaries/youtube/test_channel_full.json index 8e2db637..b6398084 100644 --- a/tests/e2e/resources/expected_downloads_summaries/youtube/test_channel_full.json +++ b/tests/e2e/resources/expected_downloads_summaries/youtube/test_channel_full.json @@ -1,5 +1,5 @@ { - ".ytdl-sub-pz-download-archive.json": "ce0ef71efbe27d9f67d80b5fbc57fc84", + ".ytdl-sub-pz-download-archive.json": "f0f692ef29653ca8d0af556415bfb065", "Season 2010/s2010.e0813 - Oblivion Mod ＂Falcor＂ p.1-thumb.jpg": "fb95b510681676e81c321171fc23143e", "Season 2010/s2010.e0813 - Oblivion Mod ＂Falcor＂ p.1.mp4": "931a705864c57d21d6fedebed4af6bbc", "Season 2010/s2010.e0813 - Oblivion Mod ＂Falcor＂ p.1.nfo": "67d8d71d048039080acbba3bce4febaa", @@ -34,6 +34,7 @@ "Season 2018/s2018.e1029 - Jesse's Minecraft Server ｜ Teaser Trailer.mp4": "82f6ee7253e1dbb83ae7215af08ffacc", "Season 2018/s2018.e1029 - Jesse's Minecraft Server ｜ Teaser Trailer.nfo": "cc7886aae3af6b7b0facd82f95390242", "Season 2018/s2018.e1102 - Jesse's Minecraft Server ｜ IP mc.jesse.id-thumb.jpg": "49cc64b25314155c1b8ab0361ac0c34f", + "Season 2018/s2018.e1102 - Jesse's Minecraft Server ｜ IP mc.jesse.id.en.srt": "3d2c4e7f65d2ca5e96da38ce7eecfc4e", "Season 2018/s2018.e1102 - Jesse's Minecraft Server ｜ IP mc.jesse.id.mp4": "e733b4cc385b953b08c8eb0f47e03c1e", "Season 2018/s2018.e1102 - Jesse's Minecraft Server ｜ IP mc.jesse.id.nfo": "2b3ccb3f1ef81ee49fe1afb88f275a09", "fanart.jpg": "c16b8b88a82cbd47d217ee80f6a8b5f3", diff --git a/tests/e2e/resources/expected_downloads_summaries/youtube/test_channel_recent.json b/tests/e2e/resources/expected_downloads_summaries/youtube/test_channel_recent.json index 7acf32f8..ebd21a90 100644 --- a/tests/e2e/resources/expected_downloads_summaries/youtube/test_channel_recent.json +++ b/tests/e2e/resources/expected_downloads_summaries/youtube/test_channel_recent.json @@ -1,9 +1,10 @@ { - ".ytdl-sub-pz-download-archive.json": "523518e883758d9d89b656f9e9966efa", + ".ytdl-sub-pz-download-archive.json": "756b60d7a6c47d8163e3283404493a8d", "Season 2018/s2018.e1029 - Jesse's Minecraft Server ｜ Teaser Trailer-thumb.jpg": "6f8f5e1e031ec2a04b0a4906c04a19ee", "Season 2018/s2018.e1029 - Jesse's Minecraft Server ｜ Teaser Trailer.mp4": "82f6ee7253e1dbb83ae7215af08ffacc", "Season 2018/s2018.e1029 - Jesse's Minecraft Server ｜ Teaser Trailer.nfo": "cc7886aae3af6b7b0facd82f95390242", "Season 2018/s2018.e1102 - Jesse's Minecraft Server ｜ IP mc.jesse.id-thumb.jpg": "49cc64b25314155c1b8ab0361ac0c34f", + "Season 2018/s2018.e1102 - Jesse's Minecraft Server ｜ IP mc.jesse.id.en.srt": "3d2c4e7f65d2ca5e96da38ce7eecfc4e", "Season 2018/s2018.e1102 - Jesse's Minecraft Server ｜ IP mc.jesse.id.mp4": "e733b4cc385b953b08c8eb0f47e03c1e", "Season 2018/s2018.e1102 - Jesse's Minecraft Server ｜ IP mc.jesse.id.nfo": "2b3ccb3f1ef81ee49fe1afb88f275a09", "fanart.jpg": "c16b8b88a82cbd47d217ee80f6a8b5f3", diff --git a/tests/e2e/resources/expected_downloads_summaries/youtube/test_channel_rolling_recent.json b/tests/e2e/resources/expected_downloads_summaries/youtube/test_channel_rolling_recent.json index faf8fab2..02c7be2d 100644 --- a/tests/e2e/resources/expected_downloads_summaries/youtube/test_channel_rolling_recent.json +++ b/tests/e2e/resources/expected_downloads_summaries/youtube/test_channel_rolling_recent.json @@ -1,6 +1,7 @@ { - ".ytdl-sub-pz-download-archive.json": "9639fffe58c44d084124c8398b46ef32", + ".ytdl-sub-pz-download-archive.json": "68e164a35d1541ce761a6d7fef19ee08", "Season 2018/s2018.e1102 - Jesse's Minecraft Server ｜ IP mc.jesse.id-thumb.jpg": "49cc64b25314155c1b8ab0361ac0c34f", + "Season 2018/s2018.e1102 - Jesse's Minecraft Server ｜ IP mc.jesse.id.en.srt": "3d2c4e7f65d2ca5e96da38ce7eecfc4e", "Season 2018/s2018.e1102 - Jesse's Minecraft Server ｜ IP mc.jesse.id.mp4": "e733b4cc385b953b08c8eb0f47e03c1e", "Season 2018/s2018.e1102 - Jesse's Minecraft Server ｜ IP mc.jesse.id.nfo": "2b3ccb3f1ef81ee49fe1afb88f275a09", "fanart.jpg": "c16b8b88a82cbd47d217ee80f6a8b5f3", diff --git a/tests/e2e/resources/transaction_log_summaries/youtube/test_channel_full.txt b/tests/e2e/resources/transaction_log_summaries/youtube/test_channel_full.txt index ea6008fd..4888db8a 100644 --- a/tests/e2e/resources/transaction_log_summaries/youtube/test_channel_full.txt +++ b/tests/e2e/resources/transaction_log_summaries/youtube/test_channel_full.txt @@ -243,6 +243,7 @@ Season 2018/s2018.e1029 - Jesse's Minecraft Server ｜ Teaser Trailer.nfo title: Jesse's Minecraft Server | Teaser Trailer year: 2018 Season 2018/s2018.e1102 - Jesse's Minecraft Server ｜ IP mc.jesse.id-thumb.jpg +Season 2018/s2018.e1102 - Jesse's Minecraft Server ｜ IP mc.jesse.id.en.srt Season 2018/s2018.e1102 - Jesse's Minecraft Server ｜ IP mc.jesse.id.mp4 Season 2018/s2018.e1102 - Jesse's Minecraft Server ｜ IP mc.jesse.id.nfo NFO tags: diff --git a/tests/e2e/resources/transaction_log_summaries/youtube/test_channel_recent.txt b/tests/e2e/resources/transaction_log_summaries/youtube/test_channel_recent.txt index dd1f46e3..e1cc52b0 100644 --- a/tests/e2e/resources/transaction_log_summaries/youtube/test_channel_recent.txt +++ b/tests/e2e/resources/transaction_log_summaries/youtube/test_channel_recent.txt @@ -22,6 +22,7 @@ Season 2018/s2018.e1029 - Jesse's Minecraft Server ｜ Teaser Trailer.nfo title: Jesse's Minecraft Server | Teaser Trailer year: 2018 Season 2018/s2018.e1102 - Jesse's Minecraft Server ｜ IP mc.jesse.id-thumb.jpg +Season 2018/s2018.e1102 - Jesse's Minecraft Server ｜ IP mc.jesse.id.en.srt Season 2018/s2018.e1102 - Jesse's Minecraft Server ｜ IP mc.jesse.id.mp4 Season 2018/s2018.e1102 - Jesse's Minecraft Server ｜ IP mc.jesse.id.nfo NFO tags: diff --git a/tests/e2e/youtube/test_channel.py b/tests/e2e/youtube/test_channel.py index f9addb76..c7ff1219 100644 --- a/tests/e2e/youtube/test_channel.py +++ b/tests/e2e/youtube/test_channel.py @@ -29,6 +29,10 @@ def channel_preset_dict(output_directory): "max_views": 100000, # do not download the popular PJ concert "break_on_reject": False, # do not break from max views }, + "subtitles": { + "subtitles_name": "{episode_name}.{lang}.{subtitles_ext}", + "allow_auto_generated_subtitles": True, + }, "overrides": {"tv_show_name": "Project / Zombie"}, } diff --git a/tests/e2e/youtube/test_playlist.py b/tests/e2e/youtube/test_playlist.py index 77c020a8..132877be 100644 --- a/tests/e2e/youtube/test_playlist.py +++ b/tests/e2e/youtube/test_playlist.py @@ -19,6 +19,10 @@ def playlist_preset_dict(output_directory): "ytdl_options": { "format": "worst[ext=mp4]", }, + "subtitles": { + "subtitles_name": "{music_video_name}.{lang}.{subtitles_ext}", + "allow_auto_generated_subtitles": True, + }, "overrides": {"artist": "JMC"}, }