[BUGFIX] Subtitles for channel + playlist downloads (#176)

* subtitle fix for playlists * download channel vids as singles * no regen tests
2022-08-13 15:47:26 -07:00 · 2022-08-13 15:47:26 -07:00 · 064dca40b2
commit 064dca40b2
parent 9a7a34f3d0
11 changed files with 58 additions and 28 deletions
--- a/src/ytdl_sub/downloaders/downloader.py
+++ b/src/ytdl_sub/downloaders/downloader.py
@ -132,7 +132,7 @@ class Downloader(DownloadArchiver, Generic[DownloaderOptionsT, DownloaderEntryT]

    def extract_info_with_retry(
        self,
-        is_downloaded_fn: Callable[[], bool],
+        is_downloaded_fn: Optional[Callable[[], bool]],
        ytdl_options_overrides: Optional[Dict] = None,
        **kwargs,
    ) -> Dict:
@ -146,7 +146,7 @@ class Downloader(DownloadArchiver, Generic[DownloaderOptionsT, DownloaderEntryT]
        Parameters
        ----------
        is_downloaded_fn
-            Function to check if the entry is downloaded
+            Optional. Function to check if the entry is downloaded
        ytdl_options_overrides
            Optional. Dict containing ytdl args to override other predefined ytdl args
        **kwargs
@ -162,7 +162,7 @@ class Downloader(DownloadArchiver, Generic[DownloaderOptionsT, DownloaderEntryT]

        while not entry_files_exist and num_tries < self._extract_entry_num_retries:
            entry_dict = self.extract_info(ytdl_options_overrides=ytdl_options_overrides, **kwargs)
-            if is_downloaded_fn():
+            if is_downloaded_fn is None or is_downloaded_fn():
                return entry_dict

            time.sleep(self._extract_entry_retry_wait_sec)
@ -298,7 +298,6 @@ class Downloader(DownloadArchiver, Generic[DownloaderOptionsT, DownloaderEntryT]
                {
                    "skip_download": True,
                    "writethumbnail": False,
-                    "writesubtitles": False,
                }
            )

--- a/src/ytdl_sub/downloaders/youtube/channel.py
+++ b/src/ytdl_sub/downloaders/youtube/channel.py
@ -160,16 +160,19 @@ class YoutubeChannelDownloader(YoutubeDownloader[YoutubeChannelDownloaderOptions
            video = YoutubeVideo(entry_dict=entry_dict, working_directory=self.working_directory)
            download_logger.info("Downloading %d/%d %s", idx, len(entry_dicts), video.title)

-            # Only do the individual download if it is not dry-run
-            if not self.is_dry_run:
-                _ = self.extract_info_with_retry(
-                    is_downloaded_fn=video.is_downloaded,
-                    ytdl_options_overrides={
-                        "playlist_items": str(video.kwargs("playlist_index")),
-                        "writeinfojson": False,
-                    },
-                    url=self.download_options.channel_url,
-                )
+            # Re-download the contents even if it's a dry-run as a single video. At this time,
+            # channels do not download subtitles or subtitle metadata
+            as_single_video_dict = self.extract_info_with_retry(
+                is_downloaded_fn=None if self.is_dry_run else video.is_downloaded,
+                ytdl_options_overrides={"writeinfojson": False, "skip_download": self.is_dry_run},
+                url=video.kwargs("webpage_url"),
+            )
+
+            # Workaround for the ytdlp issue
+            # pylint: disable=protected-access
+            video._kwargs["requested_subtitles"] = as_single_video_dict.get("requested_subtitles")
+            # pylint: enable=protected-access
+
            yield video

    def _download_thumbnail(
--- a/src/ytdl_sub/downloaders/youtube/playlist.py
+++ b/src/ytdl_sub/downloaders/youtube/playlist.py
@ -89,15 +89,17 @@ class YoutubePlaylistDownloader(
            )
            download_logger.info("Downloading %d/%d %s", idx, len(entry_dicts), video.title)

-            # Only do the individual download if it is not dry-run and downloading individually
-            if not self.is_dry_run:
-                _ = self.extract_info_with_retry(
-                    is_downloaded_fn=video.is_downloaded,
-                    ytdl_options_overrides={
-                        "playlist_items": str(entry_dict.get("playlist_index")),
-                        "writeinfojson": False,
-                    },
-                    url=self.download_options.playlist_url,
-                )
+            # Re-download the contents even if it's a dry-run as a single video. At this time,
+            # playlists do not download subtitles or subtitle metadata
+            as_single_video_dict = self.extract_info_with_retry(
+                is_downloaded_fn=None if self.is_dry_run else video.is_downloaded,
+                ytdl_options_overrides={"writeinfojson": False, "skip_download": self.is_dry_run},
+                url=video.kwargs("webpage_url"),
+            )
+
+            # Workaround for the ytdlp issue
+            # pylint: disable=protected-access
+            video._kwargs["requested_subtitles"] = as_single_video_dict.get("requested_subtitles")
+            # pylint: enable=protected-access

            yield video
--- a/src/ytdl_sub/plugins/subtitles.py
+++ b/src/ytdl_sub/plugins/subtitles.py
@ -9,6 +9,7 @@ from ytdl_sub.entries.entry import Entry
 from ytdl_sub.plugins.plugin import Plugin
 from ytdl_sub.plugins.plugin import PluginOptions
 from ytdl_sub.utils.file_handler import FileMetadata
+from ytdl_sub.utils.logger import Logger
 from ytdl_sub.validators.string_formatter_validators import StringFormatterValidator
 from ytdl_sub.validators.string_select_validator import StringSelectValidator
 from ytdl_sub.validators.validators import BoolValidator
@ -17,6 +18,9 @@ from ytdl_sub.validators.validators import StringListValidator
 SUBTITLE_EXTENSIONS: Set[str] = {"srt", "vtt", "ass", "lrc"}


+logger = Logger.get(name="subtitles")
+
+
 def _is_entry_subtitle_file(path: Path, entry: Entry) -> bool:
    if path.is_file() and path.name.startswith(entry.uid):
        for ext in SUBTITLE_EXTENSIONS:
@ -67,7 +71,9 @@ class SubtitleOptions(PluginOptions):
            key="subtitles_type", validator=SubtitlesTypeValidator, default="srt"
        ).value
        self._embed_subtitles = self._validate_key_if_present(
-            key="embed_subtitles", validator=BoolValidator
+            key="embed_subtitles",
+            validator=BoolValidator,
+            default=False,
        ).value
        self._languages = self._validate_key_if_present(
            key="languages", validator=StringListValidator, default=["en"]
@ -166,6 +172,9 @@ class SubtitlesPlugin(Plugin[SubtitleOptions]):

    def modify_entry(self, entry: Entry) -> Optional[Entry]:
        requested_subtitles = entry.kwargs("requested_subtitles")
+        if not requested_subtitles:
+            return entry
+
        languages = sorted(requested_subtitles.keys())
        entry.add_variables(
            variables_to_add={
@ -186,6 +195,10 @@ class SubtitlesPlugin(Plugin[SubtitleOptions]):
            Entry to create subtitles for
        """
        requested_subtitles = entry.kwargs("requested_subtitles")
+        if not requested_subtitles:
+            logger.info("subtitles not found for %s", entry.title)
+            return None
+
        file_metadata: Optional[FileMetadata] = None
        langs = list(requested_subtitles.keys())

--- a/tests/e2e/resources/expected_downloads_summaries/youtube/test_channel_full.json
+++ b/tests/e2e/resources/expected_downloads_summaries/youtube/test_channel_full.json
@ -1,5 +1,5 @@
 {
-  ".ytdl-sub-pz-download-archive.json": "ce0ef71efbe27d9f67d80b5fbc57fc84",
+  ".ytdl-sub-pz-download-archive.json": "f0f692ef29653ca8d0af556415bfb065",
  "Season 2010/s2010.e0813 - Oblivion Mod ＂Falcor＂ p.1-thumb.jpg": "fb95b510681676e81c321171fc23143e",
  "Season 2010/s2010.e0813 - Oblivion Mod ＂Falcor＂ p.1.mp4": "931a705864c57d21d6fedebed4af6bbc",
  "Season 2010/s2010.e0813 - Oblivion Mod ＂Falcor＂ p.1.nfo": "67d8d71d048039080acbba3bce4febaa",
@ -34,6 +34,7 @@
  "Season 2018/s2018.e1029 - Jesse's Minecraft Server ｜ Teaser Trailer.mp4": "82f6ee7253e1dbb83ae7215af08ffacc",
  "Season 2018/s2018.e1029 - Jesse's Minecraft Server ｜ Teaser Trailer.nfo": "cc7886aae3af6b7b0facd82f95390242",
  "Season 2018/s2018.e1102 - Jesse's Minecraft Server ｜ IP mc.jesse.id-thumb.jpg": "49cc64b25314155c1b8ab0361ac0c34f",
+  "Season 2018/s2018.e1102 - Jesse's Minecraft Server ｜ IP mc.jesse.id.en.srt": "3d2c4e7f65d2ca5e96da38ce7eecfc4e",
  "Season 2018/s2018.e1102 - Jesse's Minecraft Server ｜ IP mc.jesse.id.mp4": "e733b4cc385b953b08c8eb0f47e03c1e",
  "Season 2018/s2018.e1102 - Jesse's Minecraft Server ｜ IP mc.jesse.id.nfo": "2b3ccb3f1ef81ee49fe1afb88f275a09",
  "fanart.jpg": "c16b8b88a82cbd47d217ee80f6a8b5f3",
--- a/tests/e2e/resources/expected_downloads_summaries/youtube/test_channel_recent.json
+++ b/tests/e2e/resources/expected_downloads_summaries/youtube/test_channel_recent.json
@ -1,9 +1,10 @@
 {
-  ".ytdl-sub-pz-download-archive.json": "523518e883758d9d89b656f9e9966efa",
+  ".ytdl-sub-pz-download-archive.json": "756b60d7a6c47d8163e3283404493a8d",
  "Season 2018/s2018.e1029 - Jesse's Minecraft Server ｜ Teaser Trailer-thumb.jpg": "6f8f5e1e031ec2a04b0a4906c04a19ee",
  "Season 2018/s2018.e1029 - Jesse's Minecraft Server ｜ Teaser Trailer.mp4": "82f6ee7253e1dbb83ae7215af08ffacc",
  "Season 2018/s2018.e1029 - Jesse's Minecraft Server ｜ Teaser Trailer.nfo": "cc7886aae3af6b7b0facd82f95390242",
  "Season 2018/s2018.e1102 - Jesse's Minecraft Server ｜ IP mc.jesse.id-thumb.jpg": "49cc64b25314155c1b8ab0361ac0c34f",
+  "Season 2018/s2018.e1102 - Jesse's Minecraft Server ｜ IP mc.jesse.id.en.srt": "3d2c4e7f65d2ca5e96da38ce7eecfc4e",
  "Season 2018/s2018.e1102 - Jesse's Minecraft Server ｜ IP mc.jesse.id.mp4": "e733b4cc385b953b08c8eb0f47e03c1e",
  "Season 2018/s2018.e1102 - Jesse's Minecraft Server ｜ IP mc.jesse.id.nfo": "2b3ccb3f1ef81ee49fe1afb88f275a09",
  "fanart.jpg": "c16b8b88a82cbd47d217ee80f6a8b5f3",
--- a/tests/e2e/resources/expected_downloads_summaries/youtube/test_channel_rolling_recent.json
+++ b/tests/e2e/resources/expected_downloads_summaries/youtube/test_channel_rolling_recent.json
@ -1,6 +1,7 @@
 {
-  ".ytdl-sub-pz-download-archive.json": "9639fffe58c44d084124c8398b46ef32",
+  ".ytdl-sub-pz-download-archive.json": "68e164a35d1541ce761a6d7fef19ee08",
  "Season 2018/s2018.e1102 - Jesse's Minecraft Server ｜ IP mc.jesse.id-thumb.jpg": "49cc64b25314155c1b8ab0361ac0c34f",
+  "Season 2018/s2018.e1102 - Jesse's Minecraft Server ｜ IP mc.jesse.id.en.srt": "3d2c4e7f65d2ca5e96da38ce7eecfc4e",
  "Season 2018/s2018.e1102 - Jesse's Minecraft Server ｜ IP mc.jesse.id.mp4": "e733b4cc385b953b08c8eb0f47e03c1e",
  "Season 2018/s2018.e1102 - Jesse's Minecraft Server ｜ IP mc.jesse.id.nfo": "2b3ccb3f1ef81ee49fe1afb88f275a09",
  "fanart.jpg": "c16b8b88a82cbd47d217ee80f6a8b5f3",
--- a/tests/e2e/resources/transaction_log_summaries/youtube/test_channel_full.txt
+++ b/tests/e2e/resources/transaction_log_summaries/youtube/test_channel_full.txt
@ -243,6 +243,7 @@ Season 2018/s2018.e1029 - Jesse's Minecraft Server ｜ Teaser Trailer.nfo
      title: Jesse's Minecraft Server | Teaser Trailer
      year: 2018
 Season 2018/s2018.e1102 - Jesse's Minecraft Server ｜ IP mc.jesse.id-thumb.jpg
+Season 2018/s2018.e1102 - Jesse's Minecraft Server ｜ IP mc.jesse.id.en.srt
 Season 2018/s2018.e1102 - Jesse's Minecraft Server ｜ IP mc.jesse.id.mp4
 Season 2018/s2018.e1102 - Jesse's Minecraft Server ｜ IP mc.jesse.id.nfo
  NFO tags:
--- a/tests/e2e/resources/transaction_log_summaries/youtube/test_channel_recent.txt
+++ b/tests/e2e/resources/transaction_log_summaries/youtube/test_channel_recent.txt
@ -22,6 +22,7 @@ Season 2018/s2018.e1029 - Jesse's Minecraft Server ｜ Teaser Trailer.nfo
      title: Jesse's Minecraft Server | Teaser Trailer
      year: 2018
 Season 2018/s2018.e1102 - Jesse's Minecraft Server ｜ IP mc.jesse.id-thumb.jpg
+Season 2018/s2018.e1102 - Jesse's Minecraft Server ｜ IP mc.jesse.id.en.srt
 Season 2018/s2018.e1102 - Jesse's Minecraft Server ｜ IP mc.jesse.id.mp4
 Season 2018/s2018.e1102 - Jesse's Minecraft Server ｜ IP mc.jesse.id.nfo
  NFO tags:
--- a/tests/e2e/youtube/test_channel.py
+++ b/tests/e2e/youtube/test_channel.py
@ -29,6 +29,10 @@ def channel_preset_dict(output_directory):
            "max_views": 100000,  # do not download the popular PJ concert
            "break_on_reject": False,  # do not break from max views
        },
+        "subtitles": {
+            "subtitles_name": "{episode_name}.{lang}.{subtitles_ext}",
+            "allow_auto_generated_subtitles": True,
+        },
        "overrides": {"tv_show_name": "Project / Zombie"},
    }

--- a/tests/e2e/youtube/test_playlist.py
+++ b/tests/e2e/youtube/test_playlist.py
@ -19,6 +19,10 @@ def playlist_preset_dict(output_directory):
        "ytdl_options": {
            "format": "worst[ext=mp4]",
        },
+        "subtitles": {
+            "subtitles_name": "{music_video_name}.{lang}.{subtitles_ext}",
+            "allow_auto_generated_subtitles": True,
+        },
        "overrides": {"artist": "JMC"},
    }