From 9883907bf0e14314e39917b5945b361b6f4c7894 Mon Sep 17 00:00:00 2001 From: Jesse Bannon Date: Fri, 7 Oct 2022 17:16:54 -0700 Subject: [PATCH] [BUGFIX] Use yt-dlp remuxer instead of our own (#275) * [BUGFIX] Use yt-dlps remuxer instead of our own * docstring --- src/ytdl_sub/entries/entry.py | 9 ++- src/ytdl_sub/plugins/audio_extract.py | 5 +- src/ytdl_sub/plugins/file_convert.py | 75 ++++++++++++------- .../prebuilt_presets/helpers/players.yaml | 5 -- .../prebuilt_presets/tv_show/players.yaml | 3 +- .../subscription_ytdl_options.py | 10 +++ .../validators/audo_codec_validator.py | 23 ++++++ tests/e2e/plugins/test_file_convert.py | 2 +- .../plugins/file_convert/output.json | 4 +- 9 files changed, 91 insertions(+), 45 deletions(-) diff --git a/src/ytdl_sub/entries/entry.py b/src/ytdl_sub/entries/entry.py index d6bfea9c..46bbbae1 100644 --- a/src/ytdl_sub/entries/entry.py +++ b/src/ytdl_sub/entries/entry.py @@ -8,6 +8,7 @@ from typing import final from ytdl_sub.entries.base_entry import BaseEntry from ytdl_sub.entries.variables.entry_variables import EntryVariables from ytdl_sub.validators.audo_codec_validator import AUDIO_CODEC_EXTS +from ytdl_sub.validators.audo_codec_validator import VIDEO_CODEC_EXTS class Entry(EntryVariables, BaseEntry): @@ -89,11 +90,11 @@ class Entry(EntryVariables, BaseEntry): """ file_exists = os.path.isfile(self.get_download_file_path()) - # HACK: yt-dlp does not record extracted audio extensions anywhere. If the file is not - # found, try it using audio extensions + # HACK: yt-dlp does not record extracted/converted extensions anywhere. If the file is not + # found, try it using all possible extensions if not file_exists: - for audio_ext in AUDIO_CODEC_EXTS: - if os.path.isfile(self.get_download_file_path().removesuffix(self.ext) + audio_ext): + for ext in AUDIO_CODEC_EXTS.union(VIDEO_CODEC_EXTS): + if os.path.isfile(self.get_download_file_path().removesuffix(self.ext) + ext): file_exists = True break diff --git a/src/ytdl_sub/plugins/audio_extract.py b/src/ytdl_sub/plugins/audio_extract.py index a26d4bfb..0908e529 100644 --- a/src/ytdl_sub/plugins/audio_extract.py +++ b/src/ytdl_sub/plugins/audio_extract.py @@ -100,9 +100,6 @@ class AudioExtractPlugin(Plugin[AudioExtractOptions]): if not os.path.isfile(extracted_audio_file): raise FileNotDownloadedException("Failed to find the extracted audio file") - # TODO: create entry function to update kwargs - # pylint: disable=protected-access - entry._kwargs["ext"] = new_ext - # pylint: enable=protected-access + entry.add_kwargs({"ext": new_ext}) return entry diff --git a/src/ytdl_sub/plugins/file_convert.py b/src/ytdl_sub/plugins/file_convert.py index 4347a583..8b47bfc2 100644 --- a/src/ytdl_sub/plugins/file_convert.py +++ b/src/ytdl_sub/plugins/file_convert.py @@ -1,14 +1,14 @@ +import os from typing import Dict from typing import Optional from ytdl_sub.entries.entry import Entry +from ytdl_sub.entries.variables.kwargs import EXT from ytdl_sub.plugins.plugin import Plugin from ytdl_sub.plugins.plugin import PluginOptions -from ytdl_sub.plugins.plugin import PluginPriority -from ytdl_sub.utils.ffmpeg import FFMPEG -from ytdl_sub.utils.file_handler import FileHandler +from ytdl_sub.utils.exceptions import FileNotDownloadedException from ytdl_sub.utils.file_handler import FileMetadata -from ytdl_sub.validators.validators import LiteralDictValidator +from ytdl_sub.validators.audo_codec_validator import VideoCodecTypeValidator class FileConvertOptions(PluginOptions): @@ -22,55 +22,74 @@ class FileConvertOptions(PluginOptions): presets: my_example_preset: file_converter: - convert: - webm: "mp4" + convert_to: "mp4" """ - _required_keys = {"convert"} + _required_keys = {"convert_to"} def __init__(self, name, value): super().__init__(name, value) - self._convert = self._validate_key(key="convert", validator=LiteralDictValidator).dict + self._convert_to = self._validate_key( + key="convert_to", validator=VideoCodecTypeValidator + ).value @property - def convert(self) -> Dict[str, str]: + def convert_to(self) -> str: """ - Convert from one extension to another + Convert to a desired extension """ - return self._convert + return self._convert_to class FileConvertPlugin(Plugin[FileConvertOptions]): plugin_options_type = FileConvertOptions - priority = PluginPriority(modify_entry=0) - def modify_entry(self, entry: Entry) -> Entry: + def ytdl_options(self) -> Optional[Dict]: """ - If the entry is of the specified file type, convert it + Returns + ------- + ffmpeg video remuxing post processing dict """ - for convert_from, convert_to in self.plugin_options.convert.items(): - # Entry ext does not need conversion - if entry.ext != convert_from: - continue + return { + "postprocessors": [ + { + "key": "FFmpegVideoRemuxer", + "when": "post_process", + "preferedformat": self.plugin_options.convert_to, + } + ] + } - # Entry ext needs conversion - input_file_path = entry.get_download_file_path() - output_file_path = input_file_path.removesuffix(entry.ext) + convert_to + def modify_entry(self, entry: Entry) -> Optional[Entry]: + """ + Parameters + ---------- + entry + Entry with extracted audio - ffmpeg_args = ["-bitexact", "-i", input_file_path, "-c", "copy", output_file_path] + Returns + ------- + Entry with updated 'ext' source variable - if not self.is_dry_run: - FFMPEG.run(ffmpeg_args) - FileHandler.delete(input_file_path) + Raises + ------ + FileNotDownloadedException + If the audio file is not found + """ + new_ext = self.plugin_options.convert_to + converted_video_file = entry.get_download_file_path().removesuffix(entry.ext) + new_ext + if not self.is_dry_run: + if not os.path.isfile(converted_video_file): + raise FileNotDownloadedException("Failed to find the converted video file") + if entry.ext != new_ext: entry.add_kwargs( { - "ext": convert_to, - "__converted_from": convert_from, + "__converted_from": entry.ext, } ) - return entry + entry.add_kwargs({EXT: new_ext}) return entry diff --git a/src/ytdl_sub/prebuilt_presets/helpers/players.yaml b/src/ytdl_sub/prebuilt_presets/helpers/players.yaml index 3d805bd4..62a67fa8 100644 --- a/src/ytdl_sub/prebuilt_presets/helpers/players.yaml +++ b/src/ytdl_sub/prebuilt_presets/helpers/players.yaml @@ -5,8 +5,3 @@ presets: kodi_safe: True output_directory_nfo_tags: kodi_safe: True - - _plex_base: - file_convert: - convert: - webm: "mp4" \ No newline at end of file diff --git a/src/ytdl_sub/prebuilt_presets/tv_show/players.yaml b/src/ytdl_sub/prebuilt_presets/tv_show/players.yaml index be949a84..4ec155f7 100644 --- a/src/ytdl_sub/prebuilt_presets/tv_show/players.yaml +++ b/src/ytdl_sub/prebuilt_presets/tv_show/players.yaml @@ -2,12 +2,13 @@ presets: _plex_tv_show: preset: - - "_plex_base" - "_episode_video_tags" overrides: tv_show_poster_file_name: "poster.jpg" tv_show_fanart_file_name: "fanart.jpg" season_poster_file_name: "Season{season_number_padded}.jpg" + file_convert: # Plex currently does not support webm + convert_to: "mp4" _jellyfin_tv_show: preset: diff --git a/src/ytdl_sub/subscriptions/subscription_ytdl_options.py b/src/ytdl_sub/subscriptions/subscription_ytdl_options.py index b2f49919..771437ae 100644 --- a/src/ytdl_sub/subscriptions/subscription_ytdl_options.py +++ b/src/ytdl_sub/subscriptions/subscription_ytdl_options.py @@ -11,6 +11,7 @@ from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder from ytdl_sub.plugins.audio_extract import AudioExtractPlugin from ytdl_sub.plugins.chapters import ChaptersPlugin from ytdl_sub.plugins.date_range import DateRangePlugin +from ytdl_sub.plugins.file_convert import FileConvertPlugin from ytdl_sub.plugins.plugin import Plugin from ytdl_sub.plugins.subtitles import SubtitlesPlugin from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive @@ -119,6 +120,13 @@ class SubscriptionYTDLOptions: return date_range_plugin.ytdl_options() + @property + def _file_convert_options(self) -> Dict: + if not (file_convert_plugin := self._get_plugin(FileConvertPlugin)): + return {} + + return file_convert_plugin.ytdl_options() + @property def _user_ytdl_options(self) -> Dict: return self._preset.ytdl_options.dict @@ -134,6 +142,7 @@ class SubscriptionYTDLOptions: if self._dry_run: ytdl_options_builder.add( self._date_range_options, + self._file_convert_options, self._subtitle_options, self._chapter_options, self._user_ytdl_options, # user ytdl options... @@ -143,6 +152,7 @@ class SubscriptionYTDLOptions: ytdl_options_builder.add( self._output_options, self._date_range_options, + self._file_convert_options, self._subtitle_options, self._chapter_options, self._audio_extract_options, diff --git a/src/ytdl_sub/validators/audo_codec_validator.py b/src/ytdl_sub/validators/audo_codec_validator.py index b986c02f..ddc25a85 100644 --- a/src/ytdl_sub/validators/audo_codec_validator.py +++ b/src/ytdl_sub/validators/audo_codec_validator.py @@ -16,7 +16,30 @@ AUDIO_CODEC_TYPES_EXTENSION_MAPPING: Dict[str, str] = { AUDIO_CODEC_TYPES: Set[str] = set(AUDIO_CODEC_TYPES_EXTENSION_MAPPING.keys()) AUDIO_CODEC_EXTS: Set[str] = set(AUDIO_CODEC_TYPES_EXTENSION_MAPPING.values()) +VIDEO_CODEC_EXTS: Set[str] = { + "avi", + "flv", + "mkv", + "mov", + "mp4", + "webm", + "3g2", + "3gp", + "f4v", + "mk3d", + "divx", + "mpg", + "ogv", + "m4v", + "wmv", +} + class CodecTypeValidator(StringSelectValidator): _expected_value_type_name = "codec" _select_values = AUDIO_CODEC_TYPES + + +class VideoCodecTypeValidator(StringSelectValidator): + _expected_value_type_name = "codec" + _select_values = VIDEO_CODEC_EXTS diff --git a/tests/e2e/plugins/test_file_convert.py b/tests/e2e/plugins/test_file_convert.py index 0826e4bc..894cbc00 100644 --- a/tests/e2e/plugins/test_file_convert.py +++ b/tests/e2e/plugins/test_file_convert.py @@ -15,7 +15,7 @@ def preset_dict(output_directory): "ytdl_options": { "postprocessor_args": {"ffmpeg": ["-bitexact"]}, # Must add this for reproducibility }, - "file_convert": {"convert": {"webm": "mp4"}}, + "file_convert": {"convert_to": "mp4"}, } diff --git a/tests/resources/expected_downloads_summaries/plugins/file_convert/output.json b/tests/resources/expected_downloads_summaries/plugins/file_convert/output.json index a152d601..f968501a 100644 --- a/tests/resources/expected_downloads_summaries/plugins/file_convert/output.json +++ b/tests/resources/expected_downloads_summaries/plugins/file_convert/output.json @@ -1,6 +1,6 @@ { "Beyond The Guitar - When you hear Hugh Jackman is returning as Wolverine in Deadpool 3-thumb.jpg": "662fcaadf6e80d63591bac19a5fdffb0", - "Beyond The Guitar - When you hear Hugh Jackman is returning as Wolverine in Deadpool 3.info.json": "c6f7178b1efc57282dabbca958785d69", - "Beyond The Guitar - When you hear Hugh Jackman is returning as Wolverine in Deadpool 3.mp4": "3c4e0cf1cefe6bf55adbbb1d809bbd04", + "Beyond The Guitar - When you hear Hugh Jackman is returning as Wolverine in Deadpool 3.info.json": "4d51260489eca1f969107cfb4957b368", + "Beyond The Guitar - When you hear Hugh Jackman is returning as Wolverine in Deadpool 3.mp4": "390177bcc076e309d1534b8bb5afdcc7", "Beyond The Guitar - When you hear Hugh Jackman is returning as Wolverine in Deadpool 3.nfo": "cacf09ab38f9b3085da9c5af516cf22a" } \ No newline at end of file