danvics/ytdl-sub
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

subtitles seem to work, need to manually verify

Browse source
This commit is contained in:
jbannon 2022-08-12 06:45:58 +00:00
parent dae799fe12
commit 103e5aaac2
7 changed files with 180 additions and 68 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

23
src/ytdl_sub/config/preset_options.py
View file

@ -1,3 +1,4 @@
from typing import Dict
from typing import Optional
from yt_dlp.utils import DateRange
@ -74,15 +75,31 @@ class Overrides(DictFormatterValidator):
)
def apply_formatter(
self, formatter: StringFormatterValidator, entry: Optional[Entry] = None
self,
formatter: StringFormatterValidator,
entry: Optional[Entry] = None,
function_overrides: Dict[str, str] = None,
) -> str:
"""
Returns the format_string after .format has been called on it using entry (if provided) and
override values
Parameters
----------
formatter
Formatter to apply
entry
Optional. Entry to add source variables to the formatter
function_overrides
Optional. Explicit values to override the overrides themselves and source variables
Returns
-------
The format_string after .format has been called
"""
variable_dict = self.dict_with_format_strings
if entry:
variable_dict = dict(entry.to_dict(), **variable_dict)
if function_overrides:
variable_dict = dict(variable_dict, **function_overrides)
return formatter.apply_formatter(variable_dict)

73
src/ytdl_sub/plugins/subtitles.py
View file

@ -1,3 +1,4 @@
from pathlib import Path
from typing import Dict
from typing import List
from typing import Optional
@ -6,7 +7,7 @@ from typing import Set
from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder
from ytdl_sub.entries.entry import Entry
from ytdl_sub.plugins.plugin import Plugin
from ytdl_sub.validators.strict_dict_validator import StrictDictValidator
from ytdl_sub.plugins.plugin import PluginOptions
from ytdl_sub.validators.string_formatter_validators import StringFormatterValidator
from ytdl_sub.validators.string_select_validator import StringSelectValidator
from ytdl_sub.validators.validators import BoolValidator
@ -15,12 +16,20 @@ from ytdl_sub.validators.validators import StringListValidator
SUBTITLE_EXTENSIONS: Set[str] = {"srt", "vtt", "ass", "lrc"}
def _is_entry_subtitle_file(path: Path, entry: Entry) -> bool:
if path.is_file() and path.name.startswith(entry.uid):
for ext in SUBTITLE_EXTENSIONS:
if path.name.endswith(f".{ext}"):
return True
return False
class SubtitlesTypeValidator(StringSelectValidator):
_expected_value_type_name = "subtitles type"
_select_values = SUBTITLE_EXTENSIONS
class SubtitleOptions(StrictDictValidator):
class SubtitleOptions(PluginOptions):
"""
Defines how to download and store subtitles.
@ -56,10 +65,10 @@ class SubtitleOptions(StrictDictValidator):
)
self._subtitles_type = self._validate_key_if_present(
key="subtitles_type", validator=SubtitlesTypeValidator, default="srt"
)
).value
self._embed_subtitles = self._validate_key_if_present(
key="embed_subtitles", validator=BoolValidator
)
).value
self._languages = self._validate_key_if_present(
key="languages", validator=StringListValidator, default=["en"]
).list
@ -86,9 +95,9 @@ class SubtitleOptions(StrictDictValidator):
@property
def embed_subtitles(self) -> Optional[bool]:
"""
Optional. Whether to embed the subtitles into the video file.
Optional. Whether to embed the subtitles into the video file. Defaults to False.
"""
return self._subtitles_type
return self._embed_subtitles
@property
def languages(self) -> Optional[List[str]]:
@ -104,6 +113,14 @@ class SubtitleOptions(StrictDictValidator):
"""
return self._allow_auto_generated_subtitles
def added_source_variables(self) -> List[str]:
"""
Returns
-------
List of new source variables created by using the subtitles plugin
"""
return ["lang", "subtitle_ext"]
class SubtitlesPlugin(Plugin[SubtitleOptions]):
plugin_options_type = SubtitleOptions
@ -144,6 +161,18 @@ class SubtitlesPlugin(Plugin[SubtitleOptions]):
}
).to_dict()
def modify_entry(self, entry: Entry) -> Optional[Entry]:
requested_subtitles = entry.kwargs("requested_subtitles")
languages = sorted(requested_subtitles.keys())
entry.add_variables(
variables_to_add={
"subtitle_ext": self.plugin_options.subtitles_type,
"lang": ",".join(languages),
}
)
return entry
def post_process_entry(self, entry: Entry) -> None:
"""
Creates an entry's NFO file using values defined in the metadata options
@ -151,20 +180,22 @@ class SubtitlesPlugin(Plugin[SubtitleOptions]):
Parameters
----------
entry:
Entry to create an NFO file for
Entry to create subtitles for
"""
if not self.plugin_options.subtitles_name:
return
# def get_ytdlp_download_subtitle_paths(self) -> List[str]:
# possible_subtitle_exts = SUBTITLE_EXTENSIONS
# subtitle_paths: List[str] = []
#
# for ext in possible_subtitle_exts:
# for path in Path(self.working_directory()).rglob("*"):
# if (
# path.is_file()
# and path.name.startswith(self.uid)
# and path.name.endswith(f".{ext}")
# ):
# subtitle_paths.append(str(path))
#
# return subtitle_paths
requested_subtitles = entry.kwargs("requested_subtitles")
for lang in requested_subtitles.keys():
subtitle_file_name = f"{entry.uid}.{lang}.{self.plugin_options.subtitles_type}"
output_subtitle_file_name = self.overrides.apply_formatter(
formatter=self.plugin_options.subtitles_name,
entry=entry,
function_overrides={"lang": lang},
)
self.save_file(
file_name=subtitle_file_name,
output_file_name=output_subtitle_file_name,
entry=entry,
)

47
src/ytdl_sub/subscriptions/subscription_ytdl_options.py
View file

@ -67,8 +67,8 @@ class SubscriptionYTDLOptions:
def _dry_run_options(self) -> Dict:
return {
"skip_download": True,
"simulate": True,
"writethumbnail": False,
"writesubtitles": False,
}
@property
@ -90,30 +90,45 @@ class SubscriptionYTDLOptions:
# TODO: warn here
return {}
ytdl_options: Dict = {}
builder = YTDLOptionsBuilder()
subtitle_options: SubtitleOptions = subtitle_plugin.plugin_options
write_subtitle_file: bool = subtitle_options.subtitles_name is not None
if write_subtitle_file:
ytdl_options["writesubtitles"] = True
ytdl_options["postprocessors"] = [
{"key": "FFmpegSubtitlesConvertor", "format": subtitle_options.subtitles_type}
]
builder.add(
{
"writesubtitles": True,
"postprocessors": [
{
"key": "FFmpegSubtitlesConvertor",
"format": subtitle_options.subtitles_type,
}
],
}
)
if subtitle_options.embed_subtitles:
ytdl_options["postprocessors"] = [
# already_have_subtitle=True means keep the subtitle files. False means delete
{"key": "FFmpegEmbedSubtitle", "already_have_subtitle": write_subtitle_file}
]
builder.add(
{
"postprocessors": [
# already_have_subtitle=True means we downloaded the subtitle files.
{"key": "FFmpegEmbedSubtitle", "already_have_subtitle": write_subtitle_file}
]
}
)
# If neither subtitles_name or embed_subtitles is set, do not set any other flags
if not ytdl_options:
if not builder.to_dict():
return {}
ytdl_options["writeautomaticsub"] = subtitle_options.allow_auto_generated_subtitles
ytdl_options["subtitleslangs"] = subtitle_options.languages
builder.add(
{
"writeautomaticsub": subtitle_options.allow_auto_generated_subtitles,
"subtitleslangs": subtitle_options.languages,
}
)
return ytdl_options
return builder.to_dict()
@property
def _user_ytdl_options(self) -> Dict:
@ -128,7 +143,9 @@ class SubscriptionYTDLOptions:
"""
ytdl_options_builder = YTDLOptionsBuilder().add(self._global_options)
if self._dry_run:
ytdl_options_builder.add(self._dry_run_options, self._user_ytdl_options)
ytdl_options_builder.add(
self._subtitle_options, self._user_ytdl_options, self._dry_run_options
)
else:
ytdl_options_builder.add(
self._output_options, self._subtitle_options, self._user_ytdl_options

56
tests/e2e/plugins/test_subtitles.py Normal file
View file

@ -0,0 +1,56 @@
import pytest
from e2e.expected_download import assert_expected_downloads
from e2e.expected_transaction_log import assert_transaction_log_matches
from ytdl_sub.subscriptions.subscription import Subscription
@pytest.fixture
def single_video_subs_preset_dict(output_directory):
return {
"preset": "yt_music_video",
"youtube": {"video_url": "https://www.youtube.com/watch?v=2lAe1cqCOXo"},
# override the output directory with our fixture-generated dir
"output_options": {"output_directory": output_directory},
"subtitles": {
"subtitles_name": "{music_video_name}.{lang}.{subtitles_ext}",
"embed_subtitles": True,
"languages": ["en", "de"],
"allow_auto_generated_subtitles": True,
},
# download the worst format so it is fast
"ytdl_options": {
"format": "worst[ext=mp4]",
},
"overrides": {"artist": "JMC"},
}
class TestSubtitles:
@pytest.mark.parametrize("dry_run", [True, False])
def test_single_video_download_with_subtitles(
self,
music_video_config,
single_video_subs_preset_dict,
output_directory,
dry_run,
):
subscription = Subscription.from_dict(
config=music_video_config,
preset_name="subtitles_video_test",
preset_dict=single_video_subs_preset_dict,
)
transaction_log = subscription.download(dry_run=dry_run)
assert_transaction_log_matches(
output_directory=output_directory,
transaction_log=transaction_log,
transaction_log_summary_file_name="plugins/test_subtitles_video.txt",
regenerate_transaction_log=True,
)
assert_expected_downloads(
output_directory=output_directory,
dry_run=dry_run,
expected_download_summary_file_name="plugins/test_subtitles_video.json",
regenerate_expected_download_summary=True,
)

7
tests/e2e/resources/expected_downloads_summaries/plugins/test_subtitles_video.json Normal file
View file

@ -0,0 +1,7 @@
{
"JMC - YouTube Rewind 2019： For the Record ｜ #YouTubeRewind-thumb.jpg": "704246dd78074e8a0ec001dd8d03fd60",
"JMC - YouTube Rewind 2019： For the Record ｜ #YouTubeRewind.de.srt": "b343c3bb9257b7ee7ba38f570a115b37",
"JMC - YouTube Rewind 2019： For the Record ｜ #YouTubeRewind.en.srt": "fe8c6ee92cae6e059fd80fd61691adbe",
"JMC - YouTube Rewind 2019： For the Record ｜ #YouTubeRewind.mp4": "383213d2809cdc2e86e3a2c4c8deb685",
"JMC - YouTube Rewind 2019： For the Record ｜ #YouTubeRewind.nfo": "ffa10f1cbc098ace7b1c7a8fbe3097a8"
}

13
tests/e2e/resources/transaction_log_summaries/plugins/test_subtitles_video.txt Normal file
View file

@ -0,0 +1,13 @@
Files created in '{output_directory}'
----------------------------------------
JMC - YouTube Rewind 2019： For the Record ｜ #YouTubeRewind-thumb.jpg
JMC - YouTube Rewind 2019： For the Record ｜ #YouTubeRewind.de.srt
JMC - YouTube Rewind 2019： For the Record ｜ #YouTubeRewind.en.srt
JMC - YouTube Rewind 2019： For the Record ｜ #YouTubeRewind.mp4
JMC - YouTube Rewind 2019： For the Record ｜ #YouTubeRewind.nfo
NFO tags:
musicvideo:
album: Music Videos
artist: JMC
title: YouTube Rewind 2019: For the Record | #YouTubeRewind
year: 2019

29
tests/e2e/youtube/test_video.py
View file

@ -54,35 +54,6 @@ class TestYoutubeVideo:
expected_download_summary_file_name="youtube/test_video.json",
)
# @pytest.mark.parametrize("dry_run", [True, False])
# def test_single_video_download_with_subtitles(
# self,
# music_video_config,
# single_video_preset_dict,
# expected_single_video_download,
# output_directory,
# dry_run,
# ):
# single_video_preset_dict["youtube"][
# "video_url"
# ] = "https://www.youtube.com/watch?v=dQw4w9WgXcQ"
# single_video_preset_dict["subtitles"] = {}
# single_video_preset_dict["subtitles"]["subtitles_name"] = "{music_video_name}.srt"
# single_video_subscription = Subscription.from_dict(
# config=music_video_config,
# preset_name="music_video_single_video_test",
# preset_dict=single_video_preset_dict,
# )
#
# transaction_log = single_video_subscription.download(dry_run=dry_run)
# assert_transaction_log_matches(
# output_directory=output_directory,
# transaction_log=transaction_log,
# transaction_log_summary_file_name="youtube/test_video.txt",
# )
# if not dry_run:
# expected_single_video_download.assert_files_exist(relative_directory=output_directory)
@pytest.mark.parametrize("dry_run", [True, False])
def test_single_video_download_from_cli_dl(
self,