danvics/ytdl-sub
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

working again, no kwargs

Browse source
This commit is contained in:
Jesse Bannon 2023-12-13 12:10:57 -08:00
parent c3c2f03319
commit 9294bae75f
10 changed files with 88 additions and 75 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
src/ytdl_sub/config/plugin/plugin_mapping.py
View file

@ -64,6 +64,7 @@ class PluginMapping:
UrlDownloaderThumbnailPlugin,
AudioExtractPlugin,
FileConvertPlugin,
ChaptersPlugin,
SplitByChaptersPlugin,
RegexPlugin,
# add all others

4
src/ytdl_sub/downloaders/info_json/info_json_downloader.py
View file

@ -13,7 +13,7 @@ from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder
from ytdl_sub.entries.entry import YTDL_SUB_ENTRY_VARIABLES_KWARG_KEY
from ytdl_sub.entries.entry import Entry
from ytdl_sub.entries.script.variable_definitions import VARIABLES as v
from ytdl_sub.entries.script.variable_scripts import ENTRY_INJECTED_VARIABLES
from ytdl_sub.entries.script.variable_scripts import DOWNLOADER_INJECTED_VARIABLES
from ytdl_sub.entries.script.variable_scripts import VARIABLE_SCRIPTS
from ytdl_sub.utils.exceptions import ValidationException
from ytdl_sub.utils.file_handler import FileHandler
@ -115,7 +115,7 @@ class InfoJsonDownloader(SourcePlugin[InfoJsonDownloaderOptions]):
inj.variable_name,
VARIABLE_SCRIPTS[inj.variable_name],
)
for inj in ENTRY_INJECTED_VARIABLES
for inj in DOWNLOADER_INJECTED_VARIABLES
}
)
entries.append(entry)

1
src/ytdl_sub/downloaders/url/downloader.py
View file

@ -502,6 +502,7 @@ class MultiUrlDownloader(SourcePlugin[MultiUrlValidator]):
v.requested_subtitles.variable_name: download_entry.kwargs_get(
v.requested_subtitles.metadata_key
),
v.chapters.variable_name: download_entry.kwargs_get(v.chapters.metadata_key),
v.sponsorblock_chapters.variable_name: download_entry.kwargs_get(
v.sponsorblock_chapters.metadata_key
),

7
src/ytdl_sub/entries/script/variable_definitions.py
View file

@ -565,6 +565,10 @@ class VariableDefinitions:
def comments(self) -> MetadataVariable:
return MetadataVariable("comments", "comments")
@property
def chapters(self) -> MetadataVariable:
return MetadataVariable("chapters", "chapters")
@property
def sponsorblock_chapters(self) -> MetadataVariable:
return MetadataVariable("sponsorblock_chapters", "sponsorblock_chapters")
@ -577,9 +581,6 @@ class VariableDefinitions:
def ytdl_sub_input_url(self) -> Variable:
return Variable("ytdl_sub_input_url")
@property
def ytdl_sub_split_entry_parent_uid(self) -> Variable:
return Variable("ytdl_sub_split_entry_parent_uid")
@property
def download_index(self) -> Variable:

14
src/ytdl_sub/entries/script/variable_scripts.py
View file

@ -145,15 +145,15 @@ ENTRY_DEFAULT_VARIABLES: Dict[MetadataVariable, str] = {
v.playlist_uploader_id: entry_get_str(v.playlist_uploader_id, v.uploader_id),
}
# MARK AS UNRESOLVABLE UNTIL THEY ARE ADDED
ENTRY_INJECTED_VARIABLES: Dict[Variable, str] = {
# MARK AS UNRESOLVABLE UNTIL THEY ARE ADDED IN THE DOWNLOADER
DOWNLOADER_INJECTED_VARIABLES: Dict[Variable, str] = {
v.download_index: "{%int(1)}",
v.upload_date_index: "{%int(1)}",
v.comments: "",
v.comments: "{ [] }",
v.requested_subtitles: "{ {} }",
v.sponsorblock_chapters: "",
v.chapters: "{ [] }",
v.sponsorblock_chapters: "{ [] }",
v.ytdl_sub_input_url: f"{{{v.source_webpage_url.variable_name}}}",
v.ytdl_sub_split_entry_parent_uid: "",
}
ENTRY_DERIVED_VARIABLES: Dict[Variable, str] = {
@ -265,7 +265,7 @@ mergedeep.merge(
ENTRY_RELATIVE_VARIABLES,
ENTRY_REQUIRED_VARIABLES,
ENTRY_DEFAULT_VARIABLES,
ENTRY_INJECTED_VARIABLES,
DOWNLOADER_INJECTED_VARIABLES,
ENTRY_DERIVED_VARIABLES,
ENTRY_UPLOAD_DATE_VARIABLES,
ENTRY_RELEASE_DATE_VARIABLES,
@ -290,7 +290,7 @@ def _keys(*variables: Dict[Variable, str]) -> Set[str]:
UNRESOLVED_VARIABLES: Set[str] = _keys(
ENTRY_EMPTY_METADATA,
ENTRY_INJECTED_VARIABLES,
DOWNLOADER_INJECTED_VARIABLES,
)
CustomFunctions.register()

20
src/ytdl_sub/entries/variables/kwargs.py
View file

@ -1,20 +0,0 @@
from typing import List
class KwargKeys:
keys: List[str] = []
backend_keys: List[str] = []
def _(key: str, backend: bool = False) -> str:
if backend:
assert key not in KwargKeys.backend_keys
KwargKeys.backend_keys.append(key)
else:
assert key not in KwargKeys.keys
KwargKeys.keys.append(key)
return key
CHAPTERS = _("chapters", backend=True)
YTDL_SUB_CUSTOM_CHAPTERS = _("ytdl_sub_custom_chapters", backend=True)

48
src/ytdl_sub/plugins/chapters.py
View file

@ -6,12 +6,13 @@ from typing import Optional
from typing import Set
from ytdl_sub.config.plugin.plugin import Plugin
from ytdl_sub.config.plugin.plugin_operation import PluginOperation
from ytdl_sub.config.validators.options import OptionsDictValidator
from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder
from ytdl_sub.entries.entry import Entry
from ytdl_sub.entries.script.variable_definitions import VARIABLES as v
from ytdl_sub.entries.variables.kwargs import YTDL_SUB_CUSTOM_CHAPTERS
from ytdl_sub.utils.chapters import Chapters
from ytdl_sub.utils.chapters import Chapters, ytdl_sub_chapters_from_comments, \
ytdl_sub_split_by_chapters_parent_uid
from ytdl_sub.utils.ffmpeg import set_ffmpeg_metadata_chapters
from ytdl_sub.utils.file_handler import FileMetadata
from ytdl_sub.validators.regex_validator import RegexListValidator
@ -33,9 +34,7 @@ SPONSORBLOCK_CATEGORIES: Set[str] = SPONSORBLOCK_HIGHLIGHT_CATEGORIES | {
def _chapters(entry: Entry) -> List[Dict]:
if entry.kwargs_contains("chapters"):
return entry.kwargs("chapters") or []
return []
return entry.get(v.chapters, list)
def _sponsorblock_chapters(entry: Entry) -> List[Dict]:
@ -193,6 +192,13 @@ class ChaptersOptions(OptionsDictValidator):
"""
return self._force_key_frames
def added_variables(
self, resolved_variables: Set[str], unresolved_variables: Set[str]
) -> Dict[PluginOperation, Set[str]]:
return {
PluginOperation.MODIFY_ENTRY: {"ytdl_sub_chapters_from_comments"}
}
class ChaptersPlugin(Plugin[ChaptersOptions]):
plugin_options_type = ChaptersOptions
@ -298,6 +304,8 @@ class ChaptersPlugin(Plugin[ChaptersOptions]):
-------
entry
"""
has_chapters_from_comments = False
# If there are no embedded chapters, and comment chapters are allowed...
if not _contains_any_chapters(entry) and self.plugin_options.allow_chapters_from_comments:
chapters = Chapters.from_empty()
@ -310,7 +318,14 @@ class ChaptersPlugin(Plugin[ChaptersOptions]):
# If some are actually found, add a special kwarg and embed them
if chapters.contains_any_chapters():
entry.add_kwargs({YTDL_SUB_CUSTOM_CHAPTERS: chapters.to_file_metadata_dict()})
has_chapters_from_comments = True
entry.add(
{
ytdl_sub_chapters_from_comments.variable_name: (
chapters.to_yt_dlp_chapter_metadata()
)
}
)
if not self.is_dry_run:
set_ffmpeg_metadata_chapters(
@ -319,6 +334,9 @@ class ChaptersPlugin(Plugin[ChaptersOptions]):
file_duration_sec=entry.kwargs("duration"),
)
if not has_chapters_from_comments:
entry.add({ytdl_sub_chapters_from_comments.variable_name: []})
return entry
def post_process_entry(self, entry: Entry) -> Optional[FileMetadata]:
@ -332,12 +350,9 @@ class ChaptersPlugin(Plugin[ChaptersOptions]):
-------
FileMetadata outlining which chapters/SponsorBlock segments got removed
"""
if custom_chapters_metadata := entry.kwargs_get(YTDL_SUB_CUSTOM_CHAPTERS):
title: str = "Chapters from comments"
return FileMetadata.from_dict(
value_dict=custom_chapters_metadata,
title=title,
sort_dict=False, # timestamps + titles are already sorted
if custom_chapters := entry.get(ytdl_sub_chapters_from_comments, list):
return Chapters.from_yt_dlp_chapters(custom_chapters).to_file_metadata(
title="Chapters from comments"
)
if self.plugin_options.embed_chapters:
@ -354,9 +369,10 @@ class ChaptersPlugin(Plugin[ChaptersOptions]):
if removed_sponsorblock:
metadata_dict["Removed SponsorBlock Category Count(s)"] = removed_sponsorblock
# TODO: check if file actually has embedded chapters
return FileMetadata.from_dict(
value_dict=metadata_dict, title="Embedded Chapters", sort_dict=False
)
# If the entry wasn't split on embedded chapters, report it in the file metadata
if not entry.try_get(ytdl_sub_split_by_chapters_parent_uid, str):
return FileMetadata.from_dict(
value_dict=metadata_dict, title="Embedded Chapters", sort_dict=False
)
return None

14
src/ytdl_sub/plugins/split_by_chapters.py
View file

@ -12,8 +12,7 @@ from ytdl_sub.config.plugin.plugin_operation import PluginOperation
from ytdl_sub.config.validators.options import OptionsDictValidator
from ytdl_sub.entries.entry import Entry
from ytdl_sub.entries.script.variable_definitions import VARIABLES as v
from ytdl_sub.entries.variables.kwargs import CHAPTERS
from ytdl_sub.utils.chapters import Chapters
from ytdl_sub.utils.chapters import Chapters, ytdl_sub_split_by_chapters_parent_uid
from ytdl_sub.utils.chapters import Timestamp
from ytdl_sub.utils.exceptions import ValidationException
from ytdl_sub.utils.ffmpeg import FFMPEG
@ -105,7 +104,7 @@ class SplitByChaptersOptions(OptionsDictValidator):
return {
PluginOperation.MODIFY_ENTRY: {
v.uid.variable_name,
v.ytdl_sub_split_entry_parent_uid.variable_name,
ytdl_sub_split_by_chapters_parent_uid.variable_name,
}
}
@ -121,7 +120,7 @@ class SplitByChaptersPlugin(SplitPlugin[SplitByChaptersOptions]):
"chapter_index_padded": "01",
"chapter_count": 1,
v.uid.variable_name: entry.uid,
v.ytdl_sub_split_entry_parent_uid.variable_name: entry.uid,
ytdl_sub_split_by_chapters_parent_uid.variable_name: entry.uid,
}
)
return entry
@ -141,11 +140,6 @@ class SplitByChaptersPlugin(SplitPlugin[SplitByChaptersOptions]):
}
)
# pylint: disable=protected-access
if new_entry.kwargs_contains(CHAPTERS):
del new_entry._kwargs[CHAPTERS]
# pylint: enable=protected-access
timestamp_begin = chapters.timestamps[idx].readable_str
timestamp_end = Timestamp(new_entry.kwargs("duration")).readable_str
if idx + 1 < len(chapters.timestamps):
@ -195,7 +189,7 @@ class SplitByChaptersPlugin(SplitPlugin[SplitByChaptersOptions]):
new_entry.add(
{
v.uid.variable_name: new_uid,
v.ytdl_sub_split_entry_parent_uid.variable_name: entry.uid,
ytdl_sub_split_by_chapters_parent_uid.variable_name: entry.uid,
}
)
new_entry.add_kwargs({v.uid.metadata_key: new_uid})

51
src/ytdl_sub/utils/chapters.py
View file

@ -1,14 +1,18 @@
import re
from typing import Dict
from typing import List
from typing import Optional
from typing import Tuple
from ytdl_sub.entries.entry import Entry
from ytdl_sub.entries.variables.kwargs import CHAPTERS
from ytdl_sub.entries.variables.kwargs import YTDL_SUB_CUSTOM_CHAPTERS
from ytdl_sub.entries.script.variable_definitions import VARIABLES, Variable
from ytdl_sub.entries.script.variable_definitions import VariableDefinitions
from ytdl_sub.utils.file_handler import FileMetadata
v: VariableDefinitions = VARIABLES
ytdl_sub_chapters_from_comments = Variable("ytdl_sub_chapters_from_comments")
ytdl_sub_split_by_chapters_parent_uid = Variable("ytdl_sub_split_by_chapters_parent_uid")
class Timestamp:
@ -157,6 +161,17 @@ class Chapters:
"""
return self.timestamps[0].timestamp_sec == 0
def to_yt_dlp_chapter_metadata(self) -> List[Dict[str, str | float]]:
"""
Returns
-------
Metadata dict
"""
return [
{"start_time": ts.timestamp_sec, "title": title}
for ts, title in zip(self.timestamps, self.titles)
]
def to_file_metadata_dict(self) -> Dict:
"""
Returns
@ -165,7 +180,7 @@ class Chapters:
"""
return {ts.readable_str: title for ts, title in zip(self.timestamps, self.titles)}
def to_file_metadata(self, title: Optional[str] = None) -> FileMetadata:
def to_file_metadata(self, title: str) -> FileMetadata:
"""
Parameters
----------
@ -219,6 +234,17 @@ class Chapters:
# Otherwise return empty chapters
return Chapters(timestamps=[], titles=[])
@classmethod
def from_yt_dlp_chapters(cls, chapters: List[Dict[str, str | float]]):
timestamps: List[Timestamp] = []
titles: List[str] = []
for chapter in chapters:
timestamps.append(Timestamp.from_seconds(int(float(chapter["start_time"]))))
titles.append(chapter["title"])
return Chapters(timestamps=timestamps, titles=titles)
@classmethod
def from_entry_chapters(cls, entry: Entry) -> "Chapters":
"""
@ -231,19 +257,12 @@ class Chapters:
-------
Chapters object
"""
timestamps: List[Timestamp] = []
titles: List[str] = []
if chapters := (
entry.get(ytdl_sub_chapters_from_comments, list) or entry.get(v.chapters, list)
):
return cls.from_yt_dlp_chapters(chapters)
if entry.kwargs_contains(CHAPTERS):
for chapter in entry.kwargs_get(CHAPTERS, []):
timestamps.append(Timestamp.from_seconds(int(float(chapter["start_time"]))))
titles.append(chapter["title"])
elif entry.kwargs_contains(YTDL_SUB_CUSTOM_CHAPTERS):
for start_time, title in entry.kwargs_get(YTDL_SUB_CUSTOM_CHAPTERS, {}).items():
timestamps.append(Timestamp.from_str(start_time))
titles.append(title)
return Chapters(timestamps=timestamps, titles=titles)
return Chapters(timestamps=[], titles=[])
@classmethod
def from_empty(cls) -> "Chapters":

3
src/ytdl_sub/ytdl_additions/enhanced_download_archive.py
View file

@ -15,6 +15,7 @@ from yt_dlp.utils import make_archive_id
from ytdl_sub.entries.entry import Entry
from ytdl_sub.entries.script.variable_definitions import VARIABLES as v
from ytdl_sub.utils.chapters import ytdl_sub_split_by_chapters_parent_uid
from ytdl_sub.utils.file_handler import FileHandler
from ytdl_sub.utils.file_handler import FileHandlerTransactionLog
from ytdl_sub.utils.file_handler import FileMetadata
@ -220,7 +221,7 @@ class DownloadMappings:
self
"""
uid = entry.uid
if parent_uid := entry.try_get(v.ytdl_sub_split_entry_parent_uid, str):
if parent_uid := entry.try_get(ytdl_sub_split_by_chapters_parent_uid, str):
uid = parent_uid
if uid not in self.entry_ids: