diff --git a/src/ytdl_sub/config/plugin/plugin_mapping.py b/src/ytdl_sub/config/plugin/plugin_mapping.py index 896e1b6c..110a154c 100644 --- a/src/ytdl_sub/config/plugin/plugin_mapping.py +++ b/src/ytdl_sub/config/plugin/plugin_mapping.py @@ -64,6 +64,7 @@ class PluginMapping: UrlDownloaderThumbnailPlugin, AudioExtractPlugin, FileConvertPlugin, + ChaptersPlugin, SplitByChaptersPlugin, RegexPlugin, # add all others diff --git a/src/ytdl_sub/downloaders/info_json/info_json_downloader.py b/src/ytdl_sub/downloaders/info_json/info_json_downloader.py index a339eaed..6625c595 100644 --- a/src/ytdl_sub/downloaders/info_json/info_json_downloader.py +++ b/src/ytdl_sub/downloaders/info_json/info_json_downloader.py @@ -13,7 +13,7 @@ from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder from ytdl_sub.entries.entry import YTDL_SUB_ENTRY_VARIABLES_KWARG_KEY from ytdl_sub.entries.entry import Entry from ytdl_sub.entries.script.variable_definitions import VARIABLES as v -from ytdl_sub.entries.script.variable_scripts import ENTRY_INJECTED_VARIABLES +from ytdl_sub.entries.script.variable_scripts import DOWNLOADER_INJECTED_VARIABLES from ytdl_sub.entries.script.variable_scripts import VARIABLE_SCRIPTS from ytdl_sub.utils.exceptions import ValidationException from ytdl_sub.utils.file_handler import FileHandler @@ -115,7 +115,7 @@ class InfoJsonDownloader(SourcePlugin[InfoJsonDownloaderOptions]): inj.variable_name, VARIABLE_SCRIPTS[inj.variable_name], ) - for inj in ENTRY_INJECTED_VARIABLES + for inj in DOWNLOADER_INJECTED_VARIABLES } ) entries.append(entry) diff --git a/src/ytdl_sub/downloaders/url/downloader.py b/src/ytdl_sub/downloaders/url/downloader.py index fa5f7cff..02eb5251 100644 --- a/src/ytdl_sub/downloaders/url/downloader.py +++ b/src/ytdl_sub/downloaders/url/downloader.py @@ -502,6 +502,7 @@ class MultiUrlDownloader(SourcePlugin[MultiUrlValidator]): v.requested_subtitles.variable_name: download_entry.kwargs_get( v.requested_subtitles.metadata_key ), + v.chapters.variable_name: download_entry.kwargs_get(v.chapters.metadata_key), v.sponsorblock_chapters.variable_name: download_entry.kwargs_get( v.sponsorblock_chapters.metadata_key ), diff --git a/src/ytdl_sub/entries/script/variable_definitions.py b/src/ytdl_sub/entries/script/variable_definitions.py index 2e236e91..7b0def05 100644 --- a/src/ytdl_sub/entries/script/variable_definitions.py +++ b/src/ytdl_sub/entries/script/variable_definitions.py @@ -565,6 +565,10 @@ class VariableDefinitions: def comments(self) -> MetadataVariable: return MetadataVariable("comments", "comments") + @property + def chapters(self) -> MetadataVariable: + return MetadataVariable("chapters", "chapters") + @property def sponsorblock_chapters(self) -> MetadataVariable: return MetadataVariable("sponsorblock_chapters", "sponsorblock_chapters") @@ -577,9 +581,6 @@ class VariableDefinitions: def ytdl_sub_input_url(self) -> Variable: return Variable("ytdl_sub_input_url") - @property - def ytdl_sub_split_entry_parent_uid(self) -> Variable: - return Variable("ytdl_sub_split_entry_parent_uid") @property def download_index(self) -> Variable: diff --git a/src/ytdl_sub/entries/script/variable_scripts.py b/src/ytdl_sub/entries/script/variable_scripts.py index 5ce33aed..a35ac83a 100644 --- a/src/ytdl_sub/entries/script/variable_scripts.py +++ b/src/ytdl_sub/entries/script/variable_scripts.py @@ -145,15 +145,15 @@ ENTRY_DEFAULT_VARIABLES: Dict[MetadataVariable, str] = { v.playlist_uploader_id: entry_get_str(v.playlist_uploader_id, v.uploader_id), } -# MARK AS UNRESOLVABLE UNTIL THEY ARE ADDED -ENTRY_INJECTED_VARIABLES: Dict[Variable, str] = { +# MARK AS UNRESOLVABLE UNTIL THEY ARE ADDED IN THE DOWNLOADER +DOWNLOADER_INJECTED_VARIABLES: Dict[Variable, str] = { v.download_index: "{%int(1)}", v.upload_date_index: "{%int(1)}", - v.comments: "", + v.comments: "{ [] }", v.requested_subtitles: "{ {} }", - v.sponsorblock_chapters: "", + v.chapters: "{ [] }", + v.sponsorblock_chapters: "{ [] }", v.ytdl_sub_input_url: f"{{{v.source_webpage_url.variable_name}}}", - v.ytdl_sub_split_entry_parent_uid: "", } ENTRY_DERIVED_VARIABLES: Dict[Variable, str] = { @@ -265,7 +265,7 @@ mergedeep.merge( ENTRY_RELATIVE_VARIABLES, ENTRY_REQUIRED_VARIABLES, ENTRY_DEFAULT_VARIABLES, - ENTRY_INJECTED_VARIABLES, + DOWNLOADER_INJECTED_VARIABLES, ENTRY_DERIVED_VARIABLES, ENTRY_UPLOAD_DATE_VARIABLES, ENTRY_RELEASE_DATE_VARIABLES, @@ -290,7 +290,7 @@ def _keys(*variables: Dict[Variable, str]) -> Set[str]: UNRESOLVED_VARIABLES: Set[str] = _keys( ENTRY_EMPTY_METADATA, - ENTRY_INJECTED_VARIABLES, + DOWNLOADER_INJECTED_VARIABLES, ) CustomFunctions.register() diff --git a/src/ytdl_sub/entries/variables/kwargs.py b/src/ytdl_sub/entries/variables/kwargs.py deleted file mode 100644 index 7a2c8351..00000000 --- a/src/ytdl_sub/entries/variables/kwargs.py +++ /dev/null @@ -1,20 +0,0 @@ -from typing import List - - -class KwargKeys: - keys: List[str] = [] - backend_keys: List[str] = [] - - -def _(key: str, backend: bool = False) -> str: - if backend: - assert key not in KwargKeys.backend_keys - KwargKeys.backend_keys.append(key) - else: - assert key not in KwargKeys.keys - KwargKeys.keys.append(key) - return key - - -CHAPTERS = _("chapters", backend=True) -YTDL_SUB_CUSTOM_CHAPTERS = _("ytdl_sub_custom_chapters", backend=True) diff --git a/src/ytdl_sub/plugins/chapters.py b/src/ytdl_sub/plugins/chapters.py index 7e353028..54d22893 100644 --- a/src/ytdl_sub/plugins/chapters.py +++ b/src/ytdl_sub/plugins/chapters.py @@ -6,12 +6,13 @@ from typing import Optional from typing import Set from ytdl_sub.config.plugin.plugin import Plugin +from ytdl_sub.config.plugin.plugin_operation import PluginOperation from ytdl_sub.config.validators.options import OptionsDictValidator from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder from ytdl_sub.entries.entry import Entry from ytdl_sub.entries.script.variable_definitions import VARIABLES as v -from ytdl_sub.entries.variables.kwargs import YTDL_SUB_CUSTOM_CHAPTERS -from ytdl_sub.utils.chapters import Chapters +from ytdl_sub.utils.chapters import Chapters, ytdl_sub_chapters_from_comments, \ + ytdl_sub_split_by_chapters_parent_uid from ytdl_sub.utils.ffmpeg import set_ffmpeg_metadata_chapters from ytdl_sub.utils.file_handler import FileMetadata from ytdl_sub.validators.regex_validator import RegexListValidator @@ -33,9 +34,7 @@ SPONSORBLOCK_CATEGORIES: Set[str] = SPONSORBLOCK_HIGHLIGHT_CATEGORIES | { def _chapters(entry: Entry) -> List[Dict]: - if entry.kwargs_contains("chapters"): - return entry.kwargs("chapters") or [] - return [] + return entry.get(v.chapters, list) def _sponsorblock_chapters(entry: Entry) -> List[Dict]: @@ -193,6 +192,13 @@ class ChaptersOptions(OptionsDictValidator): """ return self._force_key_frames + def added_variables( + self, resolved_variables: Set[str], unresolved_variables: Set[str] + ) -> Dict[PluginOperation, Set[str]]: + return { + PluginOperation.MODIFY_ENTRY: {"ytdl_sub_chapters_from_comments"} + } + class ChaptersPlugin(Plugin[ChaptersOptions]): plugin_options_type = ChaptersOptions @@ -298,6 +304,8 @@ class ChaptersPlugin(Plugin[ChaptersOptions]): ------- entry """ + has_chapters_from_comments = False + # If there are no embedded chapters, and comment chapters are allowed... if not _contains_any_chapters(entry) and self.plugin_options.allow_chapters_from_comments: chapters = Chapters.from_empty() @@ -310,7 +318,14 @@ class ChaptersPlugin(Plugin[ChaptersOptions]): # If some are actually found, add a special kwarg and embed them if chapters.contains_any_chapters(): - entry.add_kwargs({YTDL_SUB_CUSTOM_CHAPTERS: chapters.to_file_metadata_dict()}) + has_chapters_from_comments = True + entry.add( + { + ytdl_sub_chapters_from_comments.variable_name: ( + chapters.to_yt_dlp_chapter_metadata() + ) + } + ) if not self.is_dry_run: set_ffmpeg_metadata_chapters( @@ -319,6 +334,9 @@ class ChaptersPlugin(Plugin[ChaptersOptions]): file_duration_sec=entry.kwargs("duration"), ) + if not has_chapters_from_comments: + entry.add({ytdl_sub_chapters_from_comments.variable_name: []}) + return entry def post_process_entry(self, entry: Entry) -> Optional[FileMetadata]: @@ -332,12 +350,9 @@ class ChaptersPlugin(Plugin[ChaptersOptions]): ------- FileMetadata outlining which chapters/SponsorBlock segments got removed """ - if custom_chapters_metadata := entry.kwargs_get(YTDL_SUB_CUSTOM_CHAPTERS): - title: str = "Chapters from comments" - return FileMetadata.from_dict( - value_dict=custom_chapters_metadata, - title=title, - sort_dict=False, # timestamps + titles are already sorted + if custom_chapters := entry.get(ytdl_sub_chapters_from_comments, list): + return Chapters.from_yt_dlp_chapters(custom_chapters).to_file_metadata( + title="Chapters from comments" ) if self.plugin_options.embed_chapters: @@ -354,9 +369,10 @@ class ChaptersPlugin(Plugin[ChaptersOptions]): if removed_sponsorblock: metadata_dict["Removed SponsorBlock Category Count(s)"] = removed_sponsorblock - # TODO: check if file actually has embedded chapters - return FileMetadata.from_dict( - value_dict=metadata_dict, title="Embedded Chapters", sort_dict=False - ) + # If the entry wasn't split on embedded chapters, report it in the file metadata + if not entry.try_get(ytdl_sub_split_by_chapters_parent_uid, str): + return FileMetadata.from_dict( + value_dict=metadata_dict, title="Embedded Chapters", sort_dict=False + ) return None diff --git a/src/ytdl_sub/plugins/split_by_chapters.py b/src/ytdl_sub/plugins/split_by_chapters.py index 386b067f..892a199e 100644 --- a/src/ytdl_sub/plugins/split_by_chapters.py +++ b/src/ytdl_sub/plugins/split_by_chapters.py @@ -12,8 +12,7 @@ from ytdl_sub.config.plugin.plugin_operation import PluginOperation from ytdl_sub.config.validators.options import OptionsDictValidator from ytdl_sub.entries.entry import Entry from ytdl_sub.entries.script.variable_definitions import VARIABLES as v -from ytdl_sub.entries.variables.kwargs import CHAPTERS -from ytdl_sub.utils.chapters import Chapters +from ytdl_sub.utils.chapters import Chapters, ytdl_sub_split_by_chapters_parent_uid from ytdl_sub.utils.chapters import Timestamp from ytdl_sub.utils.exceptions import ValidationException from ytdl_sub.utils.ffmpeg import FFMPEG @@ -105,7 +104,7 @@ class SplitByChaptersOptions(OptionsDictValidator): return { PluginOperation.MODIFY_ENTRY: { v.uid.variable_name, - v.ytdl_sub_split_entry_parent_uid.variable_name, + ytdl_sub_split_by_chapters_parent_uid.variable_name, } } @@ -121,7 +120,7 @@ class SplitByChaptersPlugin(SplitPlugin[SplitByChaptersOptions]): "chapter_index_padded": "01", "chapter_count": 1, v.uid.variable_name: entry.uid, - v.ytdl_sub_split_entry_parent_uid.variable_name: entry.uid, + ytdl_sub_split_by_chapters_parent_uid.variable_name: entry.uid, } ) return entry @@ -141,11 +140,6 @@ class SplitByChaptersPlugin(SplitPlugin[SplitByChaptersOptions]): } ) - # pylint: disable=protected-access - if new_entry.kwargs_contains(CHAPTERS): - del new_entry._kwargs[CHAPTERS] - # pylint: enable=protected-access - timestamp_begin = chapters.timestamps[idx].readable_str timestamp_end = Timestamp(new_entry.kwargs("duration")).readable_str if idx + 1 < len(chapters.timestamps): @@ -195,7 +189,7 @@ class SplitByChaptersPlugin(SplitPlugin[SplitByChaptersOptions]): new_entry.add( { v.uid.variable_name: new_uid, - v.ytdl_sub_split_entry_parent_uid.variable_name: entry.uid, + ytdl_sub_split_by_chapters_parent_uid.variable_name: entry.uid, } ) new_entry.add_kwargs({v.uid.metadata_key: new_uid}) diff --git a/src/ytdl_sub/utils/chapters.py b/src/ytdl_sub/utils/chapters.py index 0875e3bc..f961f5c3 100644 --- a/src/ytdl_sub/utils/chapters.py +++ b/src/ytdl_sub/utils/chapters.py @@ -1,14 +1,18 @@ import re from typing import Dict from typing import List -from typing import Optional from typing import Tuple from ytdl_sub.entries.entry import Entry -from ytdl_sub.entries.variables.kwargs import CHAPTERS -from ytdl_sub.entries.variables.kwargs import YTDL_SUB_CUSTOM_CHAPTERS +from ytdl_sub.entries.script.variable_definitions import VARIABLES, Variable +from ytdl_sub.entries.script.variable_definitions import VariableDefinitions from ytdl_sub.utils.file_handler import FileMetadata +v: VariableDefinitions = VARIABLES + +ytdl_sub_chapters_from_comments = Variable("ytdl_sub_chapters_from_comments") +ytdl_sub_split_by_chapters_parent_uid = Variable("ytdl_sub_split_by_chapters_parent_uid") + class Timestamp: @@ -157,6 +161,17 @@ class Chapters: """ return self.timestamps[0].timestamp_sec == 0 + def to_yt_dlp_chapter_metadata(self) -> List[Dict[str, str | float]]: + """ + Returns + ------- + Metadata dict + """ + return [ + {"start_time": ts.timestamp_sec, "title": title} + for ts, title in zip(self.timestamps, self.titles) + ] + def to_file_metadata_dict(self) -> Dict: """ Returns @@ -165,7 +180,7 @@ class Chapters: """ return {ts.readable_str: title for ts, title in zip(self.timestamps, self.titles)} - def to_file_metadata(self, title: Optional[str] = None) -> FileMetadata: + def to_file_metadata(self, title: str) -> FileMetadata: """ Parameters ---------- @@ -219,6 +234,17 @@ class Chapters: # Otherwise return empty chapters return Chapters(timestamps=[], titles=[]) + @classmethod + def from_yt_dlp_chapters(cls, chapters: List[Dict[str, str | float]]): + timestamps: List[Timestamp] = [] + titles: List[str] = [] + + for chapter in chapters: + timestamps.append(Timestamp.from_seconds(int(float(chapter["start_time"])))) + titles.append(chapter["title"]) + + return Chapters(timestamps=timestamps, titles=titles) + @classmethod def from_entry_chapters(cls, entry: Entry) -> "Chapters": """ @@ -231,19 +257,12 @@ class Chapters: ------- Chapters object """ - timestamps: List[Timestamp] = [] - titles: List[str] = [] + if chapters := ( + entry.get(ytdl_sub_chapters_from_comments, list) or entry.get(v.chapters, list) + ): + return cls.from_yt_dlp_chapters(chapters) - if entry.kwargs_contains(CHAPTERS): - for chapter in entry.kwargs_get(CHAPTERS, []): - timestamps.append(Timestamp.from_seconds(int(float(chapter["start_time"])))) - titles.append(chapter["title"]) - elif entry.kwargs_contains(YTDL_SUB_CUSTOM_CHAPTERS): - for start_time, title in entry.kwargs_get(YTDL_SUB_CUSTOM_CHAPTERS, {}).items(): - timestamps.append(Timestamp.from_str(start_time)) - titles.append(title) - - return Chapters(timestamps=timestamps, titles=titles) + return Chapters(timestamps=[], titles=[]) @classmethod def from_empty(cls) -> "Chapters": diff --git a/src/ytdl_sub/ytdl_additions/enhanced_download_archive.py b/src/ytdl_sub/ytdl_additions/enhanced_download_archive.py index ce3b60e3..86d5c0f5 100644 --- a/src/ytdl_sub/ytdl_additions/enhanced_download_archive.py +++ b/src/ytdl_sub/ytdl_additions/enhanced_download_archive.py @@ -15,6 +15,7 @@ from yt_dlp.utils import make_archive_id from ytdl_sub.entries.entry import Entry from ytdl_sub.entries.script.variable_definitions import VARIABLES as v +from ytdl_sub.utils.chapters import ytdl_sub_split_by_chapters_parent_uid from ytdl_sub.utils.file_handler import FileHandler from ytdl_sub.utils.file_handler import FileHandlerTransactionLog from ytdl_sub.utils.file_handler import FileMetadata @@ -220,7 +221,7 @@ class DownloadMappings: self """ uid = entry.uid - if parent_uid := entry.try_get(v.ytdl_sub_split_entry_parent_uid, str): + if parent_uid := entry.try_get(ytdl_sub_split_by_chapters_parent_uid, str): uid = parent_uid if uid not in self.entry_ids: