From b9888a969e3e6c657841c4f0ef4b9f4daaa066c8 Mon Sep 17 00:00:00 2001 From: Jesse Bannon Date: Wed, 13 Dec 2023 09:25:34 -0800 Subject: [PATCH] only two more regex ones --- src/ytdl_sub/config/plugin/plugin_mapping.py | 8 ++- src/ytdl_sub/downloaders/url/downloader.py | 1 + src/ytdl_sub/entries/entry.py | 2 +- .../entries/script/variable_definitions.py | 4 ++ .../entries/script/variable_scripts.py | 1 + src/ytdl_sub/entries/variables/kwargs.py | 2 - src/ytdl_sub/plugins/regex.py | 4 +- src/ytdl_sub/plugins/split_by_chapters.py | 72 +++++++++++-------- .../subscriptions/subscription_download.py | 4 +- .../enhanced_download_archive.py | 21 +++--- 10 files changed, 67 insertions(+), 52 deletions(-) diff --git a/src/ytdl_sub/config/plugin/plugin_mapping.py b/src/ytdl_sub/config/plugin/plugin_mapping.py index 43f09e42..896e1b6c 100644 --- a/src/ytdl_sub/config/plugin/plugin_mapping.py +++ b/src/ytdl_sub/config/plugin/plugin_mapping.py @@ -8,6 +8,8 @@ from ytdl_sub.config.plugin.plugin import Plugin from ytdl_sub.config.plugin.plugin import SplitPlugin from ytdl_sub.config.plugin.plugin_operation import PluginOperation from ytdl_sub.config.validators.options import OptionsValidator +from ytdl_sub.downloaders.url.downloader import UrlDownloaderCollectionVariablePlugin +from ytdl_sub.downloaders.url.downloader import UrlDownloaderThumbnailPlugin from ytdl_sub.plugins.audio_extract import AudioExtractPlugin from ytdl_sub.plugins.chapters import ChaptersPlugin from ytdl_sub.plugins.date_range import DateRangePlugin @@ -53,11 +55,13 @@ class PluginMapping: # All other plugins are added after the defined ordered ones _ORDER_MODIFY_ENTRY_METADATA: List[Type[Plugin]] = [ ThrottleProtectionPlugin, + UrlDownloaderCollectionVariablePlugin, SubtitlesPlugin, # add all others ] _ORDER_MODIFY_ENTRY: List[Type[Plugin]] = [ + UrlDownloaderThumbnailPlugin, AudioExtractPlugin, FileConvertPlugin, SplitByChaptersPlugin, @@ -142,11 +146,11 @@ class PluginMapping: ordered_plugins = [ plugin for plugin in ordered_plugins if not isinstance(plugin, SplitPlugin) ] - if before_split is False: + if before_split: return [ plugin for plugin in ordered_plugins if not cls._is_modified_after_split(plugin) ] - else: # before_split is True + else: # before_split is False return [plugin for plugin in ordered_plugins if cls._is_modified_after_split(plugin)] @classmethod diff --git a/src/ytdl_sub/downloaders/url/downloader.py b/src/ytdl_sub/downloaders/url/downloader.py index 23cd5400..fa5f7cff 100644 --- a/src/ytdl_sub/downloaders/url/downloader.py +++ b/src/ytdl_sub/downloaders/url/downloader.py @@ -23,6 +23,7 @@ from ytdl_sub.entries.entry import Entry from ytdl_sub.entries.entry_parent import EntryParent from ytdl_sub.entries.script.variable_definitions import VARIABLES as v from ytdl_sub.utils.file_handler import FileHandler +from ytdl_sub.utils.file_handler import FileMetadata from ytdl_sub.utils.logger import Logger from ytdl_sub.utils.thumbnail import ThumbnailTypes from ytdl_sub.utils.thumbnail import download_and_convert_url_thumbnail diff --git a/src/ytdl_sub/entries/entry.py b/src/ytdl_sub/entries/entry.py index 5628e96a..7d0f730a 100644 --- a/src/ytdl_sub/entries/entry.py +++ b/src/ytdl_sub/entries/entry.py @@ -96,7 +96,7 @@ class Entry(BaseEntry, Scriptable): ------- The download thumbnail's file name """ - return f"{self.get_str(v.uid)}.{self.get_str(v.thumbnail_ext)}" + return f"{self.uid}.{self.get_str(v.thumbnail_ext)}" def get_download_thumbnail_path(self) -> str: """Returns the entry's thumbnail's file path to where it was downloaded""" diff --git a/src/ytdl_sub/entries/script/variable_definitions.py b/src/ytdl_sub/entries/script/variable_definitions.py index 9aaa2d7e..d80b5729 100644 --- a/src/ytdl_sub/entries/script/variable_definitions.py +++ b/src/ytdl_sub/entries/script/variable_definitions.py @@ -637,6 +637,10 @@ class _Variables: def ytdl_sub_input_url(self) -> Variable: return Variable("ytdl_sub_input_url") + @property + def ytdl_sub_split_entry_parent_uid(self) -> Variable: + return Variable("ytdl_sub_split_entry_parent_uid") + @property def download_index(self) -> Variable: """ diff --git a/src/ytdl_sub/entries/script/variable_scripts.py b/src/ytdl_sub/entries/script/variable_scripts.py index 615c5cdb..30928067 100644 --- a/src/ytdl_sub/entries/script/variable_scripts.py +++ b/src/ytdl_sub/entries/script/variable_scripts.py @@ -153,6 +153,7 @@ ENTRY_INJECTED_VARIABLES: Dict[Variable, str] = { v.requested_subtitles: "{ {} }", v.sponsorblock_chapters: "", v.ytdl_sub_input_url: f"{{{v.source_webpage_url.variable_name}}}", + v.ytdl_sub_split_entry_parent_uid: "", } ENTRY_DERIVED_VARIABLES: Dict[Variable, str] = { diff --git a/src/ytdl_sub/entries/variables/kwargs.py b/src/ytdl_sub/entries/variables/kwargs.py index 53dff9e1..3456bd60 100644 --- a/src/ytdl_sub/entries/variables/kwargs.py +++ b/src/ytdl_sub/entries/variables/kwargs.py @@ -18,8 +18,6 @@ def _(key: str, backend: bool = False) -> str: CHAPTERS = _("chapters", backend=True) YTDL_SUB_CUSTOM_CHAPTERS = _("ytdl_sub_custom_chapters", backend=True) -SPLIT_BY_CHAPTERS_PARENT_ENTRY = _("split_by_chapters_parent_entry", backend=True) -COMMENTS = _("comments", backend=True) UID = _("id") EXTRACTOR = _("extractor") EXTRACTOR_KEY = _("extractor_key") diff --git a/src/ytdl_sub/plugins/regex.py b/src/ytdl_sub/plugins/regex.py index af450406..d9962e0c 100644 --- a/src/ytdl_sub/plugins/regex.py +++ b/src/ytdl_sub/plugins/regex.py @@ -11,7 +11,7 @@ from ytdl_sub.config.plugin.plugin_operation import PluginOperation from ytdl_sub.config.validators.options import OptionsDictValidator from ytdl_sub.entries.entry import Entry from ytdl_sub.script.parser import parse -from ytdl_sub.script.utils.exceptions import ScriptVariableNotResolved +from ytdl_sub.script.utils.exceptions import RuntimeException from ytdl_sub.utils.exceptions import RegexNoMatchException from ytdl_sub.utils.logger import Logger from ytdl_sub.validators.regex_validator import RegexListValidator @@ -308,7 +308,7 @@ class RegexPlugin(Plugin[RegexOptions]): _ = entry.script.get(variable_name) return True # If it can not from missing variables (from post-metadata stage), return False - except ScriptVariableNotResolved: + except RuntimeException: return False def _modify_entry_metadata(self, entry: Entry, is_metadata_stage: bool) -> Optional[Entry]: diff --git a/src/ytdl_sub/plugins/split_by_chapters.py b/src/ytdl_sub/plugins/split_by_chapters.py index 6d4a5def..76d7edf2 100644 --- a/src/ytdl_sub/plugins/split_by_chapters.py +++ b/src/ytdl_sub/plugins/split_by_chapters.py @@ -7,15 +7,12 @@ from typing import Optional from typing import Set from typing import Tuple -from yt_dlp.utils import sanitize_filename - from ytdl_sub.config.plugin.plugin import SplitPlugin from ytdl_sub.config.plugin.plugin_operation import PluginOperation from ytdl_sub.config.validators.options import OptionsDictValidator from ytdl_sub.entries.entry import Entry +from ytdl_sub.entries.script.variable_definitions import VARIABLES as v from ytdl_sub.entries.variables.kwargs import CHAPTERS -from ytdl_sub.entries.variables.kwargs import SPLIT_BY_CHAPTERS_PARENT_ENTRY -from ytdl_sub.entries.variables.kwargs import UID from ytdl_sub.utils.chapters import Chapters from ytdl_sub.utils.chapters import Timestamp from ytdl_sub.utils.exceptions import ValidationException @@ -104,22 +101,40 @@ class SplitByChaptersOptions(OptionsDictValidator): """ return self._when_no_chapters + def modified_variables(self) -> Dict[PluginOperation, Set[str]]: + return { + PluginOperation.MODIFY_ENTRY: { + v.uid.variable_name, + v.ytdl_sub_split_entry_parent_uid.variable_name, + } + } + class SplitByChaptersPlugin(SplitPlugin[SplitByChaptersOptions]): plugin_options_type = SplitByChaptersOptions + def modify_entry(self, entry: Entry) -> Optional[Entry]: + entry.add( + { + "chapter_title": f"{{ {v.title.variable_name} }}", + "chapter_index": 1, + "chapter_index_padded": "01", + "chapter_count": 1, + v.uid.variable_name: entry.uid, + v.ytdl_sub_split_entry_parent_uid.variable_name: entry.uid, + } + ) + return entry + def _create_split_entry( - self, source_entry: Entry, title: str, idx: int, chapters: Chapters + self, new_entry: Entry, title: str, idx: int, chapters: Chapters ) -> Tuple[Entry, FileMetadata]: """ Runs ffmpeg to create the split video """ - entry = copy.deepcopy(source_entry) - - entry.add_variables( + new_entry.add( { "chapter_title": title, - "chapter_title_sanitized": sanitize_filename(title), "chapter_index": idx + 1, "chapter_index_padded": f"{(idx + 1):02d}", "chapter_count": len(chapters.timestamps), @@ -127,19 +142,12 @@ class SplitByChaptersPlugin(SplitPlugin[SplitByChaptersOptions]): ) # pylint: disable=protected-access - entry.add_kwargs( - { - UID: _split_video_uid(source_uid=entry.uid, idx=idx), - SPLIT_BY_CHAPTERS_PARENT_ENTRY: source_entry._kwargs, - } - ) - - if entry.kwargs_contains(CHAPTERS): - del entry._kwargs[CHAPTERS] + if new_entry.kwargs_contains(CHAPTERS): + del new_entry._kwargs[CHAPTERS] # pylint: enable=protected-access timestamp_begin = chapters.timestamps[idx].readable_str - timestamp_end = Timestamp(entry.kwargs("duration")).readable_str + timestamp_end = Timestamp(new_entry.kwargs("duration")).readable_str if idx + 1 < len(chapters.timestamps): timestamp_end = chapters.timestamps[idx + 1].readable_str @@ -149,7 +157,7 @@ class SplitByChaptersPlugin(SplitPlugin[SplitByChaptersOptions]): "Warning" ] = "Dry-run assumes embedded chapters with no modifications" - metadata_value_dict["Source Title"] = entry.title + metadata_value_dict["Source Title"] = new_entry.title metadata_value_dict["Segment"] = f"{timestamp_begin} - {timestamp_end}" metadata = FileMetadata.from_dict( @@ -158,7 +166,7 @@ class SplitByChaptersPlugin(SplitPlugin[SplitByChaptersOptions]): sort_dict=False, ) - return entry, metadata + return new_entry, metadata def split(self, entry: Entry) -> Optional[List[Tuple[Entry, FileMetadata]]]: """ @@ -170,15 +178,8 @@ class SplitByChaptersPlugin(SplitPlugin[SplitByChaptersOptions]): # If no chapters, do not split anything if not chapters.contains_any_chapters(): if self.plugin_options.when_no_chapters == "pass": - entry.add_variables( - { - "chapter_title": entry.title, - "chapter_index": 1, - "chapter_index_padded": "01", - "chapter_count": 1, - } - ) return [(entry, FileMetadata())] + if self.plugin_options.when_no_chapters == "drop": return [] @@ -187,8 +188,17 @@ class SplitByChaptersPlugin(SplitPlugin[SplitByChaptersOptions]): ) for idx, title in enumerate(chapters.titles): + new_entry = copy.deepcopy(entry) new_uid = _split_video_uid(source_uid=entry.uid, idx=idx) + new_entry.add( + { + v.uid.variable_name: new_uid, + v.ytdl_sub_split_entry_parent_uid.variable_name: entry.uid, + } + ) + new_entry.add_kwargs({v.uid.metadata_key: new_uid}) + if not self.is_dry_run: # Get the input/output file paths input_file = entry.get_download_file_path() @@ -210,13 +220,13 @@ class SplitByChaptersPlugin(SplitPlugin[SplitByChaptersOptions]): FileHandler.copy( src_file_path=entry.get_download_thumbnail_path(), dst_file_path=Path(self.working_directory) - / f"{new_uid}.{entry.thumbnail_ext}", + / f"{new_uid}.{entry.get_str(v.thumbnail_ext)}", ) # Format the split video split_videos_and_metadata.append( self._create_split_entry( - source_entry=entry, + new_entry=new_entry, title=title, idx=idx, chapters=chapters, diff --git a/src/ytdl_sub/subscriptions/subscription_download.py b/src/ytdl_sub/subscriptions/subscription_download.py index 7bbd09ca..47be6c25 100644 --- a/src/ytdl_sub/subscriptions/subscription_download.py +++ b/src/ytdl_sub/subscriptions/subscription_download.py @@ -289,7 +289,9 @@ class SubscriptionDownload(BaseSubscription, ABC): self._cleanup_entry_files(split_entry) - self._cleanup_entry_files(entry) + # Have the split_plugin modify the parent entry before sending it to deletion + # This is needed to resolve any variables that may be needed to delete + self._cleanup_entry_files(split_plugin.modify_entry(entry)) def _process_subscription( self, diff --git a/src/ytdl_sub/ytdl_additions/enhanced_download_archive.py b/src/ytdl_sub/ytdl_additions/enhanced_download_archive.py index 4035abbd..ce3b60e3 100644 --- a/src/ytdl_sub/ytdl_additions/enhanced_download_archive.py +++ b/src/ytdl_sub/ytdl_additions/enhanced_download_archive.py @@ -15,7 +15,6 @@ from yt_dlp.utils import make_archive_id from ytdl_sub.entries.entry import Entry from ytdl_sub.entries.script.variable_definitions import VARIABLES as v -from ytdl_sub.entries.variables.kwargs import SPLIT_BY_CHAPTERS_PARENT_ENTRY from ytdl_sub.utils.file_handler import FileHandler from ytdl_sub.utils.file_handler import FileHandlerTransactionLog from ytdl_sub.utils.file_handler import FileMetadata @@ -220,10 +219,14 @@ class DownloadMappings: ------- self """ - if entry.uid not in self.entry_ids: - self._entry_mappings[entry.uid] = DownloadMapping.from_entry(entry=entry) + uid = entry.uid + if parent_uid := entry.try_get(v.ytdl_sub_split_entry_parent_uid, str): + uid = parent_uid - self._entry_mappings[entry.uid].file_names.add(entry_file_path) + if uid not in self.entry_ids: + self._entry_mappings[uid] = DownloadMapping.from_entry(entry=entry) + + self._entry_mappings[uid].file_names.add(entry_file_path) return self def remove_entry(self, entry_id: str) -> "DownloadMappings": @@ -647,15 +650,7 @@ class EnhancedDownloadArchive: if output_file_name is None: output_file_name = file_name - # If the entry is created from splitting via chapters, store it to the mapping - # using its parent entry - if entry and entry.kwargs_contains(SPLIT_BY_CHAPTERS_PARENT_ENTRY): - parent_entry = Entry( - entry_dict=entry.kwargs(SPLIT_BY_CHAPTERS_PARENT_ENTRY), - working_directory=entry.working_directory(), - ) - self.mapping.add_entry(parent_entry, entry_file_path=output_file_name) - elif entry: + if entry: self.mapping.add_entry(entry=entry, entry_file_path=output_file_name) is_modified = self._file_handler.move_file_to_output_directory(