danvics/ytdl-sub
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

only two more regex ones

Browse source
This commit is contained in:
Jesse Bannon 2023-12-13 09:25:34 -08:00
parent 6401fcc26e
commit b9888a969e
10 changed files with 67 additions and 52 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
src/ytdl_sub/config/plugin/plugin_mapping.py
View file

@ -8,6 +8,8 @@ from ytdl_sub.config.plugin.plugin import Plugin
from ytdl_sub.config.plugin.plugin import SplitPlugin
from ytdl_sub.config.plugin.plugin_operation import PluginOperation
from ytdl_sub.config.validators.options import OptionsValidator
from ytdl_sub.downloaders.url.downloader import UrlDownloaderCollectionVariablePlugin
from ytdl_sub.downloaders.url.downloader import UrlDownloaderThumbnailPlugin
from ytdl_sub.plugins.audio_extract import AudioExtractPlugin
from ytdl_sub.plugins.chapters import ChaptersPlugin
from ytdl_sub.plugins.date_range import DateRangePlugin
@ -53,11 +55,13 @@ class PluginMapping:
# All other plugins are added after the defined ordered ones
_ORDER_MODIFY_ENTRY_METADATA: List[Type[Plugin]] = [
ThrottleProtectionPlugin,
UrlDownloaderCollectionVariablePlugin,
SubtitlesPlugin,
# add all others
]
_ORDER_MODIFY_ENTRY: List[Type[Plugin]] = [
UrlDownloaderThumbnailPlugin,
AudioExtractPlugin,
FileConvertPlugin,
SplitByChaptersPlugin,
@ -142,11 +146,11 @@ class PluginMapping:
ordered_plugins = [
plugin for plugin in ordered_plugins if not isinstance(plugin, SplitPlugin)
]
if before_split is False:
if before_split:
return [
plugin for plugin in ordered_plugins if not cls._is_modified_after_split(plugin)
]
else: # before_split is True
else: # before_split is False
return [plugin for plugin in ordered_plugins if cls._is_modified_after_split(plugin)]
@classmethod

1
src/ytdl_sub/downloaders/url/downloader.py
View file

@ -23,6 +23,7 @@ from ytdl_sub.entries.entry import Entry
from ytdl_sub.entries.entry_parent import EntryParent
from ytdl_sub.entries.script.variable_definitions import VARIABLES as v
from ytdl_sub.utils.file_handler import FileHandler
from ytdl_sub.utils.file_handler import FileMetadata
from ytdl_sub.utils.logger import Logger
from ytdl_sub.utils.thumbnail import ThumbnailTypes
from ytdl_sub.utils.thumbnail import download_and_convert_url_thumbnail

2
src/ytdl_sub/entries/entry.py
View file

@ -96,7 +96,7 @@ class Entry(BaseEntry, Scriptable):
-------
The download thumbnail's file name
"""
return f"{self.get_str(v.uid)}.{self.get_str(v.thumbnail_ext)}"
return f"{self.uid}.{self.get_str(v.thumbnail_ext)}"
def get_download_thumbnail_path(self) -> str:
"""Returns the entry's thumbnail's file path to where it was downloaded"""

4
src/ytdl_sub/entries/script/variable_definitions.py
View file

@ -637,6 +637,10 @@ class _Variables:
def ytdl_sub_input_url(self) -> Variable:
return Variable("ytdl_sub_input_url")
@property
def ytdl_sub_split_entry_parent_uid(self) -> Variable:
return Variable("ytdl_sub_split_entry_parent_uid")
@property
def download_index(self) -> Variable:
"""

1
src/ytdl_sub/entries/script/variable_scripts.py
View file

@ -153,6 +153,7 @@ ENTRY_INJECTED_VARIABLES: Dict[Variable, str] = {
v.requested_subtitles: "{ {} }",
v.sponsorblock_chapters: "",
v.ytdl_sub_input_url: f"{{{v.source_webpage_url.variable_name}}}",
v.ytdl_sub_split_entry_parent_uid: "",
}
ENTRY_DERIVED_VARIABLES: Dict[Variable, str] = {

2
src/ytdl_sub/entries/variables/kwargs.py
View file

@ -18,8 +18,6 @@ def _(key: str, backend: bool = False) -> str:
CHAPTERS = _("chapters", backend=True)
YTDL_SUB_CUSTOM_CHAPTERS = _("ytdl_sub_custom_chapters", backend=True)
SPLIT_BY_CHAPTERS_PARENT_ENTRY = _("split_by_chapters_parent_entry", backend=True)
COMMENTS = _("comments", backend=True)
UID = _("id")
EXTRACTOR = _("extractor")
EXTRACTOR_KEY = _("extractor_key")

4
src/ytdl_sub/plugins/regex.py
View file

@ -11,7 +11,7 @@ from ytdl_sub.config.plugin.plugin_operation import PluginOperation
from ytdl_sub.config.validators.options import OptionsDictValidator
from ytdl_sub.entries.entry import Entry
from ytdl_sub.script.parser import parse
from ytdl_sub.script.utils.exceptions import ScriptVariableNotResolved
from ytdl_sub.script.utils.exceptions import RuntimeException
from ytdl_sub.utils.exceptions import RegexNoMatchException
from ytdl_sub.utils.logger import Logger
from ytdl_sub.validators.regex_validator import RegexListValidator
@ -308,7 +308,7 @@ class RegexPlugin(Plugin[RegexOptions]):
_ = entry.script.get(variable_name)
return True
# If it can not from missing variables (from post-metadata stage), return False
except ScriptVariableNotResolved:
except RuntimeException:
return False
def _modify_entry_metadata(self, entry: Entry, is_metadata_stage: bool) -> Optional[Entry]:

72
src/ytdl_sub/plugins/split_by_chapters.py
View file

@ -7,15 +7,12 @@ from typing import Optional
from typing import Set
from typing import Tuple
from yt_dlp.utils import sanitize_filename
from ytdl_sub.config.plugin.plugin import SplitPlugin
from ytdl_sub.config.plugin.plugin_operation import PluginOperation
from ytdl_sub.config.validators.options import OptionsDictValidator
from ytdl_sub.entries.entry import Entry
from ytdl_sub.entries.script.variable_definitions import VARIABLES as v
from ytdl_sub.entries.variables.kwargs import CHAPTERS
from ytdl_sub.entries.variables.kwargs import SPLIT_BY_CHAPTERS_PARENT_ENTRY
from ytdl_sub.entries.variables.kwargs import UID
from ytdl_sub.utils.chapters import Chapters
from ytdl_sub.utils.chapters import Timestamp
from ytdl_sub.utils.exceptions import ValidationException
@ -104,22 +101,40 @@ class SplitByChaptersOptions(OptionsDictValidator):
"""
return self._when_no_chapters
def modified_variables(self) -> Dict[PluginOperation, Set[str]]:
return {
PluginOperation.MODIFY_ENTRY: {
v.uid.variable_name,
v.ytdl_sub_split_entry_parent_uid.variable_name,
}
}
class SplitByChaptersPlugin(SplitPlugin[SplitByChaptersOptions]):
plugin_options_type = SplitByChaptersOptions
def modify_entry(self, entry: Entry) -> Optional[Entry]:
entry.add(
{
"chapter_title": f"{{ {v.title.variable_name} }}",
"chapter_index": 1,
"chapter_index_padded": "01",
"chapter_count": 1,
v.uid.variable_name: entry.uid,
v.ytdl_sub_split_entry_parent_uid.variable_name: entry.uid,
}
)
return entry
def _create_split_entry(
self, source_entry: Entry, title: str, idx: int, chapters: Chapters
self, new_entry: Entry, title: str, idx: int, chapters: Chapters
) -> Tuple[Entry, FileMetadata]:
"""
Runs ffmpeg to create the split video
"""
entry = copy.deepcopy(source_entry)
entry.add_variables(
new_entry.add(
{
"chapter_title": title,
"chapter_title_sanitized": sanitize_filename(title),
"chapter_index": idx + 1,
"chapter_index_padded": f"{(idx + 1):02d}",
"chapter_count": len(chapters.timestamps),
@ -127,19 +142,12 @@ class SplitByChaptersPlugin(SplitPlugin[SplitByChaptersOptions]):
)
# pylint: disable=protected-access
entry.add_kwargs(
{
UID: _split_video_uid(source_uid=entry.uid, idx=idx),
SPLIT_BY_CHAPTERS_PARENT_ENTRY: source_entry._kwargs,
}
)
if entry.kwargs_contains(CHAPTERS):
del entry._kwargs[CHAPTERS]
if new_entry.kwargs_contains(CHAPTERS):
del new_entry._kwargs[CHAPTERS]
# pylint: enable=protected-access
timestamp_begin = chapters.timestamps[idx].readable_str
timestamp_end = Timestamp(entry.kwargs("duration")).readable_str
timestamp_end = Timestamp(new_entry.kwargs("duration")).readable_str
if idx + 1 < len(chapters.timestamps):
timestamp_end = chapters.timestamps[idx + 1].readable_str
@ -149,7 +157,7 @@ class SplitByChaptersPlugin(SplitPlugin[SplitByChaptersOptions]):
"Warning"
] = "Dry-run assumes embedded chapters with no modifications"
metadata_value_dict["Source Title"] = entry.title
metadata_value_dict["Source Title"] = new_entry.title
metadata_value_dict["Segment"] = f"{timestamp_begin} - {timestamp_end}"
metadata = FileMetadata.from_dict(
@ -158,7 +166,7 @@ class SplitByChaptersPlugin(SplitPlugin[SplitByChaptersOptions]):
sort_dict=False,
)
return entry, metadata
return new_entry, metadata
def split(self, entry: Entry) -> Optional[List[Tuple[Entry, FileMetadata]]]:
"""
@ -170,15 +178,8 @@ class SplitByChaptersPlugin(SplitPlugin[SplitByChaptersOptions]):
# If no chapters, do not split anything
if not chapters.contains_any_chapters():
if self.plugin_options.when_no_chapters == "pass":
entry.add_variables(
{
"chapter_title": entry.title,
"chapter_index": 1,
"chapter_index_padded": "01",
"chapter_count": 1,
}
)
return [(entry, FileMetadata())]
if self.plugin_options.when_no_chapters == "drop":
return []
@ -187,8 +188,17 @@ class SplitByChaptersPlugin(SplitPlugin[SplitByChaptersOptions]):
)
for idx, title in enumerate(chapters.titles):
new_entry = copy.deepcopy(entry)
new_uid = _split_video_uid(source_uid=entry.uid, idx=idx)
new_entry.add(
{
v.uid.variable_name: new_uid,
v.ytdl_sub_split_entry_parent_uid.variable_name: entry.uid,
}
)
new_entry.add_kwargs({v.uid.metadata_key: new_uid})
if not self.is_dry_run:
# Get the input/output file paths
input_file = entry.get_download_file_path()
@ -210,13 +220,13 @@ class SplitByChaptersPlugin(SplitPlugin[SplitByChaptersOptions]):
FileHandler.copy(
src_file_path=entry.get_download_thumbnail_path(),
dst_file_path=Path(self.working_directory)
/ f"{new_uid}.{entry.thumbnail_ext}",
/ f"{new_uid}.{entry.get_str(v.thumbnail_ext)}",
)
# Format the split video
split_videos_and_metadata.append(
self._create_split_entry(
source_entry=entry,
new_entry=new_entry,
title=title,
idx=idx,
chapters=chapters,

4
src/ytdl_sub/subscriptions/subscription_download.py
View file

@ -289,7 +289,9 @@ class SubscriptionDownload(BaseSubscription, ABC):
self._cleanup_entry_files(split_entry)
self._cleanup_entry_files(entry)
# Have the split_plugin modify the parent entry before sending it to deletion
# This is needed to resolve any variables that may be needed to delete
self._cleanup_entry_files(split_plugin.modify_entry(entry))
def _process_subscription(
self,

21
src/ytdl_sub/ytdl_additions/enhanced_download_archive.py
View file

@ -15,7 +15,6 @@ from yt_dlp.utils import make_archive_id
from ytdl_sub.entries.entry import Entry
from ytdl_sub.entries.script.variable_definitions import VARIABLES as v
from ytdl_sub.entries.variables.kwargs import SPLIT_BY_CHAPTERS_PARENT_ENTRY
from ytdl_sub.utils.file_handler import FileHandler
from ytdl_sub.utils.file_handler import FileHandlerTransactionLog
from ytdl_sub.utils.file_handler import FileMetadata
@ -220,10 +219,14 @@ class DownloadMappings:
-------
self
"""
if entry.uid not in self.entry_ids:
self._entry_mappings[entry.uid] = DownloadMapping.from_entry(entry=entry)
uid = entry.uid
if parent_uid := entry.try_get(v.ytdl_sub_split_entry_parent_uid, str):
uid = parent_uid
self._entry_mappings[entry.uid].file_names.add(entry_file_path)
if uid not in self.entry_ids:
self._entry_mappings[uid] = DownloadMapping.from_entry(entry=entry)
self._entry_mappings[uid].file_names.add(entry_file_path)
return self
def remove_entry(self, entry_id: str) -> "DownloadMappings":
@ -647,15 +650,7 @@ class EnhancedDownloadArchive:
if output_file_name is None:
output_file_name = file_name
# If the entry is created from splitting via chapters, store it to the mapping
# using its parent entry
if entry and entry.kwargs_contains(SPLIT_BY_CHAPTERS_PARENT_ENTRY):
parent_entry = Entry(
entry_dict=entry.kwargs(SPLIT_BY_CHAPTERS_PARENT_ENTRY),
working_directory=entry.working_directory(),
)
self.mapping.add_entry(parent_entry, entry_file_path=output_file_name)
elif entry:
if entry:
self.mapping.add_entry(entry=entry, entry_file_path=output_file_name)
is_modified = self._file_handler.move_file_to_output_directory(