From 5c031acd9ef4ad863fc739fdee2ffae7e3df456a Mon Sep 17 00:00:00 2001 From: Jesse Bannon Date: Tue, 12 Dec 2023 15:01:35 -0800 Subject: [PATCH] audio extract test working --- src/ytdl_sub/entries/entry.py | 20 ++++++++------ src/ytdl_sub/entries/variables/kwargs.py | 1 - src/ytdl_sub/plugins/audio_extract.py | 13 ++++++++- src/ytdl_sub/plugins/chapters.py | 1 - src/ytdl_sub/plugins/file_convert.py | 35 ++++++++++++++++++------ src/ytdl_sub/plugins/music_tags.py | 5 ++-- src/ytdl_sub/script/script_output.py | 9 ++++-- tests/unit/conftest.py | 3 +- 8 files changed, 62 insertions(+), 25 deletions(-) diff --git a/src/ytdl_sub/entries/entry.py b/src/ytdl_sub/entries/entry.py index 24c92a9b..5628e96a 100644 --- a/src/ytdl_sub/entries/entry.py +++ b/src/ytdl_sub/entries/entry.py @@ -4,14 +4,14 @@ import os from pathlib import Path from typing import Dict from typing import Optional -from typing import Set from typing import Type from typing import TypeVar from typing import final from ytdl_sub.entries.base_entry import BaseEntry -from ytdl_sub.entries.script.variable_definitions import VARIABLES +from ytdl_sub.entries.script.variable_definitions import VARIABLES as v from ytdl_sub.entries.script.variable_definitions import Variable +from ytdl_sub.script.utils.exceptions import ScriptVariableNotResolved from ytdl_sub.utils.script import ScriptUtils from ytdl_sub.utils.scriptable import Scriptable from ytdl_sub.validators.audo_codec_validator import AUDIO_CODEC_EXTS @@ -33,10 +33,8 @@ class Entry(BaseEntry, Scriptable): def _add_entry_kwargs_to_script(self) -> None: # Add entry metadata, but avoid the `.add()` helper since it also adds sanitized - self.unresolvable.remove(VARIABLES.entry_metadata.variable_name) - self.script.add( - {VARIABLES.entry_metadata.variable_name: ScriptUtils.to_script(self._kwargs)} - ) + self.unresolvable.remove(v.entry_metadata.variable_name) + self.script.add({v.entry_metadata.variable_name: ScriptUtils.to_script(self._kwargs)}) self.update_script() def initialize_script(self, other: Optional[Scriptable] = None) -> "Entry": @@ -53,6 +51,12 @@ class Entry(BaseEntry, Scriptable): out = self.script.resolve(unresolvable=self.unresolvable).get_native(variable.variable_name) return expected_type(out) + def try_get(self, variable: Variable, expected_type: Type[TType]) -> Optional[TType]: + try: + return self.get(variable=variable, expected_type=expected_type) + except ScriptVariableNotResolved: + return None + def get_str(self, variable: Variable) -> str: return self.get(variable, str) @@ -66,7 +70,7 @@ class Entry(BaseEntry, Scriptable): This is not reflected in the entry. See if the mkv file exists and return "mkv" if so, otherwise, return the original extension. """ - ext = self.get_str(VARIABLES.ext) + ext = self.try_get(v.ext, str) or self.kwargs(key=v.ext.metadata_key) for possible_ext in [ext, "mkv"]: file_path = str(Path(self.working_directory()) / f"{self.uid}.{possible_ext}") if os.path.isfile(file_path): @@ -92,7 +96,7 @@ class Entry(BaseEntry, Scriptable): ------- The download thumbnail's file name """ - return f"{self.get_str(VARIABLES.uid)}.{self.get_str(VARIABLES.thumbnail_ext)}" + return f"{self.get_str(v.uid)}.{self.get_str(v.thumbnail_ext)}" def get_download_thumbnail_path(self) -> str: """Returns the entry's thumbnail's file path to where it was downloaded""" diff --git a/src/ytdl_sub/entries/variables/kwargs.py b/src/ytdl_sub/entries/variables/kwargs.py index a1f6ecae..53dff9e1 100644 --- a/src/ytdl_sub/entries/variables/kwargs.py +++ b/src/ytdl_sub/entries/variables/kwargs.py @@ -28,7 +28,6 @@ EPOCH = _("epoch") CHANNEL = _("channel") CHANNEL_ID = _("channel_id") CREATOR = _("creator") -EXT = _("ext") TITLE = _("title") DESCRIPTION = _("description") WEBPAGE_URL = _("webpage_url") diff --git a/src/ytdl_sub/plugins/audio_extract.py b/src/ytdl_sub/plugins/audio_extract.py index 750429de..c0b975e8 100644 --- a/src/ytdl_sub/plugins/audio_extract.py +++ b/src/ytdl_sub/plugins/audio_extract.py @@ -2,11 +2,14 @@ import os.path from typing import Any from typing import Dict from typing import Optional +from typing import Set from ytdl_sub.config.plugin import Plugin from ytdl_sub.config.preset_options import OptionsDictValidator +from ytdl_sub.config.preset_options import PluginOperation from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder from ytdl_sub.entries.entry import Entry +from ytdl_sub.entries.script.variable_definitions import VARIABLES as v from ytdl_sub.utils.exceptions import FileNotDownloadedException from ytdl_sub.utils.file_handler import FileMetadata from ytdl_sub.validators.audo_codec_validator import AUDIO_CODEC_EXTS @@ -65,6 +68,14 @@ class AudioExtractOptions(OptionsDictValidator): return self._quality.value return None + def added_source_variables( + self, unresolved_variables: Set[str] + ) -> Dict[PluginOperation, Set[str]]: + """ + Possibly changes ``ext``, so do not resolve until this has run + """ + return {PluginOperation.MODIFY_ENTRY: {v.ext.variable_name}} + class AudioExtractPlugin(Plugin[AudioExtractOptions]): plugin_options_type = AudioExtractOptions @@ -125,7 +136,7 @@ class AudioExtractPlugin(Plugin[AudioExtractOptions]): new_ext = AUDIO_CODEC_TYPES_EXTENSION_MAPPING[self.plugin_options.codec] extracted_audio_file = entry.get_download_file_path().removesuffix(entry.ext) + new_ext - entry.add_kwargs({"ext": new_ext}) + entry.add({v.ext.variable_name: new_ext}) if not self.is_dry_run: if not os.path.isfile(extracted_audio_file): diff --git a/src/ytdl_sub/plugins/chapters.py b/src/ytdl_sub/plugins/chapters.py index a9d5dcec..18330764 100644 --- a/src/ytdl_sub/plugins/chapters.py +++ b/src/ytdl_sub/plugins/chapters.py @@ -9,7 +9,6 @@ from ytdl_sub.config.plugin import Plugin from ytdl_sub.config.preset_options import OptionsDictValidator from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder from ytdl_sub.entries.entry import Entry -from ytdl_sub.entries.script.variable_definitions import VARIABLES from ytdl_sub.entries.script.variable_definitions import VARIABLES as v from ytdl_sub.entries.variables.kwargs import YTDL_SUB_CUSTOM_CHAPTERS from ytdl_sub.utils.chapters import Chapters diff --git a/src/ytdl_sub/plugins/file_convert.py b/src/ytdl_sub/plugins/file_convert.py index f99b374d..91862c5c 100644 --- a/src/ytdl_sub/plugins/file_convert.py +++ b/src/ytdl_sub/plugins/file_convert.py @@ -2,12 +2,15 @@ import os from typing import Any from typing import Dict from typing import Optional +from typing import Set +from ytdl_sub.config.overrides import Overrides from ytdl_sub.config.plugin import Plugin from ytdl_sub.config.plugin import PluginPriority from ytdl_sub.config.preset_options import OptionsDictValidator +from ytdl_sub.config.preset_options import PluginOperation from ytdl_sub.entries.entry import Entry -from ytdl_sub.entries.variables.kwargs import EXT +from ytdl_sub.entries.script.variable_definitions import VARIABLES as v from ytdl_sub.utils.exceptions import FileNotDownloadedException from ytdl_sub.utils.exceptions import ValidationException from ytdl_sub.utils.ffmpeg import FFMPEG @@ -16,6 +19,7 @@ from ytdl_sub.utils.file_handler import FileMetadata from ytdl_sub.validators.audo_codec_validator import FileTypeValidator from ytdl_sub.validators.string_formatter_validators import OverridesStringFormatterValidator from ytdl_sub.validators.string_select_validator import StringSelectValidator +from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive class FileConvertWithValidator(StringSelectValidator): @@ -115,6 +119,11 @@ class FileConvertOptions(OptionsDictValidator): """ return self._ffmpeg_post_process_args + def added_source_variables( + self, unresolved_variables: Set[str] + ) -> Dict[PluginOperation, Set[str]]: + return {PluginOperation.MODIFY_ENTRY: {v.ext.variable_name}} + class FileConvertPlugin(Plugin[FileConvertOptions]): plugin_options_type = FileConvertOptions @@ -123,6 +132,20 @@ class FileConvertPlugin(Plugin[FileConvertOptions]): modify_entry=PluginPriority.MODIFY_ENTRY_AFTER_SPLIT + 1 ) + def __init__( + self, + options: FileConvertOptions, + overrides: Overrides, + enhanced_download_archive: EnhancedDownloadArchive, + ): + super().__init__( + options=options, + overrides=overrides, + enhanced_download_archive=enhanced_download_archive, + ) + # Lookup of entry id to what it was converted from for logging + self._converted_from_lookup: Dict[str, str] = {} + def ytdl_options(self) -> Optional[Dict]: """ Returns @@ -198,13 +221,9 @@ class FileConvertPlugin(Plugin[FileConvertOptions]): FileHandler.delete(tmp_output_file) if original_ext != new_ext: - entry.add_kwargs( - { - "__converted_from": original_ext, - } - ) + self._converted_from_lookup[entry.ytdl_uid()] = original_ext - entry.add_kwargs({EXT: new_ext}) + entry.add({v.ext.variable_name: new_ext}) return entry @@ -212,7 +231,7 @@ class FileConvertPlugin(Plugin[FileConvertOptions]): """ Add metadata about conversion if it happened """ - if converted_from := entry.kwargs_get("__converted_from"): + if converted_from := self._converted_from_lookup.get(entry.ytdl_uid()): return FileMetadata(f"Converted from {converted_from}") return None diff --git a/src/ytdl_sub/plugins/music_tags.py b/src/ytdl_sub/plugins/music_tags.py index e7bb92e4..f0de6158 100644 --- a/src/ytdl_sub/plugins/music_tags.py +++ b/src/ytdl_sub/plugins/music_tags.py @@ -9,6 +9,7 @@ import mediafile from ytdl_sub.config.plugin import Plugin from ytdl_sub.config.preset_options import OptionsDictValidator from ytdl_sub.entries.entry import Entry +from ytdl_sub.entries.script.variable_definitions import VARIABLES as v from ytdl_sub.utils.file_handler import FileMetadata from ytdl_sub.utils.logger import Logger from ytdl_sub.validators.audo_codec_validator import AUDIO_CODEC_EXTS @@ -132,9 +133,9 @@ class MusicTagsPlugin(Plugin[MusicTagsOptions]): """ Tags the entry's audio file using values defined in the metadata options """ - if entry.ext not in AUDIO_CODEC_EXTS: + if (ext := entry.get_str(v.ext)) not in AUDIO_CODEC_EXTS: raise self.plugin_options.validation_exception( - f"music_tags plugin received a video with the extension '{entry.ext}'. Only audio " + f"music_tags plugin received a video with the extension '{ext}'. Only audio " f"files are supported for setting music tags. Ensure you are converting the video " f"to audio using the audio_extract plugin." ) diff --git a/src/ytdl_sub/script/script_output.py b/src/ytdl_sub/script/script_output.py index 1ca3f47b..68a240d2 100644 --- a/src/ytdl_sub/script/script_output.py +++ b/src/ytdl_sub/script/script_output.py @@ -3,6 +3,7 @@ from typing import Any from typing import Dict from ytdl_sub.script.types.resolvable import Resolvable +from ytdl_sub.script.utils.exceptions import ScriptVariableNotResolved @dataclass(frozen=True) @@ -16,13 +17,17 @@ class ScriptOutput: return {name: out.native for name, out in self.output.items()} def get(self, name: str) -> Resolvable: + if name not in self.output: + raise ScriptVariableNotResolved( + f"Tried to access resolved variable {name}, but it has not resolved" + ) return self.output[name] def get_native(self, name: str) -> Any: - return self.output[name].native + return self.get(name).native def get_str(self, name: str) -> str: - return str(self.output[name]) + return str(self.get(name)) def get_int(self, name: str) -> int: out = self.get_native(name) diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index 4441076e..5668feac 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -15,7 +15,6 @@ from ytdl_sub.downloaders.ytdlp import YTDLP from ytdl_sub.entries.script.variable_definitions import VARIABLES as v from ytdl_sub.entries.variables.kwargs import DESCRIPTION from ytdl_sub.entries.variables.kwargs import EPOCH -from ytdl_sub.entries.variables.kwargs import EXT from ytdl_sub.entries.variables.kwargs import EXTRACTOR from ytdl_sub.entries.variables.kwargs import EXTRACTOR_KEY from ytdl_sub.entries.variables.kwargs import TITLE @@ -68,7 +67,7 @@ def mock_entry_dict_factory(mock_downloaded_file_path) -> Callable: EXTRACTOR: "mock-entry-dict", EXTRACTOR_KEY: "mock-extractor-key", TITLE: f"Mock Entry {uid}", - EXT: "mp4", + "ext": "mp4", UPLOAD_DATE: upload_date, WEBPAGE_URL: f"https://{uid}.com", v.playlist_metadata.metadata_key: {"thumbnails": []},