From 5705a28842eaa91f064a550cb661c9e67eb82732 Mon Sep 17 00:00:00 2001 From: Jesse Bannon Date: Sat, 19 Nov 2022 16:56:45 -0800 Subject: [PATCH] [FEATURE] Add `ytdl-sub view URL` to view source variables (#340) * [FEATURE] Add `ytdl-sub view URL` to view source variables * update file path --- .gitignore | 2 + src/ytdl_sub/cli/main.py | 46 +++++++++++--- src/ytdl_sub/cli/main_args_parser.py | 28 ++++++++- src/ytdl_sub/config/preset_class_mappings.py | 2 + .../entries/variables/entry_variables.py | 5 +- src/ytdl_sub/plugins/internal/__init__.py | 0 src/ytdl_sub/plugins/internal/view.py | 62 +++++++++++++++++++ src/ytdl_sub/plugins/plugin.py | 4 +- .../prebuilt_presets/internal/__init__.py | 0 .../prebuilt_presets/internal/view.yaml | 20 ++++++ .../subscriptions/subscription_download.py | 2 +- tests/e2e/plugins/internal/__init__.py | 0 tests/e2e/plugins/internal/test_view.py | 33 ++++++++++ tests/e2e/plugins/test_chapters.py | 4 +- 14 files changed, 193 insertions(+), 15 deletions(-) create mode 100644 src/ytdl_sub/plugins/internal/__init__.py create mode 100644 src/ytdl_sub/plugins/internal/view.py create mode 100644 src/ytdl_sub/prebuilt_presets/internal/__init__.py create mode 100644 src/ytdl_sub/prebuilt_presets/internal/view.yaml create mode 100644 tests/e2e/plugins/internal/__init__.py create mode 100644 tests/e2e/plugins/internal/test_view.py diff --git a/.gitignore b/.gitignore index dce67c93..41f140cf 100644 --- a/.gitignore +++ b/.gitignore @@ -142,3 +142,5 @@ dmypy.json docker/*.whl docker/root/*.whl docker/root/defaults/examples + +.local/ diff --git a/src/ytdl_sub/cli/main.py b/src/ytdl_sub/cli/main.py index c7c85496..1aae4069 100644 --- a/src/ytdl_sub/cli/main.py +++ b/src/ytdl_sub/cli/main.py @@ -19,6 +19,10 @@ from ytdl_sub.utils.logger import Logger logger = Logger.get() +# View is a command to run a simple dry-run on a URL using the `_view` preset. +# Use ytdl-sub dl arguments to use the preset +_VIEW_EXTRA_ARGS_FORMATTER = "--preset _view --overrides.url {}" + def _download_subscriptions_from_yaml_files( config: ConfigFile, args: argparse.Namespace @@ -60,7 +64,7 @@ def _download_subscriptions_from_yaml_files( def _download_subscription_from_cli( - config: ConfigFile, args: argparse.Namespace, extra_args: List[str] + config: ConfigFile, dry_run: bool, extra_args: List[str] ) -> Tuple[Subscription, FileHandlerTransactionLog]: """ Downloads a one-off subscription using the CLI @@ -69,8 +73,8 @@ def _download_subscription_from_cli( ---------- config Configuration file - args - Arguments from argparse + dry_run + Whether this is a dry-run extra_args Extra arguments from argparse that contain dynamic subscription options @@ -88,8 +92,30 @@ def _download_subscription_from_cli( config=config, preset_name=subscription_name, preset_dict=subscription_args_dict ) - logger.info("Beginning CLI %s", ("dry run" if args.dry_run else "download")) - return subscription, subscription.download(dry_run=args.dry_run) + logger.info("Beginning CLI %s", ("dry run" if dry_run else "download")) + return subscription, subscription.download(dry_run=dry_run) + + +def _view_url_from_cli( + config: ConfigFile, url: str, split_chapters: bool +) -> Tuple[Subscription, FileHandlerTransactionLog]: + """ + `ytdl-sub view` dry-runs a URL to print its source variables. Use the pre-built `_view` preset, + inject the URL argument, and dry-run. + """ + preset = "_view_split_chapters" if split_chapters else "_view" + subscription = Subscription.from_dict( + config=config, + preset_name="ytdl-sub-view", + preset_dict={"preset": preset, "overrides": {"url": url}}, + ) + + logger.info( + "Viewing source variables for URL '%s'%s", + url, + " with split chapters" if split_chapters else "", + ) + return subscription, subscription.download(dry_run=True) @contextmanager @@ -146,9 +172,15 @@ def main() -> List[Tuple[Subscription, FileHandlerTransactionLog]]: transaction_logs = _download_subscriptions_from_yaml_files(config=config, args=args) # One-off download - if args.subparser == "dl": + elif args.subparser == "dl": transaction_logs.append( - _download_subscription_from_cli(config=config, args=args, extra_args=extra_args) + _download_subscription_from_cli( + config=config, dry_run=args.dry_run, extra_args=extra_args + ) + ) + elif args.subparser == "view": + transaction_logs.append( + _view_url_from_cli(config=config, url=args.url, split_chapters=args.split_chapters) ) for subscription, transaction_log in transaction_logs: diff --git a/src/ytdl_sub/cli/main_args_parser.py b/src/ytdl_sub/cli/main_args_parser.py index 6c712166..c77d354f 100644 --- a/src/ytdl_sub/cli/main_args_parser.py +++ b/src/ytdl_sub/cli/main_args_parser.py @@ -47,9 +47,10 @@ parser.add_argument( choices=LoggerLevels.names(), dest="ytdl_sub_log_level", ) + +subparsers = parser.add_subparsers(dest="subparser") ################################################################################################### # SUBSCRIPTION PARSER -subparsers = parser.add_subparsers(dest="subparser") subscription_parser = subparsers.add_parser("sub") subscription_parser.add_argument( "subscription_paths", @@ -61,3 +62,28 @@ subscription_parser.add_argument( ################################################################################################### # DOWNLOAD PARSER download_parser = subparsers.add_parser("dl") +################################################################################################### +# VIEW PARSER + + +class ViewArgs(Enum): + SPLIT_CHAPTERS = "--split-chapters" + + @classmethod + def all(cls) -> List[str]: + """ + Returns + ------- + List of all args used in main CLI + """ + return list(map(lambda arg: arg.value, cls)) + + +view_parser = subparsers.add_parser("view") +view_parser.add_argument( + "-sc", + ViewArgs.SPLIT_CHAPTERS.value, + action="store_true", + help="View source variables after splitting by chapters", +) +view_parser.add_argument("url", help="URL to view source variables for") diff --git a/src/ytdl_sub/config/preset_class_mappings.py b/src/ytdl_sub/config/preset_class_mappings.py index 2c8e003b..0912776d 100644 --- a/src/ytdl_sub/config/preset_class_mappings.py +++ b/src/ytdl_sub/config/preset_class_mappings.py @@ -14,6 +14,7 @@ from ytdl_sub.plugins.audio_extract import AudioExtractPlugin from ytdl_sub.plugins.chapters import ChaptersPlugin from ytdl_sub.plugins.date_range import DateRangePlugin from ytdl_sub.plugins.file_convert import FileConvertPlugin +from ytdl_sub.plugins.internal.view import ViewPlugin from ytdl_sub.plugins.music_tags import MusicTagsPlugin from ytdl_sub.plugins.nfo_tags import NfoTagsPlugin from ytdl_sub.plugins.output_directory_nfo_tags import OutputDirectoryNfoTagsPlugin @@ -116,6 +117,7 @@ class PluginMapping: """ _MAPPING: Dict[str, Type[Plugin]] = { + "_view": ViewPlugin, "audio_extract": AudioExtractPlugin, "date_range": DateRangePlugin, "file_convert": FileConvertPlugin, diff --git a/src/ytdl_sub/entries/variables/entry_variables.py b/src/ytdl_sub/entries/variables/entry_variables.py index 2a6e4957..61bcf0ae 100644 --- a/src/ytdl_sub/entries/variables/entry_variables.py +++ b/src/ytdl_sub/entries/variables/entry_variables.py @@ -1,3 +1,4 @@ +from datetime import datetime from typing import Union from yt_dlp.utils import sanitize_filename @@ -447,9 +448,9 @@ class EntryVariables(BaseEntryVariables): Returns ------- str - The entry's uploaded date, in YYYYMMDD format. + The entry's uploaded date, in YYYYMMDD format. If not present, return today's date. """ - return self.kwargs(UPLOAD_DATE) + return self.kwargs_get(UPLOAD_DATE, datetime.now().strftime("%Y%m%d")) @property def upload_year(self: Self) -> int: diff --git a/src/ytdl_sub/plugins/internal/__init__.py b/src/ytdl_sub/plugins/internal/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/ytdl_sub/plugins/internal/view.py b/src/ytdl_sub/plugins/internal/view.py new file mode 100644 index 00000000..66bef3e5 --- /dev/null +++ b/src/ytdl_sub/plugins/internal/view.py @@ -0,0 +1,62 @@ +import copy +from typing import Optional + +from ytdl_sub.config.preset_options import Overrides +from ytdl_sub.entries.entry import Entry +from ytdl_sub.plugins.output_directory_nfo_tags import OutputDirectoryNfoTagsOptions +from ytdl_sub.plugins.plugin import Plugin +from ytdl_sub.plugins.plugin import PluginOptions +from ytdl_sub.plugins.plugin import PluginPriority +from ytdl_sub.utils.file_handler import FileMetadata +from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive + + +class ViewOptions(PluginOptions): + """ + INTERNAL PLUGIN. Do not expose in documentation + """ + + _optional_keys = {"_placeholder"} + + +class ViewPlugin(Plugin[ViewOptions]): + plugin_options_type = ViewOptions + priority: PluginPriority = PluginPriority(modify_entry=PluginPriority.MODIFY_ENTRY_AFTER_SPLIT) + + _MAX_LINE_WIDTH: int = 80 + + def __init__( + self, + plugin_options: OutputDirectoryNfoTagsOptions, + overrides: Overrides, + enhanced_download_archive: EnhancedDownloadArchive, + ): + super().__init__(plugin_options, overrides, enhanced_download_archive) + self._first_entry: Optional[Entry] = None + + @classmethod + def _truncate_value(cls, value: str) -> str: + """No new lines and cut off after 80 characters""" + value = " ".join(str(value).split("\n")) + if len(value) > cls._MAX_LINE_WIDTH: + value = value[:77] + "..." + + return value + + def modify_entry(self, entry: Entry) -> Optional[Entry]: + """Only process the first entry received""" + if self._first_entry is None: + self._first_entry = entry + return entry + + return None + + def post_process_entry(self, entry: Entry) -> Optional[FileMetadata]: + """ + Adds all source variables to the entry + """ + source_var_dict = copy.deepcopy(entry.to_dict()) + for key in source_var_dict.keys(): + source_var_dict[key] = self._truncate_value(source_var_dict[key]) + + return FileMetadata.from_dict(title="Source Variables", value_dict=source_var_dict) diff --git a/src/ytdl_sub/plugins/plugin.py b/src/ytdl_sub/plugins/plugin.py index b1ecf705..2b046dfd 100644 --- a/src/ytdl_sub/plugins/plugin.py +++ b/src/ytdl_sub/plugins/plugin.py @@ -56,7 +56,7 @@ class Plugin(DownloadArchiver, Generic[PluginOptionsT], ABC): plugin_options_type: Type[PluginOptionsT] = NotImplemented priority: PluginPriority = PluginPriority() - # If the plugin creates multile entries from a single entry + # If the plugin creates multiple entries from a single entry is_split_plugin: bool = False def __init__( @@ -95,7 +95,7 @@ class Plugin(DownloadArchiver, Generic[PluginOptionsT], ABC): """ return [] - def modify_entry(self, entry: Entry) -> Optional[Entry | Tuple[Entry, FileMetadata]]: + def modify_entry(self, entry: Entry) -> Optional[Entry]: """ For each entry downloaded, modify the entry in some way before sending it to post-processing. diff --git a/src/ytdl_sub/prebuilt_presets/internal/__init__.py b/src/ytdl_sub/prebuilt_presets/internal/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/ytdl_sub/prebuilt_presets/internal/view.yaml b/src/ytdl_sub/prebuilt_presets/internal/view.yaml new file mode 100644 index 00000000..3a745ca3 --- /dev/null +++ b/src/ytdl_sub/prebuilt_presets/internal/view.yaml @@ -0,0 +1,20 @@ +presets: + + _view: + download: + download_strategy: "url" + url: "{url}" + output_options: + output_directory: "/tmp/ytdl-sub-view" + file_name: "{uid}.{ext}" + thumbnail_name: "{uid}.{thumbnail_ext}" + ytdl_options: + max_downloads: 2 + _view: + _placeholder: "" + + _view_split_chapters: + preset: + - "_view" + split_by_chapters: + when_no_chapters: "pass" diff --git a/src/ytdl_sub/subscriptions/subscription_download.py b/src/ytdl_sub/subscriptions/subscription_download.py index d321db38..380fd155 100644 --- a/src/ytdl_sub/subscriptions/subscription_download.py +++ b/src/ytdl_sub/subscriptions/subscription_download.py @@ -217,7 +217,7 @@ class SubscriptionDownload(BaseSubscription, ABC): # If split_entry is None from modify_entry, do not post process if split_entry: - self._process_entry( + self._post_process_entry( plugins=plugins, dry_run=dry_run, entry=split_entry, diff --git a/tests/e2e/plugins/internal/__init__.py b/tests/e2e/plugins/internal/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/e2e/plugins/internal/test_view.py b/tests/e2e/plugins/internal/test_view.py new file mode 100644 index 00000000..08d92154 --- /dev/null +++ b/tests/e2e/plugins/internal/test_view.py @@ -0,0 +1,33 @@ +import pytest +from e2e.conftest import mock_run_from_cli + + +class TestView: + @pytest.mark.parametrize("split_chapters", [True, False]) + def test_view_from_cli( + self, + music_video_config_path, + output_directory, + split_chapters, + ): + + args = f"--config {music_video_config_path} view " + args += "--split-chapters " if split_chapters else "" + args += f"https://www.youtube.com/playlist?list=PLBsm_SagFMmdWnCnrNtLjA9kzfrRkto4i" + subscription_transaction_log = mock_run_from_cli(args=args) + + assert len(subscription_transaction_log) == 1 + transaction_log = subscription_transaction_log[0][1] + + # Ensure the video and thumbnail are recognized + assert len(transaction_log.files_created) == 2 + + file_name = f"avUT-zd9v68{'___0' if split_chapters else ''}.webm" + video_file = transaction_log.files_created.get(file_name) + assert video_file is not None + + assert "Source Variables:" in video_file.metadata + if split_chapters: + assert " chapter_index: 1" in video_file.metadata + else: + assert " chapter_index: 1" not in video_file.metadata diff --git a/tests/e2e/plugins/test_chapters.py b/tests/e2e/plugins/test_chapters.py index 144ea060..032f13d5 100644 --- a/tests/e2e/plugins/test_chapters.py +++ b/tests/e2e/plugins/test_chapters.py @@ -68,7 +68,7 @@ class TestChapters: dry_run=dry_run, expected_download_summary_file_name="plugins/test_chapters_sb_and_embedded_subs.json", ignore_md5_hashes_for=[ - "JMC - This GPU SLIDES into this Case! - Silverstone SUGO 16 ITX Case.mp4" + "JMC/JMC - This GPU SLIDES into this Case! - Silverstone SUGO 16 ITX Case.mp4" ], ) @@ -107,6 +107,6 @@ class TestChapters: dry_run=dry_run, expected_download_summary_file_name="plugins/test_chapters_from_ts_with_subs.json", ignore_md5_hashes_for=[ - "JMC - This GPU SLIDES into this Case! - Silverstone SUGO 16 ITX Case.mp4" + "JMC/JMC - This GPU SLIDES into this Case! - Silverstone SUGO 16 ITX Case.mp4" ], )