From 38946254793c2365246216da5f50fdf3110b2f67 Mon Sep 17 00:00:00 2001 From: Jesse Bannon Date: Wed, 14 Sep 2022 22:03:44 -0700 Subject: [PATCH] [BACKEND] Remove split_video download strategy (#232) --- docs/config.rst | 13 +- src/ytdl_sub/config/preset_class_mappings.py | 2 - .../downloaders/youtube/split_video.py | 194 ------------------ .../youtube/test_split_video.txt | 68 ------ tests/e2e/youtube/test_split_video.py | 67 ------ 5 files changed, 1 insertion(+), 343 deletions(-) delete mode 100644 src/ytdl_sub/downloaders/youtube/split_video.py delete mode 100644 tests/e2e/resources/transaction_log_summaries/youtube/test_split_video.txt delete mode 100644 tests/e2e/youtube/test_split_video.py diff --git a/docs/config.rst b/docs/config.rst index ada53acb..d09d1418 100644 --- a/docs/config.rst +++ b/docs/config.rst @@ -83,17 +83,6 @@ _____ ------------------------------------------------------------------------------- -split_video -___________ -.. autoclass:: ytdl_sub.downloaders.youtube.split_video.YoutubeSplitVideoDownloaderOptions() - :members: - :member-order: bysource - :inherited-members: - -.. autofunction:: ytdl_sub.downloaders.youtube.split_video.YoutubeSplitVideoDownloader.ytdl_option_defaults() - -------------------------------------------------------------------------------- - merge_playlist ______________ .. autoclass:: ytdl_sub.downloaders.youtube.merge_playlist.YoutubeMergePlaylistDownloaderOptions() @@ -315,7 +304,7 @@ the ``{playlist_name}`` variable since the preset did not. Source Variables ---------------- -.. autoclass:: ytdl_sub.entries.variables.entry_variables.SourceVariables +.. autoclass:: ytdl_sub.entries.variables.entry_variables.EntryVariables .. _youtube-variables: diff --git a/src/ytdl_sub/config/preset_class_mappings.py b/src/ytdl_sub/config/preset_class_mappings.py index c42b0bf3..a129bf01 100644 --- a/src/ytdl_sub/config/preset_class_mappings.py +++ b/src/ytdl_sub/config/preset_class_mappings.py @@ -7,7 +7,6 @@ from ytdl_sub.downloaders.soundcloud.albums_and_singles import SoundcloudAlbumsA from ytdl_sub.downloaders.youtube.channel import YoutubeChannelDownloader from ytdl_sub.downloaders.youtube.merge_playlist import YoutubeMergePlaylistDownloader from ytdl_sub.downloaders.youtube.playlist import YoutubePlaylistDownloader -from ytdl_sub.downloaders.youtube.split_video import YoutubeSplitVideoDownloader from ytdl_sub.downloaders.youtube.video import YoutubeVideoDownloader from ytdl_sub.plugins.audio_extract import AudioExtractPlugin from ytdl_sub.plugins.chapters import ChaptersPlugin @@ -32,7 +31,6 @@ class DownloadStrategyMapping: "video": YoutubeVideoDownloader, "playlist": YoutubePlaylistDownloader, "channel": YoutubeChannelDownloader, - "split_video": YoutubeSplitVideoDownloader, "merge_playlist": YoutubeMergePlaylistDownloader, }, "soundcloud": { diff --git a/src/ytdl_sub/downloaders/youtube/split_video.py b/src/ytdl_sub/downloaders/youtube/split_video.py deleted file mode 100644 index 99440fa7..00000000 --- a/src/ytdl_sub/downloaders/youtube/split_video.py +++ /dev/null @@ -1,194 +0,0 @@ -import copy -import re -from pathlib import Path -from typing import Dict -from typing import List -from typing import Tuple - -from ytdl_sub.downloaders.youtube.abc import YoutubeDownloader -from ytdl_sub.downloaders.youtube.video import YoutubeVideoDownloaderOptions -from ytdl_sub.entries.youtube import YoutubePlaylistVideo -from ytdl_sub.entries.youtube import YoutubeVideo -from ytdl_sub.plugins.split_by_chapters import _split_video_ffmpeg_cmd -from ytdl_sub.utils.chapters import Chapters -from ytdl_sub.utils.chapters import Timestamp -from ytdl_sub.utils.ffmpeg import FFMPEG - -# Captures the following formats: -# 0:00 title -# 00:00 title -# 1:00:00 title -# 01:00:00 title -# where capture group 1 and 2 are the timestamp and title, respectively -from ytdl_sub.utils.file_handler import FileHandler -from ytdl_sub.utils.file_handler import FileMetadata -from ytdl_sub.utils.thumbnail import convert_download_thumbnail -from ytdl_sub.validators.validators import StringValidator - -_SPLIT_TIMESTAMP_REGEX = re.compile(r"^((?:\d\d:)?(?:\d:)?(?:\d)?\d:\d\d) (.+)$") - - -def _split_video_uid(source_uid: str, idx: int) -> str: - return f"{source_uid}___{idx}" - - -class YoutubeSplitVideoDownloaderOptions(YoutubeVideoDownloaderOptions): - r""" - DEPRECATED: Will be removed in v0.5.0. Use the ``split_by_chapters`` plugin instead. - - Downloads a single youtube video, then splits in to separate videos using a file containing - timestamps. Each separate video will be formatted as if it was downloaded from a playlist. - This download strategy is intended for CLI usage performing a one-time download of a video, - not a subscription. - - Usage: - - .. code-block:: yaml - - presets: - example_preset: - youtube: - # required - download_strategy: "split_video" - video_url: "youtube.com/watch?v=VMAPTo7RVDo" - split_timestamps: path/to/timestamps.txt - - - CLI usage: - - .. code-block:: bash - - ytdl-sub dl \ - --preset "example_preset" \ - --youtube.video_url "youtube.com/watch?v=VMAPTo7RVDo" \ - --youtube.split_timestamps "path/to/timestamps.txt" - - ``split_timestamps`` file format: - - .. code-block:: markdown - - 0:00 Intro - 0:24 Blackwater Park - 10:23 Bleak - 16:39 Jokes - 1:02:23 Ending - - The above will create 5 videos in total. The last timestamp, in this example, would create a - video starting at ``1:02:23`` and end at Youtube video's ending. - """ - - _required_keys = {"video_url", "split_timestamps"} - - def __init__(self, name, value): - super().__init__(name, value) - self._split_timestamps = self._validate_key("split_timestamps", StringValidator).value - - @property - def split_timestamps(self) -> str: - """ - Required. The path to the file containing the split timestamps. - """ - return self._split_timestamps - - -class YoutubeSplitVideoDownloader( - YoutubeDownloader[YoutubeSplitVideoDownloaderOptions, YoutubePlaylistVideo] -): - downloader_options_type = YoutubeSplitVideoDownloaderOptions - downloader_entry_type = YoutubePlaylistVideo - - supports_download_archive = False - supports_subtitles = False - - @classmethod - def ytdl_option_defaults(cls) -> Dict: - """ - Default `ytdl_options`_ for ``split_video`` - - .. code-block:: yaml - - ytdl_options: - ignoreerrors: True # ignore errors like hidden videos, age restriction, etc - """ - return dict( - super().ytdl_option_defaults(), - **{"break_on_existing": True}, - ) - - def _create_split_video_entry( - self, source_entry_dict: Dict, title: str, idx: int, chapters: Chapters - ) -> Tuple[YoutubePlaylistVideo, FileMetadata]: - """ - Runs ffmpeg to create the split video - """ - entry_dict = copy.deepcopy(source_entry_dict) - entry_dict["title"] = title - entry_dict["playlist_index"] = idx + 1 - entry_dict["playlist_count"] = len(chapters.timestamps) - entry_dict["id"] = _split_video_uid(source_uid=entry_dict["id"], idx=idx) - - # Remove track and artist since its now split - if "track" in entry_dict: - del entry_dict["track"] - if "artist" in entry_dict: - del entry_dict["artist"] - - timestamp_begin = chapters.timestamps[idx].readable_str - timestamp_end = Timestamp(source_entry_dict["duration"]).readable_str - if idx + 1 < len(chapters.timestamps): - timestamp_end = chapters.timestamps[idx + 1].readable_str - - metadata = FileMetadata(metadata=f"{timestamp_begin} - {timestamp_end}") - return ( - YoutubePlaylistVideo(entry_dict=entry_dict, working_directory=self.working_directory), - metadata, - ) - - def download(self) -> List[Tuple[YoutubePlaylistVideo, FileMetadata]]: - """Download a single Youtube video, then split it into multiple videos""" - split_videos_and_metadata: List[Tuple[YoutubePlaylistVideo, FileMetadata]] = [] - - chapters = Chapters.from_timestamps_file( - chapters_file_path=self.download_options.split_timestamps - ) - entry_dict = self.extract_info(url=self.download_options.video_url) - - entry = YoutubeVideo(entry_dict=entry_dict, working_directory=self.working_directory) - # convert the entry thumbnail early so we do not have to guess the thumbnail extension - # when copying it - if not self.is_dry_run: - convert_download_thumbnail(entry=entry) - - for idx, title in enumerate(chapters.titles): - new_uid = _split_video_uid(source_uid=entry.uid, idx=idx) - - if not self.is_dry_run: - # Get the input/output file paths - input_file = entry.get_download_file_path() - output_file = str(Path(self.working_directory) / f"{new_uid}.{entry.ext}") - - # Run ffmpeg to create the split the video - FFMPEG.run( - _split_video_ffmpeg_cmd( - input_file=input_file, - output_file=output_file, - timestamps=chapters.timestamps, - idx=idx, - ) - ) - - # Copy the original vid thumbnail to the working directory with the new uid. This so - # downstream logic thinks this split video has its own thumbnail - FileHandler.copy( - src_file_path=entry.get_download_thumbnail_path(), - dst_file_path=Path(self.working_directory) / f"{new_uid}.{entry.thumbnail_ext}", - ) - - # Format the split video as a YoutubePlaylistVideo - split_videos_and_metadata.append( - self._create_split_video_entry( - source_entry_dict=entry_dict, title=title, idx=idx, chapters=chapters - ) - ) - - return split_videos_and_metadata diff --git a/tests/e2e/resources/transaction_log_summaries/youtube/test_split_video.txt b/tests/e2e/resources/transaction_log_summaries/youtube/test_split_video.txt deleted file mode 100644 index e8113d7d..00000000 --- a/tests/e2e/resources/transaction_log_summaries/youtube/test_split_video.txt +++ /dev/null @@ -1,68 +0,0 @@ -Files created in '{output_directory}' ----------------------------------------- -Project Zombie - 1-6.Intro.mp4 - 0:00 - 0:10 -Project Zombie - 2-6.Part 1.mp4 - 0:10 - 0:20 -Project Zombie - 3-6.Part 2.mp4 - 0:20 - 0:30 -Project Zombie - 4-6.Part 3.mp4 - 0:30 - 0:40 -Project Zombie - 5-6.Part 4.mp4 - 0:40 - 1:01 -Project Zombie - 6-6.Part 5.mp4 - 1:01 - 1:28 -Project Zombie - Intro-thumb.jpg -Project Zombie - Intro.info.json -Project Zombie - Intro.nfo - NFO tags: - musicvideo: - album: Music Videos - artist: Project Zombie - title: Intro - year: 2010 -Project Zombie - Part 1-thumb.jpg -Project Zombie - Part 1.info.json -Project Zombie - Part 1.nfo - NFO tags: - musicvideo: - album: Music Videos - artist: Project Zombie - title: Part 1 - year: 2010 -Project Zombie - Part 2-thumb.jpg -Project Zombie - Part 2.info.json -Project Zombie - Part 2.nfo - NFO tags: - musicvideo: - album: Music Videos - artist: Project Zombie - title: Part 2 - year: 2010 -Project Zombie - Part 3-thumb.jpg -Project Zombie - Part 3.info.json -Project Zombie - Part 3.nfo - NFO tags: - musicvideo: - album: Music Videos - artist: Project Zombie - title: Part 3 - year: 2010 -Project Zombie - Part 4-thumb.jpg -Project Zombie - Part 4.info.json -Project Zombie - Part 4.nfo - NFO tags: - musicvideo: - album: Music Videos - artist: Project Zombie - title: Part 4 - year: 2010 -Project Zombie - Part 5-thumb.jpg -Project Zombie - Part 5.info.json -Project Zombie - Part 5.nfo - NFO tags: - musicvideo: - album: Music Videos - artist: Project Zombie - title: Part 5 - year: 2010 \ No newline at end of file diff --git a/tests/e2e/youtube/test_split_video.py b/tests/e2e/youtube/test_split_video.py deleted file mode 100644 index 6d1f75d8..00000000 --- a/tests/e2e/youtube/test_split_video.py +++ /dev/null @@ -1,67 +0,0 @@ -import pytest -from e2e.expected_download import assert_expected_downloads -from e2e.expected_transaction_log import assert_transaction_log_matches - -from ytdl_sub.config.preset import Preset -from ytdl_sub.subscriptions.subscription import Subscription - - -@pytest.fixture -def subscription_dict(output_directory, timestamps_file_path): - return { - "preset": "yt_music_video", - "youtube": { - "download_strategy": "split_video", - "video_url": "https://youtube.com/watch?v=HKTNxEqsN3Q", - "split_timestamps": timestamps_file_path, - }, - # override the output directory with our fixture-generated dir - "output_options": { - "output_directory": output_directory, - "file_name": "{channel_sanitized} - {playlist_index}-{playlist_size}" - ".{title_sanitized}.{ext}", - }, - # download the worst format so it is fast - "ytdl_options": { - "format": "worst[ext=mp4]", - }, - } - - -#################################################################################################### -# SINGLE VIDEO FIXTURES - - -@pytest.fixture -def single_video_subscription(music_video_config, subscription_dict): - single_video_preset = Preset.from_dict( - config=music_video_config, - preset_name="split_video_test", - preset_dict=subscription_dict, - ) - - return Subscription.from_preset( - preset=single_video_preset, - config=music_video_config, - ) - - -class TestPlaylistAsKodiMusicVideo: - """ - Downloads my old minecraft youtube channel, pretends they are music videos. Ensure the above - files exist and have the expected md5 file hashes. - """ - - @pytest.mark.parametrize("dry_run", [True, False]) - def test_split_video_download(self, single_video_subscription, output_directory, dry_run): - transaction_log = single_video_subscription.download(dry_run=dry_run) - assert_transaction_log_matches( - output_directory=output_directory, - transaction_log=transaction_log, - transaction_log_summary_file_name="youtube/test_split_video.txt", - ) - assert_expected_downloads( - output_directory=output_directory, - dry_run=dry_run, - expected_download_summary_file_name="youtube/test_split_video.json", - )