From b1d7312a77fbd8fa5e6b42138cdb78084d5d5179 Mon Sep 17 00:00:00 2001 From: jbannon Date: Fri, 17 Jun 2022 07:36:07 +0000 Subject: [PATCH] almost working, need to deal with track_title and other overwrites --- src/ytdl_sub/config/preset_class_mappings.py | 2 + .../downloaders/youtube/split_video.py | 46 +++++--- src/ytdl_sub/utils/ffmpeg.py | 8 +- tests/e2e/youtube/test_split_video.py | 110 ++++++++++++++++++ 4 files changed, 147 insertions(+), 19 deletions(-) create mode 100644 tests/e2e/youtube/test_split_video.py diff --git a/src/ytdl_sub/config/preset_class_mappings.py b/src/ytdl_sub/config/preset_class_mappings.py index 04606b36..73a668d5 100644 --- a/src/ytdl_sub/config/preset_class_mappings.py +++ b/src/ytdl_sub/config/preset_class_mappings.py @@ -4,6 +4,7 @@ from typing import Type from ytdl_sub.downloaders.downloader import Downloader from ytdl_sub.downloaders.soundcloud_downloader import SoundcloudAlbumsAndSinglesDownloader +from ytdl_sub.downloaders.youtube.split_video import YoutubeSplitVideoDownloader from ytdl_sub.downloaders.youtube_downloader import YoutubeChannelDownloader from ytdl_sub.downloaders.youtube_downloader import YoutubePlaylistDownloader from ytdl_sub.downloaders.youtube_downloader import YoutubeVideoDownloader @@ -23,6 +24,7 @@ class DownloadStrategyMapping: "video": YoutubeVideoDownloader, "playlist": YoutubePlaylistDownloader, "channel": YoutubeChannelDownloader, + "split_video": YoutubeSplitVideoDownloader, }, "soundcloud": { "albums_and_singles": SoundcloudAlbumsAndSinglesDownloader, diff --git a/src/ytdl_sub/downloaders/youtube/split_video.py b/src/ytdl_sub/downloaders/youtube/split_video.py index 7198168c..d205aa70 100644 --- a/src/ytdl_sub/downloaders/youtube/split_video.py +++ b/src/ytdl_sub/downloaders/youtube/split_video.py @@ -1,6 +1,8 @@ import copy import os.path import re +from pathlib import Path +from shutil import copyfile from typing import Dict from typing import List from typing import Tuple @@ -8,8 +10,10 @@ from typing import Tuple from ytdl_sub.downloaders.youtube_downloader import YoutubeDownloader from ytdl_sub.downloaders.youtube_downloader import YoutubeVideoDownloaderOptions from ytdl_sub.entries.youtube import YoutubePlaylistVideo +from ytdl_sub.entries.youtube import YoutubeVideo from ytdl_sub.utils.exceptions import ValidationException from ytdl_sub.utils.ffmpeg import FFMPEG +from ytdl_sub.utils.thumbnail import convert_download_thumbnail from ytdl_sub.validators.validators import StringValidator # Captures the following formats: @@ -112,7 +116,7 @@ class YoutubeSplitVideoDownloaderOptions(YoutubeVideoDownloaderOptions): end at Youtube video's ending. """ - _required_keys = super()._required_keys.union("split_timestamps") + _required_keys = {"video_url", "split_timestamps"} def __init__(self, name, value): super().__init__(name, value) @@ -155,32 +159,42 @@ class YoutubeSplitVideoDownloader( split_timestamp_path=self.download_options.split_timestamps ) entry_dict = self.extract_info(url=self.download_options.video_url) - uid = entry_dict["id"] - ext = entry_dict["ext"] - idx = 0 - for timestamp, title in timestamp_titles: + entry = YoutubeVideo(entry_dict=entry_dict, working_directory=self.working_directory) + # convert the entry thumbnail early so we do not have to guess the thumbnail extension + # when copying it + convert_download_thumbnail(entry=entry) + + for idx, timestamp_title in enumerate(timestamp_titles): + timestamp_begin, title = timestamp_title + timestamp_end = timestamp_titles[idx + 1][0] if idx + 1 < len(timestamp_titles) else "" + + new_uid = f"{entry.uid}___{idx}" + + input_file = entry.get_download_file_path() + output_file = str(Path(self.working_directory) / f"{new_uid}.{entry.ext}") + output_thumbnail_file = str( + Path(self.working_directory) / f"{new_uid}.{entry.thumbnail_ext}" + ) + entry_dict_ = copy.deepcopy(entry_dict) - new_uid = f"{uid}___{idx}" entry_dict_["title"] = title entry_dict_["playlist_index"] = idx + 1 entry_dict_["playlist_count"] = len(timestamp_titles) entry_dict_["id"] = new_uid - timestamp_begin_arg = f"-ss {timestamp}" - timestamp_end_arg = ( - f"-to {timestamp_titles[idx + 1][0]}" if idx + 1 < len(timestamp_titles) else "" - ) + cmd = ["-i", input_file, "-ss", timestamp_begin] + if timestamp_end: + cmd += ["-to", timestamp_end] + cmd += ["-vcodec", "copy", "-acodec", "copy", output_file] - FFMPEG.run( - f"-i {uid}.{ext} " - f"{timestamp_begin_arg} {timestamp_end_arg} " - f"-vcodec copy -acodec copy {new_uid}.{ext}" - ) + FFMPEG.run(cmd) + + copyfile(src=entry.get_download_thumbnail_path(), dst=output_thumbnail_file) split_videos.append( YoutubePlaylistVideo( - entry_dict=entry_dict, working_directory=self.working_directory + entry_dict=entry_dict_, working_directory=self.working_directory ) ) diff --git a/src/ytdl_sub/utils/ffmpeg.py b/src/ytdl_sub/utils/ffmpeg.py index c8f742ff..0539d2c9 100644 --- a/src/ytdl_sub/utils/ffmpeg.py +++ b/src/ytdl_sub/utils/ffmpeg.py @@ -1,4 +1,5 @@ import subprocess +from typing import List from ytdl_sub.utils.exceptions import ValidationException from ytdl_sub.utils.logger import Logger @@ -17,7 +18,7 @@ class FFMPEG: ) from subprocess_error @classmethod - def run(cls, ffmpeg_args: str) -> None: + def run(cls, ffmpeg_args: List[str]) -> None: """ Runs an ffmpeg command. Should not include 'ffmpeg' as the beginning argument. @@ -28,6 +29,7 @@ class FFMPEG: """ cls._ensure_installed() - cmd = f"ffmpeg {ffmpeg_args}" - logger.debug("Running %s", cmd) + cmd = ["ffmpeg"] + cmd.extend(ffmpeg_args) + logger.debug("Running %s", " ".join(cmd)) subprocess.run(cmd, check=True) diff --git a/tests/e2e/youtube/test_split_video.py b/tests/e2e/youtube/test_split_video.py new file mode 100644 index 00000000..cb2b170c --- /dev/null +++ b/tests/e2e/youtube/test_split_video.py @@ -0,0 +1,110 @@ +from pathlib import Path +from tempfile import NamedTemporaryFile + +import mergedeep +import pytest +from conftest import assert_debug_log +from e2e.expected_download import ExpectedDownload + +import ytdl_sub.downloaders.downloader +from ytdl_sub.config.config_file import ConfigFile +from ytdl_sub.config.preset import Preset +from ytdl_sub.subscriptions.subscription import Subscription + + +@pytest.fixture +def config_path(): + return "examples/kodi_music_videos_config.yaml" + + +@pytest.fixture +def split_timestamps_file_path(): + timestamps = [ + "0:00 Intro\n", + "00:15 Part 1\n", + "1:01 Part 2\n", + "01:24 Part 3\n", + "0:02:01 Part 4\n", + "00:02:33 Part 5\n", + ] + + with NamedTemporaryFile(mode="w", encoding="utf-8", suffix=".txt") as tmp: + tmp.writelines(timestamps) + tmp.seek(0) + yield tmp.name + + +@pytest.fixture +def subscription_name(): + return "jmc" + + +@pytest.fixture +def config(config_path): + return ConfigFile.from_file_path(config_path=config_path) + + +@pytest.fixture +def subscription_dict(output_directory, subscription_name, split_timestamps_file_path): + return { + "preset": "yt_music_video", + "youtube": { + "download_strategy": "split_video", + "video_url": "https://youtube.com/watch?v=HKTNxEqsN3Q", + "split_timestamps": split_timestamps_file_path, + }, + # override the output directory with our fixture-generated dir + "output_options": { + "output_directory": output_directory, + "file_name": "{playlist_index}.{title_sanitized}.{ext}", + }, + # download the worst format so it is fast + "ytdl_options": { + "format": "worst[ext=mp4]", + }, + } + + +#################################################################################################### +# SINGLE VIDEO FIXTURES + + +@pytest.fixture +def single_video_subscription(config, subscription_name, subscription_dict): + single_video_preset = Preset.from_dict( + config=config, + preset_name=subscription_name, + preset_dict=subscription_dict, + ) + + return Subscription.from_preset( + preset=single_video_preset, + config=config, + ) + + +@pytest.fixture +def expected_single_video_download(): + # turn off black formatter here for readability + # fmt: off + return ExpectedDownload( + expected_md5_file_hashes={ + Path("JMC - Whale & Wasp.jpg"): "b58377dfe7c39527e1990a24b36bbd77", + Path("JMC - Whale & Wasp.mp4"): "931a705864c57d21d6fedebed4af6bbc", + Path("JMC - Whale & Wasp.nfo"): "6c2f085adb847c1dcc47c19514c454d8", + } + ) + # fmt: on + + +class TestPlaylistAsKodiMusicVideo: + """ + Downloads my old minecraft youtube channel, pretends they are music videos. Ensure the above + files exist and have the expected md5 file hashes. + """ + + def test_single_video_download( + self, single_video_subscription, expected_single_video_download, output_directory + ): + single_video_subscription.download() + expected_single_video_download.assert_files_exist(relative_directory=output_directory)