diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index eee5c1e8..2125228f 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -69,6 +69,7 @@ jobs: - name: Run unit tests with coverage run: | + sudo apt-get update sudo apt-get install -y ffmpeg source /opt/env/bin/activate coverage run -m pytest tests/unit && coverage xml -o /opt/coverage/unit/coverage.xml @@ -96,6 +97,7 @@ jobs: - name: Run e2e soundcloud tests with coverage run: | + sudo apt-get update sudo apt-get install -y ffmpeg source /opt/env/bin/activate coverage run -m pytest tests/e2e/soundcloud && coverage xml -o /opt/coverage/soundcloud/coverage.xml @@ -123,6 +125,7 @@ jobs: - name: Run e2e youtube tests with coverage run: | + sudo apt-get update sudo apt-get install -y ffmpeg source /opt/env/bin/activate coverage run -m pytest tests/e2e/youtube && coverage xml -o /opt/coverage/youtube/coverage.xml @@ -150,6 +153,7 @@ jobs: - name: Run e2e plugin tests with coverage run: | + sudo apt-get update sudo apt-get install -y ffmpeg source /opt/env/bin/activate coverage run -m pytest tests/e2e/plugins && coverage xml -o /opt/coverage/plugins/coverage.xml diff --git a/README.md b/README.md index 269fb242..b61b0d0a 100644 --- a/README.md +++ b/README.md @@ -298,4 +298,6 @@ questions, submit a feature request, or pick up a bug. ## Support We are pretty active in our [Discord channel](https://discord.gg/v8j9RAHb4k) -if you have any questions. +if you have any questions. Also see our +[FAQ](https://github.com/jmbannon/ytdl-sub/wiki/FAQ) +for commonly asked questions. diff --git a/src/ytdl_sub/config/preset_class_mappings.py b/src/ytdl_sub/config/preset_class_mappings.py index c4a8eb9e..b543a151 100644 --- a/src/ytdl_sub/config/preset_class_mappings.py +++ b/src/ytdl_sub/config/preset_class_mappings.py @@ -5,11 +5,6 @@ from typing import Type from ytdl_sub.downloaders.downloader import Downloader from ytdl_sub.downloaders.generic.multi_url import MultiUrlDownloader from ytdl_sub.downloaders.generic.url import UrlDownloader -from ytdl_sub.downloaders.soundcloud.albums_and_singles import SoundcloudAlbumsAndSinglesDownloader -from ytdl_sub.downloaders.youtube.channel import YoutubeChannelDownloader -from ytdl_sub.downloaders.youtube.merge_playlist import YoutubeMergePlaylistDownloader -from ytdl_sub.downloaders.youtube.playlist import YoutubePlaylistDownloader -from ytdl_sub.downloaders.youtube.video import YoutubeVideoDownloader from ytdl_sub.plugins.audio_extract import AudioExtractPlugin from ytdl_sub.plugins.chapters import ChaptersPlugin from ytdl_sub.plugins.date_range import DateRangePlugin @@ -32,15 +27,6 @@ class DownloadStrategyMapping: """ _MAPPING: Dict[str, Dict[str, Type[Downloader]]] = { - "youtube": { - "video": YoutubeVideoDownloader, - "playlist": YoutubePlaylistDownloader, - "channel": YoutubeChannelDownloader, - "merge_playlist": YoutubeMergePlaylistDownloader, - }, - "soundcloud": { - "albums_and_singles": SoundcloudAlbumsAndSinglesDownloader, - }, "download": { "multi_url": MultiUrlDownloader, "url": UrlDownloader, diff --git a/src/ytdl_sub/config/preset_options.py b/src/ytdl_sub/config/preset_options.py index 47cf2d8c..860763ee 100644 --- a/src/ytdl_sub/config/preset_options.py +++ b/src/ytdl_sub/config/preset_options.py @@ -67,9 +67,26 @@ class YTDLOptions(LiteralDictValidator): presets: my_example_preset: ytdl_options: + # Ignore any download related errors and continue ignoreerrors: True + # Stop downloading additional metadata/videos if it + # exists in your download archive + break_on_existing: True + # Stop downloading additional metadata/videos if it + # is out of your date range + break_on_reject: True + # Path to your YouTube cookies file to download 18+ restricted content + cookiefile: "/path/to/cookies/file.txt" + # Only download this number of videos/audio + max_downloads: 10 + # Download and use English title/description/etc YouTube metadata + extractor_args: + youtube: + lang: + - "en" - where each key is a ytdl argument. + + where each key is a ytdl argument. Include in the example are some popular ytdl_options. """ diff --git a/src/ytdl_sub/downloaders/downloader.py b/src/ytdl_sub/downloaders/downloader.py index 84fa9a6f..dc019ae1 100644 --- a/src/ytdl_sub/downloaders/downloader.py +++ b/src/ytdl_sub/downloaders/downloader.py @@ -45,6 +45,7 @@ from ytdl_sub.utils.file_handler import FileHandler from ytdl_sub.utils.file_handler import FileMetadata from ytdl_sub.utils.logger import Logger from ytdl_sub.utils.thumbnail import ThumbnailTypes +from ytdl_sub.utils.thumbnail import convert_download_thumbnail from ytdl_sub.utils.thumbnail import convert_url_thumbnail from ytdl_sub.validators.strict_dict_validator import StrictDictValidator from ytdl_sub.ytdl_additions.enhanced_download_archive import DownloadArchiver @@ -625,6 +626,9 @@ class Downloader(DownloadArchiver, Generic[DownloaderOptionsT], ABC): # If latest entry, always update the thumbnail on each entry if thumbnail_id == ThumbnailTypes.LATEST_ENTRY: + # Make sure the entry's thumbnail is converted to jpg + convert_download_thumbnail(entry, error_if_not_found=False) + # always save in dry-run even if it doesn't exist... if self.is_dry_run or os.path.isfile(entry.get_download_thumbnail_path()): self.save_file( diff --git a/src/ytdl_sub/downloaders/soundcloud/__init__.py b/src/ytdl_sub/downloaders/soundcloud/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/src/ytdl_sub/downloaders/soundcloud/albums_and_singles.py b/src/ytdl_sub/downloaders/soundcloud/albums_and_singles.py deleted file mode 100644 index 74475ea6..00000000 --- a/src/ytdl_sub/downloaders/soundcloud/albums_and_singles.py +++ /dev/null @@ -1,145 +0,0 @@ -from typing import Any -from typing import Dict -from typing import Generator - -from ytdl_sub.downloaders.downloader import Downloader -from ytdl_sub.downloaders.downloader import DownloaderValidator -from ytdl_sub.downloaders.generic.validators import MultiUrlValidator -from ytdl_sub.entries.entry import Entry -from ytdl_sub.validators.url_validator import SoundcloudUsernameUrlValidator -from ytdl_sub.validators.validators import BoolValidator - - -class SoundcloudAlbumsAndSinglesDownloadOptions(DownloaderValidator): - """ - Downloads a soundcloud user's entire discography. Groups together album tracks and considers - any track not in an album as a single. Also includes any collaboration tracks. - - Usage: - - .. code-block:: yaml - - presets: - my_example_preset: - soundcloud: - # required - download_strategy: "albums_and_singles" - url: "soundcloud.com/username" - # optional - skip_premiere_tracks: True - - """ - - _required_keys = {"url"} - _optional_keys = {"skip_premiere_tracks"} - - @classmethod - def partial_validate(cls, name: str, value: Any) -> None: - """ - Partially validate a Soundcloud source - """ - if isinstance(value, dict): - value["url"] = value.get("url", "https://soundcloud.com/jessebannon") - _ = cls(name, value) - - def __init__(self, name, value): - super().__init__(name, value) - self._url = self._validate_key( - key="url", validator=SoundcloudUsernameUrlValidator - ).username_url - self._skip_premiere_tracks = self._validate_key( - "skip_premiere_tracks", BoolValidator, default=True - ) - - @property - def collection_validator(self) -> MultiUrlValidator: - """Downloads the album tracks first, then the tracks""" - return MultiUrlValidator( - name=self._name, - value={ - "urls": [ - { - "url": f"{self._url}/tracks", - "variables": { - "track_number": "1", - "track_number_padded": "01", - "track_count": "1", - "album": "{title}", - "album_sanitized": "{title_sanitized}", - "album_year": "{upload_year}", - }, - }, - { - "url": f"{self._url}/albums", - "variables": { - "track_number": "{playlist_index}", - "track_number_padded": "{playlist_index_padded}", - "track_count": "{playlist_count}", - "album": "{playlist_title}", - "album_sanitized": "{playlist_title_sanitized}", - "album_year": "{playlist_max_upload_year}", - }, - }, - ] - }, - ) - - @property - def skip_premiere_tracks(self) -> bool: - """ - Optional. True to skip tracks that require purchasing. False otherwise. Defaults to True. - """ - return self._skip_premiere_tracks.value - - @property - def url(self) -> str: - """ - Required. The Soundcloud user's url, i.e. ``soundcloud.com/the_username`` - """ - return self._url - - -class SoundcloudAlbumsAndSinglesDownloader(Downloader[SoundcloudAlbumsAndSinglesDownloadOptions]): - downloader_options_type = SoundcloudAlbumsAndSinglesDownloadOptions - downloader_entry_type = Entry - - supports_subtitles = False - supports_chapters = False - - @classmethod - def ytdl_option_defaults(cls) -> Dict: - """ - Default `ytdl_options`_ for ``albums_and_singles`` - - .. code-block:: yaml - - ytdl_options: - ignoreerrors: True # ignore errors like hidden videos, age restriction, etc - format: "bestaudio[ext=mp3]" # download format the best possible mp3 - """ - return dict( - super().ytdl_option_defaults(), - **{ - "format": "bestaudio[ext=mp3]", - }, - ) - - def _should_skip(self, entry: Entry) -> bool: - if not self.download_options.skip_premiere_tracks: - return False - - for url in [entry.kwargs_get("url", ""), entry.webpage_url]: - if "/preview/" in url: - return True - - return False - - def download(self) -> Generator[Entry, None, None]: - """ - Soundcloud subscription to download albums and tracks as singles. - """ - for entry in super().download(): - if self._should_skip(entry): - continue - - yield entry diff --git a/src/ytdl_sub/downloaders/youtube/__init__.py b/src/ytdl_sub/downloaders/youtube/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/src/ytdl_sub/downloaders/youtube/channel.py b/src/ytdl_sub/downloaders/youtube/channel.py deleted file mode 100644 index 6ae77994..00000000 --- a/src/ytdl_sub/downloaders/youtube/channel.py +++ /dev/null @@ -1,139 +0,0 @@ -from typing import Any -from typing import Dict -from typing import List -from typing import Optional - -from ytdl_sub.downloaders.downloader import Downloader -from ytdl_sub.downloaders.downloader import DownloaderValidator -from ytdl_sub.downloaders.generic.validators import MultiUrlValidator -from ytdl_sub.validators.string_formatter_validators import OverridesStringFormatterValidator -from ytdl_sub.validators.url_validator import YoutubeChannelUrlValidator - - -class YoutubeChannelDownloaderOptions(DownloaderValidator): - """ - Downloads all videos from a youtube channel. - - Usage: - - .. code-block:: yaml - - presets: - my_example_preset: - youtube: - # required - download_strategy: "channel" - channel_url: "UCsvn_Po0SmunchJYtttWpOxMg" - # optional - channel_avatar_path: "poster.jpg" - channel_banner_path: "fanart.jpg" - """ - - _required_keys = {"channel_url"} - _optional_keys = { - "channel_avatar_path", - "channel_banner_path", - } - - @classmethod - def partial_validate(cls, name: str, value: Any) -> None: - """ - Partially validate a YouTube channel source - """ - if isinstance(value, dict): - value["channel_url"] = value.get( - "channel_url", "https://www.youtube.com/c/ProjectZombie603" - ) - _ = cls(name, value) - - def __init__(self, name, value): - super().__init__(name, value) - self._channel_url = self._validate_key( - "channel_url", YoutubeChannelUrlValidator - ).channel_url - self._channel_avatar_path = self._validate_key_if_present( - "channel_avatar_path", OverridesStringFormatterValidator - ) - self._channel_banner_path = self._validate_key_if_present( - "channel_banner_path", OverridesStringFormatterValidator - ) - - @property - def collection_validator(self) -> MultiUrlValidator: - """Download from the channel url""" - playlist_thumbnails: List[Dict] = [] - if self._channel_avatar_path: - playlist_thumbnails.append( - { - "name": self._channel_avatar_path.format_string, - "uid": "avatar_uncropped", - } - ) - if self._channel_banner_path: - playlist_thumbnails.append( - { - "name": self._channel_banner_path.format_string, - "uid": "banner_uncropped", - } - ) - - return MultiUrlValidator( - name=self._name, - value={ - "urls": [ - { - "url": self.channel_url, - "playlist_thumbnails": playlist_thumbnails, - "variables": {"playlist_size": "{playlist_count}"}, - } - ] - }, - ) - - @property - def channel_url(self) -> str: - """ - Required. The channel's url, i.e. - ``https://www.youtube.com/channel/UCsvn_Po0SmunchJYOWpOxMg``. URLs with ``/username`` or - ``/c`` are valid to use. - """ - return self._channel_url - - @property - def channel_avatar_path(self) -> Optional[OverridesStringFormatterValidator]: - """ - Optional. Path to store the channel's avatar thumbnail image to. - """ - return self._channel_avatar_path - - @property - def channel_banner_path(self) -> Optional[OverridesStringFormatterValidator]: - """ - Optional. Path to store the channel's banner image to. - """ - return self._channel_banner_path - - -class YoutubeChannelDownloader(Downloader[YoutubeChannelDownloaderOptions]): - downloader_options_type = YoutubeChannelDownloaderOptions - - # pylint: disable=line-too-long - @classmethod - def ytdl_option_defaults(cls) -> Dict: - """ - Default `ytdl_options`_ for ``channel`` - - .. code-block:: yaml - - ytdl_options: - ignoreerrors: True # ignore errors like hidden videos, age restriction, etc - break_on_existing: True # stop downloads (newest to oldest) if a video is already downloaded - """ - return dict( - super().ytdl_option_defaults(), - **{ - "break_on_existing": True, - }, - ) - - # pylint: enable=line-too-long diff --git a/src/ytdl_sub/downloaders/youtube/merge_playlist.py b/src/ytdl_sub/downloaders/youtube/merge_playlist.py deleted file mode 100644 index 48d1eae9..00000000 --- a/src/ytdl_sub/downloaders/youtube/merge_playlist.py +++ /dev/null @@ -1,155 +0,0 @@ -from typing import Dict -from typing import List -from typing import Optional -from typing import Tuple - -from ytdl_sub.downloaders.downloader import Downloader -from ytdl_sub.downloaders.youtube.playlist import YoutubePlaylistDownloaderOptions -from ytdl_sub.entries.entry import Entry -from ytdl_sub.utils.chapters import Chapters -from ytdl_sub.utils.chapters import Timestamp -from ytdl_sub.utils.ffmpeg import set_ffmpeg_metadata_chapters -from ytdl_sub.utils.file_handler import FileMetadata -from ytdl_sub.validators.validators import BoolValidator - - -class YoutubeMergePlaylistDownloaderOptions(YoutubePlaylistDownloaderOptions): - r""" - Downloads all videos in a playlist and merges them into a single video. - - Usage: - - .. code-block:: yaml - - presets: - example_preset: - youtube: - # required - download_strategy: "merge_playlist" - playlist_url: "https://www.youtube.com/playlist?list=UCsvn_Po0SmunchJYtttWpOxMg" - # optional - add_chapters: False - - CLI usage: - - .. code-block:: bash - - ytdl-sub dl \ - --preset "example_preset" \ - --youtube.playlist_url "https://www.youtube.com/playlist?list=UCsvn_Po0SmunchJYtttWpOxMg" \ - --youtube.add_chapters True - """ - - _required_keys = {"playlist_url"} - _optional_keys = {"add_chapters"} - - def __init__(self, name, value): - super().__init__(name, value) - self._add_chapters = self._validate_key_if_present( - "add_chapters", validator=BoolValidator, default=False - ).value - - @property - def add_chapters(self) -> Optional[bool]: - """ - Optional. Whether to add chapters using each video's title in the merged playlist. - Defaults to False. - """ - return self._add_chapters - - -class YoutubeMergePlaylistDownloader(Downloader[YoutubeMergePlaylistDownloaderOptions]): - downloader_options_type = YoutubeMergePlaylistDownloaderOptions - supports_download_archive = False - supports_subtitles = False - supports_chapters = False - - @classmethod - def ytdl_option_defaults(cls) -> Dict: - """ - Default `ytdl_options`_ for ``merge_playlist`` - - .. code-block:: yaml - - ytdl_options: - ignoreerrors: True # ignore errors like hidden videos, age restriction, etc - postprocessors: - # Convert the videos to mkv format - - key: "FFmpegVideoConvertor" - when: "post_process" - preferedformat: "mkv" - # Concatenate all the playlist videos into a single file - - key: "FFmpegConcat" - when: "playlist" - """ - return dict( - super().ytdl_option_defaults(), - **{ - "postprocessors": [ - { - "key": "FFmpegVideoConvertor", - "when": "post_process", - "preferedformat": "mkv", - }, - { - "key": "FFmpegConcat", - "when": "playlist", - }, - ], - }, - ) - - def _get_chapters(self, merged_video: Entry, add_chapters: bool) -> FileMetadata: - titles: List[str] = [] - timestamps: List[Timestamp] = [] - - current_timestamp_sec = 0 - for video_entry in merged_video.kwargs("entries"): - timestamps.append(Timestamp(current_timestamp_sec)) - titles.append(video_entry["title"]) - - current_timestamp_sec += video_entry["duration"] - - chapters = Chapters(timestamps=timestamps, titles=titles) - - if not self.is_dry_run and add_chapters: - set_ffmpeg_metadata_chapters( - file_path=merged_video.get_download_file_path(), - chapters=chapters, - file_duration_sec=merged_video.kwargs("duration"), - ) - - return chapters.to_file_metadata(title="Timestamps of playlist videos in the merged file") - - def _to_merged_video(self, entry_dict: Dict) -> Entry: - """ - Adds a few entries not included in a playlist entry to make it look like a merged video - entry_dict - """ - # Set the upload date to be the latest playlist video date - entry_dict["upload_date"] = max( - playlist_entry["upload_date"] for playlist_entry in entry_dict["entries"] - ) - entry_dict["duration"] = sum( - playlist_entry["duration"] for playlist_entry in entry_dict["entries"] - ) - entry_dict["ext"] = ( - entry_dict["requested_downloads"][0]["ext"] - if "requested_downloads" in entry_dict - else "mkv" - ) - entry_dict["webpage_url"] = self.download_options.playlist_url - - return Entry(entry_dict=entry_dict, working_directory=self.working_directory) - - def download(self) -> List[Tuple[Entry, FileMetadata]]: - """Download a single Youtube video, then split it into multiple videos""" - url = self.overrides.apply_formatter(self.collection.urls.list[0].url) - - entry_dict = self.extract_info(url=url, ytdl_options_overrides=self.download_ytdl_options) - merged_video = self._to_merged_video(entry_dict=entry_dict) - - merged_video_metadata = self._get_chapters( - merged_video=merged_video, add_chapters=self.download_options.add_chapters - ) - return [(merged_video, merged_video_metadata)] diff --git a/src/ytdl_sub/downloaders/youtube/playlist.py b/src/ytdl_sub/downloaders/youtube/playlist.py deleted file mode 100644 index 0adac218..00000000 --- a/src/ytdl_sub/downloaders/youtube/playlist.py +++ /dev/null @@ -1,116 +0,0 @@ -from typing import Any -from typing import Dict -from typing import List -from typing import Optional - -from ytdl_sub.downloaders.downloader import Downloader -from ytdl_sub.downloaders.downloader import DownloaderValidator -from ytdl_sub.downloaders.generic.validators import MultiUrlValidator -from ytdl_sub.utils.thumbnail import ThumbnailTypes -from ytdl_sub.validators.string_formatter_validators import OverridesStringFormatterValidator -from ytdl_sub.validators.url_validator import YoutubePlaylistUrlValidator - - -class YoutubePlaylistDownloaderOptions(DownloaderValidator): - """ - Downloads all videos from a youtube playlist. - - Usage: - - .. code-block:: yaml - - presets: - my_example_preset: - youtube: - # required - download_strategy: "playlist" - playlist_url: "https://www.youtube.com/playlist?list=UCsvn_Po0SmunchJYtttWpOxMg" - # optional - playlist_thumbnail_name: "poster.jpg" - """ - - _required_keys = {"playlist_url"} - _optional_keys = {"playlist_thumbnail_name"} - - @classmethod - def partial_validate(cls, name: str, value: Any) -> None: - """ - Partially validate a YouTube playlist source - """ - if isinstance(value, dict): - value["playlist_url"] = value.get( - "playlist_url", "https://www.youtube.com/playlist?list=UCsvn_Po0SmunchJYtttWpOxMg" - ) - _ = cls(name, value) - - def __init__(self, name, value): - super().__init__(name, value) - self._playlist_url = self._validate_key( - "playlist_url", YoutubePlaylistUrlValidator - ).playlist_url - self._playlist_thumbnail_name = self._validate_key_if_present( - "playlist_thumbnail_name", OverridesStringFormatterValidator - ) - - @property - def collection_validator(self) -> MultiUrlValidator: - """Downloads the playlist url""" - playlist_thumbnails: List[Dict] = [] - if self.playlist_thumbnail_name: - playlist_thumbnails.append( - { - "name": self.playlist_thumbnail_name.format_string, - "uid": ThumbnailTypes.LATEST_ENTRY, - } - ) - - return MultiUrlValidator( - name=self._name, - value={ - "urls": [ - { - "url": self.playlist_url, - "playlist_thumbnails": playlist_thumbnails, - "variables": {"playlist_size": "{playlist_count}"}, - } - ] - }, - ) - - @property - def playlist_url(self) -> str: - """ - Required. The playlist's url, i.e. - ``https://www.youtube.com/playlist?list=UCsvn_Po0SmunchJYtttWpOxMg``. - """ - return self._playlist_url - - @property - def playlist_thumbnail_name(self) -> Optional[OverridesStringFormatterValidator]: - """ - Optional. Path to store the playlist's thumbnail - """ - return self._playlist_thumbnail_name - - -class YoutubePlaylistDownloader(Downloader[YoutubePlaylistDownloaderOptions]): - downloader_options_type = YoutubePlaylistDownloaderOptions - - # pylint: disable=line-too-long - @classmethod - def ytdl_option_defaults(cls) -> Dict: - """ - Default `ytdl_options`_ for ``playlist`` - - .. code-block:: yaml - - ytdl_options: - ignoreerrors: True # ignore errors like hidden videos, age restriction, etc - break_on_existing: True # stop downloads (newest to oldest) if a video is already downloaded - """ - return dict( - super().ytdl_option_defaults(), - **{"break_on_existing": True}, - ) - - # pylint: enable=line-too-long diff --git a/src/ytdl_sub/downloaders/youtube/video.py b/src/ytdl_sub/downloaders/youtube/video.py deleted file mode 100644 index 1a7f59c5..00000000 --- a/src/ytdl_sub/downloaders/youtube/video.py +++ /dev/null @@ -1,80 +0,0 @@ -from typing import Any -from typing import Dict - -from ytdl_sub.downloaders.downloader import Downloader -from ytdl_sub.downloaders.downloader import DownloaderValidator -from ytdl_sub.downloaders.generic.url import UrlDownloadOptions -from ytdl_sub.downloaders.generic.validators import MultiUrlValidator -from ytdl_sub.validators.url_validator import YoutubeVideoUrlValidator - - -class YoutubeVideoDownloaderOptions(DownloaderValidator): - """ - Downloads a single youtube video. This download strategy is intended for CLI usage performing - a one-time download of a video, not a subscription. - - Usage: - - .. code-block:: yaml - - presets: - example_preset: - youtube: - # required - download_strategy: "video" - video_url: "youtube.com/watch?v=VMAPTo7RVDo" - - CLI usage: - - .. code-block:: bash - - ytdl-sub dl --preset "example_preset" --youtube.video_url "youtube.com/watch?v=VMAPTo7RVDo" - """ - - _required_keys = {"video_url"} - - @classmethod - def partial_validate(cls, name: str, value: Any) -> None: - """ - Partially validate a YouTube video source - """ - if isinstance(value, dict): - value["video_url"] = value.get("video_url", "youtube.com/watch?v=VMAPTo7RVDo") - _ = cls(name, value) - - def __init__(self, name, value): - super().__init__(name, value) - self._video_url = self._validate_key("video_url", YoutubeVideoUrlValidator).video_url - - @property - def collection_validator(self) -> MultiUrlValidator: - """Downloads the video url""" - return UrlDownloadOptions( - name=self._name, value={"url": self.video_url} - ).collection_validator - - @property - def video_url(self) -> str: - """ - Required. The url of the video, i.e. ``youtube.com/watch?v=VMAPTo7RVDo``. - """ - return self._video_url - - -class YoutubeVideoDownloader(Downloader[YoutubeVideoDownloaderOptions]): - downloader_options_type = YoutubeVideoDownloaderOptions - - @classmethod - def ytdl_option_defaults(cls) -> Dict: - """ - Default `ytdl_options`_ for ``video`` - - .. code-block:: yaml - - ytdl_options: - ignoreerrors: True # ignore errors like hidden videos, age restriction, etc - """ - return dict( - super().ytdl_option_defaults(), - **{"break_on_existing": True}, - ) diff --git a/src/ytdl_sub/entries/variables/entry_variables.py b/src/ytdl_sub/entries/variables/entry_variables.py index 61bcf0ae..91762245 100644 --- a/src/ytdl_sub/entries/variables/entry_variables.py +++ b/src/ytdl_sub/entries/variables/entry_variables.py @@ -300,6 +300,16 @@ class EntryVariables(BaseEntryVariables): """ return self.kwargs_get(PLAYLIST_UPLOADER, self.uploader) + @property + def playlist_uploader_sanitized(self: Self) -> str: + """ + Returns + ------- + str + The playlist uploader, sanitized. + """ + return sanitize_filename(self.playlist_uploader) + @property def playlist_uploader_url(self: Self) -> str: """ diff --git a/src/ytdl_sub/plugins/music_tags.py b/src/ytdl_sub/plugins/music_tags.py index f66b20a6..90b7889a 100644 --- a/src/ytdl_sub/plugins/music_tags.py +++ b/src/ytdl_sub/plugins/music_tags.py @@ -11,6 +11,7 @@ from ytdl_sub.plugins.plugin import PluginOptions from ytdl_sub.utils.file_handler import FileMetadata from ytdl_sub.utils.logger import Logger from ytdl_sub.utils.thumbnail import convert_download_thumbnail +from ytdl_sub.validators.audo_codec_validator import AUDIO_CODEC_EXTS from ytdl_sub.validators.strict_dict_validator import StrictDictValidator from ytdl_sub.validators.string_formatter_validators import ListFormatterValidator from ytdl_sub.validators.string_formatter_validators import StringFormatterValidator @@ -116,6 +117,13 @@ class MusicTagsPlugin(Plugin[MusicTagsOptions]): """ Tags the entry's audio file using values defined in the metadata options """ + if entry.ext not in AUDIO_CODEC_EXTS: + raise self.plugin_options.validation_exception( + f"music_tags plugin received a video with the extension '{entry.ext}'. Only audio " + f"files are supported for setting music tags. Ensure you are converting the video " + f"to audio using the audio_extract plugin." + ) + # Resolve the tags into this dict tags_to_write: Dict[str, List[str]] = defaultdict(list) for tag_name, tag_formatters in self.plugin_options.tags.as_lists.items(): diff --git a/src/ytdl_sub/plugins/plugin.py b/src/ytdl_sub/plugins/plugin.py index 2b046dfd..0f32cafb 100644 --- a/src/ytdl_sub/plugins/plugin.py +++ b/src/ytdl_sub/plugins/plugin.py @@ -10,6 +10,7 @@ from typing import TypeVar from ytdl_sub.config.preset_options import AddsVariablesMixin from ytdl_sub.config.preset_options import Overrides from ytdl_sub.entries.entry import Entry +from ytdl_sub.utils.exceptions import ValidationException from ytdl_sub.utils.file_handler import FileMetadata from ytdl_sub.utils.logger import Logger from ytdl_sub.validators.strict_dict_validator import StrictDictValidator @@ -44,6 +45,23 @@ class PluginOptions(StrictDictValidator, AddsVariablesMixin, ABC): Class that defines the parameters to a plugin """ + def validation_exception( + self, + error_message: str | Exception, + ) -> ValidationException: + """ + Parameters + ---------- + error_message + Error message to include in the validation exception + + Returns + ------- + Validation exception that points to the location in the config. To be used for plugins + to throw good validation exceptions at runtime. + """ + return self._validation_exception(error_message=error_message) + PluginOptionsT = TypeVar("PluginOptionsT", bound=PluginOptions) diff --git a/src/ytdl_sub/plugins/regex.py b/src/ytdl_sub/plugins/regex.py index 9f20b560..51269083 100644 --- a/src/ytdl_sub/plugins/regex.py +++ b/src/ytdl_sub/plugins/regex.py @@ -104,7 +104,10 @@ class FromSourceVariablesRegex(StrictDictValidator): class RegexOptions(PluginOptions): r""" Performs regex matching on an entry's source variables. Regex can be used to filter entries - from proceeding with download or capture groups to create new source variables. + from proceeding with download or capture groups to create new source variables. NOTE to + use backslashes anywhere in your regex, i.e. ``\d``, you must add another backslash escape. This + means ``\d`` should be written as ``\\d``. This is because YAML requires an escape for any + backslash usage. Usage: @@ -113,34 +116,37 @@ class RegexOptions(PluginOptions): presets: my_example_preset: regex: - # By default, if any match fails and has no defaults, the entry will be skipped. - # If set to False, ytdl-sub will error and stop all downloads from proceeding. + # By default, if any match fails and has no defaults, the entry will + # be skipped. If False, ytdl-sub will error and stop all downloads + # from proceeding. skip_if_match_fails: True from: # For each entry's `title` value... title: # Perform this regex match on it to act as a filter. - # This will only download videos with "Official Video" in it. + # This will only download videos with "[Official Video]" in it. Note that we + # double backslash to make YAML happy match: - - '\[Official Video\]' + - '\\[Official Video\\]' # For each entry's `description` value... description: # Match with capture groups and defaults. - # This tries to scrape a date from the description and produce new source variables + # This tries to scrape a date from the description and produce new + # source variables match: - "([0-9]{4})-([0-9]{2})-([0-9]{2})" - # Each capture group creates these new source variables, respectively, as well - # a sanitized version, i.e. `captured_upload_year_sanitized` + # Each capture group creates these new source variables, respectively, + # as well a sanitized version, i.e. `captured_upload_year_sanitized` capture_group_names: - "captured_upload_year" - "captured_upload_month" - "captured_upload_day" - # And if the string does not match, use these as respective default values for the - # new source variables. + # And if the string does not match, use these as respective default + # values for the new source variables. capture_group_defaults: - "{upload_year}" - "{upload_month}" diff --git a/src/ytdl_sub/utils/ffmpeg.py b/src/ytdl_sub/utils/ffmpeg.py index affed520..2d63de8e 100644 --- a/src/ytdl_sub/utils/ffmpeg.py +++ b/src/ytdl_sub/utils/ffmpeg.py @@ -1,4 +1,3 @@ -import shutil import subprocess import tempfile from typing import Dict @@ -7,6 +6,7 @@ from typing import Optional from ytdl_sub.utils.chapters import Chapters from ytdl_sub.utils.exceptions import ValidationException +from ytdl_sub.utils.file_handler import FileHandler from ytdl_sub.utils.logger import Logger logger = Logger.get(name="ffmpeg") @@ -32,6 +32,25 @@ class FFMPEG: "Trying to use a feature which requires ffmpeg, but it cannot be found" ) from subprocess_error + @classmethod + def tmp_file_path(cls, relative_file_path: str, extension: Optional[str] = None) -> str: + """ + Parameters + ---------- + relative_file_path + Path of input file that is going to be modified + extension + Desired output extension. Defaults to input file's extension + + Returns + ------- + Temporary file path for ffmpeg output + """ + if extension is None: + extension = relative_file_path.split(".")[-1] + + return f"{relative_file_path}.out.{extension}" + @classmethod def run(cls, ffmpeg_args: List[str]) -> None: """ @@ -110,8 +129,7 @@ def set_ffmpeg_metadata_chapters( if chapters: lines += _create_metadata_chapters(chapters=chapters, file_duration_sec=file_duration_sec) - file_path_ext = file_path.split(".")[-1] - output_file_path = f"{file_path}.out.{file_path_ext}" + tmp_file_path = FFMPEG.tmp_file_path(relative_file_path=file_path) with tempfile.NamedTemporaryFile(mode="w", suffix=".txt", encoding="utf-8") as metadata_file: metadata_file.write("\n".join(lines)) metadata_file.flush() @@ -129,11 +147,10 @@ def set_ffmpeg_metadata_chapters( "-bitexact", # for reproducibility "-codec", "copy", - output_file_path, + tmp_file_path, ] ) - - shutil.move(src=output_file_path, dst=file_path) + FileHandler.move(tmp_file_path, file_path) def add_ffmpeg_metadata_key_values(file_path: str, key_values: Dict[str, str]) -> None: @@ -145,13 +162,12 @@ def add_ffmpeg_metadata_key_values(file_path: str, key_values: Dict[str, str]) - key_values The key/values to add """ - file_path_ext = file_path.split(".")[-1] - output_file_path = f"{file_path}.out.{file_path_ext}" + tmp_file_path = FFMPEG.tmp_file_path(file_path) ffmpeg_args = ["-i", file_path, "-map", "0"] for key, value in key_values.items(): ffmpeg_args.extend(["-metadata", f"{key}={value}"]) - ffmpeg_args.extend(["-codec", "copy", output_file_path]) + ffmpeg_args.extend(["-codec", "copy", tmp_file_path]) FFMPEG.run(ffmpeg_args) - shutil.move(src=output_file_path, dst=file_path) + FileHandler.move(tmp_file_path, file_path) diff --git a/src/ytdl_sub/utils/file_handler.py b/src/ytdl_sub/utils/file_handler.py index ddabadd4..6a3dc6c9 100644 --- a/src/ytdl_sub/utils/file_handler.py +++ b/src/ytdl_sub/utils/file_handler.py @@ -353,8 +353,23 @@ class FileHandler: Source file dst_file_path Destination file + + Raises + ------ + OSError + Cross-device link workaround """ - shutil.move(src=src_file_path, dst=dst_file_path) + try: + shutil.move(src=src_file_path, dst=dst_file_path) + except OSError as os_error_exc: + # Invalid cross-device link + # Can happen from using os.rename under the hood, which requires the two file on the + # same filesystem. Work around it by copying and deleting the file + if os_error_exc.errno == 18: + cls.copy(src_file_path, dst_file_path) + cls.delete(src_file_path) + else: + raise @classmethod def delete(cls, file_path: Union[str, Path]): diff --git a/src/ytdl_sub/utils/thumbnail.py b/src/ytdl_sub/utils/thumbnail.py index 549853d5..9ba26244 100644 --- a/src/ytdl_sub/utils/thumbnail.py +++ b/src/ytdl_sub/utils/thumbnail.py @@ -68,6 +68,14 @@ def convert_url_thumbnail(thumbnail_url: str, output_thumbnail_path: str) -> Opt thumbnail.write(file.read()) os.makedirs(os.path.dirname(output_thumbnail_path), exist_ok=True) - FFMPEG.run(["-bitexact", "-i", thumbnail.name, output_thumbnail_path]) + + tmp_output_path = FFMPEG.tmp_file_path( + relative_file_path=thumbnail.name, extension="jpg" + ) + FFMPEG.run(["-bitexact", "-i", thumbnail.name, tmp_output_path]) + + # Have FileHandler handle the move to a potential cross-device + FileHandler.move(tmp_output_path, output_thumbnail_path) + FileHandler.delete(tmp_output_path) return True diff --git a/tests/e2e/plugins/internal/test_view.py b/tests/e2e/plugins/internal/test_view.py index 08d92154..590bfc6e 100644 --- a/tests/e2e/plugins/internal/test_view.py +++ b/tests/e2e/plugins/internal/test_view.py @@ -1,6 +1,10 @@ +from typing import Optional + import pytest from e2e.conftest import mock_run_from_cli +from ytdl_sub.utils.file_handler import FileMetadata + class TestView: @pytest.mark.parametrize("split_chapters", [True, False]) @@ -22,12 +26,15 @@ class TestView: # Ensure the video and thumbnail are recognized assert len(transaction_log.files_created) == 2 - file_name = f"avUT-zd9v68{'___0' if split_chapters else ''}.webm" - video_file = transaction_log.files_created.get(file_name) - assert video_file is not None + video_metadata: Optional[FileMetadata] = None + for file_name, metadata in transaction_log.files_created.items(): + if file_name.endswith("webm"): + video_metadata = metadata + break - assert "Source Variables:" in video_file.metadata + assert video_metadata is not None + assert "Source Variables:" in video_metadata.metadata if split_chapters: - assert " chapter_index: 1" in video_file.metadata + assert " chapter_index: 1" in video_metadata.metadata else: - assert " chapter_index: 1" not in video_file.metadata + assert " chapter_index: 1" not in video_metadata.metadata diff --git a/tests/e2e/plugins/test_music_tags.py b/tests/e2e/plugins/test_music_tags.py new file mode 100644 index 00000000..2cec1b9b --- /dev/null +++ b/tests/e2e/plugins/test_music_tags.py @@ -0,0 +1,51 @@ +import re + +import pytest +from expected_download import assert_expected_downloads +from expected_transaction_log import assert_transaction_log_matches + +from ytdl_sub.subscriptions.subscription import Subscription +from ytdl_sub.utils.exceptions import ValidationException + + +@pytest.fixture +def single_song_video_dict(output_directory): + return { + "download": { + "download_strategy": "url", + "url": "https://www.youtube.com/watch?v=2lAe1cqCOXo", + }, + "output_options": {"output_directory": output_directory, "file_name": "will_error.mp4"}, + # test multi-tags + "music_tags": {"embed_thumbnail": True, "tags": {"genres": ["multi_tag_1", "multi_tag_2"]}}, + # download the worst format so it is fast + "ytdl_options": { + "format": "worst[ext=mp4]", + "postprocessor_args": {"ffmpeg": ["-bitexact"]}, # Must add this for reproducibility + }, + } + + +class TestMusicTags: + def test_music_tags_errors_on_video( + self, + youtube_audio_config, + single_song_video_dict, + output_directory, + ): + subscription = Subscription.from_dict( + config=youtube_audio_config, + preset_name="single_song_test", + preset_dict=single_song_video_dict, + ) + + with pytest.raises( + ValidationException, + match=re.escape( + "Validation error in single_song_test.music_tags: music_tags plugin received a " + "video with the extension 'mp4'. Only audio files are supported for setting music " + "tags. Ensure you are converting the video to audio using the audio_extract " + "plugin." + ), + ): + subscription.download(dry_run=True) diff --git a/tests/e2e/youtube/test_playlist.py b/tests/e2e/youtube/test_playlist.py index 658d5872..e2785951 100644 --- a/tests/e2e/youtube/test_playlist.py +++ b/tests/e2e/youtube/test_playlist.py @@ -11,13 +11,11 @@ from ytdl_sub.subscriptions.subscription import Subscription @pytest.fixture def playlist_preset_dict(output_directory): return { - "preset": "music_video", - "download": { - "url": "https://youtube.com/playlist?list=PL5BC0FC26BECA5A35", - "playlist_thumbnails": {"name": "poster.jpg", "uid": "latest_entry"}, - }, - # override the output directory with our fixture-generated dir - "output_options": {"output_directory": output_directory}, + "preset": [ + "jellyfin_tv_show_collection", + "season_by_collection__episode_by_year_month_day", + "collection_season_1", + ], # download the worst format so it is fast "ytdl_options": { "format": "worst[ext=mp4]", @@ -38,10 +36,15 @@ def playlist_preset_dict(output_directory): } }, "subtitles": { - "subtitles_name": "{music_video_name}.{lang}.{subtitles_ext}", + "subtitles_name": "{episode_file_path}.{lang}.{subtitles_ext}", "allow_auto_generated_subtitles": True, }, - "overrides": {"artist": "JMC"}, + "overrides": { + "tv_show_name": "JMC", + "tv_show_directory": output_directory, + "collection_season_1_url": "https://youtube.com/playlist?list=PL5BC0FC26BECA5A35", + "collection_season_1_name": "JMC - Season 1", + }, } diff --git a/tests/resources/expected_downloads_summaries/plugins/chapters/test_chapters_from_comments.json b/tests/resources/expected_downloads_summaries/plugins/chapters/test_chapters_from_comments.json index 42ea6952..67e3fd84 100644 --- a/tests/resources/expected_downloads_summaries/plugins/chapters/test_chapters_from_comments.json +++ b/tests/resources/expected_downloads_summaries/plugins/chapters/test_chapters_from_comments.json @@ -1,7 +1,7 @@ { ".ytdl-sub-chapters_from_comments-download-archive.json": "122723ce8d257eebb05178daa26141f6", "JMC/JMC - Move 78 - Automated Improvisation [Full Album]-thumb.jpg": "c12e6a6f242680d1096a1a99d74a62c6", - "JMC/JMC - Move 78 - Automated Improvisation [Full Album].info.json": "f6045c28cc588187167a3f42e6f37f0c", + "JMC/JMC - Move 78 - Automated Improvisation [Full Album].info.json": "ee8437e52186219ff510214138f00e28", "JMC/JMC - Move 78 - Automated Improvisation [Full Album].mp4": "8b8a9a731a19bc37feb091559f97ebbc", "JMC/JMC - Move 78 - Automated Improvisation [Full Album].nfo": "7a65b184d24c68fc0ec5380432250f5b" -} \ No newline at end of file +} diff --git a/tests/resources/expected_downloads_summaries/youtube/test_channel_full.json b/tests/resources/expected_downloads_summaries/youtube/test_channel_full.json index b0d22ec7..a4dc9b7b 100644 --- a/tests/resources/expected_downloads_summaries/youtube/test_channel_full.json +++ b/tests/resources/expected_downloads_summaries/youtube/test_channel_full.json @@ -25,7 +25,7 @@ "Project ⧸ Zombie/Season 2011/s2011.e112101 - qbMJh2df1M4.mp4": "60c3221125afbef78990bdcead78a82d", "Project ⧸ Zombie/Season 2011/s2011.e112101 - qbMJh2df1M4.nfo": "3c5e407a428161b75474eea13a82e009", "Project ⧸ Zombie/Season 2012/s2012.e012301 - y5-3ovwQQ_U-thumb.jpg": "54ebe9df801b278fdd17b21afa8373a6", - "Project ⧸ Zombie/Season 2012/s2012.e012301 - y5-3ovwQQ_U.mp4": "cdf5d458e148bbadba6dfed5b05e378b", + "Project ⧸ Zombie/Season 2012/s2012.e012301 - y5-3ovwQQ_U.mp4": "2141f0f928d55e94fef7f7d3c7a4aca0", "Project ⧸ Zombie/Season 2012/s2012.e012301 - y5-3ovwQQ_U.nfo": "a0d59e102f06b93718ab20e0238b4f95", "Project ⧸ Zombie/Season 2013/s2013.e071901 - c_PZdc0Zi7M-thumb.jpg": "e29d49433175de8a761af35c5307791f", "Project ⧸ Zombie/Season 2013/s2013.e071901 - c_PZdc0Zi7M.mp4": "3fb239f0a646e515bbaaeb6f6d5cdd69", diff --git a/tests/resources/expected_downloads_summaries/youtube/test_playlist.json b/tests/resources/expected_downloads_summaries/youtube/test_playlist.json index e6b6803a..e580bf5c 100644 --- a/tests/resources/expected_downloads_summaries/youtube/test_playlist.json +++ b/tests/resources/expected_downloads_summaries/youtube/test_playlist.json @@ -1,17 +1,14 @@ { - ".ytdl-sub-music_video_playlist_test-download-archive.json": "e7f7c35e33c132a96d1f12b18f388b35", - "JMC/JMC - Jesse's Minecraft Server [Trailer - Feb.1]-thumb.jpg": "b232d253df621aa770b780c1301d364d", - "JMC/JMC - Jesse's Minecraft Server [Trailer - Feb.1].info.json": "c719512a82647a676fa04d388c837e03", - "JMC/JMC - Jesse's Minecraft Server [Trailer - Feb.1].mp4": "7bfa1475504a7c8a4aaa8c89f9306331", - "JMC/JMC - Jesse's Minecraft Server [Trailer - Feb.1].nfo": "de6f9a17a5ebf92aa02b14f7342b8d36", - "JMC/JMC - Jesse's Minecraft Server [Trailer - Feb.27]-thumb.jpg": "d17c379ea8b362f5b97c6b213b0342cb", - "JMC/JMC - Jesse's Minecraft Server [Trailer - Feb.27].info.json": "d450357aa6856b11d079f1f11bbd92c5", - "JMC/JMC - Jesse's Minecraft Server [Trailer - Feb.27].mp4": "04ab5cb3cc12325d0c96a7cd04a8b91d", - "JMC/JMC - Jesse's Minecraft Server [Trailer - Feb.27].nfo": "9a4a57bb91f1ac97b588fdf66bac4c9b", - "JMC/JMC - Jesse's Minecraft Server [Trailer - Mar.21]-thumb.jpg": "e7830aa8a64b0cde65ba3f7e5fc56530", - "JMC/JMC - Jesse's Minecraft Server [Trailer - Mar.21].info.json": "8fdf1c054a6b335e9624690e08a33f1c", - "JMC/JMC - Jesse's Minecraft Server [Trailer - Mar.21].mp4": "025de6099a5c98e6397153c7a62d517d", - "JMC/JMC - Jesse's Minecraft Server [Trailer - Mar.21].nfo": "b98f1fa3e76b4592aeeaed4f6e220809", - "poster.jpg": "e7830aa8a64b0cde65ba3f7e5fc56530", - "tvshow.nfo": "e4123860532466ed5e0ebf2c9e44eb18" + "JMC/.ytdl-sub-music_video_playlist_test-download-archive.json": "8c8c22bd8a9206fa0b3c9f2eff9827fc", + "JMC/Season 01/s01.e11020101 - 0SVukUyys10-thumb.jpg": "b232d253df621aa770b780c1301d364d", + "JMC/Season 01/s01.e11020101 - 0SVukUyys10.mp4": "95f3abaabccdd76461be5dace92e9489", + "JMC/Season 01/s01.e11020101 - 0SVukUyys10.nfo": "d0f2d843b091834c62ccf33e2d2e700f", + "JMC/Season 01/s01.e11022701 - qPybBrXspds-thumb.jpg": "d17c379ea8b362f5b97c6b213b0342cb", + "JMC/Season 01/s01.e11022701 - qPybBrXspds.mp4": "5465a5e5712351945e98667006b08747", + "JMC/Season 01/s01.e11022701 - qPybBrXspds.nfo": "7b230eb8560735cc3ca9598b7b1b2cb7", + "JMC/Season 01/s01.e11032101 - DBjFvs6HafU-thumb.jpg": "e7830aa8a64b0cde65ba3f7e5fc56530", + "JMC/Season 01/s01.e11032101 - DBjFvs6HafU.mp4": "c79ed62c72feb49bd02595322c6e1b89", + "JMC/Season 01/s01.e11032101 - DBjFvs6HafU.nfo": "a0d7fd6752ca56fe4206c4c711b5765d", + "JMC/season01-poster.jpg": "e7830aa8a64b0cde65ba3f7e5fc56530", + "JMC/tvshow.nfo": "f7892df46b4176dde645e23b8a4b653e" } \ No newline at end of file diff --git a/tests/resources/transaction_log_summaries/youtube/test_playlist.txt b/tests/resources/transaction_log_summaries/youtube/test_playlist.txt index 64f9caff..aa7eeb65 100644 --- a/tests/resources/transaction_log_summaries/youtube/test_playlist.txt +++ b/tests/resources/transaction_log_summaries/youtube/test_playlist.txt @@ -2,47 +2,213 @@ Files created: ---------------------------------------- {output_directory} .ytdl-sub-music_video_playlist_test-download-archive.json - poster.jpg + season01-poster.jpg tvshow.nfo NFO tags: test: + genre: ytdl-sub + namedseason: + attributes: + number: 1 + tag: JMC - Season 1 playlist_description: Trailers, Updates, etc playlist_title: Jesse's Minecraft Server playlist_uploader: Project Zombie -{output_directory}/JMC - JMC - Jesse's Minecraft Server [Trailer - Feb.1]-thumb.jpg - JMC - Jesse's Minecraft Server [Trailer - Feb.1].info.json - JMC - Jesse's Minecraft Server [Trailer - Feb.1].mp4 - JMC - Jesse's Minecraft Server [Trailer - Feb.1].nfo + title: JMC +{output_directory}/Season 01 + s01.e11020101 - 0SVukUyys10-thumb.jpg + s01.e11020101 - 0SVukUyys10.mp4 + Video Tags: + date: 2011-02-01 + episode_id: 11020101 + genre: ytdl-sub + show: JMC + synopsis: + https://www.youtube.com/watch?v=0SVukUyys10 + + To join the server, you must apply at: + http://www.jesseminecraft.webs.com/ + + This is just a brief video of the server as of Feb. 1, 2011. + + Texture Pack I Use: + http://www.minecraftforum.net/viewtopic.php?f=25&t=29164 + title: 2011-02-01 - Jesse's Minecraft Server [Trailer - Feb.1] + year: 2011 + s01.e11020101 - 0SVukUyys10.nfo NFO tags: - musicvideo: - album: Music Videos - artist: JMC + episodedetails: + aired: 2011-02-01 + episode: 11020101 + genre: ytdl-sub playlist_count: 3 playlist_index: 3 - title: Jesse's Minecraft Server [Trailer - Feb.1] + plot: + https://www.youtube.com/watch?v=0SVukUyys10 + + To join the server, you must apply at: + http://www.jesseminecraft.webs.com/ + + This is just a brief video of the server as of Feb. 1, 2011. + + Texture Pack I Use: + http://www.minecraftforum.net/viewtopic.php?f=25&t=29164 + season: 1 + title: 2011-02-01 - Jesse's Minecraft Server [Trailer - Feb.1] year: 2011 - JMC - Jesse's Minecraft Server [Trailer - Feb.27]-thumb.jpg - JMC - Jesse's Minecraft Server [Trailer - Feb.27].info.json - JMC - Jesse's Minecraft Server [Trailer - Feb.27].mp4 - JMC - Jesse's Minecraft Server [Trailer - Feb.27].nfo + s01.e11022701 - qPybBrXspds-thumb.jpg + s01.e11022701 - qPybBrXspds.mp4 + Video Tags: + date: 2011-02-27 + episode_id: 11022701 + genre: ytdl-sub + show: JMC + synopsis: + https://www.youtube.com/watch?v=qPybBrXspds + + Website Link: + http://jesseminecraft.webs.com/ + + All you have to do is read the rules, and fill out a quick, little application to join the Server so we know you read them. We do this to keep the griefers/newbs out, it only takes three minutes, it's not that big of a deal. + + Jesse's Minecraft Server is a City Roleplay/Survival Server, there is a currency, public transportation, strict rules for immersive gameplay, and has a decent size population. Over 750 people have logged onto the server. It is very important you read the rules, we don't tolerate idiots. + + There are over 200 empty properties that are ready for anyone to own! Join now! + + This is the server state as of Feb. 27, 2011. + ---------------------------------------------------------------------------------- + + Texture Pack: + http://www.minecraftforum.net/viewtopic.php?f=25&t=29164 + + Recording Software: + http://www.fraps.com/download.php + + Video Editing Software: + http://explore.live.com/windows-live-movie-maker?os=other + + Song: + Given to Fly - Pearl Jam + (Off of the 'Yield' album) + + I claim no ownership of this song, all the credit goes to Pearl Jam and their producers. + title: 2011-02-27 - Jesse's Minecraft Server [Trailer - Feb.27] + year: 2011 + s01.e11022701 - qPybBrXspds.nfo NFO tags: - musicvideo: - album: Music Videos - artist: JMC + episodedetails: + aired: 2011-02-27 + episode: 11022701 + genre: ytdl-sub playlist_count: 3 playlist_index: 2 - title: Jesse's Minecraft Server [Trailer - Feb.27] + plot: + https://www.youtube.com/watch?v=qPybBrXspds + + Website Link: + http://jesseminecraft.webs.com/ + + All you have to do is read the rules, and fill out a quick, little application to join the Server so we know you read them. We do this to keep the griefers/newbs out, it only takes three minutes, it's not that big of a deal. + + Jesse's Minecraft Server is a City Roleplay/Survival Server, there is a currency, public transportation, strict rules for immersive gameplay, and has a decent size population. Over 750 people have logged onto the server. It is very important you read the rules, we don't tolerate idiots. + + There are over 200 empty properties that are ready for anyone to own! Join now! + + This is the server state as of Feb. 27, 2011. + ---------------------------------------------------------------------------------- + + Texture Pack: + http://www.minecraftforum.net/viewtopic.php?f=25&t=29164 + + Recording Software: + http://www.fraps.com/download.php + + Video Editing Software: + http://explore.live.com/windows-live-movie-maker?os=other + + Song: + Given to Fly - Pearl Jam + (Off of the 'Yield' album) + + I claim no ownership of this song, all the credit goes to Pearl Jam and their producers. + season: 1 + title: 2011-02-27 - Jesse's Minecraft Server [Trailer - Feb.27] year: 2011 - JMC - Jesse's Minecraft Server [Trailer - Mar.21]-thumb.jpg - JMC - Jesse's Minecraft Server [Trailer - Mar.21].info.json - JMC - Jesse's Minecraft Server [Trailer - Mar.21].mp4 - JMC - Jesse's Minecraft Server [Trailer - Mar.21].nfo + s01.e11032101 - DBjFvs6HafU-thumb.jpg + s01.e11032101 - DBjFvs6HafU.mp4 + Video Tags: + date: 2011-03-21 + episode_id: 11032101 + genre: ytdl-sub + show: JMC + synopsis: + https://www.youtube.com/watch?v=DBjFvs6HafU + + Website Link: + http://jesseminecraft.webs.com/ + + To get on the whitelist, please look at the website linked above (^^^). Due to the overwhelming amount of people trying to join, I've made it so it costs $2 to become a member through paypal. All of it is explained on the website. + + Jesse's Minecraft Server is a City Roleplay/Survival Server, there is a currency, public transportation, strict rules for immersive gameplay, and has a decent size population. Over 1000 people have logged onto the server. It is very important you read the rules, we don't tolerate idiots. + + There are over 300 empty properties that are ready for anyone to own! Join now! + + This is the server state as of Mar. 21, 2011. + --------------------------------------------------------------------------------­-- + + Texture Pack: + http://www.minecraftforum.net/viewtopic.php?f=25&t=29164 + + Recording Software: + http://www.fraps.com/download.php + + Video Editing Software: + http://explore.live.com/windows-live-movie-maker?os=other + + Song: + Indifference - Pearl Jam + (Off of the 'Vs.' album) + + I claim no ownership of this song, all the credit goes to Pearl Jam and their producers. + title: 2011-03-21 - Jesse's Minecraft Server [Trailer - Mar.21] + year: 2011 + s01.e11032101 - DBjFvs6HafU.nfo NFO tags: - musicvideo: - album: Music Videos - artist: JMC + episodedetails: + aired: 2011-03-21 + episode: 11032101 + genre: ytdl-sub playlist_count: 3 playlist_index: 1 - title: Jesse's Minecraft Server [Trailer - Mar.21] + plot: + https://www.youtube.com/watch?v=DBjFvs6HafU + + Website Link: + http://jesseminecraft.webs.com/ + + To get on the whitelist, please look at the website linked above (^^^). Due to the overwhelming amount of people trying to join, I've made it so it costs $2 to become a member through paypal. All of it is explained on the website. + + Jesse's Minecraft Server is a City Roleplay/Survival Server, there is a currency, public transportation, strict rules for immersive gameplay, and has a decent size population. Over 1000 people have logged onto the server. It is very important you read the rules, we don't tolerate idiots. + + There are over 300 empty properties that are ready for anyone to own! Join now! + + This is the server state as of Mar. 21, 2011. + --------------------------------------------------------------------------------­-- + + Texture Pack: + http://www.minecraftforum.net/viewtopic.php?f=25&t=29164 + + Recording Software: + http://www.fraps.com/download.php + + Video Editing Software: + http://explore.live.com/windows-live-movie-maker?os=other + + Song: + Indifference - Pearl Jam + (Off of the 'Vs.' album) + + I claim no ownership of this song, all the credit goes to Pearl Jam and their producers. + season: 1 + title: 2011-03-21 - Jesse's Minecraft Server [Trailer - Mar.21] year: 2011 \ No newline at end of file diff --git a/tests/unit/config/conftest.py b/tests/unit/config/conftest.py index 480428bf..2e01937d 100644 --- a/tests/unit/config/conftest.py +++ b/tests/unit/config/conftest.py @@ -48,6 +48,6 @@ def output_options() -> Dict: @pytest.fixture def youtube_video() -> Dict: return { - "download_strategy": "video", - "video_url": "youtube.com/watch?v=123abc", + "download_strategy": "url", + "url": "youtube.com/watch?v=123abc", } diff --git a/tests/unit/config/test_config_file.py b/tests/unit/config/test_config_file.py index a08b35a9..0584ed8a 100644 --- a/tests/unit/config/test_config_file.py +++ b/tests/unit/config/test_config_file.py @@ -66,12 +66,13 @@ class TestConfigFilePartiallyValidatesPresets: f"Allowed fields: {', '.join(sorted(PRESET_KEYS))}", ) - def test_error__multiple_sources(self): - self._partial_validate( - preset_dict={"youtube": {}, "download": {}}, - expected_error_message="Validation error in partial_preset: " - "Contains the sources download, youtube but can only have one", - ) + # TODO: Update with future source + # def test_error__multiple_sources(self): + # self._partial_validate( + # preset_dict={"youtube": {}, "download": {}}, + # expected_error_message="Validation error in partial_preset: " + # "Contains the sources download, youtube but can only have one", + # ) def test_error__no_download_strategy(self): self._partial_validate( diff --git a/tests/unit/config/test_preset.py b/tests/unit/config/test_preset.py index d924c82c..4fc87fe0 100644 --- a/tests/unit/config/test_preset.py +++ b/tests/unit/config/test_preset.py @@ -10,18 +10,18 @@ class TestPreset: @pytest.mark.parametrize( "source, download_strategy", [ - ("youtube", {"download_strategy": "video", "video_url": "youtube.com/watch?v=123abc"}), + ("download", {"download_strategy": "url", "url": "youtube.com/watch?v=123abc"}), ( - "youtube", + "download", { - "download_strategy": "playlist", - "playlist_url": "youtube.com/playlist?list=123abc", + "download_strategy": "url", + "url": "youtube.com/playlist?list=123abc", }, ), - ("youtube", {"download_strategy": "channel", "channel_url": "youtube.com/c/123abc"}), + ("download", {"download_strategy": "url", "url": "youtube.com/c/123abc"}), ( - "soundcloud", - {"download_strategy": "albums_and_singles", "url": "soundcloud.com/123abc"}, + "download", + {"download_strategy": "url", "url": "soundcloud.com/123abc"}, ), ], ) @@ -37,7 +37,7 @@ class TestPreset: config=config_file, name="test", value={ - "youtube": youtube_video, + "download": youtube_video, "output_options": {"output_directory": "dir", "file_name": "{dne_var}"}, "overrides": {"dne_var": "not dne"}, }, @@ -49,7 +49,7 @@ class TestPreset: name="test", value={ "preset": "parent_preset_1", - "youtube": youtube_video, + "download": youtube_video, "output_options": output_options, "nfo_tags": {"tags": {"key-2": "this-preset"}}, }, @@ -74,7 +74,7 @@ class TestPreset: name="test", value={ "preset": preset_value, - "youtube": youtube_video, + "download": youtube_video, "output_options": output_options, "nfo_tags": {"tags": {"key-3": "this-preset"}}, }, @@ -97,7 +97,7 @@ class TestPreset: config=config_file, name="test", value={ - "youtube": youtube_video, + "download": youtube_video, "output_options": dict( output_options, **{"maintain_download_archive": True, "keep_files_after": "today-{ttl}"}, @@ -122,7 +122,7 @@ class TestPreset: name="test", value={ "preset": parent_preset, - "youtube": youtube_video, + "download": youtube_video, "output_options": output_options, }, ) @@ -138,7 +138,7 @@ class TestPreset: config=config_file, name="test", value={ - "youtube": youtube_video, + "download": youtube_video, "output_options": {"output_directory": "dir", "file_name": "{dne_var}"}, }, ) @@ -154,7 +154,7 @@ class TestPreset: config=config_file, name="test", value={ - "youtube": youtube_video, + "download": youtube_video, "output_options": {"output_directory": "{dne_var}", "file_name": "file"}, }, ) @@ -170,7 +170,7 @@ class TestPreset: config=config_file, name="test", value={ - "youtube": youtube_video, + "download": youtube_video, "output_options": {"output_directory": "dir", "file_name": "file"}, "nfo_tags": { "nfo_name": "the nfo name", @@ -191,7 +191,7 @@ class TestPreset: config=config_file, name="test", value={ - "youtube": youtube_video, + "download": youtube_video, "output_options": {"output_directory": "dir", "file_name": "file"}, "output_directory_nfo_tags": { "nfo_name": "the nfo name", diff --git a/tests/unit/config/test_subscription.py b/tests/unit/config/test_subscription.py index 1eff02f3..512a34d0 100644 --- a/tests/unit/config/test_subscription.py +++ b/tests/unit/config/test_subscription.py @@ -12,7 +12,7 @@ from ytdl_sub.subscriptions.subscription import Subscription def preset_file(youtube_video: Dict, output_options: Dict) -> Dict: return { "__preset__": { - "youtube": youtube_video, + "download": youtube_video, "output_options": output_options, "nfo_tags": { "tags": {"key-3": "file_preset"}, diff --git a/tests/unit/entries/conftest.py b/tests/unit/entries/conftest.py index e3ccaedc..6f003ebe 100644 --- a/tests/unit/entries/conftest.py +++ b/tests/unit/entries/conftest.py @@ -116,6 +116,7 @@ def mock_entry_to_dict( "playlist_webpage_url": "https://yourname.here", "playlist_uid": "abc123", "playlist_uploader": "abc123", + "playlist_uploader_sanitized": "abc123", "playlist_uploader_id": "abc123", "playlist_uploader_url": "https://yourname.here", "source_count": 1,