From aa3643cfb47f7c490f6683101f85e5e35419c7e8 Mon Sep 17 00:00:00 2001 From: Jesse Bannon Date: Wed, 31 Aug 2022 23:49:05 -0700 Subject: [PATCH] beter docs, add more overrides while we are at it --- docs/config.rst | 4 +++ src/ytdl_sub/config/preset.py | 2 +- src/ytdl_sub/downloaders/downloader.py | 20 +++++++------- src/ytdl_sub/downloaders/youtube/abc.py | 9 ------- src/ytdl_sub/downloaders/youtube/channel.py | 24 +++++++++++++---- src/ytdl_sub/downloaders/youtube/playlist.py | 26 ++++++++++++++++--- src/ytdl_sub/downloaders/youtube/video.py | 3 --- .../youtube/test_playlist.json | 8 +++--- .../youtube/test_channel_full.txt | 1 + .../youtube/test_playlist.txt | 4 ++- tests/e2e/youtube/test_channel.py | 5 ++++ tests/e2e/youtube/test_playlist.py | 6 ++++- 12 files changed, 73 insertions(+), 39 deletions(-) diff --git a/docs/config.rst b/docs/config.rst index eba07f73..27af4263 100644 --- a/docs/config.rst +++ b/docs/config.rst @@ -53,6 +53,8 @@ _______ :inherited-members: :exclude-members: get_date_range +.. autofunction:: ytdl_sub.downloaders.youtube.channel.YoutubeChannelDownloader.added_override_variables() + .. autofunction:: ytdl_sub.downloaders.youtube.channel.YoutubeChannelDownloader.ytdl_option_defaults() ------------------------------------------------------------------------------- @@ -66,6 +68,8 @@ ________ :member-order: bysource :inherited-members: +.. autofunction:: ytdl_sub.downloaders.youtube.playlist.YoutubePlaylistDownloader.added_override_variables() + .. autofunction:: ytdl_sub.downloaders.youtube.playlist.YoutubePlaylistDownloader.ytdl_option_defaults() ------------------------------------------------------------------------------- diff --git a/src/ytdl_sub/config/preset.py b/src/ytdl_sub/config/preset.py index 81b2b9d1..3de9d7ea 100644 --- a/src/ytdl_sub/config/preset.py +++ b/src/ytdl_sub/config/preset.py @@ -112,7 +112,7 @@ class Preset(StrictDictValidator): @property def _added_override_variables(self) -> List[str]: - return self.downloader_options.added_override_variables() + return self.downloader.added_override_variables() def __validate_and_get_downloader(self, downloader_source: str) -> Type[Downloader]: return self._validate_key(key=downloader_source, validator=DownloadStrategyValidator).get( diff --git a/src/ytdl_sub/downloaders/downloader.py b/src/ytdl_sub/downloaders/downloader.py index f7810c5b..074fe867 100644 --- a/src/ytdl_sub/downloaders/downloader.py +++ b/src/ytdl_sub/downloaders/downloader.py @@ -40,17 +40,6 @@ class DownloaderValidator(StrictDictValidator, ABC): Placeholder class to define downloader options """ - # pylint: disable=no-self-use - def added_override_variables(self) -> List[str]: - """ - Returns - ------- - List of override variables that this downloader adds - """ - return [] - - # pylint: enable=no-self-use - DownloaderOptionsT = TypeVar("DownloaderOptionsT", bound=DownloaderValidator) DownloaderEntryT = TypeVar("DownloaderEntryT", bound=Entry) @@ -83,6 +72,15 @@ class Downloader(DownloadArchiver, Generic[DownloaderOptionsT, DownloaderEntryT] """ return {"ignoreerrors": True} + @classmethod + def added_override_variables(cls) -> List[str]: + """ + Returns + ------- + List of override variables that this downloader adds + """ + return [] + def __init__( self, download_options: DownloaderOptionsT, diff --git a/src/ytdl_sub/downloaders/youtube/abc.py b/src/ytdl_sub/downloaders/youtube/abc.py index e0726e99..3ceec884 100644 --- a/src/ytdl_sub/downloaders/youtube/abc.py +++ b/src/ytdl_sub/downloaders/youtube/abc.py @@ -1,6 +1,5 @@ from abc import ABC from typing import Generic -from typing import List from typing import TypeVar from ytdl_sub.downloaders.downloader import Downloader @@ -13,14 +12,6 @@ class YoutubeDownloaderOptions(DownloaderValidator, ABC): Abstract source validator for all soundcloud sources. """ - def added_override_variables(self) -> List[str]: - """ - Returns - ------- - List of override variables that every youtube downloader should add - """ - return ["source_description"] - YoutubeDownloaderOptionsT = TypeVar("YoutubeDownloaderOptionsT", bound=YoutubeDownloaderOptions) YoutubeVideoT = TypeVar("YoutubeVideoT", bound=YoutubeVideo) diff --git a/src/ytdl_sub/downloaders/youtube/channel.py b/src/ytdl_sub/downloaders/youtube/channel.py index 7d0d8db5..701506b5 100644 --- a/src/ytdl_sub/downloaders/youtube/channel.py +++ b/src/ytdl_sub/downloaders/youtube/channel.py @@ -37,10 +37,6 @@ class YoutubeChannelDownloaderOptions(YoutubeDownloaderOptions): # optional channel_avatar_path: "poster.jpg" channel_banner_path: "fanart.jpg" - before: "now" - after: "today-2weeks" - - Adds the override variable ``source_description``, which contains the channel's description. """ _required_keys = {"channel_url"} @@ -130,6 +126,20 @@ class YoutubeChannelDownloader(YoutubeDownloader[YoutubeChannelDownloaderOptions }, ) + @classmethod + def added_override_variables(cls) -> List[str]: + """ + Adds the following :ref:`override ` variables: + + .. code-block:: yaml + + overrides: + source_uploader: # The channel's name + source_title: # The channel's name + source_description: # The channel's description + """ + return ["source_uploader", "source_title", "source_description"] + # pylint: enable=line-too-long def __init__( @@ -186,7 +196,11 @@ class YoutubeChannelDownloader(YoutubeDownloader[YoutubeChannelDownloaderOptions entries_to_download = self._get_channel_videos(entry_dicts=entry_dicts) self.overrides.add_override_variables( - variables_to_add={"source_description": self.channel.kwargs_get("description", "")} + variables_to_add={ + "source_uploader": self.channel.kwargs("uploader"), + "source_title": self.channel.kwargs("title"), + "source_description": self.channel.kwargs_get("description", ""), + } ) # Iterate in descending order to process older videos first. In case an error occurs and a diff --git a/src/ytdl_sub/downloaders/youtube/playlist.py b/src/ytdl_sub/downloaders/youtube/playlist.py index b5394a2f..2cd81e31 100644 --- a/src/ytdl_sub/downloaders/youtube/playlist.py +++ b/src/ytdl_sub/downloaders/youtube/playlist.py @@ -1,5 +1,6 @@ from typing import Dict from typing import Generator +from typing import List from ytdl_sub.downloaders.downloader import download_logger from ytdl_sub.downloaders.youtube.abc import YoutubeDownloader @@ -22,8 +23,6 @@ class YoutubePlaylistDownloaderOptions(YoutubeDownloaderOptions): # required download_strategy: "playlist" playlist_url: "https://www.youtube.com/playlist?list=UCsvn_Po0SmunchJYtttWpOxMg" - - Adds the override variable ``source_description``, which contains the playlist's description. """ _required_keys = {"playlist_url"} @@ -66,6 +65,20 @@ class YoutubePlaylistDownloader( **{"break_on_existing": True}, ) + @classmethod + def added_override_variables(cls) -> List[str]: + """ + Adds the following :ref:`override ` variables: + + .. code-block:: yaml + + overrides: + source_uploader: # The playlist's owner's channel name + source_title: # The playlist's title + source_description: # The playlist's description + """ + return ["source_uploader", "source_title", "source_description"] + # pylint: enable=line-too-long def download(self) -> Generator[YoutubePlaylistVideo, None, None]: @@ -82,14 +95,19 @@ class YoutubePlaylistDownloader( ) playlist = self._filter_entry_dicts(entry_dicts, extractor="youtube:tab")[0] + playlist_videos = self._filter_entry_dicts(entry_dicts, sort_by="playlist_index") + self.overrides.add_override_variables( - variables_to_add={"source_description": playlist.get("description", "")} + variables_to_add={ + "source_title": playlist["title"], + "source_uploader": playlist["uploader"], + "source_description": playlist.get("description", ""), + } ) # Iterate in reverse order to process older videos first. In case an error occurs and a # the playlist must be redownloaded, it will fetch most recent metadata first, and break # on the older video that's been processed and is in the download archive. - playlist_videos = self._filter_entry_dicts(entry_dicts, sort_by="playlist_index") for idx, entry_dict in enumerate(reversed(playlist_videos), start=1): video = YoutubePlaylistVideo( entry_dict=entry_dict, working_directory=self.working_directory diff --git a/src/ytdl_sub/downloaders/youtube/video.py b/src/ytdl_sub/downloaders/youtube/video.py index a60a63d8..ac3948ad 100644 --- a/src/ytdl_sub/downloaders/youtube/video.py +++ b/src/ytdl_sub/downloaders/youtube/video.py @@ -28,8 +28,6 @@ class YoutubeVideoDownloaderOptions(YoutubeDownloaderOptions): .. code-block:: bash ytdl-sub dl --preset "example_preset" --youtube.video_url "youtube.com/watch?v=VMAPTo7RVDo" - - Adds the override variable ``source_description``, which contains the video's description. """ _required_keys = {"video_url"} @@ -69,6 +67,5 @@ class YoutubeVideoDownloader(YoutubeDownloader[YoutubeVideoDownloaderOptions, Yo """Download a single Youtube video""" entry_dict = self.extract_info(url=self.download_options.video_url) video = YoutubeVideo(entry_dict=entry_dict, working_directory=self.working_directory) - self.overrides.add_override_variables({"source_description": video.description}) return [video] diff --git a/tests/e2e/resources/expected_downloads_summaries/youtube/test_playlist.json b/tests/e2e/resources/expected_downloads_summaries/youtube/test_playlist.json index 2fefa7ec..a5e97f22 100644 --- a/tests/e2e/resources/expected_downloads_summaries/youtube/test_playlist.json +++ b/tests/e2e/resources/expected_downloads_summaries/youtube/test_playlist.json @@ -1,16 +1,16 @@ { ".ytdl-sub-music_video_playlist_test-download-archive.json": "25b8e44961343116436584e341c7fe9b", "JMC - Jesse's Minecraft Server [Trailer - Feb.1]-thumb.jpg": "b232d253df621aa770b780c1301d364d", - "JMC - Jesse's Minecraft Server [Trailer - Feb.1].info.json": "41b5c510308bd4f7c37b6f629c3e1fdf", + "JMC - Jesse's Minecraft Server [Trailer - Feb.1].info.json": "149149c22855a42532bc11384a04d407", "JMC - Jesse's Minecraft Server [Trailer - Feb.1].mp4": "e66287b9832277b6a4d1554e29d9fdcc", "JMC - Jesse's Minecraft Server [Trailer - Feb.1].nfo": "f8fd72bb97ed03938487494ad9094ca0", "JMC - Jesse's Minecraft Server [Trailer - Feb.27]-thumb.jpg": "d17c379ea8b362f5b97c6b213b0342cb", - "JMC - Jesse's Minecraft Server [Trailer - Feb.27].info.json": "bceb3f579c6917c68528afd08c2459f9", + "JMC - Jesse's Minecraft Server [Trailer - Feb.27].info.json": "bb7a96dce625354e25ab7482a6e59912", "JMC - Jesse's Minecraft Server [Trailer - Feb.27].mp4": "04ab5cb3cc12325d0c96a7cd04a8b91d", "JMC - Jesse's Minecraft Server [Trailer - Feb.27].nfo": "6de4d997cfb300356072b4ebb09cbe38", "JMC - Jesse's Minecraft Server [Trailer - Mar.21]-thumb.jpg": "e7830aa8a64b0cde65ba3f7e5fc56530", - "JMC - Jesse's Minecraft Server [Trailer - Mar.21].info.json": "0ff8b4391f1bd55e27ff8f8349c2844b", + "JMC - Jesse's Minecraft Server [Trailer - Mar.21].info.json": "dda22d8d8dfdc4fb49c0deab4ab214b0", "JMC - Jesse's Minecraft Server [Trailer - Mar.21].mp4": "025de6099a5c98e6397153c7a62d517d", "JMC - Jesse's Minecraft Server [Trailer - Mar.21].nfo": "f000a6ed8caacb62a134a6ca81e3f308", - "tvshow.nfo": "228e93a278468b0a6a924259461a6d66" + "tvshow.nfo": "792b0594defdfd6642086b76fcc6a91b" } \ No newline at end of file diff --git a/tests/e2e/resources/transaction_log_summaries/youtube/test_channel_full.txt b/tests/e2e/resources/transaction_log_summaries/youtube/test_channel_full.txt index 04a7ce3e..685e186c 100644 --- a/tests/e2e/resources/transaction_log_summaries/youtube/test_channel_full.txt +++ b/tests/e2e/resources/transaction_log_summaries/youtube/test_channel_full.txt @@ -284,4 +284,5 @@ tvshow.nfo NFO tags: tvshow: plot: Plugin and map updates for the server Project Zombie. + source_uploader: Project Zombie title: Project / Zombie \ No newline at end of file diff --git a/tests/e2e/resources/transaction_log_summaries/youtube/test_playlist.txt b/tests/e2e/resources/transaction_log_summaries/youtube/test_playlist.txt index 65fb2abf..14e1f17f 100644 --- a/tests/e2e/resources/transaction_log_summaries/youtube/test_playlist.txt +++ b/tests/e2e/resources/transaction_log_summaries/youtube/test_playlist.txt @@ -34,4 +34,6 @@ JMC - Jesse's Minecraft Server [Trailer - Mar.21].nfo tvshow.nfo NFO tags: test: - source_desc: Trailers, Updates, etc \ No newline at end of file + source_description: Trailers, Updates, etc + source_title: Jesse's Minecraft Server + source_uploader: Project Zombie \ No newline at end of file diff --git a/tests/e2e/youtube/test_channel.py b/tests/e2e/youtube/test_channel.py index 5eab30fc..8cbde89e 100644 --- a/tests/e2e/youtube/test_channel.py +++ b/tests/e2e/youtube/test_channel.py @@ -22,6 +22,11 @@ def channel_preset_dict(output_directory): "subtitles_name": "{episode_name}.{lang}.{subtitles_ext}", "allow_auto_generated_subtitles": True, }, + "output_directory_nfo_tags": { + "tags": { + "source_uploader": "{source_uploader}", + } + }, "overrides": {"tv_show_name": "Project / Zombie"}, } diff --git a/tests/e2e/youtube/test_playlist.py b/tests/e2e/youtube/test_playlist.py index 3335925e..2f98df38 100644 --- a/tests/e2e/youtube/test_playlist.py +++ b/tests/e2e/youtube/test_playlist.py @@ -22,7 +22,11 @@ def playlist_preset_dict(output_directory): "output_directory_nfo_tags": { "nfo_name": "tvshow.nfo", "nfo_root": "test", - "tags": {"source_desc": "{source_description}"}, + "tags": { + "source_title": "{source_title}", + "source_uploader": "{source_uploader}", + "source_description": "{source_description}", + }, }, "subtitles": { "subtitles_name": "{music_video_name}.{lang}.{subtitles_ext}",