From a1ccb0669ecfa7039aabf7bcce4abcf995cd51c2 Mon Sep 17 00:00:00 2001 From: jbannon Date: Sat, 9 Apr 2022 07:42:24 +0000 Subject: [PATCH] channel is close --- config.yaml | 28 ++++++++++++ subscriptions.yaml | 24 +++++++--- .../downloaders/youtube_downloader.py | 28 +++++++++++- ytdl_subscribe/entries/entry.py | 24 ++++++++++ ytdl_subscribe/subscriptions/subscription.py | 45 +++++++++---------- ytdl_subscribe/subscriptions/youtube.py | 12 ++++- .../metadata_options_validator.py | 5 ++- .../download_strategy_validators.py | 3 +- .../source_options/youtube_validators.py | 8 ++++ .../config/subscription_validator.py | 8 ++-- 10 files changed, 148 insertions(+), 37 deletions(-) diff --git a/config.yaml b/config.yaml index 22845859..a03a7b6d 100644 --- a/config.yaml +++ b/config.yaml @@ -43,3 +43,31 @@ presets: title: "{title}" album: "Music Videos" year: "{upload_year}" + + yt_channel: + youtube: + ytdl_options: + format: 'best' + ignoreerrors: True + output_options: + file_name: "{output_file_name}.{ext}" + convert_thumbnail: "jpeg" + thumbnail_name: "{output_file_name}.jpeg" + metadata_options: + nfo: + nfo_name: "{output_file_name}.nfo" + nfo_root: "episodedetails" + tags: + title: "{title}" + season: "{upload_year}" + episode: "{upload_month}{upload_day_padded}" + plot: "{description}" + year: "{upload_year}" + aired: "{upload_year}-{upload_month_padded}-{upload_day_padded}" + output_directory_nfo: + nfo_name: "tvshow.nfo" + nfo_root: "tvshow" + tags: + title: "{tv_show_name}" + overrides: + output_file_name: "Season {upload_year}/s{upload_year}.e{upload_month_padded}{upload_day_padded} - {sanitized_title}" \ No newline at end of file diff --git a/subscriptions.yaml b/subscriptions.yaml index 57db03dc..f1745adc 100644 --- a/subscriptions.yaml +++ b/subscriptions.yaml @@ -48,12 +48,24 @@ # overrides: # artist: "Rammstein" -gogo_video_test: - preset: "music_videos" +#gogo_video_test: +# preset: "music_videos" +# youtube: +# download_strategy: "video" +# video_id: "2RHTiXvELNg" +# output_options: +# output_directory: "/tmp/gogos" +# overrides: +# artist: "The Gogos" + +recent_channel_test: + preset: "yt_channel" + ytdl_options: + daterange: now-2weeks youtube: - download_strategy: "video" - video_id: "2RHTiXvELNg" + download_strategy: "channel" + channel_id: "UCRcCVDu_4oARsAKFkKYydAQ" output_options: - output_directory: "/tmp/gogos" + output_directory: "/tmp/dunkey" overrides: - artist: "The Gogos" \ No newline at end of file + tv_show_name: "Youtube: Dunkey" \ No newline at end of file diff --git a/ytdl_subscribe/downloaders/youtube_downloader.py b/ytdl_subscribe/downloaders/youtube_downloader.py index 758b8596..ab1555f5 100644 --- a/ytdl_subscribe/downloaders/youtube_downloader.py +++ b/ytdl_subscribe/downloaders/youtube_downloader.py @@ -23,6 +23,11 @@ class YoutubeDownloader(Downloader): """Returns full video url""" return f"https://youtube.com/watch?v={video_id}" + @classmethod + def channel_url(cls, channel_id: str) -> str: + """Returns full channel url""" + return f"https://youtube.com/channel/{channel_id}" + def _download_with_metadata(self, url: str) -> None: """ Do not get entries from the extract info, let it write to the info.json file and load @@ -36,12 +41,13 @@ class YoutubeDownloader(Downloader): _ = self.extract_info(ytdl_options_overrides=ytdl_metadata_override, url=url) def download_video(self, video_id: str) -> YoutubeVideo: + """Download a single Youtube video""" entry = self.extract_info(url=self.video_url(video_id)) return YoutubeVideo(**entry) def download_playlist(self, playlist_id: str) -> List[YoutubeVideo]: """ - Downloads all videos in a playlist + Downloads all videos in a Youtube playlist """ playlist_url = self.playlist_url(playlist_id=playlist_id) @@ -57,3 +63,23 @@ class YoutubeDownloader(Downloader): entries.append(YoutubeVideo(**json.load(file))) return entries + + def download_channel(self, channel_id: str) -> List[YoutubeVideo]: + """ + Downloads all videos from a channel + TODO: Add caching via ids in the metadata. Scrape output directory for any vid ids and + TODO: include it in the archive + """ + self._download_with_metadata(url=self.channel_url(channel_id)) + + # Load the entries from info.json + entries: List[YoutubeVideo] = [] + + # Load the entries from info.json + # TODO dupe code between this and playlist + for file_name in os.listdir(self.output_directory): + if file_name.endswith(".info.json") and not file_name.startswith(channel_id): + with open(Path(self.output_directory) / file_name, "r", encoding="utf-8") as file: + entries.append(YoutubeVideo(**json.load(file))) + + return entries diff --git a/ytdl_subscribe/entries/entry.py b/ytdl_subscribe/entries/entry.py index 83ab3fac..18406a32 100644 --- a/ytdl_subscribe/entries/entry.py +++ b/ytdl_subscribe/entries/entry.py @@ -62,6 +62,30 @@ class Entry: """Returns the entry's upload year""" return int(self.upload_date[:4]) + @property + def upload_month_padded(self) -> str: + """Returns the entry's upload month, padded""" + return self.upload_date[4:6] + + @property + def upload_day_padded(self) -> str: + """Returns the entry's upload day, padded""" + return self.upload_date[6:8] + + @property + def upload_month(self) -> int: + """Returns the entry's upload month as an int""" + return int(self.upload_month_padded.lstrip("0")) + + @property + def upload_day(self) -> int: + """Returns the entry's upload month as an int""" + return int(self.upload_day_padded.lstrip("0")) + + @property + def description(self) -> str: + return self.kwargs("description") + @property def thumbnail(self) -> str: """Returns the entry's thumbnail url""" diff --git a/ytdl_subscribe/subscriptions/subscription.py b/ytdl_subscribe/subscriptions/subscription.py index 8fe2200f..a9790b55 100644 --- a/ytdl_subscribe/subscriptions/subscription.py +++ b/ytdl_subscribe/subscriptions/subscription.py @@ -2,7 +2,7 @@ import os from abc import ABC from pathlib import Path from shutil import copyfile -from typing import Generic +from typing import Generic, overload, Optional from typing import Type from typing import TypeVar @@ -19,6 +19,7 @@ from ytdl_subscribe.validators.config.config_options.config_options_validator im from ytdl_subscribe.validators.config.metadata_options.metadata_options_validator import ( MetadataOptionsValidator, ) +from ytdl_subscribe.validators.config.metadata_options.nfo_validator import NFOValidator from ytdl_subscribe.validators.config.output_options.output_options_validator import ( OutputOptionsValidator, ) @@ -81,20 +82,14 @@ class Subscription(Generic[S], ABC): """Returns the directory that the downloader saves files to""" return str(Path(self.__config_options.working_directory.value) / Path(self.name)) - def _apply_formatter(self, entry: Entry, formatter: StringFormatterValidator) -> str: + def _apply_formatter(self, formatter: StringFormatterValidator, entry: Optional[Entry] = None) -> str: """ - Parameters - ---------- - entry - Entry with values to use in the formatter - formatter - The formatter itself - - Returns - ------- - The format_string after .format has been called on it using entry and override values + Returns the format_string after .format has been called on it using entry (if provided) and + override values """ - variable_dict = dict(entry.to_dict(), **self.overrides.dict_with_format_strings) + variable_dict = self.overrides.dict_with_format_strings + if entry: + variable_dict = dict(entry.to_dict(), **variable_dict) return formatter.apply_formatter(variable_dict) def _post_process_tagging(self, entry: Entry): @@ -104,21 +99,20 @@ class Subscription(Generic[S], ABC): id3_options = self.metadata_options.id3 audio_file = music_tag.load_file(entry.file_path(relative_directory=self.working_directory)) for tag, tag_formatter in id3_options.tags.dict.items(): - audio_file[tag] = self._apply_formatter(entry=entry, formatter=tag_formatter) + audio_file[tag] = self._apply_formatter(formatter=tag_formatter, entry=entry) audio_file.save() - def _post_process_nfo(self, entry: Entry): + def _post_process_nfo(self, nfo_options: NFOValidator, entry: Optional[Entry] = None): """ Creates an entry's NFO file using values defined in the metadata options """ nfo = {} - nfo_options = self.metadata_options.nfo for tag, tag_formatter in nfo_options.tags.dict.items(): - nfo[tag] = self._apply_formatter(entry=entry, formatter=tag_formatter) + nfo[tag] = self._apply_formatter(formatter=tag_formatter, entry=entry) # Write the nfo tags to XML with the nfo_root - nfo_root = self._apply_formatter(entry=entry, formatter=nfo_options.nfo_root) + nfo_root = self._apply_formatter(formatter=nfo_options.nfo_root, entry=entry) xml = dicttoxml.dicttoxml( obj=nfo, root=True, # We assume all NFOs have a root. Maybe we should not? @@ -126,9 +120,9 @@ class Subscription(Generic[S], ABC): attr_type=False, ) - nfo_file_name = self._apply_formatter(entry=entry, formatter=nfo_options.nfo_name) + nfo_file_name = self._apply_formatter(formatter=nfo_options.nfo_name, entry=entry) output_directory = self._apply_formatter( - entry=entry, formatter=self.output_options.output_directory + formatter=self.output_options.output_directory, entry=entry ) # Save the nfo's XML to file @@ -150,11 +144,11 @@ class Subscription(Generic[S], ABC): entry_source_file_path = entry.file_path(relative_directory=self.working_directory) output_directory = self._apply_formatter( - entry=entry, formatter=self.output_options.output_directory + formatter=self.output_options.output_directory, entry=entry ) output_file_name = self._apply_formatter( - entry=entry, formatter=self.output_options.file_name + formatter=self.output_options.file_name, entry=entry ) entry_destination_file_path = Path(output_directory) / Path(output_file_name) @@ -166,7 +160,7 @@ class Subscription(Generic[S], ABC): source_thumbnail_path = entry.thumbnail_path(relative_directory=self.working_directory) output_thumbnail_name = self._apply_formatter( - entry=entry, formatter=self.output_options.thumbnail_name + formatter=self.output_options.thumbnail_name, entry=entry ) output_thumbnail_path = Path(output_directory) / Path(output_thumbnail_name) @@ -185,4 +179,7 @@ class Subscription(Generic[S], ABC): copyfile(source_thumbnail_path, output_thumbnail_path) if self.metadata_options.nfo: - self._post_process_nfo(entry) + self._post_process_nfo(nfo_options=self.metadata_options.nfo, entry=entry) + + if self.metadata_options.output_directory_nfo: + self._post_process_nfo(nfo_options=self.metadata_options.output_directory_nfo) \ No newline at end of file diff --git a/ytdl_subscribe/subscriptions/youtube.py b/ytdl_subscribe/subscriptions/youtube.py index 80521c98..747ed919 100644 --- a/ytdl_subscribe/subscriptions/youtube.py +++ b/ytdl_subscribe/subscriptions/youtube.py @@ -1,7 +1,7 @@ from ytdl_subscribe.downloaders.youtube_downloader import YoutubeDownloader from ytdl_subscribe.subscriptions.subscription import Subscription from ytdl_subscribe.validators.config.source_options.youtube_validators import ( - YoutubePlaylistSourceValidator, + YoutubePlaylistSourceValidator, YoutubeChannelSourceValidator, ) from ytdl_subscribe.validators.config.source_options.youtube_validators import ( YoutubeVideoSourceValidator, @@ -18,6 +18,16 @@ class YoutubePlaylistSubscription(Subscription[YoutubePlaylistSourceValidator]): self.post_process_entry(entry) +class YoutubeChannelSubscription(Subscription[YoutubeChannelSourceValidator]): + def extract_info(self): + entries = self.get_downloader(YoutubeDownloader).download_channel( + channel_id=self.source_options.channel_id.value + ) + + for entry in entries: + self.post_process_entry(entry) + + class YoutubeVideoSubscription(Subscription[YoutubeVideoSourceValidator]): def extract_info(self): entry = self.get_downloader(YoutubeDownloader).download_video( diff --git a/ytdl_subscribe/validators/config/metadata_options/metadata_options_validator.py b/ytdl_subscribe/validators/config/metadata_options/metadata_options_validator.py index 8f5b9599..27b097b1 100644 --- a/ytdl_subscribe/validators/config/metadata_options/metadata_options_validator.py +++ b/ytdl_subscribe/validators/config/metadata_options/metadata_options_validator.py @@ -4,10 +4,13 @@ from ytdl_subscribe.validators.config.metadata_options.nfo_validator import NFOV class MetadataOptionsValidator(StrictDictValidator): - _optional_keys = {"id3", "nfo"} + _optional_keys = {"id3", "nfo", "output_directory_nfo"} def __init__(self, name, value): super().__init__(name, value) self.id3 = self._validate_key_if_present(key="id3", validator=Id3Validator) self.nfo = self._validate_key_if_present(key="nfo", validator=NFOValidator) + + # TODO: Ensure this does not depend on entry variables, only overrides + self.output_directory_nfo = self._validate_key_if_present(key="output_directory_nfo", validator=NFOValidator) diff --git a/ytdl_subscribe/validators/config/source_options/download_strategy_validators.py b/ytdl_subscribe/validators/config/source_options/download_strategy_validators.py index a1d0289b..27282b8c 100644 --- a/ytdl_subscribe/validators/config/source_options/download_strategy_validators.py +++ b/ytdl_subscribe/validators/config/source_options/download_strategy_validators.py @@ -11,7 +11,7 @@ from ytdl_subscribe.validators.config.source_options.soundcloud_validators impor ) from ytdl_subscribe.validators.config.source_options.source_validators import SourceValidator from ytdl_subscribe.validators.config.source_options.youtube_validators import ( - YoutubePlaylistSourceValidator, + YoutubePlaylistSourceValidator, YoutubeChannelSourceValidator, ) from ytdl_subscribe.validators.config.source_options.youtube_validators import ( YoutubeVideoSourceValidator, @@ -65,6 +65,7 @@ class SoundcloudDownloadStrategyValidator(DownloadStrategyValidator): class YoutubeDownloadStrategyValidator(DownloadStrategyValidator): _download_strategy_to_source_mapping = { + "channel": YoutubeChannelSourceValidator, "playlist": YoutubePlaylistSourceValidator, "video": YoutubeVideoSourceValidator, } diff --git a/ytdl_subscribe/validators/config/source_options/youtube_validators.py b/ytdl_subscribe/validators/config/source_options/youtube_validators.py index 9eda02ab..f4a8f60b 100644 --- a/ytdl_subscribe/validators/config/source_options/youtube_validators.py +++ b/ytdl_subscribe/validators/config/source_options/youtube_validators.py @@ -16,3 +16,11 @@ class YoutubeVideoSourceValidator(YoutubeSourceValidator): def __init__(self, name, value): super().__init__(name, value) self.video_id = self._validate_key("video_id", StringValidator) + + +class YoutubeChannelSourceValidator(YoutubeSourceValidator): + _required_keys = {"channel_id"} + + def __init__(self, name, value): + super().__init__(name, value) + self.channel_id = self._validate_key("channel_id", StringValidator) \ No newline at end of file diff --git a/ytdl_subscribe/validators/config/subscription_validator.py b/ytdl_subscribe/validators/config/subscription_validator.py index 862f8106..bc83d680 100644 --- a/ytdl_subscribe/validators/config/subscription_validator.py +++ b/ytdl_subscribe/validators/config/subscription_validator.py @@ -9,7 +9,8 @@ from mergedeep import mergedeep from ytdl_subscribe.subscriptions.soundcloud import SoundcloudAlbumsAndSinglesSubscription from ytdl_subscribe.subscriptions.subscription import Subscription -from ytdl_subscribe.subscriptions.youtube import YoutubePlaylistSubscription +from ytdl_subscribe.subscriptions.youtube import YoutubePlaylistSubscription, \ + YoutubeChannelSubscription from ytdl_subscribe.subscriptions.youtube import YoutubeVideoSubscription from ytdl_subscribe.validators.base.strict_dict_validator import StrictDictValidator from ytdl_subscribe.validators.base.validators import StringValidator @@ -22,7 +23,7 @@ from ytdl_subscribe.validators.config.source_options.soundcloud_validators impor SoundcloudAlbumsAndSinglesSourceValidator, ) from ytdl_subscribe.validators.config.source_options.youtube_validators import ( - YoutubePlaylistSourceValidator, + YoutubePlaylistSourceValidator, YoutubeChannelSourceValidator, ) from ytdl_subscribe.validators.config.source_options.youtube_validators import ( YoutubeVideoSourceValidator, @@ -78,7 +79,8 @@ class SubscriptionValidator(StrictDictValidator): subscription_class = YoutubePlaylistSubscription elif isinstance(self.preset.subscription_source, YoutubeVideoSourceValidator): subscription_class = YoutubeVideoSubscription - + elif isinstance(self.preset.subscription_source, YoutubeChannelSourceValidator): + subscription_class = YoutubeChannelSubscription if subscription_class is None: raise ValueError("subscription source class not found")