diff --git a/config.yaml b/config.yaml index d7a35ae1..36e37c47 100644 --- a/config.yaml +++ b/config.yaml @@ -56,7 +56,6 @@ presets: convert_thumbnail: "jpeg" thumbnail_name: "{output_file_name}.jpg" maintain_download_archive: True - maintain_stale_file_deletion: True convert_thumbnail: to: "jpg" nfo_tags: diff --git a/ytdl_subscribe/config/preset.py b/ytdl_subscribe/config/preset.py index 4683f6fd..2f024401 100644 --- a/ytdl_subscribe/config/preset.py +++ b/ytdl_subscribe/config/preset.py @@ -1,4 +1,3 @@ -from abc import ABC from typing import Any from typing import Dict from typing import List @@ -6,25 +5,21 @@ from typing import Optional from typing import Tuple from typing import Type -from yt_dlp.utils import sanitize_filename - from ytdl_subscribe.config.preset_class_mappings import DownloadStrategyMapping from ytdl_subscribe.config.preset_class_mappings import PluginMapping +from ytdl_subscribe.config.preset_options import DownloadStrategyValidator +from ytdl_subscribe.config.preset_options import OutputOptions +from ytdl_subscribe.config.preset_options import Overrides +from ytdl_subscribe.config.preset_options import YTDLOptions from ytdl_subscribe.downloaders.downloader import Downloader from ytdl_subscribe.downloaders.downloader import DownloaderValidator -from ytdl_subscribe.entries.entry import Entry from ytdl_subscribe.plugins.plugin import Plugin from ytdl_subscribe.plugins.plugin import PluginOptions from ytdl_subscribe.utils.exceptions import StringFormattingVariableNotFoundException from ytdl_subscribe.utils.exceptions import ValidationException from ytdl_subscribe.validators.strict_dict_validator import StrictDictValidator -from ytdl_subscribe.validators.string_formatter_validators import DictFormatterValidator from ytdl_subscribe.validators.string_formatter_validators import OverridesStringFormatterValidator -from ytdl_subscribe.validators.string_formatter_validators import StringFormatterValidator -from ytdl_subscribe.validators.validators import BoolValidator from ytdl_subscribe.validators.validators import DictValidator -from ytdl_subscribe.validators.validators import LiteralDictValidator -from ytdl_subscribe.validators.validators import StringValidator from ytdl_subscribe.validators.validators import Validator PRESET_REQUIRED_KEYS = {"output_options"} @@ -36,100 +31,7 @@ PRESET_OPTIONAL_KEYS = { } -class YTDLOptions(LiteralDictValidator): - """Ensures `ytdl_options` is a dict""" - - -class Overrides(DictFormatterValidator): - """Ensures `overrides` is a dict""" - - def __init__(self, name, value): - super().__init__(name, value) - for key in self._keys: - sanitized_key_name = f"sanitized_{key}" - # First, sanitize the format string - self._value[sanitized_key_name] = sanitize_filename(self._value[key].format_string) - - # Then, convert it into a StringFormatterValidator - self._value[sanitized_key_name] = StringFormatterValidator( - name="__should_never_fail__", - value=self._value[sanitized_key_name], - ) - - def apply_formatter( - self, formatter: StringFormatterValidator, entry: Optional[Entry] = None - ) -> str: - """ - Returns the format_string after .format has been called on it using entry (if provided) and - override values - """ - variable_dict = self.dict_with_format_strings - if entry: - variable_dict = dict(entry.to_dict(), **variable_dict) - return formatter.apply_formatter(variable_dict) - - -class OutputOptions(StrictDictValidator): - """Where to output the final files and thumbnails""" - - _required_keys = {"output_directory", "file_name"} - _optional_keys = { - "thumbnail_name", - "maintain_download_archive", - "maintain_stale_file_deletion", - } - - def __init__(self, name, value): - super().__init__(name, value) - - # Output directory should resolve without any entry variables. - # This is to check the directory for any download-archives before any downloads begin - self.output_directory: OverridesStringFormatterValidator = self._validate_key( - key="output_directory", validator=OverridesStringFormatterValidator - ) - - # file name and thumbnails however can use entry variables - self.file_name: StringFormatterValidator = self._validate_key( - key="file_name", validator=StringFormatterValidator - ) - self.thumbnail_name = self._validate_key_if_present( - key="thumbnail_name", validator=StringFormatterValidator - ) - - self.maintain_download_archive = self._validate_key_if_present( - key="maintain_download_archive", validator=BoolValidator, default=False - ) - self.maintain_stale_file_deletion = self._validate_key_if_present( - key="maintain_stale_file_deletion", validator=BoolValidator, default=False - ) - - if self.maintain_stale_file_deletion.value and not self.maintain_download_archive.value: - raise self._validation_exception( - "maintain_stale_file_deletion requires maintain_download_archive set to True" - ) - - -class DownloadStrategyValidator(StrictDictValidator, ABC): - """ - Ensures a download strategy exists for a source. Does not validate any more than that. - The respective Downloader's option validator will do that. - """ - - # All media sources must define a download strategy - _required_keys = {"download_strategy"} - - # Extra fields will be strict-validated using other StictDictValidators - _allow_extra_keys = True - - def __init__(self, name: str, value: Any): - super().__init__(name=name, value=value) - self.name = self._validate_key( - key="download_strategy", - validator=StringValidator, - ).value - - -class PresetValidator(StrictDictValidator): +class Preset(StrictDictValidator): _required_keys = PRESET_REQUIRED_KEYS _optional_keys = PRESET_OPTIONAL_KEYS diff --git a/ytdl_subscribe/config/preset_options.py b/ytdl_subscribe/config/preset_options.py new file mode 100644 index 00000000..63f82bf1 --- /dev/null +++ b/ytdl_subscribe/config/preset_options.py @@ -0,0 +1,108 @@ +from abc import ABC +from typing import Any +from typing import Optional + +from yt_dlp.utils import sanitize_filename + +from ytdl_subscribe.entries.entry import Entry +from ytdl_subscribe.validators.date_range_validator import DateRangeValidator +from ytdl_subscribe.validators.strict_dict_validator import StrictDictValidator +from ytdl_subscribe.validators.string_formatter_validators import DictFormatterValidator +from ytdl_subscribe.validators.string_formatter_validators import OverridesStringFormatterValidator +from ytdl_subscribe.validators.string_formatter_validators import StringFormatterValidator +from ytdl_subscribe.validators.validators import BoolValidator +from ytdl_subscribe.validators.validators import LiteralDictValidator +from ytdl_subscribe.validators.validators import StringValidator + + +class YTDLOptions(LiteralDictValidator): + """Ensures `ytdl_options` is a dict""" + + +class Overrides(DictFormatterValidator): + """Ensures `overrides` is a dict""" + + def __init__(self, name, value): + super().__init__(name, value) + for key in self._keys: + sanitized_key_name = f"sanitized_{key}" + # First, sanitize the format string + self._value[sanitized_key_name] = sanitize_filename(self._value[key].format_string) + + # Then, convert it into a StringFormatterValidator + self._value[sanitized_key_name] = StringFormatterValidator( + name="__should_never_fail__", + value=self._value[sanitized_key_name], + ) + + def apply_formatter( + self, formatter: StringFormatterValidator, entry: Optional[Entry] = None + ) -> str: + """ + Returns the format_string after .format has been called on it using entry (if provided) and + override values + """ + variable_dict = self.dict_with_format_strings + if entry: + variable_dict = dict(entry.to_dict(), **variable_dict) + return formatter.apply_formatter(variable_dict) + + +class OutputOptions(StrictDictValidator): + """Where to output the final files and thumbnails""" + + _required_keys = {"output_directory", "file_name"} + _optional_keys = { + "thumbnail_name", + "maintain_download_archive", + "maintain_stale_file_deletion", + } + + def __init__(self, name, value): + super().__init__(name, value) + + # Output directory should resolve without any entry variables. + # This is to check the directory for any download-archives before any downloads begin + self.output_directory: OverridesStringFormatterValidator = self._validate_key( + key="output_directory", validator=OverridesStringFormatterValidator + ) + + # file name and thumbnails however can use entry variables + self.file_name: StringFormatterValidator = self._validate_key( + key="file_name", validator=StringFormatterValidator + ) + self.thumbnail_name = self._validate_key_if_present( + key="thumbnail_name", validator=StringFormatterValidator + ) + + self.maintain_download_archive = self._validate_key_if_present( + key="maintain_download_archive", validator=BoolValidator, default=False + ) + self.maintain_stale_file_deletion = self._validate_key_if_present( + key="maintain_stale_file_deletion", validator=DateRangeValidator, default=False + ) + + if self.maintain_stale_file_deletion and not self.maintain_download_archive: + raise self._validation_exception( + "maintain_stale_file_deletion requires maintain_download_archive set to True" + ) + + +class DownloadStrategyValidator(StrictDictValidator, ABC): + """ + Ensures a download strategy exists for a source. Does not validate any more than that. + The respective Downloader's option validator will do that. + """ + + # All media sources must define a download strategy + _required_keys = {"download_strategy"} + + # Extra fields will be strict-validated using other StictDictValidators + _allow_extra_keys = True + + def __init__(self, name: str, value: Any): + super().__init__(name=name, value=value) + self.name = self._validate_key( + key="download_strategy", + validator=StringValidator, + ).value diff --git a/ytdl_subscribe/config/subscription.py b/ytdl_subscribe/config/subscription.py index e679022f..0a5f4add 100644 --- a/ytdl_subscribe/config/subscription.py +++ b/ytdl_subscribe/config/subscription.py @@ -9,8 +9,8 @@ from mergedeep import mergedeep from ytdl_subscribe.config.config_file import ConfigFile from ytdl_subscribe.config.preset import PRESET_OPTIONAL_KEYS from ytdl_subscribe.config.preset import PRESET_REQUIRED_KEYS -from ytdl_subscribe.config.preset import Overrides -from ytdl_subscribe.config.preset import PresetValidator +from ytdl_subscribe.config.preset import Preset +from ytdl_subscribe.config.preset_options import Overrides from ytdl_subscribe.subscriptions.subscription import Subscription from ytdl_subscribe.validators.strict_dict_validator import StrictDictValidator from ytdl_subscribe.validators.validators import StringValidator @@ -52,7 +52,7 @@ class SubscriptionValidator(StrictDictValidator): preset_dict = mergedeep.merge(preset_dict, self._dict, strategy=mergedeep.Strategy.REPLACE) del preset_dict["preset"] - self.preset = PresetValidator( + self.preset = Preset( name=f"{self._name}.{preset_name}", value=preset_dict, ) diff --git a/ytdl_subscribe/downloaders/youtube_downloader.py b/ytdl_subscribe/downloaders/youtube_downloader.py index bf73ece1..a7fce100 100644 --- a/ytdl_subscribe/downloaders/youtube_downloader.py +++ b/ytdl_subscribe/downloaders/youtube_downloader.py @@ -13,7 +13,7 @@ from yt_dlp.utils import RejectedVideoReached from ytdl_subscribe.downloaders.downloader import Downloader from ytdl_subscribe.downloaders.downloader import DownloaderValidator from ytdl_subscribe.entries.youtube import YoutubeVideo -from ytdl_subscribe.validators.date_range_validator import DownloadDateRangeSource +from ytdl_subscribe.validators.date_range_validator import DateRangeValidator from ytdl_subscribe.validators.validators import StringValidator ############################################################################### @@ -54,14 +54,19 @@ class YoutubeDownloader( """Returns full channel url""" return f"https://youtube.com/channel/{channel_id}" - def _download_with_metadata( - self, url: str, ytdl_options_overrides: Optional[Dict] = None - ) -> None: + def _download_using_metadata( + self, + url: str, + ignore_prefix: str, + ytdl_options_overrides: Optional[Dict] = None, + ) -> List[YoutubeVideo]: """ Do not get entries from the extract info, let it write to the info.json file and load that instead. This is because if the video is already downloaded in a playlist, it will not fetch the metadata (maybe there is a way??) """ + entries: List[YoutubeVideo] = [] + ytdl_overrides = { "writeinfojson": True, } @@ -73,6 +78,14 @@ class YoutubeDownloader( except RejectedVideoReached: pass + # Load the entries from info.json, ignore the playlist entry + for file_name in os.listdir(self.working_directory): + if file_name.endswith(".info.json") and not file_name.startswith(ignore_prefix): + with open(Path(self.working_directory) / file_name, "r", encoding="utf-8") as file: + entries.append(YoutubeVideo(**json.load(file))) + + return entries + def download_video(self, video_id: str) -> YoutubeVideo: """Download a single Youtube video""" entry = self.extract_info(url=self.video_url(video_id)) @@ -82,20 +95,9 @@ class YoutubeDownloader( """ Downloads all videos in a Youtube playlist """ - playlist_url = self.playlist_url(playlist_id=playlist_id) - - self._download_with_metadata(url=playlist_url) - - # Load the entries from info.json, ignore the playlist entry - entries: List[YoutubeVideo] = [] - - # Load the entries from info.json, ignore the playlist entry - for file_name in os.listdir(self.working_directory): - if file_name.endswith(".info.json") and not file_name.startswith(playlist_id): - with open(Path(self.working_directory) / file_name, "r", encoding="utf-8") as file: - entries.append(YoutubeVideo(**json.load(file))) - - return entries + return self._download_using_metadata( + url=self.playlist_url(playlist_id=playlist_id), ignore_prefix=playlist_id + ) def download_channel( self, channel_id: str, ytdl_options_overrides: Optional[Dict] = None @@ -103,22 +105,12 @@ class YoutubeDownloader( """ Downloads all videos from a channel """ - self._download_with_metadata( - url=self.channel_url(channel_id), ytdl_options_overrides=ytdl_options_overrides + return self._download_using_metadata( + url=self.channel_url(channel_id), + ignore_prefix=channel_id, + ytdl_options_overrides=ytdl_options_overrides, ) - # Load the entries from info.json - entries: List[YoutubeVideo] = [] - - # Load the entries from info.json - # TODO dupe code between this and playlist - for file_name in os.listdir(self.working_directory): - if file_name.endswith(".info.json") and not file_name.startswith(channel_id): - with open(Path(self.working_directory) / file_name, "r", encoding="utf-8") as file: - entries.append(YoutubeVideo(**json.load(file))) - - return entries - ############################################################################### # Youtube single video downloader + options @@ -163,13 +155,13 @@ class YoutubePlaylistDownloader(YoutubeDownloader[YoutubePlaylistDownloaderOptio # Youtube channel downloader + options -class YoutubeChannelDownloaderOptions(YoutubeDownloaderOptions, DownloadDateRangeSource): +class YoutubeChannelDownloaderOptions(YoutubeDownloaderOptions, DateRangeValidator): _required_keys = {"channel_id"} _optional_keys = {"before", "after"} def __init__(self, name, value): YoutubeDownloaderOptions.__init__(self, name, value) - DownloadDateRangeSource.__init__(self, name, value) + DateRangeValidator.__init__(self, name, value) self.channel_id = self._validate_key("channel_id", StringValidator) diff --git a/ytdl_subscribe/plugins/plugin.py b/ytdl_subscribe/plugins/plugin.py index 7fc34457..8caebdd5 100644 --- a/ytdl_subscribe/plugins/plugin.py +++ b/ytdl_subscribe/plugins/plugin.py @@ -4,7 +4,7 @@ from typing import Type from typing import TypeVar from typing import final -from ytdl_subscribe.config.preset import Overrides +from ytdl_subscribe.config.preset_options import Overrides from ytdl_subscribe.entries.entry import Entry from ytdl_subscribe.validators.strict_dict_validator import StrictDictValidator from ytdl_subscribe.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive diff --git a/ytdl_subscribe/subscriptions/subscription.py b/ytdl_subscribe/subscriptions/subscription.py index 0aba1071..d7efe03b 100644 --- a/ytdl_subscribe/subscriptions/subscription.py +++ b/ytdl_subscribe/subscriptions/subscription.py @@ -3,23 +3,20 @@ import os import shutil from pathlib import Path from shutil import copyfile -from typing import Dict from typing import List -from typing import Optional from typing import Tuple from typing import Type from ytdl_subscribe.config.config_file import ConfigOptions -from ytdl_subscribe.config.preset import OutputOptions -from ytdl_subscribe.config.preset import Overrides -from ytdl_subscribe.config.preset import PresetValidator -from ytdl_subscribe.config.preset import YTDLOptions +from ytdl_subscribe.config.preset import Preset +from ytdl_subscribe.config.preset_options import OutputOptions +from ytdl_subscribe.config.preset_options import Overrides +from ytdl_subscribe.config.preset_options import YTDLOptions from ytdl_subscribe.downloaders.downloader import Downloader from ytdl_subscribe.downloaders.downloader import DownloaderValidator from ytdl_subscribe.entries.entry import Entry from ytdl_subscribe.plugins.plugin import Plugin from ytdl_subscribe.plugins.plugin import PluginOptions -from ytdl_subscribe.validators.date_range_validator import DownloadDateRangeSource from ytdl_subscribe.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive @@ -41,7 +38,7 @@ class Subscription: self, name: str, config_options: ConfigOptions, - preset_options: PresetValidator, + preset_options: Preset, ): """ Parameters @@ -49,7 +46,7 @@ class Subscription: name: str Name of the subscription config_options: ConfigOptions - preset_options: PresetValidator + preset_options: Preset """ self.name = name self.__config_options = config_options @@ -143,15 +140,14 @@ class Subscription: yield - # TODO: add date range in output options - # date_range = None - # if isinstance(self.source_options, DownloadDateRangeSource): - # date_range = self.source_options.get_date_range() - # - # if date_range and self.output_options.maintain_stale_file_deletion: - # self._enhanced_download_archive.remove_stale_files(date_range=date_range) - # - # self._enhanced_download_archive.save_download_archive() + # If output options maintains stale file deletion, perform the delete here prior to saving + # the download archive + if self.output_options.maintain_stale_file_deletion: + self._enhanced_download_archive.remove_stale_files( + date_range=self.output_options.maintain_stale_file_deletion.get_date_range() + ) + + self._enhanced_download_archive.save_download_archive() def _initialize_plugins(self) -> List[Plugin]: plugins: List[Plugin] = [] diff --git a/ytdl_subscribe/validators/date_range_validator.py b/ytdl_subscribe/validators/date_range_validator.py index 564dbce9..89d0cdaf 100644 --- a/ytdl_subscribe/validators/date_range_validator.py +++ b/ytdl_subscribe/validators/date_range_validator.py @@ -7,7 +7,7 @@ from ytdl_subscribe.validators.strict_dict_validator import StrictDictValidator from ytdl_subscribe.validators.string_datetime import StringDatetimeValidator -class DownloadDateRangeSource(StrictDictValidator, ABC): +class DateRangeValidator(StrictDictValidator, ABC): optional_keys = {"before", "after"} def __init__(self, name, value):