diff --git a/config.yaml b/config.yaml index d6f16c20..4051c0c1 100644 --- a/config.yaml +++ b/config.yaml @@ -54,6 +54,7 @@ presets: convert_thumbnail: "jpeg" thumbnail_name: "{output_file_name}.jpeg" maintain_download_archive: True + maintain_stale_file_deletion: True metadata_options: nfo: nfo_name: "{output_file_name}.nfo" diff --git a/ytdl_subscribe/subscriptions/subscription.py b/ytdl_subscribe/subscriptions/subscription.py index dd2c7fba..6c562fb0 100644 --- a/ytdl_subscribe/subscriptions/subscription.py +++ b/ytdl_subscribe/subscriptions/subscription.py @@ -29,7 +29,9 @@ from ytdl_subscribe.validators.config.output_options.output_options_validator im ) from ytdl_subscribe.validators.config.overrides.overrides_validator import OverridesValidator from ytdl_subscribe.validators.config.preset_validator import PresetValidator +from ytdl_subscribe.validators.config.source_options.mixins import DownloadDateRangeSource from ytdl_subscribe.validators.config.source_options.source_validators import SourceValidator +from ytdl_subscribe.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive S = TypeVar("S", bound=SourceValidator) D = TypeVar("D", bound=Downloader) @@ -54,26 +56,17 @@ class Subscription(Generic[S], ABC): self.__config_options = config_options self.__preset_options = preset_options + self._enhanced_download_archive = EnhancedDownloadArchive( + subscription_name=name, + working_directory=self.working_directory, + output_directory=self.output_directory, + ) + @property def source_options(self) -> S: """Returns the source options defined for this subscription""" return self.__preset_options.subscription_source - def get_downloader( - self, downloader_type: Type[D], source_ytdl_options: Optional[Dict] = None - ) -> D: - """Returns the downloader that will be used to download media for this subscription""" - # if source_ytdl_options are present, override the ytdl_options with them - ytdl_options = self.__preset_options.ytdl_options.dict - if source_ytdl_options: - ytdl_options = dict(ytdl_options, **source_ytdl_options) - - return downloader_type( - output_directory=self.working_directory, - ytdl_options=ytdl_options, - download_archive_file_name=self.download_archive_file_name, - ) - @property def output_options(self) -> OutputOptionsValidator: """Returns the output options defined for this subscription""" @@ -98,9 +91,26 @@ class Subscription(Generic[S], ABC): def output_directory(self): return self._apply_formatter(formatter=self.output_options.output_directory) - @property - def download_archive_file_name(self): - return f".ytdl-subscribe-{self.name}-download-archive.txt" + def _archive_entry_file_name(self, entry: Optional[Entry], relative_file_name: str) -> None: + if entry: + self._enhanced_download_archive.mapping.add_entry( + entry=entry, entry_file_path=relative_file_name + ) + + def get_downloader( + self, downloader_type: Type[D], source_ytdl_options: Optional[Dict] = None + ) -> D: + """Returns the downloader that will be used to download media for this subscription""" + # if source_ytdl_options are present, override the ytdl_options with them + ytdl_options = self.__preset_options.ytdl_options.dict + if source_ytdl_options: + ytdl_options = dict(ytdl_options, **source_ytdl_options) + + return downloader_type( + output_directory=self.working_directory, + ytdl_options=ytdl_options, + download_archive_file_name=self._enhanced_download_archive.archive_file_name, + ) def _apply_formatter( self, formatter: StringFormatterValidator, entry: Optional[Entry] = None @@ -156,20 +166,26 @@ class Subscription(Generic[S], ABC): with open(nfo_file_path, "wb") as nfo_file: nfo_file.write(xml) + # Archive the nfo's file name + self._archive_entry_file_name(entry=entry, relative_file_name=nfo_file_name) + @contextlib.contextmanager def _maintain_archive_file(self): if not self.output_options.maintain_download_archive: return - existing_download_archive = Path(self.output_directory) / self.download_archive_file_name - working_directory_archive = Path(self.working_directory) / self.download_archive_file_name - - if os.path.exists(existing_download_archive): - shutil.copy(existing_download_archive, working_directory_archive) + self._enhanced_download_archive.prepare_download_archive() yield - shutil.copy(working_directory_archive, existing_download_archive) + date_range = None + if isinstance(self.source_options, DownloadDateRangeSource): + date_range = self.source_options.get_date_range() + + if date_range and self.output_options.maintain_stale_file_deletion: + self._enhanced_download_archive.remove_stale_files(date_range=date_range) + + self._enhanced_download_archive.save_download_archive() def download(self): """ @@ -203,6 +219,9 @@ class Subscription(Generic[S], ABC): os.makedirs(os.path.dirname(entry_destination_file_path), exist_ok=True) copyfile(entry_source_file_path, entry_destination_file_path) + self._archive_entry_file_name(entry=entry, relative_file_name=output_file_name) + + # TODO: move thumbnail to separate function # Download the thumbnail if its present if self.output_options.thumbnail_name: source_thumbnail_path = entry.thumbnail_path(relative_directory=self.working_directory) @@ -239,6 +258,9 @@ class Subscription(Generic[S], ABC): else: copyfile(source_thumbnail_path, output_thumbnail_path) + # Archive the thumbnail file name + self._archive_entry_file_name(entry=entry, relative_file_name=output_thumbnail_name) + if self.metadata_options.nfo: self._post_process_nfo(nfo_options=self.metadata_options.nfo, entry=entry) diff --git a/ytdl_subscribe/subscriptions/youtube.py b/ytdl_subscribe/subscriptions/youtube.py index e33df6c0..2b6eabd1 100644 --- a/ytdl_subscribe/subscriptions/youtube.py +++ b/ytdl_subscribe/subscriptions/youtube.py @@ -26,11 +26,9 @@ class YoutubePlaylistSubscription(Subscription[YoutubePlaylistSourceValidator]): class YoutubeChannelSubscription(Subscription[YoutubeChannelSourceValidator]): def _extract_info(self): source_ytdl_options = {} - if self.source_options.before or self.source_options.after: - source_ytdl_options["daterange"] = DateRange( - start=self.source_options.after.datetime_str if self.source_options.after else None, - end=self.source_options.before.datetime_str if self.source_options.before else None, - ) + source_date_range = self.source_options.get_date_range() + if source_date_range: + source_ytdl_options["daterange"] = source_ytdl_options downloader = self.get_downloader(YoutubeDownloader, source_ytdl_options=source_ytdl_options) entries = downloader.download_channel(channel_id=self.source_options.channel_id.value) diff --git a/ytdl_subscribe/validators/config/output_options/output_options_validator.py b/ytdl_subscribe/validators/config/output_options/output_options_validator.py index 2a845e2d..6aec32f3 100644 --- a/ytdl_subscribe/validators/config/output_options/output_options_validator.py +++ b/ytdl_subscribe/validators/config/output_options/output_options_validator.py @@ -17,7 +17,12 @@ class OutputOptionsValidator(StrictDictValidator): """Where to output the final files and thumbnails""" _required_keys = {"output_directory", "file_name"} - _optional_keys = {"convert_thumbnail", "thumbnail_name", "maintain_download_archive"} + _optional_keys = { + "convert_thumbnail", + "thumbnail_name", + "maintain_download_archive", + "maintain_stale_file_deletion", + } def __init__(self, name, value): super().__init__(name, value) @@ -40,5 +45,13 @@ class OutputOptionsValidator(StrictDictValidator): ) self.maintain_download_archive = self._validate_key_if_present( - key="maintain_download_archive", validator=BoolValidator + key="maintain_download_archive", validator=BoolValidator, default=False ) + self.maintain_stale_file_deletion = self._validate_key_if_present( + key="maintain_stale_file_deletion", validator=BoolValidator, default=False + ) + + if self.maintain_stale_file_deletion.value and not self.maintain_download_archive.value: + raise self._validation_exception( + "maintain_stale_file_deletion requires maintain_download_archive set to True" + ) diff --git a/ytdl_subscribe/validators/config/source_options/mixins.py b/ytdl_subscribe/validators/config/source_options/mixins.py new file mode 100644 index 00000000..d46fc5b4 --- /dev/null +++ b/ytdl_subscribe/validators/config/source_options/mixins.py @@ -0,0 +1,24 @@ +from abc import ABC +from typing import Optional + +from yt_dlp import DateRange + +from ytdl_subscribe.validators.base.strict_dict_validator import StrictDictValidator +from ytdl_subscribe.validators.base.string_datetime import StringDatetimeValidator + + +class DownloadDateRangeSource(StrictDictValidator, ABC): + optional_keys = {"before", "after"} + + def __init__(self, name, value): + super().__init__(name, value) + self.before = self._validate_key_if_present("before", StringDatetimeValidator) + self.after = self._validate_key_if_present("after", StringDatetimeValidator) + + def get_date_range(self) -> Optional[DateRange]: + if self.before or self.after: + return DateRange( + start=self.after.datetime_str if self.after else None, + end=self.before.datetime_str if self.before else None, + ) + return None diff --git a/ytdl_subscribe/validators/config/source_options/youtube_validators.py b/ytdl_subscribe/validators/config/source_options/youtube_validators.py index 2fe4585f..c1f8bdfe 100644 --- a/ytdl_subscribe/validators/config/source_options/youtube_validators.py +++ b/ytdl_subscribe/validators/config/source_options/youtube_validators.py @@ -1,5 +1,6 @@ from ytdl_subscribe.validators.base.string_datetime import StringDatetimeValidator from ytdl_subscribe.validators.base.validators import StringValidator +from ytdl_subscribe.validators.config.source_options.mixins import DownloadDateRangeSource from ytdl_subscribe.validators.config.source_options.source_validators import YoutubeSourceValidator @@ -19,12 +20,11 @@ class YoutubeVideoSourceValidator(YoutubeSourceValidator): self.video_id = self._validate_key("video_id", StringValidator) -class YoutubeChannelSourceValidator(YoutubeSourceValidator): +class YoutubeChannelSourceValidator(YoutubeSourceValidator, DownloadDateRangeSource): _required_keys = {"channel_id"} _optional_keys = {"before", "after"} def __init__(self, name, value): - super().__init__(name, value) + YoutubeSourceValidator.__init__(self, name, value) + DownloadDateRangeSource.__init__(self, name, value) self.channel_id = self._validate_key("channel_id", StringValidator) - self.before = self._validate_key_if_present("before", StringDatetimeValidator) - self.after = self._validate_key_if_present("after", StringDatetimeValidator) diff --git a/ytdl_subscribe/ytdl_additions/enhanced_download_archive.py b/ytdl_subscribe/ytdl_additions/enhanced_download_archive.py index d81e2c9d..0a8e298b 100644 --- a/ytdl_subscribe/ytdl_additions/enhanced_download_archive.py +++ b/ytdl_subscribe/ytdl_additions/enhanced_download_archive.py @@ -163,7 +163,7 @@ class EnhancedDownloadArchive: self._download_mapping: Optional[DownloadMappings] = None @property - def _archive_file_name(self) -> str: + def archive_file_name(self) -> str: """ :return: The download archive's file name (no path) """ @@ -210,7 +210,7 @@ class EnhancedDownloadArchive: raise ValueError("Tried to use download mapping before it was loaded") return self._download_mapping - def load(self) -> "EnhancedDownloadArchive": + def _load(self) -> "EnhancedDownloadArchive": download_archive_is_file = os.path.isfile(self._mapping_output_file_path) download_mapping_is_file = os.path.isfile(self._archive_output_file_path) @@ -243,7 +243,7 @@ class EnhancedDownloadArchive: return self - def copy_to_working_directory(self) -> "EnhancedDownloadArchive": + def _copy_to_working_directory(self) -> "EnhancedDownloadArchive": # If no download archive was found, then there is nothing to copy if not self._download_archive: return self @@ -253,6 +253,11 @@ class EnhancedDownloadArchive: return self + def prepare_download_archive(self) -> "EnhancedDownloadArchive": + self._load() + self._copy_to_working_directory() + return self + def remove_stale_files(self, date_range: DateRange) -> "EnhancedDownloadArchive": stale_mappings: Dict[str, DownloadMapping] = self.mapping.get_entries_out_of_range( date_range=date_range