From dc60f26fb12206156865c26d4f6d470f5ff117b5 Mon Sep 17 00:00:00 2001 From: Jesse Bannon Date: Fri, 17 Nov 2023 00:30:37 -0800 Subject: [PATCH] [FEATURE] Add --- src/ytdl_sub/config/preset_options.py | 21 ++++++- .../subscriptions/subscription_download.py | 6 +- .../enhanced_download_archive.py | 39 +++++++++---- tests/e2e/youtube/test_channel.py | 29 +++++++++- .../test_channel_full_keep_max_files.txt | 56 +++++++++++++++++++ 5 files changed, 136 insertions(+), 15 deletions(-) create mode 100644 tests/resources/transaction_log_summaries/youtube/test_channel_full_keep_max_files.txt diff --git a/src/ytdl_sub/config/preset_options.py b/src/ytdl_sub/config/preset_options.py index ff07ce92..052af1a3 100644 --- a/src/ytdl_sub/config/preset_options.py +++ b/src/ytdl_sub/config/preset_options.py @@ -19,6 +19,7 @@ from ytdl_sub.validators.string_formatter_validators import DictFormatterValidat from ytdl_sub.validators.string_formatter_validators import OverridesStringFormatterValidator from ytdl_sub.validators.string_formatter_validators import StringFormatterValidator from ytdl_sub.validators.validators import BoolValidator +from ytdl_sub.validators.validators import IntValidator from ytdl_sub.validators.validators import LiteralDictValidator from ytdl_sub.validators.validators import Validator @@ -253,6 +254,7 @@ class OutputOptions(StrictDictValidator): "maintain_download_archive", "keep_files_before", "keep_files_after", + "keep_max_files", } @classmethod @@ -307,6 +309,7 @@ class OutputOptions(StrictDictValidator): self._keep_files_after = self._validate_key_if_present( "keep_files_after", StringDatetimeValidator ) + self._keep_max_files = self._validate_key_if_present("keep_max_files", IntValidator) if ( self._keep_files_before or self._keep_files_after @@ -387,7 +390,8 @@ class OutputOptions(StrictDictValidator): Optional. Requires ``maintain_download_archive`` set to True. Only keeps files that are uploaded before this datetime. By default, ytdl-sub will keep - files before ``now``, which implies all files. + files before ``now``, which implies all files. Can be used in conjunction with + ``keep_max_files``. """ return self._keep_files_before @@ -397,6 +401,19 @@ class OutputOptions(StrictDictValidator): Optional. Requires ``maintain_download_archive`` set to True. Only keeps files that are uploaded after this datetime. By default, ytdl-sub will keep - files after ``19000101``, which implies all files. + files after ``19000101``, which implies all files. Can be used in conjunction with + ``keep_max_files``. """ return self._keep_files_after + + @property + def keep_max_files(self) -> Optional[int]: + """ + Optional. Requires ``maintain_download_archive`` set to True. + + Only keeps N most recently uploaded videos. Can be used in conjunction with + ``keep_files_before`` and ``keep_files_after``. + """ + if self._keep_max_files: + return self._keep_max_files.value + return None diff --git a/src/ytdl_sub/subscriptions/subscription_download.py b/src/ytdl_sub/subscriptions/subscription_download.py index 29bd2165..fd284e59 100644 --- a/src/ytdl_sub/subscriptions/subscription_download.py +++ b/src/ytdl_sub/subscriptions/subscription_download.py @@ -145,8 +145,10 @@ class SubscriptionDownload(BaseSubscription, ABC): after=self.output_options.keep_files_after, overrides=self.overrides, ) - if date_range_to_keep: - self._enhanced_download_archive.remove_stale_files(date_range=date_range_to_keep) + if date_range_to_keep or self.output_options.keep_max_files is not None: + self._enhanced_download_archive.remove_stale_files( + date_range=date_range_to_keep, keep_max_files=self.output_options.keep_max_files + ) self._enhanced_download_archive.save_download_mappings() FileHandler.delete(self._enhanced_download_archive.working_file_path) diff --git a/src/ytdl_sub/ytdl_additions/enhanced_download_archive.py b/src/ytdl_sub/ytdl_additions/enhanced_download_archive.py index ea5777e1..9fe14454 100644 --- a/src/ytdl_sub/ytdl_additions/enhanced_download_archive.py +++ b/src/ytdl_sub/ytdl_additions/enhanced_download_archive.py @@ -537,7 +537,16 @@ class EnhancedDownloadArchive: return self - def remove_stale_files(self, date_range: DateRange) -> "EnhancedDownloadArchive": + def _remove_entry(self, uid: str, mapping: DownloadMapping) -> None: + for file_name in mapping.file_names: + self._file_handler.delete_file_from_output_directory(file_name=file_name) + + self.mapping.remove_entry(entry_id=uid) + self.num_entries_removed += 1 + + def remove_stale_files( + self, date_range: Optional[DateRange], keep_max_files: Optional[int] + ) -> "EnhancedDownloadArchive": """ Checks all entries within the mappings. If any entries' upload dates are not within the provided date range, delete them. @@ -545,22 +554,32 @@ class EnhancedDownloadArchive: Parameters ---------- date_range - Date range the upload date must be in to not get deleted + Optional. Date range the upload date must be in to not get deleted + keep_max_files + Optional. Max number of files to keep Returns ------- self """ - stale_mappings: Dict[str, DownloadMapping] = self.mapping.get_entries_out_of_range( - date_range=date_range - ) + if date_range is not None: + stale_mappings: Dict[str, DownloadMapping] = self.mapping.get_entries_out_of_range( + date_range=date_range + ) - for uid, mapping in stale_mappings.items(): - for file_name in mapping.file_names: - self._file_handler.delete_file_from_output_directory(file_name=file_name) + for uid, mapping in stale_mappings.items(): + self._remove_entry(uid=uid, mapping=mapping) - self.mapping.remove_entry(entry_id=uid) - self.num_entries_removed += 1 + if keep_max_files is not None: + num_files = 0 + for uid, mapping in sorted( + self.mapping.entry_mappings.items(), + key=lambda kv_: kv_[1].upload_date, + reverse=True, + ): + num_files += 1 + if num_files > keep_max_files: + self._remove_entry(uid=uid, mapping=mapping) return self diff --git a/tests/e2e/youtube/test_channel.py b/tests/e2e/youtube/test_channel.py index 82962f8a..f8341b12 100644 --- a/tests/e2e/youtube/test_channel.py +++ b/tests/e2e/youtube/test_channel.py @@ -77,6 +77,28 @@ class TestChannel: channel_preset_dict: Dict, ): subscription_name = "pz" + tv_show_name = channel_preset_dict["overrides"]["tv_show_name"] + archive_file_name = f".ytdl-sub-{subscription_name}-download-archive.json" + + pz_channel_mock_downloaded_with_archive_factory( + tv_show_name=tv_show_name, archive_file_name=archive_file_name + ) + + full_channel_subscription = Subscription.from_dict( + config=tv_show_config, preset_name=subscription_name, preset_dict=channel_preset_dict + ) + transaction_log = full_channel_subscription.download(dry_run=True) + assert transaction_log.is_empty + + def test_full_channel_existing_archive_keep_max_files( + self, + pz_channel_mock_downloaded_with_archive_factory: Callable, + tv_show_config: ConfigFile, + channel_preset_dict: Dict, + output_directory: str, + ): + subscription_name = "pz" + channel_preset_dict = dict(channel_preset_dict, **{"output_options": {"keep_max_files": 1}}) full_channel_subscription = Subscription.from_dict( config=tv_show_config, preset_name=subscription_name, preset_dict=channel_preset_dict ) @@ -86,5 +108,10 @@ class TestChannel: pz_channel_mock_downloaded_with_archive_factory( tv_show_name=tv_show_name, archive_file_name=archive_file_name ) + transaction_log = full_channel_subscription.download(dry_run=True) - assert transaction_log.is_empty + assert_transaction_log_matches( + output_directory=output_directory, + transaction_log=transaction_log, + transaction_log_summary_file_name="youtube/test_channel_full_keep_max_files.txt", + ) diff --git a/tests/resources/transaction_log_summaries/youtube/test_channel_full_keep_max_files.txt b/tests/resources/transaction_log_summaries/youtube/test_channel_full_keep_max_files.txt new file mode 100644 index 00000000..75fd0729 --- /dev/null +++ b/tests/resources/transaction_log_summaries/youtube/test_channel_full_keep_max_files.txt @@ -0,0 +1,56 @@ +Files modified: +---------------------------------------- +{output_directory} + .ytdl-sub-pz-download-archive.json + +Files removed: +---------------------------------------- +{output_directory}/Season 2010 + s2010.e081301 - Oblivion Mod ＂Falcor＂ p.1-thumb.jpg + s2010.e081301 - Oblivion Mod ＂Falcor＂ p.1.info.json + s2010.e081301 - Oblivion Mod ＂Falcor＂ p.1.mp4 + s2010.e081301 - Oblivion Mod ＂Falcor＂ p.1.nfo + s2010.e120201 - Oblivion Mod ＂Falcor＂ p.2-thumb.jpg + s2010.e120201 - Oblivion Mod ＂Falcor＂ p.2.info.json + s2010.e120201 - Oblivion Mod ＂Falcor＂ p.2.mp4 + s2010.e120201 - Oblivion Mod ＂Falcor＂ p.2.nfo +{output_directory}/Season 2011 + s2011.e020101 - Jesse's Minecraft Server [Trailer - Feb.1]-thumb.jpg + s2011.e020101 - Jesse's Minecraft Server [Trailer - Feb.1].info.json + s2011.e020101 - Jesse's Minecraft Server [Trailer - Feb.1].mp4 + s2011.e020101 - Jesse's Minecraft Server [Trailer - Feb.1].nfo + s2011.e022701 - Jesse's Minecraft Server [Trailer - Feb.27]-thumb.jpg + s2011.e022701 - Jesse's Minecraft Server [Trailer - Feb.27].info.json + s2011.e022701 - Jesse's Minecraft Server [Trailer - Feb.27].mp4 + s2011.e022701 - Jesse's Minecraft Server [Trailer - Feb.27].nfo + s2011.e032101 - Jesse's Minecraft Server [Trailer - Mar.21]-thumb.jpg + s2011.e032101 - Jesse's Minecraft Server [Trailer - Mar.21].info.json + s2011.e032101 - Jesse's Minecraft Server [Trailer - Mar.21].mp4 + s2011.e032101 - Jesse's Minecraft Server [Trailer - Mar.21].nfo + s2011.e052901 - Project Zombie ｜Official Trailer｜ (IP： mc.projectzombie.beastnode.net)-thumb.jpg + s2011.e052901 - Project Zombie ｜Official Trailer｜ (IP： mc.projectzombie.beastnode.net).info.json + s2011.e052901 - Project Zombie ｜Official Trailer｜ (IP： mc.projectzombie.beastnode.net).mp4 + s2011.e052901 - Project Zombie ｜Official Trailer｜ (IP： mc.projectzombie.beastnode.net).nfo + s2011.e063001 - Project Zombie ｜Fin｜-thumb.jpg + s2011.e063001 - Project Zombie ｜Fin｜.info.json + s2011.e063001 - Project Zombie ｜Fin｜.mp4 + s2011.e063001 - Project Zombie ｜Fin｜.nfo + s2011.e112101 - Skyrim 'Ultra HD w⧸Mods' [PC]-thumb.jpg + s2011.e112101 - Skyrim 'Ultra HD w⧸Mods' [PC].info.json + s2011.e112101 - Skyrim 'Ultra HD w⧸Mods' [PC].mp4 + s2011.e112101 - Skyrim 'Ultra HD w⧸Mods' [PC].nfo +{output_directory}/Season 2012 + s2012.e012301 - Project Zombie ｜Map Trailer｜-thumb.jpg + s2012.e012301 - Project Zombie ｜Map Trailer｜.info.json + s2012.e012301 - Project Zombie ｜Map Trailer｜.mp4 + s2012.e012301 - Project Zombie ｜Map Trailer｜.nfo +{output_directory}/Season 2013 + s2013.e071901 - Project Zombie Rewind ｜Trailer｜-thumb.jpg + s2013.e071901 - Project Zombie Rewind ｜Trailer｜.info.json + s2013.e071901 - Project Zombie Rewind ｜Trailer｜.mp4 + s2013.e071901 - Project Zombie Rewind ｜Trailer｜.nfo +{output_directory}/Season 2018 + s2018.e102901 - Jesse's Minecraft Server ｜ Teaser Trailer-thumb.jpg + s2018.e102901 - Jesse's Minecraft Server ｜ Teaser Trailer.info.json + s2018.e102901 - Jesse's Minecraft Server ｜ Teaser Trailer.mp4 + s2018.e102901 - Jesse's Minecraft Server ｜ Teaser Trailer.nfo \ No newline at end of file