diff --git a/README.md b/README.md index 8f28f16e..fbc35e59 100644 --- a/README.md +++ b/README.md @@ -62,9 +62,10 @@ __preset__: # Root folder of all ytdl-sub Music Videos music_video_directory: "/music_videos" - # For 'Only Recent' preset, only keep vids uploaded in this range - date_range: "2months" - + # For 'Only Recent' preset, only keep vids within this range and limit + only_recent_date_range: "2months" + only_recent_max_files: 30 + # Pass any arg directly to yt-dlp's Python API ytdl_options: cookiefile: "/config/cookie.txt" diff --git a/docker/root/defaults/subscriptions.yaml b/docker/root/defaults/subscriptions.yaml index a63c7994..a7e129dc 100644 --- a/docker/root/defaults/subscriptions.yaml +++ b/docker/root/defaults/subscriptions.yaml @@ -14,8 +14,9 @@ __preset__: # Root folder of all ytdl-sub Music Videos music_video_directory: "/music_videos" - # For 'Only Recent' preset, only keep vids uploaded in this range - date_range: "2months" + # For 'Only Recent' preset, only keep vids within this range and limit + only_recent_date_range: "2months" + only_recent_max_files: 30 # Pass any arg directly to yt-dlp's Python API ytdl_options: diff --git a/examples/advanced/tv_show_config.yaml b/examples/advanced/tv_show_config.yaml index b2d0c694..9cfa4ca7 100644 --- a/examples/advanced/tv_show_config.yaml +++ b/examples/advanced/tv_show_config.yaml @@ -97,4 +97,5 @@ presets: - "Only Recent" overrides: - date_range: "2months" + only_recent_date_range: "2months" + only_recent_max_files: 30 \ No newline at end of file diff --git a/examples/tv_show_subscriptions.yaml b/examples/tv_show_subscriptions.yaml index e8fea998..d8b914cb 100644 --- a/examples/tv_show_subscriptions.yaml +++ b/examples/tv_show_subscriptions.yaml @@ -23,7 +23,10 @@ __preset__: overrides: tv_show_directory: "/tv_shows" # Root folder of all ytdl-sub TV Shows - date_range: "2months" # For 'Only Recent' preset, only keep vids uploaded in this range + + # For 'Only Recent' preset, only keep vids within this range and limit + only_recent_date_range: "2months" + only_recent_max_files: 30 # Choose the player you intend to use by setting the top-level key to be either: # - Plex TV Show by Date: diff --git a/src/ytdl_sub/config/preset_options.py b/src/ytdl_sub/config/preset_options.py index ff07ce92..b4738a8c 100644 --- a/src/ytdl_sub/config/preset_options.py +++ b/src/ytdl_sub/config/preset_options.py @@ -16,6 +16,7 @@ from ytdl_sub.validators.file_path_validators import StringFormatterFileNameVali from ytdl_sub.validators.strict_dict_validator import StrictDictValidator from ytdl_sub.validators.string_datetime import StringDatetimeValidator from ytdl_sub.validators.string_formatter_validators import DictFormatterValidator +from ytdl_sub.validators.string_formatter_validators import OverridesIntegerFormatterValidator from ytdl_sub.validators.string_formatter_validators import OverridesStringFormatterValidator from ytdl_sub.validators.string_formatter_validators import StringFormatterValidator from ytdl_sub.validators.validators import BoolValidator @@ -253,6 +254,7 @@ class OutputOptions(StrictDictValidator): "maintain_download_archive", "keep_files_before", "keep_files_after", + "keep_max_files", } @classmethod @@ -307,12 +309,15 @@ class OutputOptions(StrictDictValidator): self._keep_files_after = self._validate_key_if_present( "keep_files_after", StringDatetimeValidator ) + self._keep_max_files = self._validate_key_if_present( + "keep_max_files", OverridesIntegerFormatterValidator + ) if ( - self._keep_files_before or self._keep_files_after + self._keep_files_before or self._keep_files_after or self._keep_max_files ) and not self.maintain_download_archive: raise self._validation_exception( - "keep_files requires maintain_download_archive set to True" + "keep_files/keep_max requires maintain_download_archive set to True" ) @property @@ -387,7 +392,8 @@ class OutputOptions(StrictDictValidator): Optional. Requires ``maintain_download_archive`` set to True. Only keeps files that are uploaded before this datetime. By default, ytdl-sub will keep - files before ``now``, which implies all files. + files before ``now``, which implies all files. Can be used in conjunction with + ``keep_max_files``. """ return self._keep_files_before @@ -397,6 +403,17 @@ class OutputOptions(StrictDictValidator): Optional. Requires ``maintain_download_archive`` set to True. Only keeps files that are uploaded after this datetime. By default, ytdl-sub will keep - files after ``19000101``, which implies all files. + files after ``19000101``, which implies all files. Can be used in conjunction with + ``keep_max_files``. """ return self._keep_files_after + + @property + def keep_max_files(self) -> Optional[OverridesIntegerFormatterValidator]: + """ + Optional. Requires ``maintain_download_archive`` set to True. + + Only keeps N most recently uploaded videos. If set to <= 0, ``keep_max_files`` will not be + applied. Can be used in conjunction with ``keep_files_before`` and ``keep_files_after``. + """ + return self._keep_max_files diff --git a/src/ytdl_sub/prebuilt_presets/helpers/common.yaml b/src/ytdl_sub/prebuilt_presets/helpers/common.yaml index 031b2964..9278d012 100644 --- a/src/ytdl_sub/prebuilt_presets/helpers/common.yaml +++ b/src/ytdl_sub/prebuilt_presets/helpers/common.yaml @@ -22,12 +22,15 @@ presets: "Only Recent": # Only fetch videos after today minus date_range date_range: - after: "today-{date_range}" + after: "today-{only_recent_date_range}" # Only keep files uploaded after date_range output_options: - keep_files_after: "today-{date_range}" + keep_files_after: "today-{only_recent_date_range}" + keep_max_files: "{only_recent_max_files}" # Set the default date_range to 2 months overrides: - date_range: "2months" \ No newline at end of file + date_range: "2months" # keep for legacy-reasons + only_recent_date_range: "{date_range}" + only_recent_max_files: 0 \ No newline at end of file diff --git a/src/ytdl_sub/subscriptions/subscription_download.py b/src/ytdl_sub/subscriptions/subscription_download.py index 29bd2165..312fb46f 100644 --- a/src/ytdl_sub/subscriptions/subscription_download.py +++ b/src/ytdl_sub/subscriptions/subscription_download.py @@ -145,8 +145,18 @@ class SubscriptionDownload(BaseSubscription, ABC): after=self.output_options.keep_files_after, overrides=self.overrides, ) - if date_range_to_keep: - self._enhanced_download_archive.remove_stale_files(date_range=date_range_to_keep) + + keep_max_files: Optional[int] = None + if self.output_options.keep_max_files: + # validated it can be cast to int within the validator + keep_max_files = int( + self.overrides.apply_formatter(self.output_options.keep_max_files) + ) + + if date_range_to_keep or self.output_options.keep_max_files is not None: + self._enhanced_download_archive.remove_stale_files( + date_range=date_range_to_keep, keep_max_files=keep_max_files + ) self._enhanced_download_archive.save_download_mappings() FileHandler.delete(self._enhanced_download_archive.working_file_path) diff --git a/src/ytdl_sub/validators/string_formatter_validators.py b/src/ytdl_sub/validators/string_formatter_validators.py index b3a83cc1..4408680f 100644 --- a/src/ytdl_sub/validators/string_formatter_validators.py +++ b/src/ytdl_sub/validators/string_formatter_validators.py @@ -218,6 +218,20 @@ class OverridesStringFormatterValidator(StringFormatterValidator): # pylint: enable=line-too-long +class OverridesIntegerFormatterValidator(StringFormatterValidator): + _expected_value_type_name = "integer" + + def apply_formatter(self, variable_dict: Dict[str, str]) -> str: + output = super().apply_formatter(variable_dict) + try: + int(output) + except Exception as exc: + raise self._validation_exception( + f"Expected an integer, but received '{output}'" + ) from exc + return output + + class ListFormatterValidator(ListValidator[StringFormatterValidator]): _inner_list_type = StringFormatterValidator diff --git a/src/ytdl_sub/ytdl_additions/enhanced_download_archive.py b/src/ytdl_sub/ytdl_additions/enhanced_download_archive.py index ea5777e1..584f5541 100644 --- a/src/ytdl_sub/ytdl_additions/enhanced_download_archive.py +++ b/src/ytdl_sub/ytdl_additions/enhanced_download_archive.py @@ -537,7 +537,16 @@ class EnhancedDownloadArchive: return self - def remove_stale_files(self, date_range: DateRange) -> "EnhancedDownloadArchive": + def _remove_entry(self, uid: str, mapping: DownloadMapping) -> None: + for file_name in mapping.file_names: + self._file_handler.delete_file_from_output_directory(file_name=file_name) + + self.mapping.remove_entry(entry_id=uid) + self.num_entries_removed += 1 + + def remove_stale_files( + self, date_range: Optional[DateRange], keep_max_files: Optional[int] + ) -> "EnhancedDownloadArchive": """ Checks all entries within the mappings. If any entries' upload dates are not within the provided date range, delete them. @@ -545,22 +554,32 @@ class EnhancedDownloadArchive: Parameters ---------- date_range - Date range the upload date must be in to not get deleted + Optional. Date range the upload date must be in to not get deleted + keep_max_files + Optional. Max number of files to keep Returns ------- self """ - stale_mappings: Dict[str, DownloadMapping] = self.mapping.get_entries_out_of_range( - date_range=date_range - ) + if date_range is not None: + stale_mappings: Dict[str, DownloadMapping] = self.mapping.get_entries_out_of_range( + date_range=date_range + ) - for uid, mapping in stale_mappings.items(): - for file_name in mapping.file_names: - self._file_handler.delete_file_from_output_directory(file_name=file_name) + for uid, mapping in stale_mappings.items(): + self._remove_entry(uid=uid, mapping=mapping) - self.mapping.remove_entry(entry_id=uid) - self.num_entries_removed += 1 + if keep_max_files is not None and keep_max_files > 0: + num_files = 0 + for uid, mapping in sorted( + self.mapping.entry_mappings.items(), + key=lambda kv_: kv_[1].upload_date, + reverse=True, + ): + num_files += 1 + if num_files > keep_max_files: + self._remove_entry(uid=uid, mapping=mapping) return self diff --git a/tests/e2e/youtube/test_channel.py b/tests/e2e/youtube/test_channel.py index 82962f8a..2b36cf9b 100644 --- a/tests/e2e/youtube/test_channel.py +++ b/tests/e2e/youtube/test_channel.py @@ -12,7 +12,9 @@ from ytdl_sub.subscriptions.subscription import Subscription @pytest.fixture def channel_preset_dict(output_directory): return { - "preset": "TV Show Full Archive", + "preset": [ + "TV Show Full Archive", + ], "format": "worst[ext=mp4]", # download the worst format so it is fast "ytdl_options": { "max_views": 100000, # do not download the popular PJ concert @@ -77,6 +79,31 @@ class TestChannel: channel_preset_dict: Dict, ): subscription_name = "pz" + tv_show_name = channel_preset_dict["overrides"]["tv_show_name"] + archive_file_name = f".ytdl-sub-{subscription_name}-download-archive.json" + + pz_channel_mock_downloaded_with_archive_factory( + tv_show_name=tv_show_name, archive_file_name=archive_file_name + ) + + full_channel_subscription = Subscription.from_dict( + config=tv_show_config, preset_name=subscription_name, preset_dict=channel_preset_dict + ) + transaction_log = full_channel_subscription.download(dry_run=True) + assert transaction_log.is_empty + + def test_full_channel_existing_archive_keep_max_files( + self, + pz_channel_mock_downloaded_with_archive_factory: Callable, + tv_show_config: ConfigFile, + channel_preset_dict: Dict, + output_directory: str, + ): + subscription_name = "pz" + channel_preset_dict["preset"].append("Only Recent") + channel_preset_dict["overrides"]["only_recent_date_range"] = "10years" + channel_preset_dict["overrides"]["only_recent_max_files"] = 1 + full_channel_subscription = Subscription.from_dict( config=tv_show_config, preset_name=subscription_name, preset_dict=channel_preset_dict ) @@ -86,5 +113,10 @@ class TestChannel: pz_channel_mock_downloaded_with_archive_factory( tv_show_name=tv_show_name, archive_file_name=archive_file_name ) + transaction_log = full_channel_subscription.download(dry_run=True) - assert transaction_log.is_empty + assert_transaction_log_matches( + output_directory=output_directory, + transaction_log=transaction_log, + transaction_log_summary_file_name="youtube/test_channel_full_keep_max_files.txt", + ) diff --git a/tests/resources/transaction_log_summaries/youtube/test_channel_full_keep_max_files.txt b/tests/resources/transaction_log_summaries/youtube/test_channel_full_keep_max_files.txt new file mode 100644 index 00000000..75fd0729 --- /dev/null +++ b/tests/resources/transaction_log_summaries/youtube/test_channel_full_keep_max_files.txt @@ -0,0 +1,56 @@ +Files modified: +---------------------------------------- +{output_directory} + .ytdl-sub-pz-download-archive.json + +Files removed: +---------------------------------------- +{output_directory}/Season 2010 + s2010.e081301 - Oblivion Mod ＂Falcor＂ p.1-thumb.jpg + s2010.e081301 - Oblivion Mod ＂Falcor＂ p.1.info.json + s2010.e081301 - Oblivion Mod ＂Falcor＂ p.1.mp4 + s2010.e081301 - Oblivion Mod ＂Falcor＂ p.1.nfo + s2010.e120201 - Oblivion Mod ＂Falcor＂ p.2-thumb.jpg + s2010.e120201 - Oblivion Mod ＂Falcor＂ p.2.info.json + s2010.e120201 - Oblivion Mod ＂Falcor＂ p.2.mp4 + s2010.e120201 - Oblivion Mod ＂Falcor＂ p.2.nfo +{output_directory}/Season 2011 + s2011.e020101 - Jesse's Minecraft Server [Trailer - Feb.1]-thumb.jpg + s2011.e020101 - Jesse's Minecraft Server [Trailer - Feb.1].info.json + s2011.e020101 - Jesse's Minecraft Server [Trailer - Feb.1].mp4 + s2011.e020101 - Jesse's Minecraft Server [Trailer - Feb.1].nfo + s2011.e022701 - Jesse's Minecraft Server [Trailer - Feb.27]-thumb.jpg + s2011.e022701 - Jesse's Minecraft Server [Trailer - Feb.27].info.json + s2011.e022701 - Jesse's Minecraft Server [Trailer - Feb.27].mp4 + s2011.e022701 - Jesse's Minecraft Server [Trailer - Feb.27].nfo + s2011.e032101 - Jesse's Minecraft Server [Trailer - Mar.21]-thumb.jpg + s2011.e032101 - Jesse's Minecraft Server [Trailer - Mar.21].info.json + s2011.e032101 - Jesse's Minecraft Server [Trailer - Mar.21].mp4 + s2011.e032101 - Jesse's Minecraft Server [Trailer - Mar.21].nfo + s2011.e052901 - Project Zombie ｜Official Trailer｜ (IP： mc.projectzombie.beastnode.net)-thumb.jpg + s2011.e052901 - Project Zombie ｜Official Trailer｜ (IP： mc.projectzombie.beastnode.net).info.json + s2011.e052901 - Project Zombie ｜Official Trailer｜ (IP： mc.projectzombie.beastnode.net).mp4 + s2011.e052901 - Project Zombie ｜Official Trailer｜ (IP： mc.projectzombie.beastnode.net).nfo + s2011.e063001 - Project Zombie ｜Fin｜-thumb.jpg + s2011.e063001 - Project Zombie ｜Fin｜.info.json + s2011.e063001 - Project Zombie ｜Fin｜.mp4 + s2011.e063001 - Project Zombie ｜Fin｜.nfo + s2011.e112101 - Skyrim 'Ultra HD w⧸Mods' [PC]-thumb.jpg + s2011.e112101 - Skyrim 'Ultra HD w⧸Mods' [PC].info.json + s2011.e112101 - Skyrim 'Ultra HD w⧸Mods' [PC].mp4 + s2011.e112101 - Skyrim 'Ultra HD w⧸Mods' [PC].nfo +{output_directory}/Season 2012 + s2012.e012301 - Project Zombie ｜Map Trailer｜-thumb.jpg + s2012.e012301 - Project Zombie ｜Map Trailer｜.info.json + s2012.e012301 - Project Zombie ｜Map Trailer｜.mp4 + s2012.e012301 - Project Zombie ｜Map Trailer｜.nfo +{output_directory}/Season 2013 + s2013.e071901 - Project Zombie Rewind ｜Trailer｜-thumb.jpg + s2013.e071901 - Project Zombie Rewind ｜Trailer｜.info.json + s2013.e071901 - Project Zombie Rewind ｜Trailer｜.mp4 + s2013.e071901 - Project Zombie Rewind ｜Trailer｜.nfo +{output_directory}/Season 2018 + s2018.e102901 - Jesse's Minecraft Server ｜ Teaser Trailer-thumb.jpg + s2018.e102901 - Jesse's Minecraft Server ｜ Teaser Trailer.info.json + s2018.e102901 - Jesse's Minecraft Server ｜ Teaser Trailer.mp4 + s2018.e102901 - Jesse's Minecraft Server ｜ Teaser Trailer.nfo \ No newline at end of file