[FEATURE] Add output_options.keep_max_files to limit number of downloads per subscription (#820)

Adds a new mechanism to limit subscriptions' number of videos to an explicit number (only supported date ranges prior to this).

Example usage:
```
output_options:
  keep_max_files: 10
```
will only keep 10 videos at max.

In addition:
- `keep_max_files` is bundled into the `Only Recent` preset and can be used via setting the `only_recent_max_files` override variable (see examples or README for usage).
- Changed the `date_range` variable name to `only_recent_date_range` for more clarity.  Usage of `date_range` will continue to work.

README.md

@@ -62,9 +62,10 @@ __preset__:
     # Root folder of all ytdl-sub Music Videos
     music_video_directory: "/music_videos"
     
-    # For 'Only Recent' preset, only keep vids uploaded in this range
+    # For 'Only Recent' preset, only keep vids within this range and limit
-    date_range: "2months"
+    only_recent_date_range: "2months"
-  
+    only_recent_max_files: 30
+    
   # Pass any arg directly to yt-dlp's Python API
   ytdl_options:
     cookiefile: "/config/cookie.txt" 

docker/root/defaults/subscriptions.yaml

@@ -14,8 +14,9 @@ __preset__:
     # Root folder of all ytdl-sub Music Videos
     music_video_directory: "/music_videos"
 
-    # For 'Only Recent' preset, only keep vids uploaded in this range
+    # For 'Only Recent' preset, only keep vids within this range and limit
-    date_range: "2months"
+    only_recent_date_range: "2months"
+    only_recent_max_files: 30
 
   # Pass any arg directly to yt-dlp's Python API
   ytdl_options:

examples/advanced/tv_show_config.yaml

@@ -97,4 +97,5 @@ presets:
       - "Only Recent"
 
     overrides:
-      date_range: "2months"
+      only_recent_date_range: "2months"
+      only_recent_max_files: 30

examples/tv_show_subscriptions.yaml

@@ -23,7 +23,10 @@
 __preset__:
   overrides:
     tv_show_directory: "/tv_shows"  # Root folder of all ytdl-sub TV Shows
-    date_range: "2months"  # For 'Only Recent' preset, only keep vids uploaded in this range
+
+    # For 'Only Recent' preset, only keep vids within this range and limit
+    only_recent_date_range: "2months"
+    only_recent_max_files: 30
 
 # Choose the player you intend to use by setting the top-level key to be either:
 #   - Plex TV Show by Date:

src/ytdl_sub/config/preset_options.py

@@ -16,6 +16,7 @@ from ytdl_sub.validators.file_path_validators import StringFormatterFileNameVali
 from ytdl_sub.validators.strict_dict_validator import StrictDictValidator
 from ytdl_sub.validators.string_datetime import StringDatetimeValidator
 from ytdl_sub.validators.string_formatter_validators import DictFormatterValidator
+from ytdl_sub.validators.string_formatter_validators import OverridesIntegerFormatterValidator
 from ytdl_sub.validators.string_formatter_validators import OverridesStringFormatterValidator
 from ytdl_sub.validators.string_formatter_validators import StringFormatterValidator
 from ytdl_sub.validators.validators import BoolValidator
@@ -253,6 +254,7 @@ class OutputOptions(StrictDictValidator):
         "maintain_download_archive",
         "keep_files_before",
         "keep_files_after",
+        "keep_max_files",
     }
 
     @classmethod
@@ -307,12 +309,15 @@ class OutputOptions(StrictDictValidator):
         self._keep_files_after = self._validate_key_if_present(
             "keep_files_after", StringDatetimeValidator
         )
+        self._keep_max_files = self._validate_key_if_present(
+            "keep_max_files", OverridesIntegerFormatterValidator
+        )
 
         if (
-            self._keep_files_before or self._keep_files_after
+            self._keep_files_before or self._keep_files_after or self._keep_max_files
         ) and not self.maintain_download_archive:
             raise self._validation_exception(
-                "keep_files requires maintain_download_archive set to True"
+                "keep_files/keep_max requires maintain_download_archive set to True"
             )
 
     @property
@@ -387,7 +392,8 @@ class OutputOptions(StrictDictValidator):
         Optional. Requires ``maintain_download_archive`` set to True.
 
         Only keeps files that are uploaded before this datetime. By default, ytdl-sub will keep
-        files before ``now``, which implies all files.
+        files before ``now``, which implies all files. Can be used in conjunction with
+        ``keep_max_files``.
         """
         return self._keep_files_before
 
@@ -397,6 +403,17 @@ class OutputOptions(StrictDictValidator):
         Optional. Requires ``maintain_download_archive`` set to True.
 
         Only keeps files that are uploaded after this datetime. By default, ytdl-sub will keep
-        files after ``19000101``, which implies all files.
+        files after ``19000101``, which implies all files. Can be used in conjunction with
+        ``keep_max_files``.
         """
         return self._keep_files_after
+
+    @property
+    def keep_max_files(self) -> Optional[OverridesIntegerFormatterValidator]:
+        """
+        Optional. Requires ``maintain_download_archive`` set to True.
+
+        Only keeps N most recently uploaded videos. If set to <= 0, ``keep_max_files`` will not be
+        applied. Can be used in conjunction with ``keep_files_before`` and ``keep_files_after``.
+        """
+        return self._keep_max_files

src/ytdl_sub/prebuilt_presets/helpers/common.yaml

@@ -22,12 +22,15 @@ presets:
   "Only Recent":
     # Only fetch videos after today minus date_range
     date_range:
-      after: "today-{date_range}"
+      after: "today-{only_recent_date_range}"
 
     # Only keep files uploaded after date_range
     output_options:
-      keep_files_after: "today-{date_range}"
+      keep_files_after: "today-{only_recent_date_range}"
+      keep_max_files: "{only_recent_max_files}"
 
     # Set the default date_range to 2 months
     overrides:
-      date_range: "2months"
+      date_range: "2months"  # keep for legacy-reasons
+      only_recent_date_range: "{date_range}"
+      only_recent_max_files: 0

src/ytdl_sub/subscriptions/subscription_download.py

@@ -145,8 +145,18 @@ class SubscriptionDownload(BaseSubscription, ABC):
                 after=self.output_options.keep_files_after,
                 overrides=self.overrides,
             )
-            if date_range_to_keep:
+
-                self._enhanced_download_archive.remove_stale_files(date_range=date_range_to_keep)
+            keep_max_files: Optional[int] = None
+            if self.output_options.keep_max_files:
+                # validated it can be cast to int within the validator
+                keep_max_files = int(
+                    self.overrides.apply_formatter(self.output_options.keep_max_files)
+                )
+
+            if date_range_to_keep or self.output_options.keep_max_files is not None:
+                self._enhanced_download_archive.remove_stale_files(
+                    date_range=date_range_to_keep, keep_max_files=keep_max_files
+                )
 
             self._enhanced_download_archive.save_download_mappings()
             FileHandler.delete(self._enhanced_download_archive.working_file_path)

src/ytdl_sub/validators/string_formatter_validators.py

@@ -218,6 +218,20 @@ class OverridesStringFormatterValidator(StringFormatterValidator):
 # pylint: enable=line-too-long
 
 
+class OverridesIntegerFormatterValidator(StringFormatterValidator):
+    _expected_value_type_name = "integer"
+
+    def apply_formatter(self, variable_dict: Dict[str, str]) -> str:
+        output = super().apply_formatter(variable_dict)
+        try:
+            int(output)
+        except Exception as exc:
+            raise self._validation_exception(
+                f"Expected an integer, but received '{output}'"
+            ) from exc
+        return output
+
+
 class ListFormatterValidator(ListValidator[StringFormatterValidator]):
     _inner_list_type = StringFormatterValidator
 

src/ytdl_sub/ytdl_additions/enhanced_download_archive.py

@@ -537,7 +537,16 @@ class EnhancedDownloadArchive:
 
         return self
 
-    def remove_stale_files(self, date_range: DateRange) -> "EnhancedDownloadArchive":
+    def _remove_entry(self, uid: str, mapping: DownloadMapping) -> None:
+        for file_name in mapping.file_names:
+            self._file_handler.delete_file_from_output_directory(file_name=file_name)
+
+        self.mapping.remove_entry(entry_id=uid)
+        self.num_entries_removed += 1
+
+    def remove_stale_files(
+        self, date_range: Optional[DateRange], keep_max_files: Optional[int]
+    ) -> "EnhancedDownloadArchive":
         """
         Checks all entries within the mappings. If any entries' upload dates are not within the
         provided date range, delete them.
@@ -545,22 +554,32 @@ class EnhancedDownloadArchive:
         Parameters
         ----------
         date_range
-            Date range the upload date must be in to not get deleted
+            Optional. Date range the upload date must be in to not get deleted
+        keep_max_files
+            Optional. Max number of files to keep
 
         Returns
         -------
         self
         """
-        stale_mappings: Dict[str, DownloadMapping] = self.mapping.get_entries_out_of_range(
+        if date_range is not None:
-            date_range=date_range
+            stale_mappings: Dict[str, DownloadMapping] = self.mapping.get_entries_out_of_range(
-        )
+                date_range=date_range
+            )
 
-        for uid, mapping in stale_mappings.items():
+            for uid, mapping in stale_mappings.items():
-            for file_name in mapping.file_names:
+                self._remove_entry(uid=uid, mapping=mapping)
-                self._file_handler.delete_file_from_output_directory(file_name=file_name)
 
-            self.mapping.remove_entry(entry_id=uid)
+        if keep_max_files is not None and keep_max_files > 0:
-            self.num_entries_removed += 1
+            num_files = 0
+            for uid, mapping in sorted(
+                self.mapping.entry_mappings.items(),
+                key=lambda kv_: kv_[1].upload_date,
+                reverse=True,
+            ):
+                num_files += 1
+                if num_files > keep_max_files:
+                    self._remove_entry(uid=uid, mapping=mapping)
 
         return self
 

tests/e2e/youtube/test_channel.py

@@ -12,7 +12,9 @@ from ytdl_sub.subscriptions.subscription import Subscription
 @pytest.fixture
 def channel_preset_dict(output_directory):
     return {
-        "preset": "TV Show Full Archive",
+        "preset": [
+            "TV Show Full Archive",
+        ],
         "format": "worst[ext=mp4]",  # download the worst format so it is fast
         "ytdl_options": {
             "max_views": 100000,  # do not download the popular PJ concert
@@ -77,6 +79,31 @@ class TestChannel:
         channel_preset_dict: Dict,
     ):
         subscription_name = "pz"
+        tv_show_name = channel_preset_dict["overrides"]["tv_show_name"]
+        archive_file_name = f".ytdl-sub-{subscription_name}-download-archive.json"
+
+        pz_channel_mock_downloaded_with_archive_factory(
+            tv_show_name=tv_show_name, archive_file_name=archive_file_name
+        )
+
+        full_channel_subscription = Subscription.from_dict(
+            config=tv_show_config, preset_name=subscription_name, preset_dict=channel_preset_dict
+        )
+        transaction_log = full_channel_subscription.download(dry_run=True)
+        assert transaction_log.is_empty
+
+    def test_full_channel_existing_archive_keep_max_files(
+        self,
+        pz_channel_mock_downloaded_with_archive_factory: Callable,
+        tv_show_config: ConfigFile,
+        channel_preset_dict: Dict,
+        output_directory: str,
+    ):
+        subscription_name = "pz"
+        channel_preset_dict["preset"].append("Only Recent")
+        channel_preset_dict["overrides"]["only_recent_date_range"] = "10years"
+        channel_preset_dict["overrides"]["only_recent_max_files"] = 1
+
         full_channel_subscription = Subscription.from_dict(
             config=tv_show_config, preset_name=subscription_name, preset_dict=channel_preset_dict
         )
@@ -86,5 +113,10 @@ class TestChannel:
         pz_channel_mock_downloaded_with_archive_factory(
             tv_show_name=tv_show_name, archive_file_name=archive_file_name
         )
+
         transaction_log = full_channel_subscription.download(dry_run=True)
-        assert transaction_log.is_empty
+        assert_transaction_log_matches(
+            output_directory=output_directory,
+            transaction_log=transaction_log,
+            transaction_log_summary_file_name="youtube/test_channel_full_keep_max_files.txt",
+        )

tests/resources/transaction_log_summaries/youtube/test_channel_full_keep_max_files.txt

@@ -0,0 +1,56 @@
+Files modified:
+----------------------------------------
+{output_directory}
+  .ytdl-sub-pz-download-archive.json
+
+Files removed:
+----------------------------------------
+{output_directory}/Season 2010
+  s2010.e081301 - Oblivion Mod ＂Falcor＂ p.1-thumb.jpg
+  s2010.e081301 - Oblivion Mod ＂Falcor＂ p.1.info.json
+  s2010.e081301 - Oblivion Mod ＂Falcor＂ p.1.mp4
+  s2010.e081301 - Oblivion Mod ＂Falcor＂ p.1.nfo
+  s2010.e120201 - Oblivion Mod ＂Falcor＂ p.2-thumb.jpg
+  s2010.e120201 - Oblivion Mod ＂Falcor＂ p.2.info.json
+  s2010.e120201 - Oblivion Mod ＂Falcor＂ p.2.mp4
+  s2010.e120201 - Oblivion Mod ＂Falcor＂ p.2.nfo
+{output_directory}/Season 2011
+  s2011.e020101 - Jesse's Minecraft Server [Trailer - Feb.1]-thumb.jpg
+  s2011.e020101 - Jesse's Minecraft Server [Trailer - Feb.1].info.json
+  s2011.e020101 - Jesse's Minecraft Server [Trailer - Feb.1].mp4
+  s2011.e020101 - Jesse's Minecraft Server [Trailer - Feb.1].nfo
+  s2011.e022701 - Jesse's Minecraft Server [Trailer - Feb.27]-thumb.jpg
+  s2011.e022701 - Jesse's Minecraft Server [Trailer - Feb.27].info.json
+  s2011.e022701 - Jesse's Minecraft Server [Trailer - Feb.27].mp4
+  s2011.e022701 - Jesse's Minecraft Server [Trailer - Feb.27].nfo
+  s2011.e032101 - Jesse's Minecraft Server [Trailer - Mar.21]-thumb.jpg
+  s2011.e032101 - Jesse's Minecraft Server [Trailer - Mar.21].info.json
+  s2011.e032101 - Jesse's Minecraft Server [Trailer - Mar.21].mp4
+  s2011.e032101 - Jesse's Minecraft Server [Trailer - Mar.21].nfo
+  s2011.e052901 - Project Zombie ｜Official Trailer｜ (IP： mc.projectzombie.beastnode.net)-thumb.jpg
+  s2011.e052901 - Project Zombie ｜Official Trailer｜ (IP： mc.projectzombie.beastnode.net).info.json
+  s2011.e052901 - Project Zombie ｜Official Trailer｜ (IP： mc.projectzombie.beastnode.net).mp4
+  s2011.e052901 - Project Zombie ｜Official Trailer｜ (IP： mc.projectzombie.beastnode.net).nfo
+  s2011.e063001 - Project Zombie ｜Fin｜-thumb.jpg
+  s2011.e063001 - Project Zombie ｜Fin｜.info.json
+  s2011.e063001 - Project Zombie ｜Fin｜.mp4
+  s2011.e063001 - Project Zombie ｜Fin｜.nfo
+  s2011.e112101 - Skyrim 'Ultra HD w⧸Mods' [PC]-thumb.jpg
+  s2011.e112101 - Skyrim 'Ultra HD w⧸Mods' [PC].info.json
+  s2011.e112101 - Skyrim 'Ultra HD w⧸Mods' [PC].mp4
+  s2011.e112101 - Skyrim 'Ultra HD w⧸Mods' [PC].nfo
+{output_directory}/Season 2012
+  s2012.e012301 - Project Zombie ｜Map Trailer｜-thumb.jpg
+  s2012.e012301 - Project Zombie ｜Map Trailer｜.info.json
+  s2012.e012301 - Project Zombie ｜Map Trailer｜.mp4
+  s2012.e012301 - Project Zombie ｜Map Trailer｜.nfo
+{output_directory}/Season 2013
+  s2013.e071901 - Project Zombie Rewind ｜Trailer｜-thumb.jpg
+  s2013.e071901 - Project Zombie Rewind ｜Trailer｜.info.json
+  s2013.e071901 - Project Zombie Rewind ｜Trailer｜.mp4
+  s2013.e071901 - Project Zombie Rewind ｜Trailer｜.nfo
+{output_directory}/Season 2018
+  s2018.e102901 - Jesse's Minecraft Server ｜ Teaser Trailer-thumb.jpg
+  s2018.e102901 - Jesse's Minecraft Server ｜ Teaser Trailer.info.json
+  s2018.e102901 - Jesse's Minecraft Server ｜ Teaser Trailer.mp4
+  s2018.e102901 - Jesse's Minecraft Server ｜ Teaser Trailer.nfo