[FEATURE] Add output_options.keep_max_files to limit number of downloads per subscription (#820)
Adds a new mechanism to limit subscriptions' number of videos to an explicit number (only supported date ranges prior to this). Example usage: ``` output_options: keep_max_files: 10 ``` will only keep 10 videos at max. In addition: - `keep_max_files` is bundled into the `Only Recent` preset and can be used via setting the `only_recent_max_files` override variable (see examples or README for usage). - Changed the `date_range` variable name to `only_recent_date_range` for more clarity. Usage of `date_range` will continue to work.
This commit is contained in:
Jesse Bannon
GitHub
parent
52d1a5887e
commit
544af2207b
11 changed files with 185 additions and 28 deletions
|
|
@ -62,9 +62,10 @@ __preset__:
|
|||
# Root folder of all ytdl-sub Music Videos
|
||||
music_video_directory: "/music_videos"
|
||||
|
||||
# For 'Only Recent' preset, only keep vids uploaded in this range
|
||||
date_range: "2months"
|
||||
|
||||
# For 'Only Recent' preset, only keep vids within this range and limit
|
||||
only_recent_date_range: "2months"
|
||||
only_recent_max_files: 30
|
||||
|
||||
# Pass any arg directly to yt-dlp's Python API
|
||||
ytdl_options:
|
||||
cookiefile: "/config/cookie.txt"
|
||||
|
|
|
|||
|
|
@ -14,8 +14,9 @@ __preset__:
|
|||
# Root folder of all ytdl-sub Music Videos
|
||||
music_video_directory: "/music_videos"
|
||||
|
||||
# For 'Only Recent' preset, only keep vids uploaded in this range
|
||||
date_range: "2months"
|
||||
# For 'Only Recent' preset, only keep vids within this range and limit
|
||||
only_recent_date_range: "2months"
|
||||
only_recent_max_files: 30
|
||||
|
||||
# Pass any arg directly to yt-dlp's Python API
|
||||
ytdl_options:
|
||||
|
|
|
|||
|
|
@ -97,4 +97,5 @@ presets:
|
|||
- "Only Recent"
|
||||
|
||||
overrides:
|
||||
date_range: "2months"
|
||||
only_recent_date_range: "2months"
|
||||
only_recent_max_files: 30
|
||||
|
|
@ -23,7 +23,10 @@
|
|||
__preset__:
|
||||
overrides:
|
||||
tv_show_directory: "/tv_shows" # Root folder of all ytdl-sub TV Shows
|
||||
date_range: "2months" # For 'Only Recent' preset, only keep vids uploaded in this range
|
||||
|
||||
# For 'Only Recent' preset, only keep vids within this range and limit
|
||||
only_recent_date_range: "2months"
|
||||
only_recent_max_files: 30
|
||||
|
||||
# Choose the player you intend to use by setting the top-level key to be either:
|
||||
# - Plex TV Show by Date:
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@ from ytdl_sub.validators.file_path_validators import StringFormatterFileNameVali
|
|||
from ytdl_sub.validators.strict_dict_validator import StrictDictValidator
|
||||
from ytdl_sub.validators.string_datetime import StringDatetimeValidator
|
||||
from ytdl_sub.validators.string_formatter_validators import DictFormatterValidator
|
||||
from ytdl_sub.validators.string_formatter_validators import OverridesIntegerFormatterValidator
|
||||
from ytdl_sub.validators.string_formatter_validators import OverridesStringFormatterValidator
|
||||
from ytdl_sub.validators.string_formatter_validators import StringFormatterValidator
|
||||
from ytdl_sub.validators.validators import BoolValidator
|
||||
|
|
@ -253,6 +254,7 @@ class OutputOptions(StrictDictValidator):
|
|||
"maintain_download_archive",
|
||||
"keep_files_before",
|
||||
"keep_files_after",
|
||||
"keep_max_files",
|
||||
}
|
||||
|
||||
@classmethod
|
||||
|
|
@ -307,12 +309,15 @@ class OutputOptions(StrictDictValidator):
|
|||
self._keep_files_after = self._validate_key_if_present(
|
||||
"keep_files_after", StringDatetimeValidator
|
||||
)
|
||||
self._keep_max_files = self._validate_key_if_present(
|
||||
"keep_max_files", OverridesIntegerFormatterValidator
|
||||
)
|
||||
|
||||
if (
|
||||
self._keep_files_before or self._keep_files_after
|
||||
self._keep_files_before or self._keep_files_after or self._keep_max_files
|
||||
) and not self.maintain_download_archive:
|
||||
raise self._validation_exception(
|
||||
"keep_files requires maintain_download_archive set to True"
|
||||
"keep_files/keep_max requires maintain_download_archive set to True"
|
||||
)
|
||||
|
||||
@property
|
||||
|
|
@ -387,7 +392,8 @@ class OutputOptions(StrictDictValidator):
|
|||
Optional. Requires ``maintain_download_archive`` set to True.
|
||||
|
||||
Only keeps files that are uploaded before this datetime. By default, ytdl-sub will keep
|
||||
files before ``now``, which implies all files.
|
||||
files before ``now``, which implies all files. Can be used in conjunction with
|
||||
``keep_max_files``.
|
||||
"""
|
||||
return self._keep_files_before
|
||||
|
||||
|
|
@ -397,6 +403,17 @@ class OutputOptions(StrictDictValidator):
|
|||
Optional. Requires ``maintain_download_archive`` set to True.
|
||||
|
||||
Only keeps files that are uploaded after this datetime. By default, ytdl-sub will keep
|
||||
files after ``19000101``, which implies all files.
|
||||
files after ``19000101``, which implies all files. Can be used in conjunction with
|
||||
``keep_max_files``.
|
||||
"""
|
||||
return self._keep_files_after
|
||||
|
||||
@property
|
||||
def keep_max_files(self) -> Optional[OverridesIntegerFormatterValidator]:
|
||||
"""
|
||||
Optional. Requires ``maintain_download_archive`` set to True.
|
||||
|
||||
Only keeps N most recently uploaded videos. If set to <= 0, ``keep_max_files`` will not be
|
||||
applied. Can be used in conjunction with ``keep_files_before`` and ``keep_files_after``.
|
||||
"""
|
||||
return self._keep_max_files
|
||||
|
|
|
|||
|
|
@ -22,12 +22,15 @@ presets:
|
|||
"Only Recent":
|
||||
# Only fetch videos after today minus date_range
|
||||
date_range:
|
||||
after: "today-{date_range}"
|
||||
after: "today-{only_recent_date_range}"
|
||||
|
||||
# Only keep files uploaded after date_range
|
||||
output_options:
|
||||
keep_files_after: "today-{date_range}"
|
||||
keep_files_after: "today-{only_recent_date_range}"
|
||||
keep_max_files: "{only_recent_max_files}"
|
||||
|
||||
# Set the default date_range to 2 months
|
||||
overrides:
|
||||
date_range: "2months"
|
||||
date_range: "2months" # keep for legacy-reasons
|
||||
only_recent_date_range: "{date_range}"
|
||||
only_recent_max_files: 0
|
||||
|
|
@ -145,8 +145,18 @@ class SubscriptionDownload(BaseSubscription, ABC):
|
|||
after=self.output_options.keep_files_after,
|
||||
overrides=self.overrides,
|
||||
)
|
||||
if date_range_to_keep:
|
||||
self._enhanced_download_archive.remove_stale_files(date_range=date_range_to_keep)
|
||||
|
||||
keep_max_files: Optional[int] = None
|
||||
if self.output_options.keep_max_files:
|
||||
# validated it can be cast to int within the validator
|
||||
keep_max_files = int(
|
||||
self.overrides.apply_formatter(self.output_options.keep_max_files)
|
||||
)
|
||||
|
||||
if date_range_to_keep or self.output_options.keep_max_files is not None:
|
||||
self._enhanced_download_archive.remove_stale_files(
|
||||
date_range=date_range_to_keep, keep_max_files=keep_max_files
|
||||
)
|
||||
|
||||
self._enhanced_download_archive.save_download_mappings()
|
||||
FileHandler.delete(self._enhanced_download_archive.working_file_path)
|
||||
|
|
|
|||
|
|
@ -218,6 +218,20 @@ class OverridesStringFormatterValidator(StringFormatterValidator):
|
|||
# pylint: enable=line-too-long
|
||||
|
||||
|
||||
class OverridesIntegerFormatterValidator(StringFormatterValidator):
|
||||
_expected_value_type_name = "integer"
|
||||
|
||||
def apply_formatter(self, variable_dict: Dict[str, str]) -> str:
|
||||
output = super().apply_formatter(variable_dict)
|
||||
try:
|
||||
int(output)
|
||||
except Exception as exc:
|
||||
raise self._validation_exception(
|
||||
f"Expected an integer, but received '{output}'"
|
||||
) from exc
|
||||
return output
|
||||
|
||||
|
||||
class ListFormatterValidator(ListValidator[StringFormatterValidator]):
|
||||
_inner_list_type = StringFormatterValidator
|
||||
|
||||
|
|
|
|||
|
|
@ -537,7 +537,16 @@ class EnhancedDownloadArchive:
|
|||
|
||||
return self
|
||||
|
||||
def remove_stale_files(self, date_range: DateRange) -> "EnhancedDownloadArchive":
|
||||
def _remove_entry(self, uid: str, mapping: DownloadMapping) -> None:
|
||||
for file_name in mapping.file_names:
|
||||
self._file_handler.delete_file_from_output_directory(file_name=file_name)
|
||||
|
||||
self.mapping.remove_entry(entry_id=uid)
|
||||
self.num_entries_removed += 1
|
||||
|
||||
def remove_stale_files(
|
||||
self, date_range: Optional[DateRange], keep_max_files: Optional[int]
|
||||
) -> "EnhancedDownloadArchive":
|
||||
"""
|
||||
Checks all entries within the mappings. If any entries' upload dates are not within the
|
||||
provided date range, delete them.
|
||||
|
|
@ -545,22 +554,32 @@ class EnhancedDownloadArchive:
|
|||
Parameters
|
||||
----------
|
||||
date_range
|
||||
Date range the upload date must be in to not get deleted
|
||||
Optional. Date range the upload date must be in to not get deleted
|
||||
keep_max_files
|
||||
Optional. Max number of files to keep
|
||||
|
||||
Returns
|
||||
-------
|
||||
self
|
||||
"""
|
||||
stale_mappings: Dict[str, DownloadMapping] = self.mapping.get_entries_out_of_range(
|
||||
date_range=date_range
|
||||
)
|
||||
if date_range is not None:
|
||||
stale_mappings: Dict[str, DownloadMapping] = self.mapping.get_entries_out_of_range(
|
||||
date_range=date_range
|
||||
)
|
||||
|
||||
for uid, mapping in stale_mappings.items():
|
||||
for file_name in mapping.file_names:
|
||||
self._file_handler.delete_file_from_output_directory(file_name=file_name)
|
||||
for uid, mapping in stale_mappings.items():
|
||||
self._remove_entry(uid=uid, mapping=mapping)
|
||||
|
||||
self.mapping.remove_entry(entry_id=uid)
|
||||
self.num_entries_removed += 1
|
||||
if keep_max_files is not None and keep_max_files > 0:
|
||||
num_files = 0
|
||||
for uid, mapping in sorted(
|
||||
self.mapping.entry_mappings.items(),
|
||||
key=lambda kv_: kv_[1].upload_date,
|
||||
reverse=True,
|
||||
):
|
||||
num_files += 1
|
||||
if num_files > keep_max_files:
|
||||
self._remove_entry(uid=uid, mapping=mapping)
|
||||
|
||||
return self
|
||||
|
||||
|
|
|
|||
|
|
@ -12,7 +12,9 @@ from ytdl_sub.subscriptions.subscription import Subscription
|
|||
@pytest.fixture
|
||||
def channel_preset_dict(output_directory):
|
||||
return {
|
||||
"preset": "TV Show Full Archive",
|
||||
"preset": [
|
||||
"TV Show Full Archive",
|
||||
],
|
||||
"format": "worst[ext=mp4]", # download the worst format so it is fast
|
||||
"ytdl_options": {
|
||||
"max_views": 100000, # do not download the popular PJ concert
|
||||
|
|
@ -77,6 +79,31 @@ class TestChannel:
|
|||
channel_preset_dict: Dict,
|
||||
):
|
||||
subscription_name = "pz"
|
||||
tv_show_name = channel_preset_dict["overrides"]["tv_show_name"]
|
||||
archive_file_name = f".ytdl-sub-{subscription_name}-download-archive.json"
|
||||
|
||||
pz_channel_mock_downloaded_with_archive_factory(
|
||||
tv_show_name=tv_show_name, archive_file_name=archive_file_name
|
||||
)
|
||||
|
||||
full_channel_subscription = Subscription.from_dict(
|
||||
config=tv_show_config, preset_name=subscription_name, preset_dict=channel_preset_dict
|
||||
)
|
||||
transaction_log = full_channel_subscription.download(dry_run=True)
|
||||
assert transaction_log.is_empty
|
||||
|
||||
def test_full_channel_existing_archive_keep_max_files(
|
||||
self,
|
||||
pz_channel_mock_downloaded_with_archive_factory: Callable,
|
||||
tv_show_config: ConfigFile,
|
||||
channel_preset_dict: Dict,
|
||||
output_directory: str,
|
||||
):
|
||||
subscription_name = "pz"
|
||||
channel_preset_dict["preset"].append("Only Recent")
|
||||
channel_preset_dict["overrides"]["only_recent_date_range"] = "10years"
|
||||
channel_preset_dict["overrides"]["only_recent_max_files"] = 1
|
||||
|
||||
full_channel_subscription = Subscription.from_dict(
|
||||
config=tv_show_config, preset_name=subscription_name, preset_dict=channel_preset_dict
|
||||
)
|
||||
|
|
@ -86,5 +113,10 @@ class TestChannel:
|
|||
pz_channel_mock_downloaded_with_archive_factory(
|
||||
tv_show_name=tv_show_name, archive_file_name=archive_file_name
|
||||
)
|
||||
|
||||
transaction_log = full_channel_subscription.download(dry_run=True)
|
||||
assert transaction_log.is_empty
|
||||
assert_transaction_log_matches(
|
||||
output_directory=output_directory,
|
||||
transaction_log=transaction_log,
|
||||
transaction_log_summary_file_name="youtube/test_channel_full_keep_max_files.txt",
|
||||
)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,56 @@
|
|||
Files modified:
|
||||
----------------------------------------
|
||||
{output_directory}
|
||||
.ytdl-sub-pz-download-archive.json
|
||||
|
||||
Files removed:
|
||||
----------------------------------------
|
||||
{output_directory}/Season 2010
|
||||
s2010.e081301 - Oblivion Mod ＂Falcor＂ p.1-thumb.jpg
|
||||
s2010.e081301 - Oblivion Mod ＂Falcor＂ p.1.info.json
|
||||
s2010.e081301 - Oblivion Mod ＂Falcor＂ p.1.mp4
|
||||
s2010.e081301 - Oblivion Mod ＂Falcor＂ p.1.nfo
|
||||
s2010.e120201 - Oblivion Mod ＂Falcor＂ p.2-thumb.jpg
|
||||
s2010.e120201 - Oblivion Mod ＂Falcor＂ p.2.info.json
|
||||
s2010.e120201 - Oblivion Mod ＂Falcor＂ p.2.mp4
|
||||
s2010.e120201 - Oblivion Mod ＂Falcor＂ p.2.nfo
|
||||
{output_directory}/Season 2011
|
||||
s2011.e020101 - Jesse's Minecraft Server [Trailer - Feb.1]-thumb.jpg
|
||||
s2011.e020101 - Jesse's Minecraft Server [Trailer - Feb.1].info.json
|
||||
s2011.e020101 - Jesse's Minecraft Server [Trailer - Feb.1].mp4
|
||||
s2011.e020101 - Jesse's Minecraft Server [Trailer - Feb.1].nfo
|
||||
s2011.e022701 - Jesse's Minecraft Server [Trailer - Feb.27]-thumb.jpg
|
||||
s2011.e022701 - Jesse's Minecraft Server [Trailer - Feb.27].info.json
|
||||
s2011.e022701 - Jesse's Minecraft Server [Trailer - Feb.27].mp4
|
||||
s2011.e022701 - Jesse's Minecraft Server [Trailer - Feb.27].nfo
|
||||
s2011.e032101 - Jesse's Minecraft Server [Trailer - Mar.21]-thumb.jpg
|
||||
s2011.e032101 - Jesse's Minecraft Server [Trailer - Mar.21].info.json
|
||||
s2011.e032101 - Jesse's Minecraft Server [Trailer - Mar.21].mp4
|
||||
s2011.e032101 - Jesse's Minecraft Server [Trailer - Mar.21].nfo
|
||||
s2011.e052901 - Project Zombie ｜Official Trailer｜ (IP： mc.projectzombie.beastnode.net)-thumb.jpg
|
||||
s2011.e052901 - Project Zombie ｜Official Trailer｜ (IP： mc.projectzombie.beastnode.net).info.json
|
||||
s2011.e052901 - Project Zombie ｜Official Trailer｜ (IP： mc.projectzombie.beastnode.net).mp4
|
||||
s2011.e052901 - Project Zombie ｜Official Trailer｜ (IP： mc.projectzombie.beastnode.net).nfo
|
||||
s2011.e063001 - Project Zombie ｜Fin｜-thumb.jpg
|
||||
s2011.e063001 - Project Zombie ｜Fin｜.info.json
|
||||
s2011.e063001 - Project Zombie ｜Fin｜.mp4
|
||||
s2011.e063001 - Project Zombie ｜Fin｜.nfo
|
||||
s2011.e112101 - Skyrim 'Ultra HD w⧸Mods' [PC]-thumb.jpg
|
||||
s2011.e112101 - Skyrim 'Ultra HD w⧸Mods' [PC].info.json
|
||||
s2011.e112101 - Skyrim 'Ultra HD w⧸Mods' [PC].mp4
|
||||
s2011.e112101 - Skyrim 'Ultra HD w⧸Mods' [PC].nfo
|
||||
{output_directory}/Season 2012
|
||||
s2012.e012301 - Project Zombie ｜Map Trailer｜-thumb.jpg
|
||||
s2012.e012301 - Project Zombie ｜Map Trailer｜.info.json
|
||||
s2012.e012301 - Project Zombie ｜Map Trailer｜.mp4
|
||||
s2012.e012301 - Project Zombie ｜Map Trailer｜.nfo
|
||||
{output_directory}/Season 2013
|
||||
s2013.e071901 - Project Zombie Rewind ｜Trailer｜-thumb.jpg
|
||||
s2013.e071901 - Project Zombie Rewind ｜Trailer｜.info.json
|
||||
s2013.e071901 - Project Zombie Rewind ｜Trailer｜.mp4
|
||||
s2013.e071901 - Project Zombie Rewind ｜Trailer｜.nfo
|
||||
{output_directory}/Season 2018
|
||||
s2018.e102901 - Jesse's Minecraft Server ｜ Teaser Trailer-thumb.jpg
|
||||
s2018.e102901 - Jesse's Minecraft Server ｜ Teaser Trailer.info.json
|
||||
s2018.e102901 - Jesse's Minecraft Server ｜ Teaser Trailer.mp4
|
||||
s2018.e102901 - Jesse's Minecraft Server ｜ Teaser Trailer.nfo
|
||||
Loading…
Reference in a new issue