[FEATURE] Add

This commit is contained in:
Jesse Bannon 2023-11-17 00:30:37 -08:00
parent 52d1a5887e
commit dc60f26fb1
5 changed files with 136 additions and 15 deletions

View file

@ -19,6 +19,7 @@ from ytdl_sub.validators.string_formatter_validators import DictFormatterValidat
from ytdl_sub.validators.string_formatter_validators import OverridesStringFormatterValidator from ytdl_sub.validators.string_formatter_validators import OverridesStringFormatterValidator
from ytdl_sub.validators.string_formatter_validators import StringFormatterValidator from ytdl_sub.validators.string_formatter_validators import StringFormatterValidator
from ytdl_sub.validators.validators import BoolValidator from ytdl_sub.validators.validators import BoolValidator
from ytdl_sub.validators.validators import IntValidator
from ytdl_sub.validators.validators import LiteralDictValidator from ytdl_sub.validators.validators import LiteralDictValidator
from ytdl_sub.validators.validators import Validator from ytdl_sub.validators.validators import Validator
@ -253,6 +254,7 @@ class OutputOptions(StrictDictValidator):
"maintain_download_archive", "maintain_download_archive",
"keep_files_before", "keep_files_before",
"keep_files_after", "keep_files_after",
"keep_max_files",
} }
@classmethod @classmethod
@ -307,6 +309,7 @@ class OutputOptions(StrictDictValidator):
self._keep_files_after = self._validate_key_if_present( self._keep_files_after = self._validate_key_if_present(
"keep_files_after", StringDatetimeValidator "keep_files_after", StringDatetimeValidator
) )
self._keep_max_files = self._validate_key_if_present("keep_max_files", IntValidator)
if ( if (
self._keep_files_before or self._keep_files_after self._keep_files_before or self._keep_files_after
@ -387,7 +390,8 @@ class OutputOptions(StrictDictValidator):
Optional. Requires ``maintain_download_archive`` set to True. Optional. Requires ``maintain_download_archive`` set to True.
Only keeps files that are uploaded before this datetime. By default, ytdl-sub will keep Only keeps files that are uploaded before this datetime. By default, ytdl-sub will keep
files before ``now``, which implies all files. files before ``now``, which implies all files. Can be used in conjunction with
``keep_max_files``.
""" """
return self._keep_files_before return self._keep_files_before
@ -397,6 +401,19 @@ class OutputOptions(StrictDictValidator):
Optional. Requires ``maintain_download_archive`` set to True. Optional. Requires ``maintain_download_archive`` set to True.
Only keeps files that are uploaded after this datetime. By default, ytdl-sub will keep Only keeps files that are uploaded after this datetime. By default, ytdl-sub will keep
files after ``19000101``, which implies all files. files after ``19000101``, which implies all files. Can be used in conjunction with
``keep_max_files``.
""" """
return self._keep_files_after return self._keep_files_after
@property
def keep_max_files(self) -> Optional[int]:
"""
Optional. Requires ``maintain_download_archive`` set to True.
Only keeps N most recently uploaded videos. Can be used in conjunction with
``keep_files_before`` and ``keep_files_after``.
"""
if self._keep_max_files:
return self._keep_max_files.value
return None

View file

@ -145,8 +145,10 @@ class SubscriptionDownload(BaseSubscription, ABC):
after=self.output_options.keep_files_after, after=self.output_options.keep_files_after,
overrides=self.overrides, overrides=self.overrides,
) )
if date_range_to_keep: if date_range_to_keep or self.output_options.keep_max_files is not None:
self._enhanced_download_archive.remove_stale_files(date_range=date_range_to_keep) self._enhanced_download_archive.remove_stale_files(
date_range=date_range_to_keep, keep_max_files=self.output_options.keep_max_files
)
self._enhanced_download_archive.save_download_mappings() self._enhanced_download_archive.save_download_mappings()
FileHandler.delete(self._enhanced_download_archive.working_file_path) FileHandler.delete(self._enhanced_download_archive.working_file_path)

View file

@ -537,7 +537,16 @@ class EnhancedDownloadArchive:
return self return self
def remove_stale_files(self, date_range: DateRange) -> "EnhancedDownloadArchive": def _remove_entry(self, uid: str, mapping: DownloadMapping) -> None:
for file_name in mapping.file_names:
self._file_handler.delete_file_from_output_directory(file_name=file_name)
self.mapping.remove_entry(entry_id=uid)
self.num_entries_removed += 1
def remove_stale_files(
self, date_range: Optional[DateRange], keep_max_files: Optional[int]
) -> "EnhancedDownloadArchive":
""" """
Checks all entries within the mappings. If any entries' upload dates are not within the Checks all entries within the mappings. If any entries' upload dates are not within the
provided date range, delete them. provided date range, delete them.
@ -545,22 +554,32 @@ class EnhancedDownloadArchive:
Parameters Parameters
---------- ----------
date_range date_range
Date range the upload date must be in to not get deleted Optional. Date range the upload date must be in to not get deleted
keep_max_files
Optional. Max number of files to keep
Returns Returns
------- -------
self self
""" """
stale_mappings: Dict[str, DownloadMapping] = self.mapping.get_entries_out_of_range( if date_range is not None:
date_range=date_range stale_mappings: Dict[str, DownloadMapping] = self.mapping.get_entries_out_of_range(
) date_range=date_range
)
for uid, mapping in stale_mappings.items(): for uid, mapping in stale_mappings.items():
for file_name in mapping.file_names: self._remove_entry(uid=uid, mapping=mapping)
self._file_handler.delete_file_from_output_directory(file_name=file_name)
self.mapping.remove_entry(entry_id=uid) if keep_max_files is not None:
self.num_entries_removed += 1 num_files = 0
for uid, mapping in sorted(
self.mapping.entry_mappings.items(),
key=lambda kv_: kv_[1].upload_date,
reverse=True,
):
num_files += 1
if num_files > keep_max_files:
self._remove_entry(uid=uid, mapping=mapping)
return self return self

View file

@ -77,6 +77,28 @@ class TestChannel:
channel_preset_dict: Dict, channel_preset_dict: Dict,
): ):
subscription_name = "pz" subscription_name = "pz"
tv_show_name = channel_preset_dict["overrides"]["tv_show_name"]
archive_file_name = f".ytdl-sub-{subscription_name}-download-archive.json"
pz_channel_mock_downloaded_with_archive_factory(
tv_show_name=tv_show_name, archive_file_name=archive_file_name
)
full_channel_subscription = Subscription.from_dict(
config=tv_show_config, preset_name=subscription_name, preset_dict=channel_preset_dict
)
transaction_log = full_channel_subscription.download(dry_run=True)
assert transaction_log.is_empty
def test_full_channel_existing_archive_keep_max_files(
self,
pz_channel_mock_downloaded_with_archive_factory: Callable,
tv_show_config: ConfigFile,
channel_preset_dict: Dict,
output_directory: str,
):
subscription_name = "pz"
channel_preset_dict = dict(channel_preset_dict, **{"output_options": {"keep_max_files": 1}})
full_channel_subscription = Subscription.from_dict( full_channel_subscription = Subscription.from_dict(
config=tv_show_config, preset_name=subscription_name, preset_dict=channel_preset_dict config=tv_show_config, preset_name=subscription_name, preset_dict=channel_preset_dict
) )
@ -86,5 +108,10 @@ class TestChannel:
pz_channel_mock_downloaded_with_archive_factory( pz_channel_mock_downloaded_with_archive_factory(
tv_show_name=tv_show_name, archive_file_name=archive_file_name tv_show_name=tv_show_name, archive_file_name=archive_file_name
) )
transaction_log = full_channel_subscription.download(dry_run=True) transaction_log = full_channel_subscription.download(dry_run=True)
assert transaction_log.is_empty assert_transaction_log_matches(
output_directory=output_directory,
transaction_log=transaction_log,
transaction_log_summary_file_name="youtube/test_channel_full_keep_max_files.txt",
)

View file

@ -0,0 +1,56 @@
Files modified:
----------------------------------------
{output_directory}
.ytdl-sub-pz-download-archive.json
Files removed:
----------------------------------------
{output_directory}/Season 2010
s2010.e081301 - Oblivion Mod Falcor p.1-thumb.jpg
s2010.e081301 - Oblivion Mod Falcor p.1.info.json
s2010.e081301 - Oblivion Mod Falcor p.1.mp4
s2010.e081301 - Oblivion Mod Falcor p.1.nfo
s2010.e120201 - Oblivion Mod Falcor p.2-thumb.jpg
s2010.e120201 - Oblivion Mod Falcor p.2.info.json
s2010.e120201 - Oblivion Mod Falcor p.2.mp4
s2010.e120201 - Oblivion Mod Falcor p.2.nfo
{output_directory}/Season 2011
s2011.e020101 - Jesse's Minecraft Server [Trailer - Feb.1]-thumb.jpg
s2011.e020101 - Jesse's Minecraft Server [Trailer - Feb.1].info.json
s2011.e020101 - Jesse's Minecraft Server [Trailer - Feb.1].mp4
s2011.e020101 - Jesse's Minecraft Server [Trailer - Feb.1].nfo
s2011.e022701 - Jesse's Minecraft Server [Trailer - Feb.27]-thumb.jpg
s2011.e022701 - Jesse's Minecraft Server [Trailer - Feb.27].info.json
s2011.e022701 - Jesse's Minecraft Server [Trailer - Feb.27].mp4
s2011.e022701 - Jesse's Minecraft Server [Trailer - Feb.27].nfo
s2011.e032101 - Jesse's Minecraft Server [Trailer - Mar.21]-thumb.jpg
s2011.e032101 - Jesse's Minecraft Server [Trailer - Mar.21].info.json
s2011.e032101 - Jesse's Minecraft Server [Trailer - Mar.21].mp4
s2011.e032101 - Jesse's Minecraft Server [Trailer - Mar.21].nfo
s2011.e052901 - Project Zombie Official Trailer (IP mc.projectzombie.beastnode.net)-thumb.jpg
s2011.e052901 - Project Zombie Official Trailer (IP mc.projectzombie.beastnode.net).info.json
s2011.e052901 - Project Zombie Official Trailer (IP mc.projectzombie.beastnode.net).mp4
s2011.e052901 - Project Zombie Official Trailer (IP mc.projectzombie.beastnode.net).nfo
s2011.e063001 - Project Zombie Fin-thumb.jpg
s2011.e063001 - Project Zombie Fin.info.json
s2011.e063001 - Project Zombie Fin.mp4
s2011.e063001 - Project Zombie Fin.nfo
s2011.e112101 - Skyrim 'Ultra HD wMods' [PC]-thumb.jpg
s2011.e112101 - Skyrim 'Ultra HD wMods' [PC].info.json
s2011.e112101 - Skyrim 'Ultra HD wMods' [PC].mp4
s2011.e112101 - Skyrim 'Ultra HD wMods' [PC].nfo
{output_directory}/Season 2012
s2012.e012301 - Project Zombie Map Trailer-thumb.jpg
s2012.e012301 - Project Zombie Map Trailer.info.json
s2012.e012301 - Project Zombie Map Trailer.mp4
s2012.e012301 - Project Zombie Map Trailer.nfo
{output_directory}/Season 2013
s2013.e071901 - Project Zombie Rewind Trailer-thumb.jpg
s2013.e071901 - Project Zombie Rewind Trailer.info.json
s2013.e071901 - Project Zombie Rewind Trailer.mp4
s2013.e071901 - Project Zombie Rewind Trailer.nfo
{output_directory}/Season 2018
s2018.e102901 - Jesse's Minecraft Server Teaser Trailer-thumb.jpg
s2018.e102901 - Jesse's Minecraft Server Teaser Trailer.info.json
s2018.e102901 - Jesse's Minecraft Server Teaser Trailer.mp4
s2018.e102901 - Jesse's Minecraft Server Teaser Trailer.nfo