[FEATURE] Add
This commit is contained in:
Jesse Bannon
parent
52d1a5887e
commit
dc60f26fb1
5 changed files with 136 additions and 15 deletions
|
|
@ -19,6 +19,7 @@ from ytdl_sub.validators.string_formatter_validators import DictFormatterValidat
|
|||
from ytdl_sub.validators.string_formatter_validators import OverridesStringFormatterValidator
|
||||
from ytdl_sub.validators.string_formatter_validators import StringFormatterValidator
|
||||
from ytdl_sub.validators.validators import BoolValidator
|
||||
from ytdl_sub.validators.validators import IntValidator
|
||||
from ytdl_sub.validators.validators import LiteralDictValidator
|
||||
from ytdl_sub.validators.validators import Validator
|
||||
|
||||
|
|
@ -253,6 +254,7 @@ class OutputOptions(StrictDictValidator):
|
|||
"maintain_download_archive",
|
||||
"keep_files_before",
|
||||
"keep_files_after",
|
||||
"keep_max_files",
|
||||
}
|
||||
|
||||
@classmethod
|
||||
|
|
@ -307,6 +309,7 @@ class OutputOptions(StrictDictValidator):
|
|||
self._keep_files_after = self._validate_key_if_present(
|
||||
"keep_files_after", StringDatetimeValidator
|
||||
)
|
||||
self._keep_max_files = self._validate_key_if_present("keep_max_files", IntValidator)
|
||||
|
||||
if (
|
||||
self._keep_files_before or self._keep_files_after
|
||||
|
|
@ -387,7 +390,8 @@ class OutputOptions(StrictDictValidator):
|
|||
Optional. Requires ``maintain_download_archive`` set to True.
|
||||
|
||||
Only keeps files that are uploaded before this datetime. By default, ytdl-sub will keep
|
||||
files before ``now``, which implies all files.
|
||||
files before ``now``, which implies all files. Can be used in conjunction with
|
||||
``keep_max_files``.
|
||||
"""
|
||||
return self._keep_files_before
|
||||
|
||||
|
|
@ -397,6 +401,19 @@ class OutputOptions(StrictDictValidator):
|
|||
Optional. Requires ``maintain_download_archive`` set to True.
|
||||
|
||||
Only keeps files that are uploaded after this datetime. By default, ytdl-sub will keep
|
||||
files after ``19000101``, which implies all files.
|
||||
files after ``19000101``, which implies all files. Can be used in conjunction with
|
||||
``keep_max_files``.
|
||||
"""
|
||||
return self._keep_files_after
|
||||
|
||||
@property
|
||||
def keep_max_files(self) -> Optional[int]:
|
||||
"""
|
||||
Optional. Requires ``maintain_download_archive`` set to True.
|
||||
|
||||
Only keeps N most recently uploaded videos. Can be used in conjunction with
|
||||
``keep_files_before`` and ``keep_files_after``.
|
||||
"""
|
||||
if self._keep_max_files:
|
||||
return self._keep_max_files.value
|
||||
return None
|
||||
|
|
|
|||
|
|
@ -145,8 +145,10 @@ class SubscriptionDownload(BaseSubscription, ABC):
|
|||
after=self.output_options.keep_files_after,
|
||||
overrides=self.overrides,
|
||||
)
|
||||
if date_range_to_keep:
|
||||
self._enhanced_download_archive.remove_stale_files(date_range=date_range_to_keep)
|
||||
if date_range_to_keep or self.output_options.keep_max_files is not None:
|
||||
self._enhanced_download_archive.remove_stale_files(
|
||||
date_range=date_range_to_keep, keep_max_files=self.output_options.keep_max_files
|
||||
)
|
||||
|
||||
self._enhanced_download_archive.save_download_mappings()
|
||||
FileHandler.delete(self._enhanced_download_archive.working_file_path)
|
||||
|
|
|
|||
|
|
@ -537,7 +537,16 @@ class EnhancedDownloadArchive:
|
|||
|
||||
return self
|
||||
|
||||
def remove_stale_files(self, date_range: DateRange) -> "EnhancedDownloadArchive":
|
||||
def _remove_entry(self, uid: str, mapping: DownloadMapping) -> None:
|
||||
for file_name in mapping.file_names:
|
||||
self._file_handler.delete_file_from_output_directory(file_name=file_name)
|
||||
|
||||
self.mapping.remove_entry(entry_id=uid)
|
||||
self.num_entries_removed += 1
|
||||
|
||||
def remove_stale_files(
|
||||
self, date_range: Optional[DateRange], keep_max_files: Optional[int]
|
||||
) -> "EnhancedDownloadArchive":
|
||||
"""
|
||||
Checks all entries within the mappings. If any entries' upload dates are not within the
|
||||
provided date range, delete them.
|
||||
|
|
@ -545,22 +554,32 @@ class EnhancedDownloadArchive:
|
|||
Parameters
|
||||
----------
|
||||
date_range
|
||||
Date range the upload date must be in to not get deleted
|
||||
Optional. Date range the upload date must be in to not get deleted
|
||||
keep_max_files
|
||||
Optional. Max number of files to keep
|
||||
|
||||
Returns
|
||||
-------
|
||||
self
|
||||
"""
|
||||
stale_mappings: Dict[str, DownloadMapping] = self.mapping.get_entries_out_of_range(
|
||||
date_range=date_range
|
||||
)
|
||||
if date_range is not None:
|
||||
stale_mappings: Dict[str, DownloadMapping] = self.mapping.get_entries_out_of_range(
|
||||
date_range=date_range
|
||||
)
|
||||
|
||||
for uid, mapping in stale_mappings.items():
|
||||
for file_name in mapping.file_names:
|
||||
self._file_handler.delete_file_from_output_directory(file_name=file_name)
|
||||
for uid, mapping in stale_mappings.items():
|
||||
self._remove_entry(uid=uid, mapping=mapping)
|
||||
|
||||
self.mapping.remove_entry(entry_id=uid)
|
||||
self.num_entries_removed += 1
|
||||
if keep_max_files is not None:
|
||||
num_files = 0
|
||||
for uid, mapping in sorted(
|
||||
self.mapping.entry_mappings.items(),
|
||||
key=lambda kv_: kv_[1].upload_date,
|
||||
reverse=True,
|
||||
):
|
||||
num_files += 1
|
||||
if num_files > keep_max_files:
|
||||
self._remove_entry(uid=uid, mapping=mapping)
|
||||
|
||||
return self
|
||||
|
||||
|
|
|
|||
|
|
@ -77,6 +77,28 @@ class TestChannel:
|
|||
channel_preset_dict: Dict,
|
||||
):
|
||||
subscription_name = "pz"
|
||||
tv_show_name = channel_preset_dict["overrides"]["tv_show_name"]
|
||||
archive_file_name = f".ytdl-sub-{subscription_name}-download-archive.json"
|
||||
|
||||
pz_channel_mock_downloaded_with_archive_factory(
|
||||
tv_show_name=tv_show_name, archive_file_name=archive_file_name
|
||||
)
|
||||
|
||||
full_channel_subscription = Subscription.from_dict(
|
||||
config=tv_show_config, preset_name=subscription_name, preset_dict=channel_preset_dict
|
||||
)
|
||||
transaction_log = full_channel_subscription.download(dry_run=True)
|
||||
assert transaction_log.is_empty
|
||||
|
||||
def test_full_channel_existing_archive_keep_max_files(
|
||||
self,
|
||||
pz_channel_mock_downloaded_with_archive_factory: Callable,
|
||||
tv_show_config: ConfigFile,
|
||||
channel_preset_dict: Dict,
|
||||
output_directory: str,
|
||||
):
|
||||
subscription_name = "pz"
|
||||
channel_preset_dict = dict(channel_preset_dict, **{"output_options": {"keep_max_files": 1}})
|
||||
full_channel_subscription = Subscription.from_dict(
|
||||
config=tv_show_config, preset_name=subscription_name, preset_dict=channel_preset_dict
|
||||
)
|
||||
|
|
@ -86,5 +108,10 @@ class TestChannel:
|
|||
pz_channel_mock_downloaded_with_archive_factory(
|
||||
tv_show_name=tv_show_name, archive_file_name=archive_file_name
|
||||
)
|
||||
|
||||
transaction_log = full_channel_subscription.download(dry_run=True)
|
||||
assert transaction_log.is_empty
|
||||
assert_transaction_log_matches(
|
||||
output_directory=output_directory,
|
||||
transaction_log=transaction_log,
|
||||
transaction_log_summary_file_name="youtube/test_channel_full_keep_max_files.txt",
|
||||
)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,56 @@
|
|||
Files modified:
|
||||
----------------------------------------
|
||||
{output_directory}
|
||||
.ytdl-sub-pz-download-archive.json
|
||||
|
||||
Files removed:
|
||||
----------------------------------------
|
||||
{output_directory}/Season 2010
|
||||
s2010.e081301 - Oblivion Mod ＂Falcor＂ p.1-thumb.jpg
|
||||
s2010.e081301 - Oblivion Mod ＂Falcor＂ p.1.info.json
|
||||
s2010.e081301 - Oblivion Mod ＂Falcor＂ p.1.mp4
|
||||
s2010.e081301 - Oblivion Mod ＂Falcor＂ p.1.nfo
|
||||
s2010.e120201 - Oblivion Mod ＂Falcor＂ p.2-thumb.jpg
|
||||
s2010.e120201 - Oblivion Mod ＂Falcor＂ p.2.info.json
|
||||
s2010.e120201 - Oblivion Mod ＂Falcor＂ p.2.mp4
|
||||
s2010.e120201 - Oblivion Mod ＂Falcor＂ p.2.nfo
|
||||
{output_directory}/Season 2011
|
||||
s2011.e020101 - Jesse's Minecraft Server [Trailer - Feb.1]-thumb.jpg
|
||||
s2011.e020101 - Jesse's Minecraft Server [Trailer - Feb.1].info.json
|
||||
s2011.e020101 - Jesse's Minecraft Server [Trailer - Feb.1].mp4
|
||||
s2011.e020101 - Jesse's Minecraft Server [Trailer - Feb.1].nfo
|
||||
s2011.e022701 - Jesse's Minecraft Server [Trailer - Feb.27]-thumb.jpg
|
||||
s2011.e022701 - Jesse's Minecraft Server [Trailer - Feb.27].info.json
|
||||
s2011.e022701 - Jesse's Minecraft Server [Trailer - Feb.27].mp4
|
||||
s2011.e022701 - Jesse's Minecraft Server [Trailer - Feb.27].nfo
|
||||
s2011.e032101 - Jesse's Minecraft Server [Trailer - Mar.21]-thumb.jpg
|
||||
s2011.e032101 - Jesse's Minecraft Server [Trailer - Mar.21].info.json
|
||||
s2011.e032101 - Jesse's Minecraft Server [Trailer - Mar.21].mp4
|
||||
s2011.e032101 - Jesse's Minecraft Server [Trailer - Mar.21].nfo
|
||||
s2011.e052901 - Project Zombie ｜Official Trailer｜ (IP： mc.projectzombie.beastnode.net)-thumb.jpg
|
||||
s2011.e052901 - Project Zombie ｜Official Trailer｜ (IP： mc.projectzombie.beastnode.net).info.json
|
||||
s2011.e052901 - Project Zombie ｜Official Trailer｜ (IP： mc.projectzombie.beastnode.net).mp4
|
||||
s2011.e052901 - Project Zombie ｜Official Trailer｜ (IP： mc.projectzombie.beastnode.net).nfo
|
||||
s2011.e063001 - Project Zombie ｜Fin｜-thumb.jpg
|
||||
s2011.e063001 - Project Zombie ｜Fin｜.info.json
|
||||
s2011.e063001 - Project Zombie ｜Fin｜.mp4
|
||||
s2011.e063001 - Project Zombie ｜Fin｜.nfo
|
||||
s2011.e112101 - Skyrim 'Ultra HD w⧸Mods' [PC]-thumb.jpg
|
||||
s2011.e112101 - Skyrim 'Ultra HD w⧸Mods' [PC].info.json
|
||||
s2011.e112101 - Skyrim 'Ultra HD w⧸Mods' [PC].mp4
|
||||
s2011.e112101 - Skyrim 'Ultra HD w⧸Mods' [PC].nfo
|
||||
{output_directory}/Season 2012
|
||||
s2012.e012301 - Project Zombie ｜Map Trailer｜-thumb.jpg
|
||||
s2012.e012301 - Project Zombie ｜Map Trailer｜.info.json
|
||||
s2012.e012301 - Project Zombie ｜Map Trailer｜.mp4
|
||||
s2012.e012301 - Project Zombie ｜Map Trailer｜.nfo
|
||||
{output_directory}/Season 2013
|
||||
s2013.e071901 - Project Zombie Rewind ｜Trailer｜-thumb.jpg
|
||||
s2013.e071901 - Project Zombie Rewind ｜Trailer｜.info.json
|
||||
s2013.e071901 - Project Zombie Rewind ｜Trailer｜.mp4
|
||||
s2013.e071901 - Project Zombie Rewind ｜Trailer｜.nfo
|
||||
{output_directory}/Season 2018
|
||||
s2018.e102901 - Jesse's Minecraft Server ｜ Teaser Trailer-thumb.jpg
|
||||
s2018.e102901 - Jesse's Minecraft Server ｜ Teaser Trailer.info.json
|
||||
s2018.e102901 - Jesse's Minecraft Server ｜ Teaser Trailer.mp4
|
||||
s2018.e102901 - Jesse's Minecraft Server ｜ Teaser Trailer.nfo
|
||||
Loading…
Reference in a new issue