[FEATURE] Add
This commit is contained in:
parent
52d1a5887e
commit
dc60f26fb1
5 changed files with 136 additions and 15 deletions
|
|
@ -19,6 +19,7 @@ from ytdl_sub.validators.string_formatter_validators import DictFormatterValidat
|
||||||
from ytdl_sub.validators.string_formatter_validators import OverridesStringFormatterValidator
|
from ytdl_sub.validators.string_formatter_validators import OverridesStringFormatterValidator
|
||||||
from ytdl_sub.validators.string_formatter_validators import StringFormatterValidator
|
from ytdl_sub.validators.string_formatter_validators import StringFormatterValidator
|
||||||
from ytdl_sub.validators.validators import BoolValidator
|
from ytdl_sub.validators.validators import BoolValidator
|
||||||
|
from ytdl_sub.validators.validators import IntValidator
|
||||||
from ytdl_sub.validators.validators import LiteralDictValidator
|
from ytdl_sub.validators.validators import LiteralDictValidator
|
||||||
from ytdl_sub.validators.validators import Validator
|
from ytdl_sub.validators.validators import Validator
|
||||||
|
|
||||||
|
|
@ -253,6 +254,7 @@ class OutputOptions(StrictDictValidator):
|
||||||
"maintain_download_archive",
|
"maintain_download_archive",
|
||||||
"keep_files_before",
|
"keep_files_before",
|
||||||
"keep_files_after",
|
"keep_files_after",
|
||||||
|
"keep_max_files",
|
||||||
}
|
}
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
|
|
@ -307,6 +309,7 @@ class OutputOptions(StrictDictValidator):
|
||||||
self._keep_files_after = self._validate_key_if_present(
|
self._keep_files_after = self._validate_key_if_present(
|
||||||
"keep_files_after", StringDatetimeValidator
|
"keep_files_after", StringDatetimeValidator
|
||||||
)
|
)
|
||||||
|
self._keep_max_files = self._validate_key_if_present("keep_max_files", IntValidator)
|
||||||
|
|
||||||
if (
|
if (
|
||||||
self._keep_files_before or self._keep_files_after
|
self._keep_files_before or self._keep_files_after
|
||||||
|
|
@ -387,7 +390,8 @@ class OutputOptions(StrictDictValidator):
|
||||||
Optional. Requires ``maintain_download_archive`` set to True.
|
Optional. Requires ``maintain_download_archive`` set to True.
|
||||||
|
|
||||||
Only keeps files that are uploaded before this datetime. By default, ytdl-sub will keep
|
Only keeps files that are uploaded before this datetime. By default, ytdl-sub will keep
|
||||||
files before ``now``, which implies all files.
|
files before ``now``, which implies all files. Can be used in conjunction with
|
||||||
|
``keep_max_files``.
|
||||||
"""
|
"""
|
||||||
return self._keep_files_before
|
return self._keep_files_before
|
||||||
|
|
||||||
|
|
@ -397,6 +401,19 @@ class OutputOptions(StrictDictValidator):
|
||||||
Optional. Requires ``maintain_download_archive`` set to True.
|
Optional. Requires ``maintain_download_archive`` set to True.
|
||||||
|
|
||||||
Only keeps files that are uploaded after this datetime. By default, ytdl-sub will keep
|
Only keeps files that are uploaded after this datetime. By default, ytdl-sub will keep
|
||||||
files after ``19000101``, which implies all files.
|
files after ``19000101``, which implies all files. Can be used in conjunction with
|
||||||
|
``keep_max_files``.
|
||||||
"""
|
"""
|
||||||
return self._keep_files_after
|
return self._keep_files_after
|
||||||
|
|
||||||
|
@property
|
||||||
|
def keep_max_files(self) -> Optional[int]:
|
||||||
|
"""
|
||||||
|
Optional. Requires ``maintain_download_archive`` set to True.
|
||||||
|
|
||||||
|
Only keeps N most recently uploaded videos. Can be used in conjunction with
|
||||||
|
``keep_files_before`` and ``keep_files_after``.
|
||||||
|
"""
|
||||||
|
if self._keep_max_files:
|
||||||
|
return self._keep_max_files.value
|
||||||
|
return None
|
||||||
|
|
|
||||||
|
|
@ -145,8 +145,10 @@ class SubscriptionDownload(BaseSubscription, ABC):
|
||||||
after=self.output_options.keep_files_after,
|
after=self.output_options.keep_files_after,
|
||||||
overrides=self.overrides,
|
overrides=self.overrides,
|
||||||
)
|
)
|
||||||
if date_range_to_keep:
|
if date_range_to_keep or self.output_options.keep_max_files is not None:
|
||||||
self._enhanced_download_archive.remove_stale_files(date_range=date_range_to_keep)
|
self._enhanced_download_archive.remove_stale_files(
|
||||||
|
date_range=date_range_to_keep, keep_max_files=self.output_options.keep_max_files
|
||||||
|
)
|
||||||
|
|
||||||
self._enhanced_download_archive.save_download_mappings()
|
self._enhanced_download_archive.save_download_mappings()
|
||||||
FileHandler.delete(self._enhanced_download_archive.working_file_path)
|
FileHandler.delete(self._enhanced_download_archive.working_file_path)
|
||||||
|
|
|
||||||
|
|
@ -537,7 +537,16 @@ class EnhancedDownloadArchive:
|
||||||
|
|
||||||
return self
|
return self
|
||||||
|
|
||||||
def remove_stale_files(self, date_range: DateRange) -> "EnhancedDownloadArchive":
|
def _remove_entry(self, uid: str, mapping: DownloadMapping) -> None:
|
||||||
|
for file_name in mapping.file_names:
|
||||||
|
self._file_handler.delete_file_from_output_directory(file_name=file_name)
|
||||||
|
|
||||||
|
self.mapping.remove_entry(entry_id=uid)
|
||||||
|
self.num_entries_removed += 1
|
||||||
|
|
||||||
|
def remove_stale_files(
|
||||||
|
self, date_range: Optional[DateRange], keep_max_files: Optional[int]
|
||||||
|
) -> "EnhancedDownloadArchive":
|
||||||
"""
|
"""
|
||||||
Checks all entries within the mappings. If any entries' upload dates are not within the
|
Checks all entries within the mappings. If any entries' upload dates are not within the
|
||||||
provided date range, delete them.
|
provided date range, delete them.
|
||||||
|
|
@ -545,22 +554,32 @@ class EnhancedDownloadArchive:
|
||||||
Parameters
|
Parameters
|
||||||
----------
|
----------
|
||||||
date_range
|
date_range
|
||||||
Date range the upload date must be in to not get deleted
|
Optional. Date range the upload date must be in to not get deleted
|
||||||
|
keep_max_files
|
||||||
|
Optional. Max number of files to keep
|
||||||
|
|
||||||
Returns
|
Returns
|
||||||
-------
|
-------
|
||||||
self
|
self
|
||||||
"""
|
"""
|
||||||
|
if date_range is not None:
|
||||||
stale_mappings: Dict[str, DownloadMapping] = self.mapping.get_entries_out_of_range(
|
stale_mappings: Dict[str, DownloadMapping] = self.mapping.get_entries_out_of_range(
|
||||||
date_range=date_range
|
date_range=date_range
|
||||||
)
|
)
|
||||||
|
|
||||||
for uid, mapping in stale_mappings.items():
|
for uid, mapping in stale_mappings.items():
|
||||||
for file_name in mapping.file_names:
|
self._remove_entry(uid=uid, mapping=mapping)
|
||||||
self._file_handler.delete_file_from_output_directory(file_name=file_name)
|
|
||||||
|
|
||||||
self.mapping.remove_entry(entry_id=uid)
|
if keep_max_files is not None:
|
||||||
self.num_entries_removed += 1
|
num_files = 0
|
||||||
|
for uid, mapping in sorted(
|
||||||
|
self.mapping.entry_mappings.items(),
|
||||||
|
key=lambda kv_: kv_[1].upload_date,
|
||||||
|
reverse=True,
|
||||||
|
):
|
||||||
|
num_files += 1
|
||||||
|
if num_files > keep_max_files:
|
||||||
|
self._remove_entry(uid=uid, mapping=mapping)
|
||||||
|
|
||||||
return self
|
return self
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -77,6 +77,28 @@ class TestChannel:
|
||||||
channel_preset_dict: Dict,
|
channel_preset_dict: Dict,
|
||||||
):
|
):
|
||||||
subscription_name = "pz"
|
subscription_name = "pz"
|
||||||
|
tv_show_name = channel_preset_dict["overrides"]["tv_show_name"]
|
||||||
|
archive_file_name = f".ytdl-sub-{subscription_name}-download-archive.json"
|
||||||
|
|
||||||
|
pz_channel_mock_downloaded_with_archive_factory(
|
||||||
|
tv_show_name=tv_show_name, archive_file_name=archive_file_name
|
||||||
|
)
|
||||||
|
|
||||||
|
full_channel_subscription = Subscription.from_dict(
|
||||||
|
config=tv_show_config, preset_name=subscription_name, preset_dict=channel_preset_dict
|
||||||
|
)
|
||||||
|
transaction_log = full_channel_subscription.download(dry_run=True)
|
||||||
|
assert transaction_log.is_empty
|
||||||
|
|
||||||
|
def test_full_channel_existing_archive_keep_max_files(
|
||||||
|
self,
|
||||||
|
pz_channel_mock_downloaded_with_archive_factory: Callable,
|
||||||
|
tv_show_config: ConfigFile,
|
||||||
|
channel_preset_dict: Dict,
|
||||||
|
output_directory: str,
|
||||||
|
):
|
||||||
|
subscription_name = "pz"
|
||||||
|
channel_preset_dict = dict(channel_preset_dict, **{"output_options": {"keep_max_files": 1}})
|
||||||
full_channel_subscription = Subscription.from_dict(
|
full_channel_subscription = Subscription.from_dict(
|
||||||
config=tv_show_config, preset_name=subscription_name, preset_dict=channel_preset_dict
|
config=tv_show_config, preset_name=subscription_name, preset_dict=channel_preset_dict
|
||||||
)
|
)
|
||||||
|
|
@ -86,5 +108,10 @@ class TestChannel:
|
||||||
pz_channel_mock_downloaded_with_archive_factory(
|
pz_channel_mock_downloaded_with_archive_factory(
|
||||||
tv_show_name=tv_show_name, archive_file_name=archive_file_name
|
tv_show_name=tv_show_name, archive_file_name=archive_file_name
|
||||||
)
|
)
|
||||||
|
|
||||||
transaction_log = full_channel_subscription.download(dry_run=True)
|
transaction_log = full_channel_subscription.download(dry_run=True)
|
||||||
assert transaction_log.is_empty
|
assert_transaction_log_matches(
|
||||||
|
output_directory=output_directory,
|
||||||
|
transaction_log=transaction_log,
|
||||||
|
transaction_log_summary_file_name="youtube/test_channel_full_keep_max_files.txt",
|
||||||
|
)
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,56 @@
|
||||||
|
Files modified:
|
||||||
|
----------------------------------------
|
||||||
|
{output_directory}
|
||||||
|
.ytdl-sub-pz-download-archive.json
|
||||||
|
|
||||||
|
Files removed:
|
||||||
|
----------------------------------------
|
||||||
|
{output_directory}/Season 2010
|
||||||
|
s2010.e081301 - Oblivion Mod "Falcor" p.1-thumb.jpg
|
||||||
|
s2010.e081301 - Oblivion Mod "Falcor" p.1.info.json
|
||||||
|
s2010.e081301 - Oblivion Mod "Falcor" p.1.mp4
|
||||||
|
s2010.e081301 - Oblivion Mod "Falcor" p.1.nfo
|
||||||
|
s2010.e120201 - Oblivion Mod "Falcor" p.2-thumb.jpg
|
||||||
|
s2010.e120201 - Oblivion Mod "Falcor" p.2.info.json
|
||||||
|
s2010.e120201 - Oblivion Mod "Falcor" p.2.mp4
|
||||||
|
s2010.e120201 - Oblivion Mod "Falcor" p.2.nfo
|
||||||
|
{output_directory}/Season 2011
|
||||||
|
s2011.e020101 - Jesse's Minecraft Server [Trailer - Feb.1]-thumb.jpg
|
||||||
|
s2011.e020101 - Jesse's Minecraft Server [Trailer - Feb.1].info.json
|
||||||
|
s2011.e020101 - Jesse's Minecraft Server [Trailer - Feb.1].mp4
|
||||||
|
s2011.e020101 - Jesse's Minecraft Server [Trailer - Feb.1].nfo
|
||||||
|
s2011.e022701 - Jesse's Minecraft Server [Trailer - Feb.27]-thumb.jpg
|
||||||
|
s2011.e022701 - Jesse's Minecraft Server [Trailer - Feb.27].info.json
|
||||||
|
s2011.e022701 - Jesse's Minecraft Server [Trailer - Feb.27].mp4
|
||||||
|
s2011.e022701 - Jesse's Minecraft Server [Trailer - Feb.27].nfo
|
||||||
|
s2011.e032101 - Jesse's Minecraft Server [Trailer - Mar.21]-thumb.jpg
|
||||||
|
s2011.e032101 - Jesse's Minecraft Server [Trailer - Mar.21].info.json
|
||||||
|
s2011.e032101 - Jesse's Minecraft Server [Trailer - Mar.21].mp4
|
||||||
|
s2011.e032101 - Jesse's Minecraft Server [Trailer - Mar.21].nfo
|
||||||
|
s2011.e052901 - Project Zombie |Official Trailer| (IP: mc.projectzombie.beastnode.net)-thumb.jpg
|
||||||
|
s2011.e052901 - Project Zombie |Official Trailer| (IP: mc.projectzombie.beastnode.net).info.json
|
||||||
|
s2011.e052901 - Project Zombie |Official Trailer| (IP: mc.projectzombie.beastnode.net).mp4
|
||||||
|
s2011.e052901 - Project Zombie |Official Trailer| (IP: mc.projectzombie.beastnode.net).nfo
|
||||||
|
s2011.e063001 - Project Zombie |Fin|-thumb.jpg
|
||||||
|
s2011.e063001 - Project Zombie |Fin|.info.json
|
||||||
|
s2011.e063001 - Project Zombie |Fin|.mp4
|
||||||
|
s2011.e063001 - Project Zombie |Fin|.nfo
|
||||||
|
s2011.e112101 - Skyrim 'Ultra HD w⧸Mods' [PC]-thumb.jpg
|
||||||
|
s2011.e112101 - Skyrim 'Ultra HD w⧸Mods' [PC].info.json
|
||||||
|
s2011.e112101 - Skyrim 'Ultra HD w⧸Mods' [PC].mp4
|
||||||
|
s2011.e112101 - Skyrim 'Ultra HD w⧸Mods' [PC].nfo
|
||||||
|
{output_directory}/Season 2012
|
||||||
|
s2012.e012301 - Project Zombie |Map Trailer|-thumb.jpg
|
||||||
|
s2012.e012301 - Project Zombie |Map Trailer|.info.json
|
||||||
|
s2012.e012301 - Project Zombie |Map Trailer|.mp4
|
||||||
|
s2012.e012301 - Project Zombie |Map Trailer|.nfo
|
||||||
|
{output_directory}/Season 2013
|
||||||
|
s2013.e071901 - Project Zombie Rewind |Trailer|-thumb.jpg
|
||||||
|
s2013.e071901 - Project Zombie Rewind |Trailer|.info.json
|
||||||
|
s2013.e071901 - Project Zombie Rewind |Trailer|.mp4
|
||||||
|
s2013.e071901 - Project Zombie Rewind |Trailer|.nfo
|
||||||
|
{output_directory}/Season 2018
|
||||||
|
s2018.e102901 - Jesse's Minecraft Server | Teaser Trailer-thumb.jpg
|
||||||
|
s2018.e102901 - Jesse's Minecraft Server | Teaser Trailer.info.json
|
||||||
|
s2018.e102901 - Jesse's Minecraft Server | Teaser Trailer.mp4
|
||||||
|
s2018.e102901 - Jesse's Minecraft Server | Teaser Trailer.nfo
|
||||||
Loading…
Reference in a new issue