update keep_files to be single level

This commit is contained in:
jbannon 2022-05-11 05:18:18 +00:00
parent 20d376b5a1
commit 389f2729a6
7 changed files with 61 additions and 36 deletions

View file

@ -87,6 +87,7 @@ output_options
.. autoclass:: ytdl_sub.config.preset_options.OutputOptions() .. autoclass:: ytdl_sub.config.preset_options.OutputOptions()
:members: :members:
:member-order: bysource :member-order: bysource
:exclude-members: get_upload_date_range_to_keep
------------------------------------------------------------------------------- -------------------------------------------------------------------------------

View file

@ -70,7 +70,7 @@ john_smith_recent_archive:
# only keep the last 14-days worth of videos, and delete the rest. # only keep the last 14-days worth of videos, and delete the rest.
# #
# The only difference between this example and the one above is # The only difference between this example and the one above is
# - output_options.keep_files.after # - output_options.keep_files_after
# This is saying "only keep files if they were uploaded in the last 14 days". # This is saying "only keep files if they were uploaded in the last 14 days".
# All other files that this subscription had previously downloaded will be # All other files that this subscription had previously downloaded will be
# deleted. # deleted.
@ -85,5 +85,4 @@ john_smith_rolling_archive:
break_on_reject: True break_on_reject: True
break_on_existing: True break_on_existing: True
output_options: output_options:
keep_files: keep_files_after: today-2weeks
after: today-2weeks

View file

@ -1,10 +1,11 @@
from typing import Optional from typing import Optional
from yt_dlp.utils import DateRange
from yt_dlp.utils import sanitize_filename from yt_dlp.utils import sanitize_filename
from ytdl_sub.entries.entry import Entry from ytdl_sub.entries.entry import Entry
from ytdl_sub.validators.date_range_validator import DateRangeValidator
from ytdl_sub.validators.strict_dict_validator import StrictDictValidator from ytdl_sub.validators.strict_dict_validator import StrictDictValidator
from ytdl_sub.validators.string_datetime import StringDatetimeValidator
from ytdl_sub.validators.string_formatter_validators import DictFormatterValidator from ytdl_sub.validators.string_formatter_validators import DictFormatterValidator
from ytdl_sub.validators.string_formatter_validators import OverridesStringFormatterValidator from ytdl_sub.validators.string_formatter_validators import OverridesStringFormatterValidator
from ytdl_sub.validators.string_formatter_validators import StringFormatterValidator from ytdl_sub.validators.string_formatter_validators import StringFormatterValidator
@ -102,16 +103,16 @@ class OutputOptions(StrictDictValidator):
# optional # optional
thumbnail_name: "{sanitized_title}.{thumbnail_ext}" thumbnail_name: "{sanitized_title}.{thumbnail_ext}"
maintain_download_archive: True maintain_download_archive: True
keep_files: keep_files_before: now
before: now keep_files_after: 19000101
after: today-2weeks
""" """
_required_keys = {"output_directory", "file_name"} _required_keys = {"output_directory", "file_name"}
_optional_keys = { _optional_keys = {
"thumbnail_name", "thumbnail_name",
"maintain_download_archive", "maintain_download_archive",
"keep_files", "keep_files_before",
"keep_files_after",
} }
def __init__(self, name, value): def __init__(self, name, value):
@ -134,11 +135,17 @@ class OutputOptions(StrictDictValidator):
self._maintain_download_archive = self._validate_key_if_present( self._maintain_download_archive = self._validate_key_if_present(
key="maintain_download_archive", validator=BoolValidator, default=False key="maintain_download_archive", validator=BoolValidator, default=False
) )
self._keep_files = self._validate_key_if_present(
key="keep_files", validator=DateRangeValidator self._keep_files_before = self._validate_key_if_present(
"keep_files_before", StringDatetimeValidator
)
self._keep_files_after = self._validate_key_if_present(
"keep_files_after", StringDatetimeValidator
) )
if self._keep_files and not self.maintain_download_archive: if (
self._keep_files_before or self._keep_files_after
) and not self.maintain_download_archive:
raise self._validation_exception( raise self._validation_exception(
"keep_files requires maintain_download_archive set to True" "keep_files requires maintain_download_archive set to True"
) )
@ -183,23 +190,34 @@ class OutputOptions(StrictDictValidator):
return self._maintain_download_archive.value return self._maintain_download_archive.value
@property @property
def keep_files(self) -> DateRangeValidator: def keep_files_before(self) -> Optional[StringDatetimeValidator]:
""" """
Optional. Requires ``maintain_download_archive`` set to True. Optional. Requires ``maintain_download_archive`` set to True.
Only keeps files that are uploaded in the defined range. ``before`` and ``after`` are Only keeps files that are uploaded before this datetime. By default, ytdl-sub will keep
date-times. A common usage of this option is to only fill in the after, such as: files before ``now``, which implies all files.
.. code-block:: yaml
presets:
my_example_preset:
output_options:
keep_files:
after: today-2weeks
Which translates to 'keep files uploaded in the last two weeks'.
By default, ytdl-sub will keep all files.
""" """
return self._keep_files return self._keep_files_before
@property
def keep_files_after(self) -> Optional[StringDatetimeValidator]:
"""
Optional. Requires ``maintain_download_archive`` set to True.
Only keeps files that are uploaded after this datetime. By default, ytdl-sub will keep
files after ``19000101``, which implies all files.
"""
return self._keep_files_after
def get_upload_date_range_to_keep(self) -> Optional[DateRange]:
"""
Returns
-------
Date range if the 'before' or 'after' is defined. None otherwise.
"""
if self.keep_files_before or self.keep_files_after:
return DateRange(
start=self.keep_files_after.datetime_str if self.keep_files_after else None,
end=self.keep_files_before.datetime_str if self.keep_files_before else None,
)
return None

View file

@ -202,10 +202,9 @@ class Subscription:
# If output options maintains stale file deletion, perform the delete here prior to saving # If output options maintains stale file deletion, perform the delete here prior to saving
# the download archive # the download archive
if self.output_options.maintain_download_archive: if self.output_options.maintain_download_archive:
if self.output_options.keep_files: date_range_to_keep = self.output_options.get_upload_date_range_to_keep()
self._enhanced_download_archive.remove_stale_files( if date_range_to_keep:
date_range=self.output_options.keep_files.get_date_range() self._enhanced_download_archive.remove_stale_files(date_range=date_range_to_keep)
)
self._enhanced_download_archive.save_download_mappings() self._enhanced_download_archive.save_download_mappings()

View file

@ -5,7 +5,14 @@ from ytdl_sub.validators.validators import Validator
class StringDatetimeValidator(Validator): class StringDatetimeValidator(Validator):
""" """
Validates a ytdl datetime string value String that contains a yt-dlp datetime. From their docs:
.. code-block:: Markdown
A string in the format YYYYMMDD or
(now|today|yesterday|date)[+-][0-9](microsecond|second|minute|hour|day|week|month|year)(s)
Valid examples are ``now-2weeks`` or ``20200101``.
""" """
_expected_value_type = str _expected_value_type = str

View file

@ -171,7 +171,8 @@ class OverridesStringFormatterValidator(StringFormatterValidator):
class DictFormatterValidator(LiteralDictValidator): class DictFormatterValidator(LiteralDictValidator):
""" """
Validates a dictionary made up of key: string_formatters A dict made up of
:class:`~ytdl_sub.validators.string_formatter_validators.StringFormatterValidator`.
""" """
_key_validator = StringFormatterValidator _key_validator = StringFormatterValidator
@ -195,8 +196,8 @@ class DictFormatterValidator(LiteralDictValidator):
class OverridesDictFormatterValidator(DictFormatterValidator): class OverridesDictFormatterValidator(DictFormatterValidator):
""" """
Validates a dictionary made up of key: string_formatters, that must be resolved by overrides A dict made up of
only. :class:`~ytdl_sub.validators.string_formatter_validators.OverridesStringFormatterValidator`.
""" """
_key_validator = OverridesStringFormatterValidator _key_validator = OverridesStringFormatterValidator

View file

@ -183,7 +183,7 @@ def expected_recent_channel_download():
@pytest.fixture @pytest.fixture
def rolling_recent_channel_subscription_dict(recent_channel_subscription_dict): def rolling_recent_channel_subscription_dict(recent_channel_subscription_dict):
return mergedeep.merge( return mergedeep.merge(
recent_channel_subscription_dict, {"output_options": {"keep_files": {"after": "20181101"}}} recent_channel_subscription_dict, {"output_options": {"keep_files_after": "20181101"}}
) )