updatead and working

This commit is contained in:
Jesse Bannon 2023-11-17 01:22:05 -08:00
parent 89c0dbd2fd
commit ffa318a74d
7 changed files with 41 additions and 15 deletions

View file

@ -97,4 +97,5 @@ presets:
- "Only Recent" - "Only Recent"
overrides: overrides:
date_range: "2months" only_recent_date_range: "2months"
only_recent_max_files: 30

View file

@ -16,10 +16,10 @@ from ytdl_sub.validators.file_path_validators import StringFormatterFileNameVali
from ytdl_sub.validators.strict_dict_validator import StrictDictValidator from ytdl_sub.validators.strict_dict_validator import StrictDictValidator
from ytdl_sub.validators.string_datetime import StringDatetimeValidator from ytdl_sub.validators.string_datetime import StringDatetimeValidator
from ytdl_sub.validators.string_formatter_validators import DictFormatterValidator from ytdl_sub.validators.string_formatter_validators import DictFormatterValidator
from ytdl_sub.validators.string_formatter_validators import OverridesIntegerFormatterValidator
from ytdl_sub.validators.string_formatter_validators import OverridesStringFormatterValidator from ytdl_sub.validators.string_formatter_validators import OverridesStringFormatterValidator
from ytdl_sub.validators.string_formatter_validators import StringFormatterValidator from ytdl_sub.validators.string_formatter_validators import StringFormatterValidator
from ytdl_sub.validators.validators import BoolValidator from ytdl_sub.validators.validators import BoolValidator
from ytdl_sub.validators.validators import IntValidator
from ytdl_sub.validators.validators import LiteralDictValidator from ytdl_sub.validators.validators import LiteralDictValidator
from ytdl_sub.validators.validators import Validator from ytdl_sub.validators.validators import Validator
@ -309,13 +309,15 @@ class OutputOptions(StrictDictValidator):
self._keep_files_after = self._validate_key_if_present( self._keep_files_after = self._validate_key_if_present(
"keep_files_after", StringDatetimeValidator "keep_files_after", StringDatetimeValidator
) )
self._keep_max_files = self._validate_key_if_present("keep_max_files", IntValidator) self._keep_max_files = self._validate_key_if_present(
"keep_max_files", OverridesIntegerFormatterValidator
)
if ( if (
self._keep_files_before or self._keep_files_after self._keep_files_before or self._keep_files_after or self._keep_max_files
) and not self.maintain_download_archive: ) and not self.maintain_download_archive:
raise self._validation_exception( raise self._validation_exception(
"keep_files requires maintain_download_archive set to True" "keep_files/keep_max requires maintain_download_archive set to True"
) )
@property @property
@ -407,13 +409,11 @@ class OutputOptions(StrictDictValidator):
return self._keep_files_after return self._keep_files_after
@property @property
def keep_max_files(self) -> Optional[int]: def keep_max_files(self) -> Optional[OverridesIntegerFormatterValidator]:
""" """
Optional. Requires ``maintain_download_archive`` set to True. Optional. Requires ``maintain_download_archive`` set to True.
Only keeps N most recently uploaded videos. If set to <= 0, ``keep_max_files`` will not be Only keeps N most recently uploaded videos. If set to <= 0, ``keep_max_files`` will not be
applied. Can be used in conjunction with ``keep_files_before`` and ``keep_files_after``. applied. Can be used in conjunction with ``keep_files_before`` and ``keep_files_after``.
""" """
if self._keep_max_files and self._keep_max_files.value > 0: return self._keep_max_files
return self._keep_max_files.value
return None

View file

@ -27,10 +27,10 @@ presets:
# Only keep files uploaded after date_range # Only keep files uploaded after date_range
output_options: output_options:
keep_files_after: "today-{only_recent_date_range}" keep_files_after: "today-{only_recent_date_range}"
keep_files_max: "{only_recent_max_files}" keep_max_files: "{only_recent_max_files}"
# Set the default date_range to 2 months # Set the default date_range to 2 months
overrides: overrides:
date_range: "2months" date_range: "2months" # keep for legacy-reasons
only_recent_date_range: "{date_range}" only_recent_date_range: "{date_range}"
only_recent_max_files: 0 only_recent_max_files: 0

View file

@ -145,9 +145,17 @@ class SubscriptionDownload(BaseSubscription, ABC):
after=self.output_options.keep_files_after, after=self.output_options.keep_files_after,
overrides=self.overrides, overrides=self.overrides,
) )
keep_max_files: Optional[int] = None
if self.output_options.keep_max_files:
# validated it can be cast to int within the validator
keep_max_files = int(
self.overrides.apply_formatter(self.output_options.keep_max_files)
)
if date_range_to_keep or self.output_options.keep_max_files is not None: if date_range_to_keep or self.output_options.keep_max_files is not None:
self._enhanced_download_archive.remove_stale_files( self._enhanced_download_archive.remove_stale_files(
date_range=date_range_to_keep, keep_max_files=self.output_options.keep_max_files date_range=date_range_to_keep, keep_max_files=keep_max_files
) )
self._enhanced_download_archive.save_download_mappings() self._enhanced_download_archive.save_download_mappings()

View file

@ -218,6 +218,18 @@ class OverridesStringFormatterValidator(StringFormatterValidator):
# pylint: enable=line-too-long # pylint: enable=line-too-long
class OverridesIntegerFormatterValidator(StringFormatterValidator):
_expected_value_type_name = "integer"
def apply_formatter(self, variable_dict: Dict[str, str]) -> str:
output = super().apply_formatter(variable_dict)
try:
int(output)
except Exception as exc:
raise self._validation_exception(f"Expected an integer, but received '{output}'") from exc
return output
class ListFormatterValidator(ListValidator[StringFormatterValidator]): class ListFormatterValidator(ListValidator[StringFormatterValidator]):
_inner_list_type = StringFormatterValidator _inner_list_type = StringFormatterValidator

View file

@ -570,7 +570,7 @@ class EnhancedDownloadArchive:
for uid, mapping in stale_mappings.items(): for uid, mapping in stale_mappings.items():
self._remove_entry(uid=uid, mapping=mapping) self._remove_entry(uid=uid, mapping=mapping)
if keep_max_files is not None: if keep_max_files is not None and keep_max_files > 0:
num_files = 0 num_files = 0
for uid, mapping in sorted( for uid, mapping in sorted(
self.mapping.entry_mappings.items(), self.mapping.entry_mappings.items(),

View file

@ -12,7 +12,9 @@ from ytdl_sub.subscriptions.subscription import Subscription
@pytest.fixture @pytest.fixture
def channel_preset_dict(output_directory): def channel_preset_dict(output_directory):
return { return {
"preset": "TV Show Full Archive", "preset": [
"TV Show Full Archive",
],
"format": "worst[ext=mp4]", # download the worst format so it is fast "format": "worst[ext=mp4]", # download the worst format so it is fast
"ytdl_options": { "ytdl_options": {
"max_views": 100000, # do not download the popular PJ concert "max_views": 100000, # do not download the popular PJ concert
@ -98,7 +100,10 @@ class TestChannel:
output_directory: str, output_directory: str,
): ):
subscription_name = "pz" subscription_name = "pz"
channel_preset_dict = dict(channel_preset_dict, **{"output_options": {"keep_max_files": 1}}) channel_preset_dict["preset"].append("Only Recent")
channel_preset_dict["overrides"]["only_recent_date_range"] = "10years"
channel_preset_dict["overrides"]["only_recent_max_files"] = 1
full_channel_subscription = Subscription.from_dict( full_channel_subscription = Subscription.from_dict(
config=tv_show_config, preset_name=subscription_name, preset_dict=channel_preset_dict config=tv_show_config, preset_name=subscription_name, preset_dict=channel_preset_dict
) )