updatead and working
This commit is contained in:
parent
89c0dbd2fd
commit
ffa318a74d
7 changed files with 41 additions and 15 deletions
|
|
@ -97,4 +97,5 @@ presets:
|
||||||
- "Only Recent"
|
- "Only Recent"
|
||||||
|
|
||||||
overrides:
|
overrides:
|
||||||
date_range: "2months"
|
only_recent_date_range: "2months"
|
||||||
|
only_recent_max_files: 30
|
||||||
|
|
@ -16,10 +16,10 @@ from ytdl_sub.validators.file_path_validators import StringFormatterFileNameVali
|
||||||
from ytdl_sub.validators.strict_dict_validator import StrictDictValidator
|
from ytdl_sub.validators.strict_dict_validator import StrictDictValidator
|
||||||
from ytdl_sub.validators.string_datetime import StringDatetimeValidator
|
from ytdl_sub.validators.string_datetime import StringDatetimeValidator
|
||||||
from ytdl_sub.validators.string_formatter_validators import DictFormatterValidator
|
from ytdl_sub.validators.string_formatter_validators import DictFormatterValidator
|
||||||
|
from ytdl_sub.validators.string_formatter_validators import OverridesIntegerFormatterValidator
|
||||||
from ytdl_sub.validators.string_formatter_validators import OverridesStringFormatterValidator
|
from ytdl_sub.validators.string_formatter_validators import OverridesStringFormatterValidator
|
||||||
from ytdl_sub.validators.string_formatter_validators import StringFormatterValidator
|
from ytdl_sub.validators.string_formatter_validators import StringFormatterValidator
|
||||||
from ytdl_sub.validators.validators import BoolValidator
|
from ytdl_sub.validators.validators import BoolValidator
|
||||||
from ytdl_sub.validators.validators import IntValidator
|
|
||||||
from ytdl_sub.validators.validators import LiteralDictValidator
|
from ytdl_sub.validators.validators import LiteralDictValidator
|
||||||
from ytdl_sub.validators.validators import Validator
|
from ytdl_sub.validators.validators import Validator
|
||||||
|
|
||||||
|
|
@ -309,13 +309,15 @@ class OutputOptions(StrictDictValidator):
|
||||||
self._keep_files_after = self._validate_key_if_present(
|
self._keep_files_after = self._validate_key_if_present(
|
||||||
"keep_files_after", StringDatetimeValidator
|
"keep_files_after", StringDatetimeValidator
|
||||||
)
|
)
|
||||||
self._keep_max_files = self._validate_key_if_present("keep_max_files", IntValidator)
|
self._keep_max_files = self._validate_key_if_present(
|
||||||
|
"keep_max_files", OverridesIntegerFormatterValidator
|
||||||
|
)
|
||||||
|
|
||||||
if (
|
if (
|
||||||
self._keep_files_before or self._keep_files_after
|
self._keep_files_before or self._keep_files_after or self._keep_max_files
|
||||||
) and not self.maintain_download_archive:
|
) and not self.maintain_download_archive:
|
||||||
raise self._validation_exception(
|
raise self._validation_exception(
|
||||||
"keep_files requires maintain_download_archive set to True"
|
"keep_files/keep_max requires maintain_download_archive set to True"
|
||||||
)
|
)
|
||||||
|
|
||||||
@property
|
@property
|
||||||
|
|
@ -407,13 +409,11 @@ class OutputOptions(StrictDictValidator):
|
||||||
return self._keep_files_after
|
return self._keep_files_after
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def keep_max_files(self) -> Optional[int]:
|
def keep_max_files(self) -> Optional[OverridesIntegerFormatterValidator]:
|
||||||
"""
|
"""
|
||||||
Optional. Requires ``maintain_download_archive`` set to True.
|
Optional. Requires ``maintain_download_archive`` set to True.
|
||||||
|
|
||||||
Only keeps N most recently uploaded videos. If set to <= 0, ``keep_max_files`` will not be
|
Only keeps N most recently uploaded videos. If set to <= 0, ``keep_max_files`` will not be
|
||||||
applied. Can be used in conjunction with ``keep_files_before`` and ``keep_files_after``.
|
applied. Can be used in conjunction with ``keep_files_before`` and ``keep_files_after``.
|
||||||
"""
|
"""
|
||||||
if self._keep_max_files and self._keep_max_files.value > 0:
|
return self._keep_max_files
|
||||||
return self._keep_max_files.value
|
|
||||||
return None
|
|
||||||
|
|
|
||||||
|
|
@ -27,10 +27,10 @@ presets:
|
||||||
# Only keep files uploaded after date_range
|
# Only keep files uploaded after date_range
|
||||||
output_options:
|
output_options:
|
||||||
keep_files_after: "today-{only_recent_date_range}"
|
keep_files_after: "today-{only_recent_date_range}"
|
||||||
keep_files_max: "{only_recent_max_files}"
|
keep_max_files: "{only_recent_max_files}"
|
||||||
|
|
||||||
# Set the default date_range to 2 months
|
# Set the default date_range to 2 months
|
||||||
overrides:
|
overrides:
|
||||||
date_range: "2months"
|
date_range: "2months" # keep for legacy-reasons
|
||||||
only_recent_date_range: "{date_range}"
|
only_recent_date_range: "{date_range}"
|
||||||
only_recent_max_files: 0
|
only_recent_max_files: 0
|
||||||
|
|
@ -145,9 +145,17 @@ class SubscriptionDownload(BaseSubscription, ABC):
|
||||||
after=self.output_options.keep_files_after,
|
after=self.output_options.keep_files_after,
|
||||||
overrides=self.overrides,
|
overrides=self.overrides,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
keep_max_files: Optional[int] = None
|
||||||
|
if self.output_options.keep_max_files:
|
||||||
|
# validated it can be cast to int within the validator
|
||||||
|
keep_max_files = int(
|
||||||
|
self.overrides.apply_formatter(self.output_options.keep_max_files)
|
||||||
|
)
|
||||||
|
|
||||||
if date_range_to_keep or self.output_options.keep_max_files is not None:
|
if date_range_to_keep or self.output_options.keep_max_files is not None:
|
||||||
self._enhanced_download_archive.remove_stale_files(
|
self._enhanced_download_archive.remove_stale_files(
|
||||||
date_range=date_range_to_keep, keep_max_files=self.output_options.keep_max_files
|
date_range=date_range_to_keep, keep_max_files=keep_max_files
|
||||||
)
|
)
|
||||||
|
|
||||||
self._enhanced_download_archive.save_download_mappings()
|
self._enhanced_download_archive.save_download_mappings()
|
||||||
|
|
|
||||||
|
|
@ -218,6 +218,18 @@ class OverridesStringFormatterValidator(StringFormatterValidator):
|
||||||
# pylint: enable=line-too-long
|
# pylint: enable=line-too-long
|
||||||
|
|
||||||
|
|
||||||
|
class OverridesIntegerFormatterValidator(StringFormatterValidator):
|
||||||
|
_expected_value_type_name = "integer"
|
||||||
|
|
||||||
|
def apply_formatter(self, variable_dict: Dict[str, str]) -> str:
|
||||||
|
output = super().apply_formatter(variable_dict)
|
||||||
|
try:
|
||||||
|
int(output)
|
||||||
|
except Exception as exc:
|
||||||
|
raise self._validation_exception(f"Expected an integer, but received '{output}'") from exc
|
||||||
|
return output
|
||||||
|
|
||||||
|
|
||||||
class ListFormatterValidator(ListValidator[StringFormatterValidator]):
|
class ListFormatterValidator(ListValidator[StringFormatterValidator]):
|
||||||
_inner_list_type = StringFormatterValidator
|
_inner_list_type = StringFormatterValidator
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -570,7 +570,7 @@ class EnhancedDownloadArchive:
|
||||||
for uid, mapping in stale_mappings.items():
|
for uid, mapping in stale_mappings.items():
|
||||||
self._remove_entry(uid=uid, mapping=mapping)
|
self._remove_entry(uid=uid, mapping=mapping)
|
||||||
|
|
||||||
if keep_max_files is not None:
|
if keep_max_files is not None and keep_max_files > 0:
|
||||||
num_files = 0
|
num_files = 0
|
||||||
for uid, mapping in sorted(
|
for uid, mapping in sorted(
|
||||||
self.mapping.entry_mappings.items(),
|
self.mapping.entry_mappings.items(),
|
||||||
|
|
|
||||||
|
|
@ -12,7 +12,9 @@ from ytdl_sub.subscriptions.subscription import Subscription
|
||||||
@pytest.fixture
|
@pytest.fixture
|
||||||
def channel_preset_dict(output_directory):
|
def channel_preset_dict(output_directory):
|
||||||
return {
|
return {
|
||||||
"preset": "TV Show Full Archive",
|
"preset": [
|
||||||
|
"TV Show Full Archive",
|
||||||
|
],
|
||||||
"format": "worst[ext=mp4]", # download the worst format so it is fast
|
"format": "worst[ext=mp4]", # download the worst format so it is fast
|
||||||
"ytdl_options": {
|
"ytdl_options": {
|
||||||
"max_views": 100000, # do not download the popular PJ concert
|
"max_views": 100000, # do not download the popular PJ concert
|
||||||
|
|
@ -98,7 +100,10 @@ class TestChannel:
|
||||||
output_directory: str,
|
output_directory: str,
|
||||||
):
|
):
|
||||||
subscription_name = "pz"
|
subscription_name = "pz"
|
||||||
channel_preset_dict = dict(channel_preset_dict, **{"output_options": {"keep_max_files": 1}})
|
channel_preset_dict["preset"].append("Only Recent")
|
||||||
|
channel_preset_dict["overrides"]["only_recent_date_range"] = "10years"
|
||||||
|
channel_preset_dict["overrides"]["only_recent_max_files"] = 1
|
||||||
|
|
||||||
full_channel_subscription = Subscription.from_dict(
|
full_channel_subscription = Subscription.from_dict(
|
||||||
config=tv_show_config, preset_name=subscription_name, preset_dict=channel_preset_dict
|
config=tv_show_config, preset_name=subscription_name, preset_dict=channel_preset_dict
|
||||||
)
|
)
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue