danvics/ytdl-sub
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

[FEATURE] Ability to customize date used for deleting old files (#1221)

Browse source 
Implements https://github.com/jmbannon/ytdl-sub/issues/1182

Adds the ability to change the `keep_files_date_eval` (traditionally upload_date_standardized) to any date, so long as its in the form of YYYY-MM-DD.

Will be able to use this to have episodes get deleted based on their release_date or epoch_date, instead of their upload_date.
This commit is contained in:
Jesse Bannon 2025-05-31 08:52:15 -07:00 committed by GitHub
parent c40b032eed
commit 987b1cd028
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
9 changed files with 137 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
docs/source/config_reference/plugins.rst
View file

@ -640,6 +640,8 @@ Defines where to output files and thumbnails after all post-processing has compl
maintain_download_archive: True
keep_files_before: now
keep_files_after: 19000101
keep_max_files: 1000
keep_files_date_eval: "{upload_date_standardized}"
``download_archive_name``
@ -690,6 +692,16 @@ Defines where to output files and thumbnails after all post-processing has compl
``keep_max_files``.
``keep_files_date_eval``
:expected type: str
:description:
Uses this standardized date in the form of YYYY-MM-DD to record in the
download archive for a given entry. Subsequently, uses this value to
perform evaluation for keep_files_before/after and keep_max_files. Defaults
to the entry's upload_date_standardized variable.
``keep_max_files``
:expected type: Optional[OverridesFormatter]

6
docs/source/config_reference/scripting/entry_variables.rst
View file

@ -659,3 +659,9 @@ ytdl_sub_input_url_index
:type: ``Integer``
:description:
The index of the input URL as defined in the subscription, top-most being the 0th index.
ytdl_sub_keep_files_date_eval
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
:type: ``String``
:description:
The standardized date variable supplied in ``output_options.keep_files_date_eval``.

36
src/ytdl_sub/config/preset_options.py
View file

@ -1,15 +1,19 @@
from typing import Any
from typing import Dict
from typing import Optional
from typing import Set
from ytdl_sub.config.defaults import DEFAULT_DOWNLOAD_ARCHIVE_NAME
from ytdl_sub.config.overrides import Overrides
from ytdl_sub.config.plugin.plugin_operation import PluginOperation
from ytdl_sub.config.validators.options import OptionsDictValidator
from ytdl_sub.entries.script.variable_definitions import VARIABLES as v
from ytdl_sub.validators.file_path_validators import OverridesStringFormatterFilePathValidator
from ytdl_sub.validators.file_path_validators import StringFormatterFileNameValidator
from ytdl_sub.validators.strict_dict_validator import StrictDictValidator
from ytdl_sub.validators.string_datetime import StringDatetimeValidator
from ytdl_sub.validators.string_formatter_validators import OverridesIntegerFormatterValidator
from ytdl_sub.validators.string_formatter_validators import OverridesStringFormatterValidator
from ytdl_sub.validators.string_formatter_validators import StandardizedDateValidator
from ytdl_sub.validators.string_formatter_validators import StringFormatterValidator
from ytdl_sub.validators.string_formatter_validators import (
UnstructuredOverridesDictFormatterValidator,
@ -65,7 +69,7 @@ class YTDLOptions(UnstructuredOverridesDictFormatterValidator):
# pylint: disable=line-too-long
class OutputOptions(StrictDictValidator):
class OutputOptions(OptionsDictValidator):
"""
Defines where to output files and thumbnails after all post-processing has completed.
@ -87,6 +91,8 @@ class OutputOptions(StrictDictValidator):
maintain_download_archive: True
keep_files_before: now
keep_files_after: 19000101
keep_max_files: 1000
keep_files_date_eval: "{upload_date_standardized}"
"""
_required_keys = {"output_directory", "file_name"}
@ -99,6 +105,8 @@ class OutputOptions(StrictDictValidator):
"keep_files_before",
"keep_files_after",
"keep_max_files",
"download_archive_standardized_date",
"keep_files_date_eval",
}
@classmethod
@ -156,6 +164,11 @@ class OutputOptions(StrictDictValidator):
self._keep_max_files = self._validate_key_if_present(
"keep_max_files", OverridesIntegerFormatterValidator
)
self._keep_files_date_eval = self._validate_key(
"keep_files_date_eval",
StandardizedDateValidator,
default=f"{{{v.upload_date_standardized.variable_name}}}",
)
if (
self._keep_files_before or self._keep_files_after or self._keep_max_files
@ -272,6 +285,18 @@ class OutputOptions(StrictDictValidator):
"""
return self._keep_files_after
@property
def keep_files_date_eval(self) -> StandardizedDateValidator:
"""
:expected type: str
:description:
Uses this standardized date in the form of YYYY-MM-DD to record in the
download archive for a given entry. Subsequently, uses this value to
perform evaluation for keep_files_before/after and keep_max_files. Defaults
to the entry's upload_date_standardized variable.
"""
return self._keep_files_date_eval
@property
def keep_max_files(self) -> Optional[OverridesIntegerFormatterValidator]:
"""
@ -283,3 +308,10 @@ class OutputOptions(StrictDictValidator):
applied. Can be used in conjunction with ``keep_files_before`` and ``keep_files_after``.
"""
return self._keep_max_files
def added_variables(self, unresolved_variables: Set[str]) -> Dict[PluginOperation, Set[str]]:
return {
# PluginOperation.MODIFY_ENTRY_METADATA: {
# VARIABLES.ytdl_sub_entry_date_eval.variable_name
# }
}

7
src/ytdl_sub/config/validators/variable_validation.py
View file

@ -57,13 +57,14 @@ def _add_dummy_overrides(overrides: Overrides) -> Dict[str, str]:
def _get_added_and_modified_variables(
plugins: PresetPlugins, downloader_options: MultiUrlValidator
plugins: PresetPlugins, downloader_options: MultiUrlValidator, output_options: OutputOptions
) -> Iterable[Tuple[OptionsValidator, Set[str], Set[str]]]:
"""
Iterates and returns the plugin options, added variables, modified variables
"""
options: List[OptionsValidator] = plugins.plugin_options
options.append(downloader_options)
options.append(output_options)
for plugin_options in options:
added_variables: Set[str] = set()
@ -117,6 +118,7 @@ class VariableValidation:
) in _get_added_and_modified_variables(
plugins=self.plugins,
downloader_options=self.downloader_options,
output_options=self.output_options,
):
for added_variable in added_variables:
@ -183,6 +185,9 @@ class VariableValidation:
self._add_variables(PluginOperation.DOWNLOADER, options=self.downloader_options)
self._add_subscription_override_variables()
# Always add output options first
self._add_variables(PluginOperation.MODIFY_ENTRY_METADATA, options=self.output_options)
# Metadata variables to be added
for plugin_options in PluginMapping.order_options_by(
self.plugins.zipped(), PluginOperation.MODIFY_ENTRY_METADATA

12
src/ytdl_sub/entries/script/variable_definitions.py
View file

@ -823,6 +823,17 @@ class YtdlSubVariableDefinitions(ABC):
variable_name="upload_date_index_reversed_padded", pad=2
)
@cached_property
def ytdl_sub_keep_files_date_eval(self: "VariableDefinitions") -> StringVariable:
"""
:description:
The standardized date variable supplied in ``output_options.keep_files_date_eval``.
"""
return StringVariable(
variable_name="ytdl_sub_entry_date_eval",
definition=f"{{%string({self.upload_date_standardized.variable_name})}}",
)
class EntryVariableDefinitions(ABC):
@cached_property
@ -1121,6 +1132,7 @@ class VariableDefinitions(
self.ytdl_sub_input_url,
self.ytdl_sub_input_url_index,
self.ytdl_sub_input_url_count,
self.ytdl_sub_keep_files_date_eval,
}
@cache

19
src/ytdl_sub/subscriptions/subscription_download.py
View file

@ -17,6 +17,7 @@ from ytdl_sub.downloaders.source_plugin import SourcePlugin
from ytdl_sub.downloaders.url.downloader import MultiUrlDownloader
from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder
from ytdl_sub.entries.entry import Entry
from ytdl_sub.entries.script.variable_definitions import VARIABLES
from ytdl_sub.subscriptions.base_subscription import BaseSubscription
from ytdl_sub.subscriptions.subscription_ytdl_options import SubscriptionYTDLOptions
from ytdl_sub.utils.datetime import to_date_range
@ -215,9 +216,23 @@ class SubscriptionDownload(BaseSubscription, ABC):
FileHandler.delete(entry.get_download_thumbnail_path())
FileHandler.delete(entry.get_download_info_json_path())
@classmethod
def _preprocess_entry(cls, plugins: List[Plugin], entry: Entry) -> Optional[Entry]:
def _preprocess_entry(self, plugins: List[Plugin], entry: Entry) -> Optional[Entry]:
maybe_entry: Optional[Entry] = entry
# Inject OutputOption variables here
entry.add(
{
VARIABLES.ytdl_sub_keep_files_date_eval: (
self.output_options.keep_files_date_eval.format_string
)
}
)
# Run it to make sure it's actually a standardized date
_ = self.overrides.apply_formatter(
formatter=self.output_options.keep_files_date_eval, entry=entry
)
for plugin in PluginMapping.order_plugins_by(
plugins, PluginOperation.MODIFY_ENTRY_METADATA
):

15
src/ytdl_sub/validators/string_formatter_validators.py
View file

@ -1,3 +1,4 @@
from datetime import datetime
from typing import Dict
from typing import Set
from typing import Union
@ -73,6 +74,20 @@ class StringFormatterValidator(StringValidator):
return resolved
class StandardizedDateValidator(StringFormatterValidator):
_expected_value_type_name = "standardized_date"
def post_process(self, resolved: str) -> str:
try:
datetime.strptime(resolved, "%Y-%m-%d")
except ValueError as exc:
raise self._validation_exception(
f"Expected a standardized date in the form of YYYY-MM-DD, but received '{resolved}'"
) from exc
return resolved
# pylint: disable=line-too-long
class OverridesStringFormatterValidator(StringFormatterValidator):
"""

2
src/ytdl_sub/ytdl_additions/enhanced_download_archive.py
View file

@ -75,7 +75,7 @@ class DownloadMapping:
DownloadMapping for the entry
"""
return DownloadMapping(
upload_date=entry.get(v.upload_date_standardized, str),
upload_date=entry.get(v.ytdl_sub_keep_files_date_eval, str),
extractor=entry.download_archive_extractor,
file_names=set(),
)

34
tests/integration/plugins/test_output_options.py
View file

@ -1,3 +1,4 @@
import re
from pathlib import Path
from typing import Dict
from unittest.mock import patch
@ -11,6 +12,7 @@ from ytdl_sub.config.config_file import ConfigFile
from ytdl_sub.downloaders.ytdlp import YTDLP
from ytdl_sub.entries.entry import Entry
from ytdl_sub.subscriptions.subscription import Subscription
from ytdl_sub.utils.exceptions import ValidationException
from ytdl_sub.utils.file_handler import FileHandler
from ytdl_sub.utils.thumbnail import try_convert_download_thumbnail
@ -223,3 +225,35 @@ class TestOutputOptions:
dry_run=False,
expected_download_summary_file_name="plugins/output_options/test_missing_thumb.json",
)
def test_invalid_keep_files_date_eval(
self,
config: ConfigFile,
subscription_name: str,
output_options_subscription_dict: Dict,
output_directory: str,
mock_download_collection_entries,
):
output_options_subscription_dict["output_options"]["keep_files_date_eval"] = "nope"
subscription = Subscription.from_dict(
config=config,
preset_name=subscription_name,
preset_dict=output_options_subscription_dict,
)
expected_error_msg = (
"Validation error in subscription_test.output_options.keep_files_date_eval: "
"Expected a standardized date in the form of YYYY-MM-DD, but received 'nope'"
)
with (
mock_download_collection_entries(
is_youtube_channel=False,
num_urls=1,
is_extracted_audio=False,
is_dry_run=True,
),
pytest.raises(ValidationException, match=re.escape(expected_error_msg)),
):
subscription.download(dry_run=True)