danvics/ytdl-sub
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

[FEATURE] Ability to choose release_date for Only Recent eval

Browse source
This commit is contained in:
Jesse Bannon 2025-05-06 09:38:07 -07:00
parent d926617d5d
commit 69705d5ac0
3 changed files with 62 additions and 12 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

31
src/ytdl_sub/config/preset_options.py
View file

@ -14,6 +14,7 @@ from ytdl_sub.validators.string_formatter_validators import StringFormatterValid
from ytdl_sub.validators.string_formatter_validators import (
UnstructuredOverridesDictFormatterValidator,
)
from ytdl_sub.validators.string_select_validator import StringSelectValidator
from ytdl_sub.validators.validators import BoolValidator
@ -64,6 +65,20 @@ class YTDLOptions(UnstructuredOverridesDictFormatterValidator):
# Disable for proper docstring formatting
# pylint: disable=line-too-long
class KeepFilesDateEvalValidator(StringSelectValidator):
UPLOAD_DATE = "upload_date"
RELEASE_DATE = "release_date"
_expected_value_type_name = "keep_files_date_eval"
_select_values = {UPLOAD_DATE, RELEASE_DATE}
@property
def is_upload_date(self) -> bool:
return self.value == self.UPLOAD_DATE
@property
def is_release_date(self) -> bool:
return self.value == self.RELEASE_DATE
class OutputOptions(StrictDictValidator):
"""
@ -87,6 +102,8 @@ class OutputOptions(StrictDictValidator):
maintain_download_archive: True
keep_files_before: now
keep_files_after: 19000101
keep_max_files: 1000
keep_files_date_eval: "upload_date"
"""
_required_keys = {"output_directory", "file_name"}
@ -99,6 +116,7 @@ class OutputOptions(StrictDictValidator):
"keep_files_before",
"keep_files_after",
"keep_max_files",
"keep_files_date_eval",
}
@classmethod
@ -156,6 +174,9 @@ class OutputOptions(StrictDictValidator):
self._keep_max_files = self._validate_key_if_present(
"keep_max_files", OverridesIntegerFormatterValidator
)
self._keep_files_date_eval = self._validate_key_if_present(
"keep_files_date_eval", KeepFilesDateEvalValidator, default=KeepFilesDateEvalValidator.UPLOAD_DATE
)
if (
self._keep_files_before or self._keep_files_after or self._keep_max_files
@ -272,6 +293,16 @@ class OutputOptions(StrictDictValidator):
"""
return self._keep_files_after
@property
def keep_files_date_eval(self) -> Optional[KeepFilesDateEvalValidator]:
"""
:expected type: str
:description:
When using keep_files_before/after, uses the date set in this field for evaluation.
Supports ``upload_date``, ``release_date``, defaults to ``upload_date``.
"""
return self._keep_files_date_eval
@property
def keep_max_files(self) -> Optional[OverridesIntegerFormatterValidator]:
"""

1
src/ytdl_sub/subscriptions/base_subscription.py
View file

@ -31,6 +31,7 @@ def _initialize_download_archive(
file_name=overrides.apply_formatter(output_options.download_archive_name),
working_directory=working_directory,
output_directory=output_directory,
entry_date_eval=output_options.keep_files_date_eval,
migrated_file_name=migrated_file_name,
).reinitialize(dry_run=True)

42
src/ytdl_sub/ytdl_additions/enhanced_download_archive.py
View file

@ -13,6 +13,7 @@ from typing import Set
from yt_dlp import DateRange
from yt_dlp.utils import make_archive_id
from ytdl_sub.config.preset_options import KeepFilesDateEvalValidator
from ytdl_sub.entries.entry import Entry
from ytdl_sub.entries.entry import ytdl_sub_split_by_chapters_parent_uid
from ytdl_sub.entries.script.variable_definitions import VARIABLES
@ -63,19 +64,21 @@ class DownloadMapping:
)
@classmethod
def from_entry(cls, entry: Entry) -> "DownloadMapping":
def from_entry(cls, entry: Entry, entry_date: str) -> "DownloadMapping":
"""
Parameters
----------
entry
Entry to create a download mapping for
entry_date:
Date to use for the entry
Returns
-------
DownloadMapping for the entry
"""
return DownloadMapping(
upload_date=entry.get(v.upload_date_standardized, str),
upload_date=entry_date,
extractor=entry.download_archive_extractor,
file_names=set(),
)
@ -151,19 +154,25 @@ class DownloadArchive:
class DownloadMappings:
_strptime_format = "%Y-%m-%d"
def __init__(self):
def __init__(self, entry_date_eval: KeepFilesDateEvalValidator):
"""
Initializes an empty mapping
entry_date_eval
Which date to use for download mapping logging
"""
self._entry_date_eval = entry_date_eval
self._entry_mappings: Dict[str, DownloadMapping] = {}
@classmethod
def from_file(cls, json_file_path: str) -> "DownloadMappings":
def from_file(cls, json_file_path: str, entry_date_eval: KeepFilesDateEvalValidator) -> "DownloadMappings":
"""
Parameters
----------
json_file_path
Path to a json file that contains download mappings
entry_date_eval
Which date to use for download mapping logging
Returns
-------
@ -177,7 +186,7 @@ class DownloadMappings:
mapping_dict=entry_mappings_json[uid]
)
download_mappings = DownloadMappings()
download_mappings = DownloadMappings(entry_date_eval=entry_date_eval)
download_mappings._entry_mappings = entry_mappings_json
return download_mappings
@ -218,7 +227,6 @@ class DownloadMappings:
Entry that this file belongs to
entry_file_path
Relative path to the file that lives in the output directory
Returns
-------
self
@ -228,7 +236,14 @@ class DownloadMappings:
uid = parent_uid
if uid not in self.entry_ids:
self._entry_mappings[uid] = DownloadMapping.from_entry(entry=entry)
if self._entry_date_eval.is_upload_date:
entry_date = entry.get(v.upload_date_standardized, str)
elif self._entry_date_eval.is_release_date:
entry_date = entry.get(v.release_date_standardized, str)
else:
raise AssertionError("Unsupported entry_date_eval. Should not reach.")
self._entry_mappings[uid] = DownloadMapping.from_entry(entry=entry, entry_date=entry_date)
self._entry_mappings[uid].file_names.add(entry_file_path)
return self
@ -370,7 +385,7 @@ class EnhancedDownloadArchive:
@classmethod
def _maybe_load_download_mappings(
cls, mapping_file_path: str, migrated_mapping_file_path: Optional[str]
cls, mapping_file_path: str, migrated_mapping_file_path: Optional[str], entry_date_eval: KeepFilesDateEvalValidator
) -> DownloadMappings:
"""
Tries to load download mappings if a file exists. Otherwise returns empty mappings.
@ -382,21 +397,22 @@ class EnhancedDownloadArchive:
"`output_options.migrated_download_archive` to "
"`output_options.download_archive`"
)
return DownloadMappings.from_file(migrated_mapping_file_path)
return DownloadMappings.from_file(json_file_path=migrated_mapping_file_path, entry_date_eval=entry_date_eval)
logger.warning(
"MIGRATION DETECTED, will write archive file to %s", migrated_mapping_file_path
)
if os.path.isfile(mapping_file_path):
return DownloadMappings.from_file(json_file_path=mapping_file_path)
return DownloadMappings()
return DownloadMappings.from_file(json_file_path=mapping_file_path, entry_date_eval=entry_date_eval)
return DownloadMappings(entry_date_eval=entry_date_eval)
def __init__(
self,
file_name: str,
working_directory: str,
output_directory: str,
entry_date_eval: KeepFilesDateEvalValidator,
dry_run: bool = False,
migrated_file_name: Optional[str] = None,
):
@ -404,7 +420,8 @@ class EnhancedDownloadArchive:
self._file_handler = FileHandler(
working_directory=working_directory, output_directory=output_directory, dry_run=dry_run
)
self._download_mapping = DownloadMappings() # gets reinitialized
self._entry_date_eval = entry_date_eval
self._download_mapping = DownloadMappings(entry_date_eval=entry_date_eval) # gets reinitialized
self._migrated_file_name = migrated_file_name
self.num_entries_added: int = 0
@ -442,6 +459,7 @@ class EnhancedDownloadArchive:
self._download_mapping = self._maybe_load_download_mappings(
mapping_file_path=self._output_file_path,
migrated_mapping_file_path=self._migrated_file_path,
entry_date_eval=self._entry_date_eval,
)
return self