diff --git a/src/ytdl_sub/config/preset_options.py b/src/ytdl_sub/config/preset_options.py index f1ba8534..3025241f 100644 --- a/src/ytdl_sub/config/preset_options.py +++ b/src/ytdl_sub/config/preset_options.py @@ -14,6 +14,7 @@ from ytdl_sub.validators.string_formatter_validators import StringFormatterValid from ytdl_sub.validators.string_formatter_validators import ( UnstructuredOverridesDictFormatterValidator, ) +from ytdl_sub.validators.string_select_validator import StringSelectValidator from ytdl_sub.validators.validators import BoolValidator @@ -64,6 +65,20 @@ class YTDLOptions(UnstructuredOverridesDictFormatterValidator): # Disable for proper docstring formatting # pylint: disable=line-too-long +class KeepFilesDateEvalValidator(StringSelectValidator): + UPLOAD_DATE = "upload_date" + RELEASE_DATE = "release_date" + _expected_value_type_name = "keep_files_date_eval" + _select_values = {UPLOAD_DATE, RELEASE_DATE} + + @property + def is_upload_date(self) -> bool: + return self.value == self.UPLOAD_DATE + + @property + def is_release_date(self) -> bool: + return self.value == self.RELEASE_DATE + class OutputOptions(StrictDictValidator): """ @@ -87,6 +102,8 @@ class OutputOptions(StrictDictValidator): maintain_download_archive: True keep_files_before: now keep_files_after: 19000101 + keep_max_files: 1000 + keep_files_date_eval: "upload_date" """ _required_keys = {"output_directory", "file_name"} @@ -99,6 +116,7 @@ class OutputOptions(StrictDictValidator): "keep_files_before", "keep_files_after", "keep_max_files", + "keep_files_date_eval", } @classmethod @@ -156,6 +174,9 @@ class OutputOptions(StrictDictValidator): self._keep_max_files = self._validate_key_if_present( "keep_max_files", OverridesIntegerFormatterValidator ) + self._keep_files_date_eval = self._validate_key_if_present( + "keep_files_date_eval", KeepFilesDateEvalValidator, default=KeepFilesDateEvalValidator.UPLOAD_DATE + ) if ( self._keep_files_before or self._keep_files_after or self._keep_max_files @@ -272,6 +293,16 @@ class OutputOptions(StrictDictValidator): """ return self._keep_files_after + @property + def keep_files_date_eval(self) -> Optional[KeepFilesDateEvalValidator]: + """ + :expected type: str + :description: + When using keep_files_before/after, uses the date set in this field for evaluation. + Supports ``upload_date``, ``release_date``, defaults to ``upload_date``. + """ + return self._keep_files_date_eval + @property def keep_max_files(self) -> Optional[OverridesIntegerFormatterValidator]: """ diff --git a/src/ytdl_sub/subscriptions/base_subscription.py b/src/ytdl_sub/subscriptions/base_subscription.py index 970f142d..c0ecd16e 100644 --- a/src/ytdl_sub/subscriptions/base_subscription.py +++ b/src/ytdl_sub/subscriptions/base_subscription.py @@ -31,6 +31,7 @@ def _initialize_download_archive( file_name=overrides.apply_formatter(output_options.download_archive_name), working_directory=working_directory, output_directory=output_directory, + entry_date_eval=output_options.keep_files_date_eval, migrated_file_name=migrated_file_name, ).reinitialize(dry_run=True) diff --git a/src/ytdl_sub/ytdl_additions/enhanced_download_archive.py b/src/ytdl_sub/ytdl_additions/enhanced_download_archive.py index ec155245..dfec1f34 100644 --- a/src/ytdl_sub/ytdl_additions/enhanced_download_archive.py +++ b/src/ytdl_sub/ytdl_additions/enhanced_download_archive.py @@ -13,6 +13,7 @@ from typing import Set from yt_dlp import DateRange from yt_dlp.utils import make_archive_id +from ytdl_sub.config.preset_options import KeepFilesDateEvalValidator from ytdl_sub.entries.entry import Entry from ytdl_sub.entries.entry import ytdl_sub_split_by_chapters_parent_uid from ytdl_sub.entries.script.variable_definitions import VARIABLES @@ -63,19 +64,21 @@ class DownloadMapping: ) @classmethod - def from_entry(cls, entry: Entry) -> "DownloadMapping": + def from_entry(cls, entry: Entry, entry_date: str) -> "DownloadMapping": """ Parameters ---------- entry Entry to create a download mapping for + entry_date: + Date to use for the entry Returns ------- DownloadMapping for the entry """ return DownloadMapping( - upload_date=entry.get(v.upload_date_standardized, str), + upload_date=entry_date, extractor=entry.download_archive_extractor, file_names=set(), ) @@ -151,19 +154,25 @@ class DownloadArchive: class DownloadMappings: _strptime_format = "%Y-%m-%d" - def __init__(self): + def __init__(self, entry_date_eval: KeepFilesDateEvalValidator): """ Initializes an empty mapping + + entry_date_eval + Which date to use for download mapping logging """ + self._entry_date_eval = entry_date_eval self._entry_mappings: Dict[str, DownloadMapping] = {} @classmethod - def from_file(cls, json_file_path: str) -> "DownloadMappings": + def from_file(cls, json_file_path: str, entry_date_eval: KeepFilesDateEvalValidator) -> "DownloadMappings": """ Parameters ---------- json_file_path Path to a json file that contains download mappings + entry_date_eval + Which date to use for download mapping logging Returns ------- @@ -177,7 +186,7 @@ class DownloadMappings: mapping_dict=entry_mappings_json[uid] ) - download_mappings = DownloadMappings() + download_mappings = DownloadMappings(entry_date_eval=entry_date_eval) download_mappings._entry_mappings = entry_mappings_json return download_mappings @@ -218,7 +227,6 @@ class DownloadMappings: Entry that this file belongs to entry_file_path Relative path to the file that lives in the output directory - Returns ------- self @@ -228,7 +236,14 @@ class DownloadMappings: uid = parent_uid if uid not in self.entry_ids: - self._entry_mappings[uid] = DownloadMapping.from_entry(entry=entry) + if self._entry_date_eval.is_upload_date: + entry_date = entry.get(v.upload_date_standardized, str) + elif self._entry_date_eval.is_release_date: + entry_date = entry.get(v.release_date_standardized, str) + else: + raise AssertionError("Unsupported entry_date_eval. Should not reach.") + + self._entry_mappings[uid] = DownloadMapping.from_entry(entry=entry, entry_date=entry_date) self._entry_mappings[uid].file_names.add(entry_file_path) return self @@ -370,7 +385,7 @@ class EnhancedDownloadArchive: @classmethod def _maybe_load_download_mappings( - cls, mapping_file_path: str, migrated_mapping_file_path: Optional[str] + cls, mapping_file_path: str, migrated_mapping_file_path: Optional[str], entry_date_eval: KeepFilesDateEvalValidator ) -> DownloadMappings: """ Tries to load download mappings if a file exists. Otherwise returns empty mappings. @@ -382,21 +397,22 @@ class EnhancedDownloadArchive: "`output_options.migrated_download_archive` to " "`output_options.download_archive`" ) - return DownloadMappings.from_file(migrated_mapping_file_path) + return DownloadMappings.from_file(json_file_path=migrated_mapping_file_path, entry_date_eval=entry_date_eval) logger.warning( "MIGRATION DETECTED, will write archive file to %s", migrated_mapping_file_path ) if os.path.isfile(mapping_file_path): - return DownloadMappings.from_file(json_file_path=mapping_file_path) - return DownloadMappings() + return DownloadMappings.from_file(json_file_path=mapping_file_path, entry_date_eval=entry_date_eval) + return DownloadMappings(entry_date_eval=entry_date_eval) def __init__( self, file_name: str, working_directory: str, output_directory: str, + entry_date_eval: KeepFilesDateEvalValidator, dry_run: bool = False, migrated_file_name: Optional[str] = None, ): @@ -404,7 +420,8 @@ class EnhancedDownloadArchive: self._file_handler = FileHandler( working_directory=working_directory, output_directory=output_directory, dry_run=dry_run ) - self._download_mapping = DownloadMappings() # gets reinitialized + self._entry_date_eval = entry_date_eval + self._download_mapping = DownloadMappings(entry_date_eval=entry_date_eval) # gets reinitialized self._migrated_file_name = migrated_file_name self.num_entries_added: int = 0 @@ -442,6 +459,7 @@ class EnhancedDownloadArchive: self._download_mapping = self._maybe_load_download_mappings( mapping_file_path=self._output_file_path, migrated_mapping_file_path=self._migrated_file_path, + entry_date_eval=self._entry_date_eval, ) return self