hmm
This commit is contained in:
parent
69705d5ac0
commit
c352d31fc7
5 changed files with 44 additions and 32 deletions
|
|
@ -8,13 +8,13 @@ from ytdl_sub.validators.file_path_validators import OverridesStringFormatterFil
|
||||||
from ytdl_sub.validators.file_path_validators import StringFormatterFileNameValidator
|
from ytdl_sub.validators.file_path_validators import StringFormatterFileNameValidator
|
||||||
from ytdl_sub.validators.strict_dict_validator import StrictDictValidator
|
from ytdl_sub.validators.strict_dict_validator import StrictDictValidator
|
||||||
from ytdl_sub.validators.string_datetime import StringDatetimeValidator
|
from ytdl_sub.validators.string_datetime import StringDatetimeValidator
|
||||||
from ytdl_sub.validators.string_formatter_validators import OverridesIntegerFormatterValidator
|
from ytdl_sub.validators.string_formatter_validators import OverridesIntegerFormatterValidator, \
|
||||||
|
OverridesStandardizedDateValidator
|
||||||
from ytdl_sub.validators.string_formatter_validators import OverridesStringFormatterValidator
|
from ytdl_sub.validators.string_formatter_validators import OverridesStringFormatterValidator
|
||||||
from ytdl_sub.validators.string_formatter_validators import StringFormatterValidator
|
from ytdl_sub.validators.string_formatter_validators import StringFormatterValidator
|
||||||
from ytdl_sub.validators.string_formatter_validators import (
|
from ytdl_sub.validators.string_formatter_validators import (
|
||||||
UnstructuredOverridesDictFormatterValidator,
|
UnstructuredOverridesDictFormatterValidator,
|
||||||
)
|
)
|
||||||
from ytdl_sub.validators.string_select_validator import StringSelectValidator
|
|
||||||
from ytdl_sub.validators.validators import BoolValidator
|
from ytdl_sub.validators.validators import BoolValidator
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -65,20 +65,6 @@ class YTDLOptions(UnstructuredOverridesDictFormatterValidator):
|
||||||
# Disable for proper docstring formatting
|
# Disable for proper docstring formatting
|
||||||
# pylint: disable=line-too-long
|
# pylint: disable=line-too-long
|
||||||
|
|
||||||
class KeepFilesDateEvalValidator(StringSelectValidator):
|
|
||||||
UPLOAD_DATE = "upload_date"
|
|
||||||
RELEASE_DATE = "release_date"
|
|
||||||
_expected_value_type_name = "keep_files_date_eval"
|
|
||||||
_select_values = {UPLOAD_DATE, RELEASE_DATE}
|
|
||||||
|
|
||||||
@property
|
|
||||||
def is_upload_date(self) -> bool:
|
|
||||||
return self.value == self.UPLOAD_DATE
|
|
||||||
|
|
||||||
@property
|
|
||||||
def is_release_date(self) -> bool:
|
|
||||||
return self.value == self.RELEASE_DATE
|
|
||||||
|
|
||||||
|
|
||||||
class OutputOptions(StrictDictValidator):
|
class OutputOptions(StrictDictValidator):
|
||||||
"""
|
"""
|
||||||
|
|
@ -116,7 +102,7 @@ class OutputOptions(StrictDictValidator):
|
||||||
"keep_files_before",
|
"keep_files_before",
|
||||||
"keep_files_after",
|
"keep_files_after",
|
||||||
"keep_max_files",
|
"keep_max_files",
|
||||||
"keep_files_date_eval",
|
"download_archive_standardized_date",
|
||||||
}
|
}
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
|
|
@ -174,8 +160,8 @@ class OutputOptions(StrictDictValidator):
|
||||||
self._keep_max_files = self._validate_key_if_present(
|
self._keep_max_files = self._validate_key_if_present(
|
||||||
"keep_max_files", OverridesIntegerFormatterValidator
|
"keep_max_files", OverridesIntegerFormatterValidator
|
||||||
)
|
)
|
||||||
self._keep_files_date_eval = self._validate_key_if_present(
|
self._entry_date_eval = self._validate_key_if_present(
|
||||||
"keep_files_date_eval", KeepFilesDateEvalValidator, default=KeepFilesDateEvalValidator.UPLOAD_DATE
|
"entry_date_eval", OverridesStandardizedDateValidator
|
||||||
)
|
)
|
||||||
|
|
||||||
if (
|
if (
|
||||||
|
|
@ -294,14 +280,16 @@ class OutputOptions(StrictDictValidator):
|
||||||
return self._keep_files_after
|
return self._keep_files_after
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def keep_files_date_eval(self) -> Optional[KeepFilesDateEvalValidator]:
|
def entry_date_eval(self) -> Optional[OverridesStandardizedDateValidator]:
|
||||||
"""
|
"""
|
||||||
:expected type: str
|
:expected type: str
|
||||||
:description:
|
:description:
|
||||||
When using keep_files_before/after, uses the date set in this field for evaluation.
|
Uses this standardized date in the form of YYYY-MM-DD to record in the
|
||||||
Supports ``upload_date``, ``release_date``, defaults to ``upload_date``.
|
download archive for a given entry. Subsequently, uses this value to
|
||||||
|
perform evaluation for keep_files_before/after and keep_max_files. Defaults
|
||||||
|
to the entry's upload_date_standardized variable.
|
||||||
"""
|
"""
|
||||||
return self._keep_files_date_eval
|
return self._entry_date_eval
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def keep_max_files(self) -> Optional[OverridesIntegerFormatterValidator]:
|
def keep_max_files(self) -> Optional[OverridesIntegerFormatterValidator]:
|
||||||
|
|
|
||||||
|
|
@ -823,6 +823,14 @@ class YtdlSubVariableDefinitions(ABC):
|
||||||
variable_name="upload_date_index_reversed_padded", pad=2
|
variable_name="upload_date_index_reversed_padded", pad=2
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@cached_property
|
||||||
|
def ytdl_sub_entry_date_eval(self: "VariableDefinitions") -> StringVariable:
|
||||||
|
"""
|
||||||
|
:description:
|
||||||
|
The standardized
|
||||||
|
"""
|
||||||
|
return StringVariable(variable_name="ytdl_sub_input_url", definition="{ %string('') }")
|
||||||
|
|
||||||
|
|
||||||
class EntryVariableDefinitions(ABC):
|
class EntryVariableDefinitions(ABC):
|
||||||
@cached_property
|
@cached_property
|
||||||
|
|
|
||||||
|
|
@ -31,7 +31,7 @@ def _initialize_download_archive(
|
||||||
file_name=overrides.apply_formatter(output_options.download_archive_name),
|
file_name=overrides.apply_formatter(output_options.download_archive_name),
|
||||||
working_directory=working_directory,
|
working_directory=working_directory,
|
||||||
output_directory=output_directory,
|
output_directory=output_directory,
|
||||||
entry_date_eval=output_options.keep_files_date_eval,
|
entry_date_eval=output_options.entry_date_eval,
|
||||||
migrated_file_name=migrated_file_name,
|
migrated_file_name=migrated_file_name,
|
||||||
).reinitialize(dry_run=True)
|
).reinitialize(dry_run=True)
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,3 +1,4 @@
|
||||||
|
from datetime import datetime
|
||||||
from typing import Dict
|
from typing import Dict
|
||||||
from typing import Set
|
from typing import Set
|
||||||
from typing import Union
|
from typing import Union
|
||||||
|
|
@ -103,6 +104,19 @@ class OverridesIntegerFormatterValidator(OverridesStringFormatterValidator):
|
||||||
return resolved
|
return resolved
|
||||||
|
|
||||||
|
|
||||||
|
class OverridesStandardizedDateValidator(OverridesStringFormatterValidator):
|
||||||
|
_expected_value_type_name = "standardized_date"
|
||||||
|
|
||||||
|
def post_process(self, resolved: str) -> str:
|
||||||
|
try:
|
||||||
|
datetime.strptime(resolved, "%Y-%m-%d")
|
||||||
|
except ValueError as exc:
|
||||||
|
raise self._validation_exception(
|
||||||
|
f"Expected a standardized date in the form of YYYY-MM-DD, but received '{resolved}'"
|
||||||
|
) from exc
|
||||||
|
|
||||||
|
return resolved
|
||||||
|
|
||||||
class OverridesBooleanFormatterValidator(OverridesStringFormatterValidator):
|
class OverridesBooleanFormatterValidator(OverridesStringFormatterValidator):
|
||||||
_expected_value_type_name = "boolean"
|
_expected_value_type_name = "boolean"
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -13,7 +13,6 @@ from typing import Set
|
||||||
from yt_dlp import DateRange
|
from yt_dlp import DateRange
|
||||||
from yt_dlp.utils import make_archive_id
|
from yt_dlp.utils import make_archive_id
|
||||||
|
|
||||||
from ytdl_sub.config.preset_options import KeepFilesDateEvalValidator
|
|
||||||
from ytdl_sub.entries.entry import Entry
|
from ytdl_sub.entries.entry import Entry
|
||||||
from ytdl_sub.entries.entry import ytdl_sub_split_by_chapters_parent_uid
|
from ytdl_sub.entries.entry import ytdl_sub_split_by_chapters_parent_uid
|
||||||
from ytdl_sub.entries.script.variable_definitions import VARIABLES
|
from ytdl_sub.entries.script.variable_definitions import VARIABLES
|
||||||
|
|
@ -22,6 +21,7 @@ from ytdl_sub.utils.file_handler import FileHandler
|
||||||
from ytdl_sub.utils.file_handler import FileHandlerTransactionLog
|
from ytdl_sub.utils.file_handler import FileHandlerTransactionLog
|
||||||
from ytdl_sub.utils.file_handler import FileMetadata
|
from ytdl_sub.utils.file_handler import FileMetadata
|
||||||
from ytdl_sub.utils.logger import Logger
|
from ytdl_sub.utils.logger import Logger
|
||||||
|
from ytdl_sub.validators.string_formatter_validators import OverridesStandardizedDateValidator
|
||||||
|
|
||||||
logger = Logger.get("archive")
|
logger = Logger.get("archive")
|
||||||
|
|
||||||
|
|
@ -154,7 +154,7 @@ class DownloadArchive:
|
||||||
class DownloadMappings:
|
class DownloadMappings:
|
||||||
_strptime_format = "%Y-%m-%d"
|
_strptime_format = "%Y-%m-%d"
|
||||||
|
|
||||||
def __init__(self, entry_date_eval: KeepFilesDateEvalValidator):
|
def __init__(self, entry_date_eval: Optional[OverridesStandardizedDateValidator]):
|
||||||
"""
|
"""
|
||||||
Initializes an empty mapping
|
Initializes an empty mapping
|
||||||
|
|
||||||
|
|
@ -165,7 +165,7 @@ class DownloadMappings:
|
||||||
self._entry_mappings: Dict[str, DownloadMapping] = {}
|
self._entry_mappings: Dict[str, DownloadMapping] = {}
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def from_file(cls, json_file_path: str, entry_date_eval: KeepFilesDateEvalValidator) -> "DownloadMappings":
|
def from_file(cls, json_file_path: str, entry_date_eval: Optional[OverridesStandardizedDateValidator]) -> "DownloadMappings":
|
||||||
"""
|
"""
|
||||||
Parameters
|
Parameters
|
||||||
----------
|
----------
|
||||||
|
|
@ -236,8 +236,9 @@ class DownloadMappings:
|
||||||
uid = parent_uid
|
uid = parent_uid
|
||||||
|
|
||||||
if uid not in self.entry_ids:
|
if uid not in self.entry_ids:
|
||||||
if self._entry_date_eval.is_upload_date:
|
|
||||||
entry_date = entry.get(v.upload_date_standardized, str)
|
entry_date = entry.get(v.upload_date_standardized, str)
|
||||||
|
if self._entry_date_eval is not None:
|
||||||
|
entry_date = entry
|
||||||
elif self._entry_date_eval.is_release_date:
|
elif self._entry_date_eval.is_release_date:
|
||||||
entry_date = entry.get(v.release_date_standardized, str)
|
entry_date = entry.get(v.release_date_standardized, str)
|
||||||
else:
|
else:
|
||||||
|
|
@ -385,7 +386,7 @@ class EnhancedDownloadArchive:
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def _maybe_load_download_mappings(
|
def _maybe_load_download_mappings(
|
||||||
cls, mapping_file_path: str, migrated_mapping_file_path: Optional[str], entry_date_eval: KeepFilesDateEvalValidator
|
cls, mapping_file_path: str, migrated_mapping_file_path: Optional[str], entry_date_eval: Optional[OverridesStandardizedDateValidator]
|
||||||
) -> DownloadMappings:
|
) -> DownloadMappings:
|
||||||
"""
|
"""
|
||||||
Tries to load download mappings if a file exists. Otherwise returns empty mappings.
|
Tries to load download mappings if a file exists. Otherwise returns empty mappings.
|
||||||
|
|
@ -412,7 +413,7 @@ class EnhancedDownloadArchive:
|
||||||
file_name: str,
|
file_name: str,
|
||||||
working_directory: str,
|
working_directory: str,
|
||||||
output_directory: str,
|
output_directory: str,
|
||||||
entry_date_eval: KeepFilesDateEvalValidator,
|
entry_date_eval: Optional[OverridesStandardizedDateValidator],
|
||||||
dry_run: bool = False,
|
dry_run: bool = False,
|
||||||
migrated_file_name: Optional[str] = None,
|
migrated_file_name: Optional[str] = None,
|
||||||
):
|
):
|
||||||
|
|
@ -421,7 +422,8 @@ class EnhancedDownloadArchive:
|
||||||
working_directory=working_directory, output_directory=output_directory, dry_run=dry_run
|
working_directory=working_directory, output_directory=output_directory, dry_run=dry_run
|
||||||
)
|
)
|
||||||
self._entry_date_eval = entry_date_eval
|
self._entry_date_eval = entry_date_eval
|
||||||
self._download_mapping = DownloadMappings(entry_date_eval=entry_date_eval) # gets reinitialized
|
# gets reinitialized
|
||||||
|
self._download_mapping = DownloadMappings(entry_date_eval=entry_date_eval)
|
||||||
self._migrated_file_name = migrated_file_name
|
self._migrated_file_name = migrated_file_name
|
||||||
|
|
||||||
self.num_entries_added: int = 0
|
self.num_entries_added: int = 0
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue