danvics/ytdl-sub
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

working?

Browse source
This commit is contained in:
Jesse Bannon 2025-05-31 08:03:21 -07:00
parent c352d31fc7
commit 9fd46bf929
7 changed files with 72 additions and 59 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

29
src/ytdl_sub/config/preset_options.py
View file

@ -1,16 +1,19 @@
from typing import Any
from typing import Dict
from typing import Optional
from typing import Set
from ytdl_sub.config.defaults import DEFAULT_DOWNLOAD_ARCHIVE_NAME
from ytdl_sub.config.overrides import Overrides
from ytdl_sub.config.plugin.plugin_operation import PluginOperation
from ytdl_sub.config.validators.options import OptionsDictValidator
from ytdl_sub.entries.script.variable_definitions import VARIABLES as v
from ytdl_sub.validators.file_path_validators import OverridesStringFormatterFilePathValidator
from ytdl_sub.validators.file_path_validators import StringFormatterFileNameValidator
from ytdl_sub.validators.strict_dict_validator import StrictDictValidator
from ytdl_sub.validators.string_datetime import StringDatetimeValidator
from ytdl_sub.validators.string_formatter_validators import OverridesIntegerFormatterValidator, \
OverridesStandardizedDateValidator
from ytdl_sub.validators.string_formatter_validators import OverridesIntegerFormatterValidator
from ytdl_sub.validators.string_formatter_validators import OverridesStringFormatterValidator
from ytdl_sub.validators.string_formatter_validators import StandardizedDateValidator
from ytdl_sub.validators.string_formatter_validators import StringFormatterValidator
from ytdl_sub.validators.string_formatter_validators import (
UnstructuredOverridesDictFormatterValidator,
@ -66,7 +69,7 @@ class YTDLOptions(UnstructuredOverridesDictFormatterValidator):
# pylint: disable=line-too-long
class OutputOptions(StrictDictValidator):
class OutputOptions(OptionsDictValidator):
"""
Defines where to output files and thumbnails after all post-processing has completed.
@ -89,7 +92,7 @@ class OutputOptions(StrictDictValidator):
keep_files_before: now
keep_files_after: 19000101
keep_max_files: 1000
keep_files_date_eval: "upload_date"
keep_files_date_eval: "{upload_date_standardized}"
"""
_required_keys = {"output_directory", "file_name"}
@ -103,6 +106,7 @@ class OutputOptions(StrictDictValidator):
"keep_files_after",
"keep_max_files",
"download_archive_standardized_date",
"entry_date_eval",
}
@classmethod
@ -160,8 +164,10 @@ class OutputOptions(StrictDictValidator):
self._keep_max_files = self._validate_key_if_present(
"keep_max_files", OverridesIntegerFormatterValidator
)
self._entry_date_eval = self._validate_key_if_present(
"entry_date_eval", OverridesStandardizedDateValidator
self._entry_date_eval = self._validate_key(
"entry_date_eval",
StandardizedDateValidator,
default=f"{{{v.upload_date_standardized.variable_name}}}"
)
if (
@ -280,7 +286,7 @@ class OutputOptions(StrictDictValidator):
return self._keep_files_after
@property
def entry_date_eval(self) -> Optional[OverridesStandardizedDateValidator]:
def entry_date_eval(self) -> StandardizedDateValidator:
"""
:expected type: str
:description:
@ -302,3 +308,10 @@ class OutputOptions(StrictDictValidator):
applied. Can be used in conjunction with ``keep_files_before`` and ``keep_files_after``.
"""
return self._keep_max_files
def added_variables(self, unresolved_variables: Set[str]) -> Dict[PluginOperation, Set[str]]:
return {
# PluginOperation.MODIFY_ENTRY_METADATA: {
# VARIABLES.ytdl_sub_entry_date_eval.variable_name
# }
}

7
src/ytdl_sub/config/validators/variable_validation.py
View file

@ -57,13 +57,14 @@ def _add_dummy_overrides(overrides: Overrides) -> Dict[str, str]:
def _get_added_and_modified_variables(
plugins: PresetPlugins, downloader_options: MultiUrlValidator
plugins: PresetPlugins, downloader_options: MultiUrlValidator, output_options: OutputOptions
) -> Iterable[Tuple[OptionsValidator, Set[str], Set[str]]]:
"""
Iterates and returns the plugin options, added variables, modified variables
"""
options: List[OptionsValidator] = plugins.plugin_options
options.append(downloader_options)
options.append(output_options)
for plugin_options in options:
added_variables: Set[str] = set()
@ -117,6 +118,7 @@ class VariableValidation:
) in _get_added_and_modified_variables(
plugins=self.plugins,
downloader_options=self.downloader_options,
output_options=self.output_options,
):
for added_variable in added_variables:
@ -183,6 +185,9 @@ class VariableValidation:
self._add_variables(PluginOperation.DOWNLOADER, options=self.downloader_options)
self._add_subscription_override_variables()
# Always add output options first
self._add_variables(PluginOperation.MODIFY_ENTRY_METADATA, options=self.output_options)
# Metadata variables to be added
for plugin_options in PluginMapping.order_options_by(
self.plugins.zipped(), PluginOperation.MODIFY_ENTRY_METADATA

8
src/ytdl_sub/entries/script/variable_definitions.py
View file

@ -827,9 +827,12 @@ class YtdlSubVariableDefinitions(ABC):
def ytdl_sub_entry_date_eval(self: "VariableDefinitions") -> StringVariable:
"""
:description:
The standardized
The standardized date variable supplied in ``output_options.entry_date_eval``
"""
return StringVariable(variable_name="ytdl_sub_input_url", definition="{ %string('') }")
return StringVariable(
variable_name="ytdl_sub_entry_date_eval",
definition=f"{{%string({self.upload_date_standardized.variable_name})}}",
)
class EntryVariableDefinitions(ABC):
@ -1129,6 +1132,7 @@ class VariableDefinitions(
self.ytdl_sub_input_url,
self.ytdl_sub_input_url_index,
self.ytdl_sub_input_url_count,
self.ytdl_sub_entry_date_eval,
}
@cache

1
src/ytdl_sub/subscriptions/base_subscription.py
View file

@ -31,7 +31,6 @@ def _initialize_download_archive(
file_name=overrides.apply_formatter(output_options.download_archive_name),
working_directory=working_directory,
output_directory=output_directory,
entry_date_eval=output_options.entry_date_eval,
migrated_file_name=migrated_file_name,
).reinitialize(dry_run=True)

15
src/ytdl_sub/subscriptions/subscription_download.py
View file

@ -17,6 +17,7 @@ from ytdl_sub.downloaders.source_plugin import SourcePlugin
from ytdl_sub.downloaders.url.downloader import MultiUrlDownloader
from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder
from ytdl_sub.entries.entry import Entry
from ytdl_sub.entries.script.variable_definitions import VARIABLES
from ytdl_sub.subscriptions.base_subscription import BaseSubscription
from ytdl_sub.subscriptions.subscription_ytdl_options import SubscriptionYTDLOptions
from ytdl_sub.utils.datetime import to_date_range
@ -215,9 +216,19 @@ class SubscriptionDownload(BaseSubscription, ABC):
FileHandler.delete(entry.get_download_thumbnail_path())
FileHandler.delete(entry.get_download_info_json_path())
@classmethod
def _preprocess_entry(cls, plugins: List[Plugin], entry: Entry) -> Optional[Entry]:
def _preprocess_entry(self, plugins: List[Plugin], entry: Entry) -> Optional[Entry]:
maybe_entry: Optional[Entry] = entry
# Inject OutputOption variables here
entry.add(
{VARIABLES.ytdl_sub_entry_date_eval: self.output_options.entry_date_eval.format_string}
)
# Run it to make sure it's actually a standardized date
_ = self.overrides.apply_formatter(
formatter=self.output_options.entry_date_eval, entry=entry
)
for plugin in PluginMapping.order_plugins_by(
plugins, PluginOperation.MODIFY_ENTRY_METADATA
):

27
src/ytdl_sub/validators/string_formatter_validators.py
View file

@ -74,6 +74,20 @@ class StringFormatterValidator(StringValidator):
return resolved
class StandardizedDateValidator(StringFormatterValidator):
_expected_value_type_name = "standardized_date"
def post_process(self, resolved: str) -> str:
try:
datetime.strptime(resolved, "%Y-%m-%d")
except ValueError as exc:
raise self._validation_exception(
f"Expected a standardized date in the form of YYYY-MM-DD, but received '{resolved}'"
) from exc
return resolved
# pylint: disable=line-too-long
class OverridesStringFormatterValidator(StringFormatterValidator):
"""
@ -104,19 +118,6 @@ class OverridesIntegerFormatterValidator(OverridesStringFormatterValidator):
return resolved
class OverridesStandardizedDateValidator(OverridesStringFormatterValidator):
_expected_value_type_name = "standardized_date"
def post_process(self, resolved: str) -> str:
try:
datetime.strptime(resolved, "%Y-%m-%d")
except ValueError as exc:
raise self._validation_exception(
f"Expected a standardized date in the form of YYYY-MM-DD, but received '{resolved}'"
) from exc
return resolved
class OverridesBooleanFormatterValidator(OverridesStringFormatterValidator):
_expected_value_type_name = "boolean"

44
src/ytdl_sub/ytdl_additions/enhanced_download_archive.py
View file

@ -21,7 +21,6 @@ from ytdl_sub.utils.file_handler import FileHandler
from ytdl_sub.utils.file_handler import FileHandlerTransactionLog
from ytdl_sub.utils.file_handler import FileMetadata
from ytdl_sub.utils.logger import Logger
from ytdl_sub.validators.string_formatter_validators import OverridesStandardizedDateValidator
logger = Logger.get("archive")
@ -64,21 +63,19 @@ class DownloadMapping:
)
@classmethod
def from_entry(cls, entry: Entry, entry_date: str) -> "DownloadMapping":
def from_entry(cls, entry: Entry) -> "DownloadMapping":
"""
Parameters
----------
entry
Entry to create a download mapping for
entry_date:
Date to use for the entry
Returns
-------
DownloadMapping for the entry
"""
return DownloadMapping(
upload_date=entry_date,
upload_date=entry.get(v.ytdl_sub_entry_date_eval, str),
extractor=entry.download_archive_extractor,
file_names=set(),
)
@ -154,25 +151,19 @@ class DownloadArchive:
class DownloadMappings:
_strptime_format = "%Y-%m-%d"
def __init__(self, entry_date_eval: Optional[OverridesStandardizedDateValidator]):
def __init__(self):
"""
Initializes an empty mapping
entry_date_eval
Which date to use for download mapping logging
"""
self._entry_date_eval = entry_date_eval
self._entry_mappings: Dict[str, DownloadMapping] = {}
@classmethod
def from_file(cls, json_file_path: str, entry_date_eval: Optional[OverridesStandardizedDateValidator]) -> "DownloadMappings":
def from_file(cls, json_file_path: str) -> "DownloadMappings":
"""
Parameters
----------
json_file_path
Path to a json file that contains download mappings
entry_date_eval
Which date to use for download mapping logging
Returns
-------
@ -186,7 +177,7 @@ class DownloadMappings:
mapping_dict=entry_mappings_json[uid]
)
download_mappings = DownloadMappings(entry_date_eval=entry_date_eval)
download_mappings = DownloadMappings()
download_mappings._entry_mappings = entry_mappings_json
return download_mappings
@ -227,6 +218,7 @@ class DownloadMappings:
Entry that this file belongs to
entry_file_path
Relative path to the file that lives in the output directory
Returns
-------
self
@ -236,15 +228,7 @@ class DownloadMappings:
uid = parent_uid
if uid not in self.entry_ids:
entry_date = entry.get(v.upload_date_standardized, str)
if self._entry_date_eval is not None:
entry_date = entry
elif self._entry_date_eval.is_release_date:
entry_date = entry.get(v.release_date_standardized, str)
else:
raise AssertionError("Unsupported entry_date_eval. Should not reach.")
self._entry_mappings[uid] = DownloadMapping.from_entry(entry=entry, entry_date=entry_date)
self._entry_mappings[uid] = DownloadMapping.from_entry(entry=entry)
self._entry_mappings[uid].file_names.add(entry_file_path)
return self
@ -386,7 +370,7 @@ class EnhancedDownloadArchive:
@classmethod
def _maybe_load_download_mappings(
cls, mapping_file_path: str, migrated_mapping_file_path: Optional[str], entry_date_eval: Optional[OverridesStandardizedDateValidator]
cls, mapping_file_path: str, migrated_mapping_file_path: Optional[str]
) -> DownloadMappings:
"""
Tries to load download mappings if a file exists. Otherwise returns empty mappings.
@ -398,22 +382,21 @@ class EnhancedDownloadArchive:
"`output_options.migrated_download_archive` to "
"`output_options.download_archive`"
)
return DownloadMappings.from_file(json_file_path=migrated_mapping_file_path, entry_date_eval=entry_date_eval)
return DownloadMappings.from_file(migrated_mapping_file_path)
logger.warning(
"MIGRATION DETECTED, will write archive file to %s", migrated_mapping_file_path
)
if os.path.isfile(mapping_file_path):
return DownloadMappings.from_file(json_file_path=mapping_file_path, entry_date_eval=entry_date_eval)
return DownloadMappings(entry_date_eval=entry_date_eval)
return DownloadMappings.from_file(json_file_path=mapping_file_path)
return DownloadMappings()
def __init__(
self,
file_name: str,
working_directory: str,
output_directory: str,
entry_date_eval: Optional[OverridesStandardizedDateValidator],
dry_run: bool = False,
migrated_file_name: Optional[str] = None,
):
@ -421,9 +404,7 @@ class EnhancedDownloadArchive:
self._file_handler = FileHandler(
working_directory=working_directory, output_directory=output_directory, dry_run=dry_run
)
self._entry_date_eval = entry_date_eval
# gets reinitialized
self._download_mapping = DownloadMappings(entry_date_eval=entry_date_eval)
self._download_mapping = DownloadMappings() # gets reinitialized
self._migrated_file_name = migrated_file_name
self.num_entries_added: int = 0
@ -461,7 +442,6 @@ class EnhancedDownloadArchive:
self._download_mapping = self._maybe_load_download_mappings(
mapping_file_path=self._output_file_path,
migrated_mapping_file_path=self._migrated_file_path,
entry_date_eval=self._entry_date_eval,
)
return self