From b76182eba1e93addd760a0600709ea66ba3994d9 Mon Sep 17 00:00:00 2001 From: Jesse Bannon Date: Thu, 15 Jan 2026 21:04:26 -0800 Subject: [PATCH] ENTER configurable resolution level when validating. Needs work including setting unresolved variables --- .../config/validators/variable_validation.py | 38 ++++++++- .../entries/script/variable_definitions.py | 7 ++ .../subscriptions/base_subscription.py | 17 +++- tests/unit/config/test_subscription.py | 78 ++++++++++++++++++- 4 files changed, 132 insertions(+), 8 deletions(-) diff --git a/src/ytdl_sub/config/validators/variable_validation.py b/src/ytdl_sub/config/validators/variable_validation.py index 82069d7c..c7d610e3 100644 --- a/src/ytdl_sub/config/validators/variable_validation.py +++ b/src/ytdl_sub/config/validators/variable_validation.py @@ -1,3 +1,4 @@ +import copy from typing import Dict from ytdl_sub.config.overrides import Overrides @@ -7,10 +8,21 @@ from ytdl_sub.config.plugin.preset_plugins import PresetPlugins from ytdl_sub.config.preset_options import OutputOptions from ytdl_sub.config.validators.options import OptionsValidator from ytdl_sub.downloaders.url.validators import MultiUrlValidator +from ytdl_sub.entries.script.variable_definitions import VARIABLES +from ytdl_sub.entries.script.variable_types import Variable +from ytdl_sub.script.script import Script from ytdl_sub.validators.string_formatter_validators import validate_formatters +class ResolutionLevel: + ORIGINAL = 0 + FILL = 1 + RESOLVE = 2 + INTERNAL = 3 + + class VariableValidation: + def __init__( self, overrides: Overrides, @@ -23,7 +35,7 @@ class VariableValidation: self.output_options = output_options self.plugins = plugins - self.script = self.overrides.script + self.script: Script = self.overrides.script self.unresolved_variables = self.plugins.get_all_variables( additional_options=[self.output_options, self.downloader_options] ) @@ -39,6 +51,30 @@ class VariableValidation: self.unresolved_variables -= added_variables | modified_variables + def set_resolution_level(self, resolution_level: int) -> "VariableValidation": + if resolution_level == ResolutionLevel.ORIGINAL: + # Do not perform any resolution of override variables + self.unresolved_variables.update(self.overrides.keys) + elif resolution_level == ResolutionLevel.FILL: + # If anything is resolvable, 'fill' it with the resolved value. + # This will happen automatically when validating + pass + elif resolution_level == ResolutionLevel.RESOLVE: + # Partial resolve everything, but not including internal variables + self.script = copy.deepcopy(self.script).resolve_partial( + unresolvable=self.unresolved_variables + | VARIABLES.variable_names(include_sanitized=True) + ) + elif resolution_level == ResolutionLevel.INTERNAL: + # Partial resolve everything including internal variables + self.script = copy.deepcopy(self.script).resolve_partial( + unresolvable=self.unresolved_variables + ) + else: + raise ValueError("Invalid resolution level") + + return self + def ensure_proper_usage(self) -> Dict: """ Validate variables resolve as plugins are executed, and return diff --git a/src/ytdl_sub/entries/script/variable_definitions.py b/src/ytdl_sub/entries/script/variable_definitions.py index 67505473..27ef0e54 100644 --- a/src/ytdl_sub/entries/script/variable_definitions.py +++ b/src/ytdl_sub/entries/script/variable_definitions.py @@ -1135,6 +1135,13 @@ class VariableDefinitions( ] } + @cache + def variable_names(self, include_sanitized: bool): + var_names: Set[str] = self.scripts().keys() + if include_sanitized: + var_names |= {f"{name}_sanitized" for name in var_names} + return var_names + @cache def injected_variables(self) -> Set[MetadataVariable]: """ diff --git a/src/ytdl_sub/subscriptions/base_subscription.py b/src/ytdl_sub/subscriptions/base_subscription.py index 656e98ed..dc09cc4f 100644 --- a/src/ytdl_sub/subscriptions/base_subscription.py +++ b/src/ytdl_sub/subscriptions/base_subscription.py @@ -8,6 +8,7 @@ from ytdl_sub.config.plugin.preset_plugins import PresetPlugins from ytdl_sub.config.preset import Preset from ytdl_sub.config.preset_options import OutputOptions from ytdl_sub.config.preset_options import YTDLOptions +from ytdl_sub.config.validators.variable_validation import ResolutionLevel from ytdl_sub.config.validators.variable_validation import VariableValidation from ytdl_sub.downloaders.url.validators import MultiUrlValidator from ytdl_sub.entries.variables.override_variables import SubscriptionVariables @@ -79,7 +80,7 @@ class BaseSubscription(ABC): ) # Validate after adding the subscription name - self._validated_dict = VariableValidation( + _ = VariableValidation( overrides=self.overrides, downloader_options=self.downloader_options, output_options=self.output_options, @@ -256,10 +257,20 @@ class BaseSubscription(ABC): """ return self._preset_options.yaml - def resolved_yaml(self) -> str: + def resolved_yaml(self, resolution_level: int = ResolutionLevel.FILL) -> str: """ Returns ------- Human-readable, condensed YAML definition of the subscription. """ - return dump_yaml(self._validated_dict) + out = ( + VariableValidation( + overrides=self.overrides, + downloader_options=self.downloader_options, + output_options=self.output_options, + plugins=self.plugins, + ) + .set_resolution_level(resolution_level) + .ensure_proper_usage() + ) + return dump_yaml(out) diff --git a/tests/unit/config/test_subscription.py b/tests/unit/config/test_subscription.py index 35942f8d..a3fbb05b 100644 --- a/tests/unit/config/test_subscription.py +++ b/tests/unit/config/test_subscription.py @@ -547,9 +547,73 @@ def test_default_docker_config_and_subscriptions( resolved_yaml_as_json = yaml.safe_load(default_subs[0].resolved_yaml()) - # Since this creates random values, ignore it for this test - assert "throttle_protection" in resolved_yaml_as_json - del resolved_yaml_as_json["throttle_protection"] + assert resolved_yaml_as_json == { + "chapters": { + "allow_chapters_from_comments": False, + "embed_chapters": True, + "enable": "True", + "force_key_frames": False, + }, + "date_range": {"breaks": "True", "enable": "True", "type": "upload_date"}, + "download": [ + { + "download_reverse": "True", + "include_sibling_metadata": False, + "playlist_thumbnails": [ + {"name": "{avatar_uncropped_thumbnail_file_name}", "uid": "avatar_uncropped"}, + {"name": "{banner_uncropped_thumbnail_file_name}", "uid": "banner_uncropped"}, + ], + "source_thumbnails": [ + {"name": "{avatar_uncropped_thumbnail_file_name}", "uid": "avatar_uncropped"}, + {"name": "{banner_uncropped_thumbnail_file_name}", "uid": "banner_uncropped"}, + ], + "url": "https://www.youtube.com/@novapbs", + "variables": {}, + "webpage_url": "{modified_webpage_url}", + "ytdl_options": {}, + } + ], + "file_convert": {"convert_to": "mp4", "convert_with": "yt-dlp", "enable": "True"}, + "format": "(bv*[ext=mp4][vcodec~='^((he|a)vc|h26[45])']+ba[ext=m4a]) / (bv[ext=mp4]*+ba[ext=m4a]/b)", + "output_options": { + "download_archive_name": ".ytdl-sub-NOVA PBS-download-archive.json", + "file_name": "{episode_file_path}.{ext}", + "info_json_name": "{episode_file_path}.{info_json_ext}", + "keep_files_date_eval": "{episode_date_standardized}", + "maintain_download_archive": True, + "output_directory": f"{output_directory}/NOVA PBS", + "preserve_mtime": False, + "thumbnail_name": "{thumbnail_file_name}", + }, + "throttle_protection": { + "enable": True, + "sleep_per_download_s": {"max": "28.4", "min": "13.8"}, + "sleep_per_request_s": {"max": "0.75", "min": "0.0"}, + "sleep_per_subscription_s": {"max": "26.1", "min": "16.3"}, + }, + "video_tags": { + "contentRating": "{episode_content_rating}", + "date": "{episode_date_standardized}", + "episode_id": "{episode_number}", + "genre": "{tv_show_genre}", + "show": "{tv_show_name}", + "synopsis": "{episode_plot}", + "title": "{episode_title}", + "year": "{episode_year}", + }, + } + + +def test_default_docker_config_and_subscriptions( + docker_default_subscription_path: Path, output_directory: str +): + default_config = ConfigFile.from_file_path("docker/root/defaults/config.yaml") + default_subs = Subscription.from_file_path( + config=default_config, subscription_path=docker_default_subscription_path + ) + assert len(default_subs) == 1 + + resolved_yaml_as_json = yaml.safe_load(default_subs[0].resolved_yaml(resolution_level=3)) assert resolved_yaml_as_json == { "chapters": { @@ -589,6 +653,12 @@ def test_default_docker_config_and_subscriptions( "preserve_mtime": False, "thumbnail_name": "{thumbnail_file_name}", }, + "throttle_protection": { + "enable": True, + "sleep_per_download_s": {"max": "28.4", "min": "13.8"}, + "sleep_per_request_s": {"max": "0.75", "min": "0.0"}, + "sleep_per_subscription_s": {"max": "26.1", "min": "16.3"}, + }, "video_tags": { "contentRating": "{episode_content_rating}", "date": "{episode_date_standardized}", @@ -599,4 +669,4 @@ def test_default_docker_config_and_subscriptions( "title": "{episode_title}", "year": "{episode_year}", }, - } + } \ No newline at end of file