diff --git a/src/ytdl_sub/config/validators/variable_validation.py b/src/ytdl_sub/config/validators/variable_validation.py index 9d427324..941d228e 100644 --- a/src/ytdl_sub/config/validators/variable_validation.py +++ b/src/ytdl_sub/config/validators/variable_validation.py @@ -12,14 +12,16 @@ from ytdl_sub.downloaders.url.validators import MultiUrlValidator from ytdl_sub.entries.script.variable_definitions import UNRESOLVED_VARIABLES from ytdl_sub.entries.script.variable_definitions import VARIABLES from ytdl_sub.script.script import Script +from ytdl_sub.script.utils.name_validation import is_function from ytdl_sub.utils.script import ScriptUtils from ytdl_sub.validators.string_formatter_validators import validate_formatters class ResolutionLevel: ORIGINAL = 0 - RESOLVE = 1 - INTERNAL = 2 + FILL = 1 + RESOLVE = 2 + INTERNAL = 3 @classmethod def name_of(cls, resolution_level: int) -> str: @@ -28,6 +30,8 @@ class ResolutionLevel: """ if resolution_level == cls.ORIGINAL: return "original" + if resolution_level == cls.FILL: + return "fill" if resolution_level == cls.RESOLVE: return "resolve" if resolution_level == cls.INTERNAL: @@ -39,7 +43,7 @@ class ResolutionLevel: """ All possible resolution levels. """ - return [cls.ORIGINAL, cls.RESOLVE, cls.INTERNAL] + return [cls.ORIGINAL, cls.FILL, cls.RESOLVE, cls.INTERNAL] class VariableValidation: @@ -54,22 +58,28 @@ class VariableValidation: } def _apply_resolution_level(self) -> None: - if self._resolution_level == ResolutionLevel.RESOLVE: + if self._resolution_level == ResolutionLevel.FILL: + self.unresolved_variables |= VARIABLES.variable_names(include_sanitized=True) + # Only partial resolve definitions that are already resolved + self.unresolved_variables |= { + name + for name in self.overrides.keys + if not is_function(name) and not self.script.definition_of(name).maybe_resolvable + } + elif self._resolution_level == ResolutionLevel.RESOLVE: # Partial resolve everything, but not including internal variables self.unresolved_variables |= VARIABLES.variable_names(include_sanitized=True) - self.script = self.script.resolve_partial( - unresolvable=self.unresolved_variables, - output_filter=self._get_resolve_partial_filter(), - ) elif self._resolution_level == ResolutionLevel.INTERNAL: # Partial resolve everything including internal variables - self.script = self.script.resolve_partial( - unresolvable=self.unresolved_variables, - output_filter=self._get_resolve_partial_filter(), - ) + pass else: raise ValueError("Invalid resolution level for validation") + self.script = self.script.resolve_partial( + unresolvable=self.unresolved_variables, + output_filter=self._get_resolve_partial_filter(), + ) + def __init__( self, overrides: Overrides, diff --git a/tests/resources.py b/tests/resources.py index a327be02..4a2b9e71 100644 --- a/tests/resources.py +++ b/tests/resources.py @@ -5,7 +5,7 @@ from pathlib import Path from typing import Dict DISABLE_YOUTUBE_TESTS: bool = True -REGENERATE_FIXTURES: bool = False +REGENERATE_FIXTURES: bool = True RESOURCE_PATH: Path = Path("tests") / "resources" _FILE_FIXTURE_PATH: Path = RESOURCE_PATH / "file_fixtures" diff --git a/tests/resources/expected_json/tv_show/inspect_sub_fill.json b/tests/resources/expected_json/tv_show/inspect_sub_fill.json new file mode 100644 index 00000000..6d1a8662 --- /dev/null +++ b/tests/resources/expected_json/tv_show/inspect_sub_fill.json @@ -0,0 +1,250 @@ +{ + "chapters": { + "allow_chapters_from_comments": false, + "embed_chapters": true, + "enable": true, + "force_key_frames": false + }, + "date_range": { + "breaks": true, + "enable": true, + "type": "upload_date" + }, + "download": [ + { + "download_reverse": true, + "include_sibling_metadata": false, + "playlist_thumbnails": [ + { + "name": "poster.jpg", + "uid": "avatar_uncropped" + }, + { + "name": "fanart.jpg", + "uid": "banner_uncropped" + } + ], + "source_thumbnails": [ + { + "name": "poster.jpg", + "uid": "avatar_uncropped" + }, + { + "name": "fanart.jpg", + "uid": "banner_uncropped" + } + ], + "url": [ + "https://www.youtube.com/@novapbs" + ], + "variables": {}, + "webpage_url": "{ modified_webpage_url }", + "ytdl_options": {} + } + ], + "file_convert": { + "convert_to": "mp4", + "convert_with": "yt-dlp", + "enable": true + }, + "format": "(bv*[ext=mp4][vcodec~='^((he|a)vc|h26[45])']+ba[ext=m4a]) / (bv[ext=mp4]*+ba[ext=m4a]/b)", + "output_options": { + "download_archive_name": ".ytdl-sub-NOVA PBS-download-archive.json", + "file_name": "{ episode_file_path }.{ ext }", + "info_json_name": "{ episode_file_path }.{ info_json_ext }", + "keep_files_date_eval": "{ episode_date_standardized }", + "maintain_download_archive": true, + "output_directory": "/var/folders/rw/hl1xmkmj68zdl2kjx3l0dwzc0000gn/T/tmpc_p6gjjw/NOVA PBS", + "preserve_mtime": false, + "thumbnail_name": "{ thumbnail_file_name }" + }, + "overrides": { + "%bilateral_url": "{ %if( %and( enable_bilateral_scraping, subscription_has_download_archive, %is_bilateral_url( $0 ) ), $0, '' ) }", + "%episode_ordering_": "{ %eq( %lower( tv_show_by_date_episode_ordering ), $0 ) }", + "%is_bilateral_url": "{ %contains( $0, \"youtube.com/playlist\" ) }", + "%ordering_pair_eq": "{ %eq( [ tv_show_by_date_season_ordering, tv_show_by_date_episode_ordering ], [ $0, $1 ] ) }", + "%season_ordering_": "{ %eq( %lower( tv_show_by_date_season_ordering ), $0 ) }", + "assert_not_collection": true, + "avatar_uncropped_thumbnail_file_name": "poster.jpg", + "banner_uncropped_thumbnail_file_name": "fanart.jpg", + "enable_bilateral_scraping": true, + "enable_resolution_assert": true, + "enable_throttle_protection": true, + "episode_content_rating": "TV-14", + "episode_date_standardized": "{ upload_date_standardized }", + "episode_file_name": "s{ upload_year }.e{ %pad_zero( %int( %concat( upload_month, upload_day_padded, upload_date_index_padded ) ), 6 ) } - { title_sanitized_plex }", + "episode_file_path": "{ %sanitize( %concat( \"Season \", upload_year ) ) }/{ %sanitize( %concat( \"s\", upload_year, \".e\", %pad_zero( %int( %concat( upload_month, upload_day_padded, upload_date_index_padded ) ), 6 ), \" - \", title_sanitized_plex ) ) }", + "episode_number": "{ upload_month }{ upload_day_padded }{ upload_date_index_padded }", + "episode_number_and_padded_": "{ [ %concat( upload_month, upload_day_padded, upload_date_index_padded ), 6 ] }", + "episode_number_padded": "{ %pad_zero( %int( %concat( upload_month, upload_day_padded, upload_date_index_padded ) ), 6 ) }", + "episode_plot": "{ webpage_url }\n\n{ description }", + "episode_title": "{ upload_date_standardized } - { title }", + "episode_year": "{ %slice( upload_date_standardized, 0, 4 ) }", + "file_title": "{ title_sanitized_plex }", + "file_uid": "{ uid_sanitized_plex }", + "include_sibling_metadata": false, + "modified_webpage_url": "{ webpage_url }", + "only_recent_date_range": "2months", + "only_recent_max_files": 30, + "resolution_assert": "{ %if( %and( enable_resolution_assert, %ne( height, 0 ), %not( resolution_assert_is_ignored ) ), %assert( %gte( height, resolution_assert_height_gte ), %concat( \"Entry \", title, \" downloaded at a low resolution (\", resolution_readable, \"), you've probably been throttled. \", \"Stopping further downloads, wait a few hours and try again. \", \"Disable using the override variable `enable_resolution_assert: False`.\" ) ), \"false is no-op\" ) }", + "resolution_assert_height_gte": 361, + "resolution_assert_ignore_titles": "{ [ ] }", + "resolution_assert_is_ignored": "{ %print_if_true( %concat( title, \" has a match in resolution_assert_ignore_titles, skipping resolution assert.\" ), %contains_any( title, [ ] ) ) }", + "resolution_assert_print": true, + "resolution_readable": "{ width }x{ height }", + "s01_name": "", + "s01_url": "", + "season_directory_name": "Season { upload_year }", + "season_number": "{ upload_year }", + "season_number_padded": "{ upload_year }", + "season_poster_file_name": "{ %sanitize( %concat( \"Season \", upload_year ) ) }/Season{ upload_year }.jpg", + "subscription_array": [ + "https://www.youtube.com/@novapbs" + ], + "subscription_indent_1": "Documentaries", + "subscription_indent_2": "TV-14", + "subscription_value": "https://www.youtube.com/@novapbs", + "subscription_value_1": "https://www.youtube.com/@novapbs", + "thumbnail_file_name": "{ %concat( %sanitize( %concat( \"Season \", upload_year ) ), \"/\", %sanitize( %concat( \"s\", upload_year, \".e\", %pad_zero( %int( %concat( upload_month, upload_day_padded, upload_date_index_padded ) ), 6 ), \" - \", title_sanitized_plex ) ) ) }-thumb.jpg", + "tv_show_by_date_episode_ordering": "upload-month-day", + "tv_show_by_date_ordering_pair_validation_": "{ [ %concat( upload_month, upload_day_padded, upload_date_index_padded ), 6 ] }", + "tv_show_by_date_season_ordering": "upload-year", + "tv_show_content_rating": "TV-14", + "tv_show_content_rating_default": "TV-14", + "tv_show_date_range_type": "upload_date", + "tv_show_directory": "/var/folders/rw/hl1xmkmj68zdl2kjx3l0dwzc0000gn/T/tmpc_p6gjjw", + "tv_show_fanart_file_name": "fanart.jpg", + "tv_show_genre": "Documentaries", + "tv_show_genre_default": "ytdl-sub", + "tv_show_name": "NOVA PBS", + "tv_show_poster_file_name": "poster.jpg", + "url": "https://www.youtube.com/@novapbs", + "url10": "", + "url100": "", + "url11": "", + "url12": "", + "url13": "", + "url14": "", + "url15": "", + "url16": "", + "url17": "", + "url18": "", + "url19": "", + "url2": "", + "url20": "", + "url21": "", + "url22": "", + "url23": "", + "url24": "", + "url25": "", + "url26": "", + "url27": "", + "url28": "", + "url29": "", + "url3": "", + "url30": "", + "url31": "", + "url32": "", + "url33": "", + "url34": "", + "url35": "", + "url36": "", + "url37": "", + "url38": "", + "url39": "", + "url4": "", + "url40": "", + "url41": "", + "url42": "", + "url43": "", + "url44": "", + "url45": "", + "url46": "", + "url47": "", + "url48": "", + "url49": "", + "url5": "", + "url50": "", + "url51": "", + "url52": "", + "url53": "", + "url54": "", + "url55": "", + "url56": "", + "url57": "", + "url58": "", + "url59": "", + "url6": "", + "url60": "", + "url61": "", + "url62": "", + "url63": "", + "url64": "", + "url65": "", + "url66": "", + "url67": "", + "url68": "", + "url69": "", + "url7": "", + "url70": "", + "url71": "", + "url72": "", + "url73": "", + "url74": "", + "url75": "", + "url76": "", + "url77": "", + "url78": "", + "url79": "", + "url8": "", + "url80": "", + "url81": "", + "url82": "", + "url83": "", + "url84": "", + "url85": "", + "url86": "", + "url87": "", + "url88": "", + "url89": "", + "url9": "", + "url90": "", + "url91": "", + "url92": "", + "url93": "", + "url94": "", + "url95": "", + "url96": "", + "url97": "", + "url98": "", + "url99": "", + "urls": [ + "https://www.youtube.com/@novapbs" + ] + }, + "throttle_protection": { + "enable": true, + "sleep_per_download_s": { + "max": 28.4, + "min": 13.8 + }, + "sleep_per_request_s": { + "max": 0.75, + "min": 0.0 + }, + "sleep_per_subscription_s": { + "max": 26.1, + "min": 16.3 + } + }, + "video_tags": { + "contentRating": "TV-14", + "date": "{ episode_date_standardized }", + "episode_id": "{ episode_number }", + "genre": "Documentaries", + "show": "NOVA PBS", + "synopsis": "{ episode_plot }", + "title": "{ episode_title }", + "year": "{ episode_year }" + } +} \ No newline at end of file diff --git a/tests/resources/expected_json/tv_show/inspect_sub_internal.json b/tests/resources/expected_json/tv_show/inspect_sub_internal.json index e7efe974..3f1c1eaf 100644 --- a/tests/resources/expected_json/tv_show/inspect_sub_internal.json +++ b/tests/resources/expected_json/tv_show/inspect_sub_internal.json @@ -54,7 +54,7 @@ "info_json_name": "{ %concat( %sanitize( %concat( \"Season \", %int( %map_get( %to_date_metadata( %map_get_non_empty( entry_metadata, \"upload_date\", %datetime_strftime( %map_get( entry_metadata, \"epoch\" ), \"%Y%m%d\" ) ) ), \"year\" ) ) ) ), \"/\", %sanitize( %concat( \"s\", %int( %map_get( %to_date_metadata( %map_get_non_empty( entry_metadata, \"upload_date\", %datetime_strftime( %map_get( entry_metadata, \"epoch\" ), \"%Y%m%d\" ) ) ), \"year\" ) ), \".e\", %pad_zero( %int( %concat( %int( %map_get( %to_date_metadata( %map_get_non_empty( entry_metadata, \"upload_date\", %datetime_strftime( %map_get( entry_metadata, \"epoch\" ), \"%Y%m%d\" ) ) ), \"month\" ) ), %string( %map_get( %to_date_metadata( %map_get_non_empty( entry_metadata, \"upload_date\", %datetime_strftime( %map_get( entry_metadata, \"epoch\" ), \"%Y%m%d\" ) ) ), \"day_padded\" ) ), %pad_zero( upload_date_index, 2 ) ) ), 6 ), \" - \", %sanitize_plex_episode( %map_get_non_empty( entry_metadata, \"title\", %map_get( entry_metadata, \"id\" ) ) ) ) ) ) }.info.json", "keep_files_date_eval": "{ %string( %map_get( %to_date_metadata( %map_get_non_empty( entry_metadata, \"upload_date\", %datetime_strftime( %map_get( entry_metadata, \"epoch\" ), \"%Y%m%d\" ) ) ), \"date_standardized\" ) ) }", "maintain_download_archive": true, - "output_directory": "/var/folders/rw/hl1xmkmj68zdl2kjx3l0dwzc0000gn/T/tmp6np_7efb/NOVA PBS", + "output_directory": "/var/folders/rw/hl1xmkmj68zdl2kjx3l0dwzc0000gn/T/tmpps6shcpt/NOVA PBS", "preserve_mtime": false, "thumbnail_name": "{ %concat( %sanitize( %concat( \"Season \", %int( %map_get( %to_date_metadata( %map_get_non_empty( entry_metadata, \"upload_date\", %datetime_strftime( %map_get( entry_metadata, \"epoch\" ), \"%Y%m%d\" ) ) ), \"year\" ) ) ) ), \"/\", %sanitize( %concat( \"s\", %int( %map_get( %to_date_metadata( %map_get_non_empty( entry_metadata, \"upload_date\", %datetime_strftime( %map_get( entry_metadata, \"epoch\" ), \"%Y%m%d\" ) ) ), \"year\" ) ), \".e\", %pad_zero( %int( %concat( %int( %map_get( %to_date_metadata( %map_get_non_empty( entry_metadata, \"upload_date\", %datetime_strftime( %map_get( entry_metadata, \"epoch\" ), \"%Y%m%d\" ) ) ), \"month\" ) ), %string( %map_get( %to_date_metadata( %map_get_non_empty( entry_metadata, \"upload_date\", %datetime_strftime( %map_get( entry_metadata, \"epoch\" ), \"%Y%m%d\" ) ) ), \"day_padded\" ) ), %pad_zero( upload_date_index, 2 ) ) ), 6 ), \" - \", %sanitize_plex_episode( %map_get_non_empty( entry_metadata, \"title\", %map_get( entry_metadata, \"id\" ) ) ) ) ) ) }-thumb.jpg" }, @@ -112,7 +112,7 @@ "tv_show_content_rating": "TV-14", "tv_show_content_rating_default": "TV-14", "tv_show_date_range_type": "upload_date", - "tv_show_directory": "/var/folders/rw/hl1xmkmj68zdl2kjx3l0dwzc0000gn/T/tmp6np_7efb", + "tv_show_directory": "/var/folders/rw/hl1xmkmj68zdl2kjx3l0dwzc0000gn/T/tmpps6shcpt", "tv_show_fanart_file_name": "fanart.jpg", "tv_show_genre": "Documentaries", "tv_show_genre_default": "ytdl-sub", diff --git a/tests/resources/expected_json/tv_show/inspect_sub_original.json b/tests/resources/expected_json/tv_show/inspect_sub_original.json index 34f3fdc1..0a7540d3 100644 --- a/tests/resources/expected_json/tv_show/inspect_sub_original.json +++ b/tests/resources/expected_json/tv_show/inspect_sub_original.json @@ -109,7 +109,7 @@ "tv_show_content_rating": "{subscription_indent_2}", "tv_show_content_rating_default": "TV-14", "tv_show_date_range_type": "{\n %if(\n %contains(tv_show_by_date_season_ordering, \"release\"),\n \"release_date\",\n \"upload_date\"\n )\n}", - "tv_show_directory": "/var/folders/rw/hl1xmkmj68zdl2kjx3l0dwzc0000gn/T/tmpwmu3ixzi", + "tv_show_directory": "/var/folders/rw/hl1xmkmj68zdl2kjx3l0dwzc0000gn/T/tmp5yx9y73k", "tv_show_fanart_file_name": "fanart.jpg", "tv_show_genre": "{subscription_indent_1}", "tv_show_genre_default": "ytdl-sub", diff --git a/tests/resources/expected_json/tv_show/inspect_sub_resolve.json b/tests/resources/expected_json/tv_show/inspect_sub_resolve.json index 45541b15..3c2898ea 100644 --- a/tests/resources/expected_json/tv_show/inspect_sub_resolve.json +++ b/tests/resources/expected_json/tv_show/inspect_sub_resolve.json @@ -54,7 +54,7 @@ "info_json_name": "{ %concat( %sanitize( %concat( \"Season \", upload_year ) ), \"/\", %sanitize( %concat( \"s\", upload_year, \".e\", %pad_zero( %int( %concat( upload_month, upload_day_padded, upload_date_index_padded ) ), 6 ), \" - \", title_sanitized_plex ) ) ) }.{ info_json_ext }", "keep_files_date_eval": "{ upload_date_standardized }", "maintain_download_archive": true, - "output_directory": "/var/folders/rw/hl1xmkmj68zdl2kjx3l0dwzc0000gn/T/tmpnhm5yd08/NOVA PBS", + "output_directory": "/var/folders/rw/hl1xmkmj68zdl2kjx3l0dwzc0000gn/T/tmp3j0cm_tw/NOVA PBS", "preserve_mtime": false, "thumbnail_name": "{ %concat( %sanitize( %concat( \"Season \", upload_year ) ), \"/\", %sanitize( %concat( \"s\", upload_year, \".e\", %pad_zero( %int( %concat( upload_month, upload_day_padded, upload_date_index_padded ) ), 6 ), \" - \", title_sanitized_plex ) ) ) }-thumb.jpg" }, @@ -112,7 +112,7 @@ "tv_show_content_rating": "TV-14", "tv_show_content_rating_default": "TV-14", "tv_show_date_range_type": "upload_date", - "tv_show_directory": "/var/folders/rw/hl1xmkmj68zdl2kjx3l0dwzc0000gn/T/tmpnhm5yd08", + "tv_show_directory": "/var/folders/rw/hl1xmkmj68zdl2kjx3l0dwzc0000gn/T/tmp3j0cm_tw", "tv_show_fanart_file_name": "fanart.jpg", "tv_show_genre": "Documentaries", "tv_show_genre_default": "ytdl-sub",