This commit is contained in:
Jesse Bannon 2026-01-16 12:15:38 -08:00
parent b76182eba1
commit 9c2fc05d41
5 changed files with 597 additions and 27 deletions

View file

@ -11,6 +11,7 @@ from ytdl_sub.downloaders.url.validators import MultiUrlValidator
from ytdl_sub.entries.script.variable_definitions import VARIABLES
from ytdl_sub.entries.script.variable_types import Variable
from ytdl_sub.script.script import Script
from ytdl_sub.utils.script import ScriptUtils
from ytdl_sub.validators.string_formatter_validators import validate_formatters
@ -23,12 +24,33 @@ class ResolutionLevel:
class VariableValidation:
def _apply_resolution_level(self) -> None:
if self._resolution_level == ResolutionLevel.FILL:
# If anything is resolvable, 'fill' it with the resolved value.
# This will happen automatically when validating
pass
elif self._resolution_level == ResolutionLevel.RESOLVE:
# Partial resolve everything, but not including internal variables
self.script = self.script.resolve_partial(
unresolvable=self.unresolved_variables
| VARIABLES.variable_names(include_sanitized=True)
)
elif self._resolution_level == ResolutionLevel.INTERNAL:
# Partial resolve everything including internal variables
self.script = self.script.resolve_partial(
unresolvable=self.unresolved_variables
)
else:
raise ValueError("Invalid resolution level for validation")
def __init__(
self,
overrides: Overrides,
downloader_options: MultiUrlValidator,
output_options: OutputOptions,
plugins: PresetPlugins,
resolution_level: int = ResolutionLevel.FILL
):
self.overrides = overrides
self.downloader_options = downloader_options
@ -39,6 +61,9 @@ class VariableValidation:
self.unresolved_variables = self.plugins.get_all_variables(
additional_options=[self.output_options, self.downloader_options]
)
self._resolution_level = resolution_level
self._apply_resolution_level()
def _add_variables(self, plugin_op: PluginOperation, options: OptionsValidator) -> None:
"""
@ -51,36 +76,12 @@ class VariableValidation:
self.unresolved_variables -= added_variables | modified_variables
def set_resolution_level(self, resolution_level: int) -> "VariableValidation":
if resolution_level == ResolutionLevel.ORIGINAL:
# Do not perform any resolution of override variables
self.unresolved_variables.update(self.overrides.keys)
elif resolution_level == ResolutionLevel.FILL:
# If anything is resolvable, 'fill' it with the resolved value.
# This will happen automatically when validating
pass
elif resolution_level == ResolutionLevel.RESOLVE:
# Partial resolve everything, but not including internal variables
self.script = copy.deepcopy(self.script).resolve_partial(
unresolvable=self.unresolved_variables
| VARIABLES.variable_names(include_sanitized=True)
)
elif resolution_level == ResolutionLevel.INTERNAL:
# Partial resolve everything including internal variables
self.script = copy.deepcopy(self.script).resolve_partial(
unresolvable=self.unresolved_variables
)
else:
raise ValueError("Invalid resolution level")
return self
def ensure_proper_usage(self) -> Dict:
"""
Validate variables resolve as plugins are executed, and return
a mock script which contains actualized added variables from the plugins
"""
resolved_subscription: Dict = {}
self._add_variables(PluginOperation.DOWNLOADER, options=self.downloader_options)
@ -126,5 +127,18 @@ class VariableValidation:
if url_output["url"]:
resolved_subscription["download"].append(url_output)
# TODO: make function
resolved_subscription['overrides'] = {}
for name in self.overrides.keys:
value = self.script.definition_of(name)
if name in self.script.function_names:
# Keep custom functions as-is
resolved_subscription['overrides'][name] = self.overrides.dict_with_format_strings[name]
elif resolved := value.maybe_resolvable:
resolved_subscription['overrides'][name] = resolved.native
else:
resolved_subscription['overrides'][name] = ScriptUtils.to_native_script(value)
assert not self.unresolved_variables
return resolved_subscription

View file

@ -679,6 +679,13 @@ class Script:
raise RuntimeException(f"Tried to get unresolved variable {variable_name}")
def definition_of(self, name: str) -> SyntaxTree:
if name.startswith('%') and name[1:] in self._functions:
return self._functions[name[1:]]
if name in self._variables:
return self._variables[name]
raise RuntimeException(f"Tried to get non-existent definition with name {name}")
@property
def variable_names(self) -> Set[str]:
"""

View file

@ -269,8 +269,8 @@ class BaseSubscription(ABC):
downloader_options=self.downloader_options,
output_options=self.output_options,
plugins=self.plugins,
resolution_level=resolution_level,
)
.set_resolution_level(resolution_level)
.ensure_proper_usage()
)
return dump_yaml(out)

View file

@ -238,6 +238,7 @@ def _validate_formatter(
mock_script: Script,
unresolved_variables: Set[str],
formatter_validator: Union[StringFormatterValidator, OverridesStringFormatterValidator],
resolve_partial: bool = True
) -> str:
parsed = formatter_validator.parsed
if resolved := parsed.maybe_resolvable:
@ -250,6 +251,12 @@ def _validate_formatter(
variable_names = {var.name for var in parsed.variables}
custom_function_names = {f"%{func.name}" for func in parsed.custom_functions}
# if resolve_partial:
# formatter_hash = str(formatter_validator.__hash__())
# mock_script.add_parsed(
# {formatter_hash: formatter_validator.parsed}
# ).resolve_partial(unresolvable=unresolved_variables).definition_of(formatter_hash)
# Add lambda functions to custom function names, if it's custom
for lambda_func in parsed.lambdas:
if lambda_func in mock_script.function_names:

View file

@ -585,6 +585,335 @@ def test_default_docker_config_and_subscriptions(
"preserve_mtime": False,
"thumbnail_name": "{thumbnail_file_name}",
},
'overrides': {'%bilateral_url': '{ %if( %and( enable_bilateral_scraping, '
'subscription_has_download_archive, '
"%is_bilateral_url( $0 ) ), $0, '' ) }",
'%episode_ordering_': '{ %eq( %lower( '
'tv_show_by_date_episode_ordering ), $0 ) '
'}',
'%is_bilateral_url': '{ %contains( $0, "youtube.com/playlist" ) '
'}',
'%ordering_pair_eq': '{ %eq( [ tv_show_by_date_season_ordering, '
'tv_show_by_date_episode_ordering ], [ $0, '
'$1 ] ) }',
'%season_ordering_': '{ %eq( %lower( '
'tv_show_by_date_season_ordering ), $0 ) }',
'assert_not_collection': '{ %assert( %and( %not( %bool( s01_url '
') ), %not( %bool( s01_name ) ) ), '
'"Provided `s01_url` or `s01_name` '
'variable to TV Show by Date preset '
'when it expects `url`. Perhaps you '
'meant to use the `TV Show Collection` '
'preset?" ) }',
'avatar_uncropped_thumbnail_file_name': '{ '
'tv_show_poster_file_name '
'}',
'banner_uncropped_thumbnail_file_name': '{ '
'tv_show_fanart_file_name '
'}',
'enable_bilateral_scraping': '{ %bool(True) }',
'enable_resolution_assert': '{ %bool(True) }',
'enable_throttle_protection': '{ %bool(True) }',
'episode_content_rating': '{ tv_show_content_rating }',
'episode_date_standardized': '{ %if( %contains( '
'tv_show_by_date_season_ordering, '
'"release" ), '
'release_date_standardized, '
'upload_date_standardized ) }',
'episode_file_name': '{ %concat( %string( "s" ), %string( '
'season_number_padded ), %string( ".e" ), '
'%string( episode_number_padded ), '
'%string( " - " ), %string( file_title ) ) '
'}',
'episode_file_path': '{ %concat( %string( '
'season_directory_name_sanitized ), '
'%string( "/" ), %string( '
'episode_file_name_sanitized ) ) }',
'episode_number': '{ %array_at( episode_number_and_padded_, 0 ) '
'}',
'episode_number_and_padded_': '{ %elif( %episode_ordering_( '
'"upload-day" ), [ %concat( '
'upload_day, '
'upload_date_index_padded ), 4 ], '
'%episode_ordering_( '
'"upload-month-day" ), [ %concat( '
'upload_month, upload_day_padded, '
'upload_date_index_padded ), 6 ], '
'%episode_ordering_( '
'"upload-month-day-reversed" ), [ '
'%concat( '
'upload_day_of_year_reversed, '
'upload_date_index_reversed_padded '
'), 5 ], %episode_ordering_( '
'"release-day" ), [ %concat( '
'release_day, '
'upload_date_index_padded ), 4 ], '
'%episode_ordering_( '
'"release-month-day" ), [ '
'%concat( release_month, '
'release_day_padded, '
'upload_date_index_padded ), 6 ], '
'%episode_ordering_( '
'"release-month-day-reversed" ), '
'[ %concat( '
'release_day_of_year_reversed, '
'upload_date_index_reversed_padded '
'), 5 ], %episode_ordering_( '
'"download-index" ), [ '
'download_index, 6 ], %throw( '
"'tv_show_by_date_episode_ordering "
'must be one of the following: '
'"upload-day", '
'"upload-month-day", '
'"upload-month-day-reversed", '
'"release-day", '
'"release-month-day", '
'"release-month-day-reversed", '
'"download-index"\' ) ) }',
'episode_number_padded': '{ %pad_zero( %int( episode_number ), '
'%int( %array_at( '
'episode_number_and_padded_, 1 ) ) ) }',
'episode_plot': '{ %concat( %string( webpage_url ), %string( "\n'
'\n'
'" ), %string( description ) ) }',
'episode_title': '{ %concat( %string( episode_date_standardized '
'), %string( " - " ), %string( title ) ) }',
'episode_year': '{ %slice( episode_date_standardized, 0, 4 ) }',
'file_title': '{ title_sanitized_plex }',
'file_uid': '{ uid_sanitized_plex }',
'include_sibling_metadata': '{ %bool(False) }',
'modified_webpage_url': '{ webpage_url }',
'music_directory': '/music',
'music_video_directory': '/music_videos',
'resolution_assert': '{ %if( %and( enable_resolution_assert, '
'%ne( height, 0 ), %not( '
'resolution_assert_is_ignored ) ), '
'%assert( %gte( height, '
'resolution_assert_height_gte ), %concat( '
'"Entry ", title, " downloaded at a low '
'resolution (", resolution_readable, "), '
'you\'ve probably been throttled. ", '
'"Stopping further downloads, wait a few '
'hours and try again. ", "Disable using '
'the override variable '
'`enable_resolution_assert: False`." ) ), '
'"false is no-op" ) }',
'resolution_assert_height_gte': '{ %int(361) }',
'resolution_assert_ignore_titles': '{ [ ] }',
'resolution_assert_is_ignored': '{ %print_if_true( %concat( '
'title, " has a match in '
'resolution_assert_ignore_titles, '
'skipping resolution assert." '
'), %contains_any( title, '
'resolution_assert_ignore_titles '
') ) }',
'resolution_assert_print': '{ %print( %if( '
'enable_resolution_assert, '
'"Resolution assert is enabled, will '
'fail on low-quality video downloads '
'and presume throttle. Disable using '
'the override variable '
'`enable_resolution_assert: False`", '
'"Resolution assert is disabled. Use '
'at your own risk!" ), '
'enable_resolution_assert, -1 ) }',
'resolution_readable': '{ %concat( %string( width ), %string( '
'"x" ), %string( height ) ) }',
's01_name': '',
's01_url': '',
'season_directory_name': '{ %concat( %string( "Season " ), '
'%string( season_number_padded ) ) }',
'season_number': '{ %elif( %season_ordering_( "upload-year" ), '
'upload_year, %season_ordering_( '
'"upload-year-month" ), %concat( upload_year, '
'upload_month_padded ), %season_ordering_( '
'"release-year" ), release_year, '
'%season_ordering_( "release-year-month" ), '
'%concat( release_year, release_month_padded '
"), %throw( 'tv_show_by_date_season_ordering "
'must be one of the following: "upload-year", '
'"upload-year-month", "release-year", '
'"release-year-month"\' ) ) }',
'season_number_padded': '{ season_number }',
'season_poster_file_name': '{ %concat( %string( '
'season_directory_name_sanitized ), '
'%string( "/Season" ), %string( '
'season_number_padded ), %string( '
'".jpg" ) ) }',
'subscription_array': '{ %from_json( '
'\'["https://www.youtube.com/@novapbs"]\' '
') }',
'subscription_indent_1': 'Documentaries',
'subscription_indent_2': '{ tv_show_content_rating_default }',
'subscription_value': 'https://www.youtube.com/@novapbs',
'subscription_value_1': 'https://www.youtube.com/@novapbs',
'thumbnail_file_name': '{ %concat( %string( episode_file_path '
'), %string( "-thumb.jpg" ) ) }',
'tv_show_by_date_episode_ordering': 'upload-month-day',
'tv_show_by_date_ordering_pair_validation_': '{ %assert_then( '
'%or( '
'%ordering_pair_eq( '
'"upload-year", '
'"upload-month-day" '
'), '
'%ordering_pair_eq( '
'"upload-year", '
'"upload-month-day-reversed" '
'), '
'%ordering_pair_eq( '
'"upload-year", '
'"download-index" '
'), '
'%ordering_pair_eq( '
'"upload-year-month", '
'"upload-day" ), '
'%ordering_pair_eq( '
'"release-year", '
'"release-month-day" '
'), '
'%ordering_pair_eq( '
'"release-year", '
'"release-month-day-reversed" '
'), '
'%ordering_pair_eq( '
'"release-year", '
'"download-index" '
'), '
'%ordering_pair_eq( '
'"release-year-month", '
'"release-day" ) '
'), '
'episode_number_and_padded_, '
'"Detected '
'incompatibility '
'between '
'tv_show_by_date_season_ordering '
'and '
'tv_show_by_date_episode_ordering. '
'Ensure you are '
'not using both '
'upload and '
'release date, and '
'that the '
'year/month/day '
'are included in '
'the combined '
'season and '
'episode." ) }',
'tv_show_by_date_season_ordering': 'upload-year',
'tv_show_content_rating': '{ subscription_indent_2 }',
'tv_show_content_rating_default': 'TV-14',
'tv_show_date_range_type': '{ %if( %contains( '
'tv_show_by_date_season_ordering, '
'"release" ), "release_date", '
'"upload_date" ) }',
'tv_show_directory': output_directory,
'tv_show_fanart_file_name': 'fanart.jpg',
'tv_show_genre': '{ subscription_indent_1 }',
'tv_show_genre_default': 'ytdl-sub',
'tv_show_name': '{ subscription_name }',
'tv_show_poster_file_name': 'poster.jpg',
'url': '{ subscription_value }',
'url10': '',
'url100': '',
'url11': '',
'url12': '',
'url13': '',
'url14': '',
'url15': '',
'url16': '',
'url17': '',
'url18': '',
'url19': '',
'url2': '',
'url20': '',
'url21': '',
'url22': '',
'url23': '',
'url24': '',
'url25': '',
'url26': '',
'url27': '',
'url28': '',
'url29': '',
'url3': '',
'url30': '',
'url31': '',
'url32': '',
'url33': '',
'url34': '',
'url35': '',
'url36': '',
'url37': '',
'url38': '',
'url39': '',
'url4': '',
'url40': '',
'url41': '',
'url42': '',
'url43': '',
'url44': '',
'url45': '',
'url46': '',
'url47': '',
'url48': '',
'url49': '',
'url5': '',
'url50': '',
'url51': '',
'url52': '',
'url53': '',
'url54': '',
'url55': '',
'url56': '',
'url57': '',
'url58': '',
'url59': '',
'url6': '',
'url60': '',
'url61': '',
'url62': '',
'url63': '',
'url64': '',
'url65': '',
'url66': '',
'url67': '',
'url68': '',
'url69': '',
'url7': '',
'url70': '',
'url71': '',
'url72': '',
'url73': '',
'url74': '',
'url75': '',
'url76': '',
'url77': '',
'url78': '',
'url79': '',
'url8': '',
'url80': '',
'url81': '',
'url82': '',
'url83': '',
'url84': '',
'url85': '',
'url86': '',
'url87': '',
'url88': '',
'url89': '',
'url9': '',
'url90': '',
'url91': '',
'url92': '',
'url93': '',
'url94': '',
'url95': '',
'url96': '',
'url97': '',
'url98': '',
'url99': '',
'urls': '{ subscription_array }'},
"throttle_protection": {
"enable": True,
"sleep_per_download_s": {"max": "28.4", "min": "13.8"},
@ -604,7 +933,7 @@ def test_default_docker_config_and_subscriptions(
}
def test_default_docker_config_and_subscriptions(
def test_default_docker_config_and_subscriptions_resolved_resolution(
docker_default_subscription_path: Path, output_directory: str
):
default_config = ConfigFile.from_file_path("docker/root/defaults/config.yaml")
@ -613,7 +942,7 @@ def test_default_docker_config_and_subscriptions(
)
assert len(default_subs) == 1
resolved_yaml_as_json = yaml.safe_load(default_subs[0].resolved_yaml(resolution_level=3))
resolved_yaml_as_json = yaml.safe_load(default_subs[0].resolved_yaml(resolution_level=2))
assert resolved_yaml_as_json == {
"chapters": {
@ -653,6 +982,219 @@ def test_default_docker_config_and_subscriptions(
"preserve_mtime": False,
"thumbnail_name": "{thumbnail_file_name}",
},
'overrides': {'%bilateral_url': '{ %if( %and( enable_bilateral_scraping, '
'subscription_has_download_archive, '
"%is_bilateral_url( $0 ) ), $0, '' ) }",
'%episode_ordering_': '{ %eq( %lower( '
'tv_show_by_date_episode_ordering ), $0 ) '
'}',
'%is_bilateral_url': '{ %contains( $0, "youtube.com/playlist" ) '
'}',
'%ordering_pair_eq': '{ %eq( [ tv_show_by_date_season_ordering, '
'tv_show_by_date_episode_ordering ], [ $0, '
'$1 ] ) }',
'%season_ordering_': '{ %eq( %lower( '
'tv_show_by_date_season_ordering ), $0 ) }',
'assert_not_collection': True,
'avatar_uncropped_thumbnail_file_name': 'poster.jpg',
'banner_uncropped_thumbnail_file_name': 'fanart.jpg',
'enable_bilateral_scraping': True,
'enable_resolution_assert': True,
'enable_throttle_protection': True,
'episode_content_rating': 'TV-14',
'episode_date_standardized': '{ upload_date_standardized }',
'episode_file_name': 's{ upload_year }.e{ %pad_zero( %int( '
'%concat( upload_month, upload_day_padded, '
'upload_date_index_padded ) ), 6 ) } - { '
'title_sanitized_plex }',
'episode_file_path': '{ %sanitize( %concat( "Season ", %string( '
'upload_year ) ) ) }/{ %sanitize( %concat( '
'"s", %string( upload_year ), ".e", '
'%string( %pad_zero( %int( %concat( '
'upload_month, upload_day_padded, '
'upload_date_index_padded ) ), 6 ) ), " - '
'", %string( title_sanitized_plex ) ) ) }',
'episode_number': '{ upload_month }{ upload_day_padded }{ '
'upload_date_index_padded }',
'episode_number_and_padded_': '{ [ %concat( upload_month, '
'upload_day_padded, '
'upload_date_index_padded ), 6 ] '
'}',
'episode_number_padded': '{ %pad_zero( %int( %concat( '
'upload_month, upload_day_padded, '
'upload_date_index_padded ) ), 6 ) }',
'episode_plot': '{ webpage_url }\n\n{ description }',
'episode_title': '{ upload_date_standardized } - { title }',
'episode_year': '{ %slice( upload_date_standardized, 0, 4 ) }',
'file_title': '{ title_sanitized_plex }',
'file_uid': '{ uid_sanitized_plex }',
'include_sibling_metadata': False,
'modified_webpage_url': '{ webpage_url }',
'music_directory': '/music',
'music_video_directory': '/music_videos',
'resolution_assert': '{ %if( %and( enable_resolution_assert, '
'%ne( height, 0 ), %not( '
'resolution_assert_is_ignored ) ), '
'%assert( %gte( height, '
'resolution_assert_height_gte ), %concat( '
'"Entry ", title, " downloaded at a low '
'resolution (", resolution_readable, "), '
'you\'ve probably been throttled. ", '
'"Stopping further downloads, wait a few '
'hours and try again. ", "Disable using '
'the override variable '
'`enable_resolution_assert: False`." ) ), '
'"false is no-op" ) }',
'resolution_assert_height_gte': 361,
'resolution_assert_ignore_titles': '{ [ ] }',
'resolution_assert_is_ignored': '{ %print_if_true( %concat( '
'title, " has a match in '
'resolution_assert_ignore_titles, '
'skipping resolution assert." '
'), %contains_any( title, '
'resolution_assert_ignore_titles '
') ) }',
'resolution_assert_print': True,
'resolution_readable': '{ width }x{ height }',
's01_name': '',
's01_url': '',
'season_directory_name': 'Season { upload_year }',
'season_number': '{ upload_year }',
'season_number_padded': '{ upload_year }',
'season_poster_file_name': '{ %sanitize( %concat( "Season ", '
'%string( upload_year ) ) ) '
'}/Season{ upload_year }.jpg',
'subscription_array': ['https://www.youtube.com/@novapbs'],
'subscription_indent_1': 'Documentaries',
'subscription_indent_2': 'TV-14',
'subscription_value': 'https://www.youtube.com/@novapbs',
'subscription_value_1': 'https://www.youtube.com/@novapbs',
'thumbnail_file_name': '{ %concat( %string( %sanitize( %concat( '
'"Season ", %string( upload_year ) ) ) '
'), "/", %string( %sanitize( %concat( '
'"s", %string( upload_year ), ".e", '
'%string( %pad_zero( %int( %concat( '
'upload_month, upload_day_padded, '
'upload_date_index_padded ) ), 6 ) ), " '
'- ", %string( title_sanitized_plex ) ) '
') ) ) }-thumb.jpg',
'tv_show_by_date_episode_ordering': 'upload-month-day',
'tv_show_by_date_ordering_pair_validation_': '{ [ %concat( '
'upload_month, '
'upload_day_padded, '
'upload_date_index_padded '
'), 6 ] }',
'tv_show_by_date_season_ordering': 'upload-year',
'tv_show_content_rating': 'TV-14',
'tv_show_content_rating_default': 'TV-14',
'tv_show_date_range_type': 'upload_date',
'tv_show_directory': output_directory,
'tv_show_fanart_file_name': 'fanart.jpg',
'tv_show_genre': 'Documentaries',
'tv_show_genre_default': 'ytdl-sub',
'tv_show_name': 'NOVA PBS',
'tv_show_poster_file_name': 'poster.jpg',
'url': 'https://www.youtube.com/@novapbs',
'url10': '',
'url100': '',
'url11': '',
'url12': '',
'url13': '',
'url14': '',
'url15': '',
'url16': '',
'url17': '',
'url18': '',
'url19': '',
'url2': '',
'url20': '',
'url21': '',
'url22': '',
'url23': '',
'url24': '',
'url25': '',
'url26': '',
'url27': '',
'url28': '',
'url29': '',
'url3': '',
'url30': '',
'url31': '',
'url32': '',
'url33': '',
'url34': '',
'url35': '',
'url36': '',
'url37': '',
'url38': '',
'url39': '',
'url4': '',
'url40': '',
'url41': '',
'url42': '',
'url43': '',
'url44': '',
'url45': '',
'url46': '',
'url47': '',
'url48': '',
'url49': '',
'url5': '',
'url50': '',
'url51': '',
'url52': '',
'url53': '',
'url54': '',
'url55': '',
'url56': '',
'url57': '',
'url58': '',
'url59': '',
'url6': '',
'url60': '',
'url61': '',
'url62': '',
'url63': '',
'url64': '',
'url65': '',
'url66': '',
'url67': '',
'url68': '',
'url69': '',
'url7': '',
'url70': '',
'url71': '',
'url72': '',
'url73': '',
'url74': '',
'url75': '',
'url76': '',
'url77': '',
'url78': '',
'url79': '',
'url8': '',
'url80': '',
'url81': '',
'url82': '',
'url83': '',
'url84': '',
'url85': '',
'url86': '',
'url87': '',
'url88': '',
'url89': '',
'url9': '',
'url90': '',
'url91': '',
'url92': '',
'url93': '',
'url94': '',
'url95': '',
'url96': '',
'url97': '',
'url98': '',
'url99': '',
'urls': ['https://www.youtube.com/@novapbs']},
"throttle_protection": {
"enable": True,
"sleep_per_download_s": {"max": "28.4", "min": "13.8"},