danvics/ytdl-sub
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

unit tests passing

Browse source
This commit is contained in:
Jesse Bannon 2023-12-08 15:10:17 -08:00
parent ec605d1b1b
commit 6d681611c4
8 changed files with 126 additions and 263 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
src/ytdl_sub/config/config_file.py
View file

@ -6,8 +6,8 @@ from ytdl_sub.config.config_validator import ConfigValidator
from ytdl_sub.config.preset import Preset
from ytdl_sub.utils.exceptions import FileNotFoundException
from ytdl_sub.utils.ffmpeg import FFMPEG
from ytdl_sub.utils.file_path import FilePathTruncater
from ytdl_sub.utils.yaml import load_yaml
from ytdl_sub.validators.file_path_validators import FilePathValidatorMixin
class ConfigFile(ConfigValidator):
@ -36,7 +36,7 @@ class ConfigFile(ConfigValidator):
ffprobe_path=self.config_options.ffprobe_path,
)
FilePathValidatorMixin.set_max_file_name_bytes(
FilePathTruncater.set_max_file_name_bytes(
max_file_name_bytes=self.config_options.file_name_max_bytes
)

13
src/ytdl_sub/entries/script/function_scripts.py
View file

@ -1,3 +1,5 @@
import os
from yt_dlp.utils import sanitize_filename
from ytdl_sub.script.functions import Functions
@ -7,6 +9,7 @@ from ytdl_sub.script.types.resolvable import Integer
from ytdl_sub.script.types.resolvable import ReturnableArgument
from ytdl_sub.script.types.resolvable import String
from ytdl_sub.script.utils.exceptions import RuntimeException
from ytdl_sub.utils.file_path import FilePathTruncater
def _pad(num: int, width: int):
@ -23,6 +26,14 @@ class CustomFunctions:
value = String(value.value.replace("{", "｛").replace("}", "｝"))
return value
@staticmethod
def to_native_filepath(filepath: String) -> String:
return String(str(os.path.realpath(filepath.value)))
@staticmethod
def truncate_filepath_if_too_long(filepath: String) -> String:
return String(FilePathTruncater.maybe_truncate_file_path(filepath.value))
@staticmethod
def sanitize(value: AnyArgument) -> String:
return String(sanitize_filename(str(value)))
@ -113,6 +124,8 @@ class CustomFunctions:
def register():
if not Functions.is_built_in("sanitize"):
Functions.register_function(CustomFunctions.legacy_bracket_safety)
Functions.register_function(CustomFunctions.truncate_filepath_if_too_long)
Functions.register_function(CustomFunctions.to_native_filepath)
Functions.register_function(CustomFunctions.sanitize)
Functions.register_function(CustomFunctions.sanitize_plex_episode)
Functions.register_function(CustomFunctions.to_date_metadata)

61
src/ytdl_sub/utils/file_path.py Normal file
View file

@ -0,0 +1,61 @@
import os
from pathlib import Path
from typing import Tuple
from ytdl_sub.config.defaults import MAX_FILE_NAME_BYTES
from ytdl_sub.utils.file_handler import get_file_extension
class FilePathTruncater:
_EXTENSION_BYTES = len("-thumb.jpg".encode("utf-8")) + 8
_DEFAULT_MAX_BASE_FILE_NAME_BYTES: int = MAX_FILE_NAME_BYTES - _EXTENSION_BYTES
_MAX_BASE_FILE_NAME_BYTES: int = _DEFAULT_MAX_BASE_FILE_NAME_BYTES
@classmethod
def set_max_file_name_bytes(cls, max_file_name_bytes: int) -> None:
"""Actually sets the max _base_ file name in bytes (excludes extension)"""
max_base_file_name_bytes = max_file_name_bytes - cls._EXTENSION_BYTES
# bound between (extension_bytes + 20, MAX_FILE_NAME_BYTES)
max_base_file_name_bytes = max(max_base_file_name_bytes, 16)
max_base_file_name_bytes = min(
max_base_file_name_bytes, MAX_FILE_NAME_BYTES - cls._EXTENSION_BYTES
)
cls._MAX_BASE_FILE_NAME_BYTES = max_base_file_name_bytes
@classmethod
def _is_file_name_too_long(cls, file_name: str) -> bool:
return len(file_name.encode("utf-8")) > cls._MAX_BASE_FILE_NAME_BYTES
@classmethod
def _get_extension_split(cls, file_name: str) -> Tuple[str, str, str]:
if file_name.endswith("-thumb.jpg"):
ext = "-thumb.jpg"
delimiter = ""
else:
ext = get_file_extension(file_name)
delimiter = "."
return file_name[: -len(ext)], ext, delimiter
@classmethod
def _truncate_file_name(cls, file_name: str) -> str:
file_sub_name, file_ext, delimiter = cls._get_extension_split(file_name)
desired_size = cls._MAX_BASE_FILE_NAME_BYTES - len(file_ext.encode("utf-8")) - 1
while len(file_sub_name.encode("utf-8")) > desired_size:
file_sub_name = file_sub_name[:-1]
return f"{file_sub_name}{delimiter}{file_ext}"
@classmethod
def maybe_truncate_file_path(cls, file_path: str) -> str:
"""Turn into a Path, then a string, to get correct directory separators"""
file_directory, file_name = os.path.split(Path(file_path))
if cls._is_file_name_too_long(file_name):
return str(Path(file_directory) / cls._truncate_file_name(file_name))
return str(file_path)

13
src/ytdl_sub/utils/scriptable.py
View file

@ -8,6 +8,7 @@ from typing import Set
from ytdl_sub.entries.script.variable_scripts import UNRESOLVED_VARIABLES
from ytdl_sub.entries.script.variable_scripts import VARIABLE_SCRIPTS
from ytdl_sub.script.parser import parse
from ytdl_sub.script.script import Script
from ytdl_sub.script.types.resolvable import Resolvable
from ytdl_sub.script.types.resolvable import String
@ -30,6 +31,18 @@ class Scriptable(ABC):
}
return dict(variables, **sanitized_variables)
@classmethod
def wrappable_format_string(cls, format_string: str) -> str:
parsed = parse(format_string)
if resolvable := parsed.maybe_resolvable:
return f"'{str(resolvable)}'"
stripped_format_string = format_string.strip()
if stripped_format_string.startswith("{") and stripped_format_string.endswith("}"):
return stripped_format_string[1:-1]
return format_string
@classmethod
def to_script(cls, value: Any) -> str:
if isinstance(value, str):

110
src/ytdl_sub/validators/file_path_validators.py
View file

@ -1,12 +1,10 @@
import os
from pathlib import Path
from typing import Any
from typing import Dict
from typing import Tuple
from ytdl_sub.config.defaults import MAX_FILE_NAME_BYTES
from ytdl_sub.utils.file_handler import get_file_extension
from ytdl_sub.utils.subtitles import SUBTITLE_EXTENSIONS
from ytdl_sub.script.parser import parse
from ytdl_sub.script.types.resolvable import String
from ytdl_sub.utils.scriptable import Scriptable
from ytdl_sub.validators.string_formatter_validators import OverridesStringFormatterValidator
from ytdl_sub.validators.string_formatter_validators import StringFormatterValidator
from ytdl_sub.validators.validators import StringValidator
@ -35,57 +33,8 @@ class FFprobeFileValidator(FFmpegFileValidator):
_ffmpeg_dependency = "ffprobe"
class FilePathValidatorMixin:
_EXTENSION_BYTES = len("-thumb.jpg".encode("utf-8")) + 8
_DEFAULT_MAX_BASE_FILE_NAME_BYTES: int = MAX_FILE_NAME_BYTES - _EXTENSION_BYTES
_MAX_BASE_FILE_NAME_BYTES: int = _DEFAULT_MAX_BASE_FILE_NAME_BYTES
@classmethod
def set_max_file_name_bytes(cls, max_file_name_bytes: int) -> None:
"""Actually sets the max _base_ file name in bytes (excludes extension)"""
max_base_file_name_bytes = max_file_name_bytes - cls._EXTENSION_BYTES
# bound between (extension_bytes + 20, MAX_FILE_NAME_BYTES)
max_base_file_name_bytes = max(max_base_file_name_bytes, 16)
max_base_file_name_bytes = min(
max_base_file_name_bytes, MAX_FILE_NAME_BYTES - cls._EXTENSION_BYTES
)
cls._MAX_BASE_FILE_NAME_BYTES = max_base_file_name_bytes
@classmethod
def _is_file_name_too_long(cls, file_name: str) -> bool:
return len(file_name.encode("utf-8")) > cls._MAX_BASE_FILE_NAME_BYTES
@classmethod
def _get_extension_split(cls, file_name: str) -> Tuple[str, str]:
ext = get_file_extension(file_name)
return file_name[: -len(ext)], ext
@classmethod
def _truncate_file_name(cls, file_name: str) -> str:
file_sub_name, file_ext = cls._get_extension_split(file_name)
desired_size = cls._MAX_BASE_FILE_NAME_BYTES - len(file_ext.encode("utf-8")) - 1
while len(file_sub_name.encode("utf-8")) > desired_size:
file_sub_name = file_sub_name[:-1]
return f"{file_sub_name}.{file_ext}"
@classmethod
def _maybe_truncate_file_path(cls, file_path: Path) -> str:
"""Turn into a Path, then a string, to get correct directory separators"""
file_directory, file_name = os.path.split(Path(file_path))
if cls._is_file_name_too_long(file_name):
return str(Path(file_directory) / cls._truncate_file_name(file_name))
return str(file_path)
# pylint: disable=line-too-long
class StringFormatterFileNameValidator(StringFormatterValidator, FilePathValidatorMixin):
class StringFormatterFileNameValidator(StringFormatterValidator):
"""
Same as a
:class:`StringFormatterValidator <ytdl_sub.validators.string_formatter_validators.StringFormatterValidator>`
@ -97,51 +46,16 @@ class StringFormatterFileNameValidator(StringFormatterValidator, FilePathValidat
_expected_value_type_name = "filepath"
@classmethod
def _is_file_name_too_long(cls, file_name: str) -> bool:
return len(file_name.encode("utf-8")) > cls._MAX_BASE_FILE_NAME_BYTES
@classmethod
def _get_extension_split(cls, file_name: str) -> Tuple[str, str]:
"""
Returns
-------
file_name, ext (including .)
"""
if file_name.endswith(".info.json"):
ext = ".info.json"
elif file_name.endswith("-thumb.jpg"):
ext = "-thumb.jpg"
elif any(file_name.endswith(f".{subtitle_ext}") for subtitle_ext in SUBTITLE_EXTENSIONS):
file_name_split = file_name.split(".")
ext = file_name_split[-1]
# Try to capture .lang.ext
if len(file_name_split) > 2 and len(file_name_split[-2]) < 6:
ext = f".{file_name_split[-2]}.{file_name_split[-1]}"
else:
ext = f".{file_name.rsplit('.', maxsplit=1)[-1]}"
return file_name[: -len(ext)], ext
@classmethod
def _truncate_file_name(cls, file_name: str) -> str:
file_sub_name, file_ext = cls._get_extension_split(file_name)
while len(file_sub_name.encode("utf-8")) > cls._MAX_BASE_FILE_NAME_BYTES:
file_sub_name = file_sub_name[:-1]
return f"{file_sub_name}{file_ext}"
def apply_formatter(self, variable_dict: Dict[str, str]) -> str:
"""Turn into a Path, then a string, to get correct directory separators"""
file_path = Path(super().apply_formatter(variable_dict))
return self._maybe_truncate_file_path(file_path)
@property
def format_string(self) -> str:
return f"{{%to_native_filepath(%truncate_filepath_if_too_long({Scriptable.wrappable_format_string(super().format_string)}))}}"
class OverridesStringFormatterFilePathValidator(OverridesStringFormatterValidator):
_expected_value_type_name = "static filepath"
def apply_formatter(self, variable_dict: Dict[str, str]) -> str:
"""Turn into a Path, then a string, to get correct directory separators"""
return os.path.realpath(super().apply_formatter(variable_dict))
@property
def format_string(self) -> str:
return (
f"{{%to_native_filepath({Scriptable.wrappable_format_string(super().format_string)})}}"
)

7
src/ytdl_sub/validators/string_formatter_validators.py
View file

@ -11,6 +11,7 @@ from ytdl_sub.entries.script.variable_definitions import VARIABLES
from ytdl_sub.script.parser import parse
from ytdl_sub.script.script import Script
from ytdl_sub.script.types.resolvable import Resolvable
from ytdl_sub.script.utils.exceptions import UserException
from ytdl_sub.utils.exceptions import InvalidVariableNameException
from ytdl_sub.utils.exceptions import StringFormattingException
from ytdl_sub.utils.exceptions import StringFormattingVariableNotFoundException
@ -81,9 +82,11 @@ class StringFormatterValidator(StringValidator):
def __init__(self, name, value: str):
super().__init__(name=name, value=value)
_ = parse(str(value))
try:
_ = parse(str(value))
except UserException as exc:
raise self._validation_exception(exc) from exc
@final
@property
def format_string(self) -> str:
"""

26
tests/unit/validators/test_file_path_validators.py
View file

@ -5,11 +5,13 @@ import pytest
from ytdl_sub.config.config_file import ConfigFile
from ytdl_sub.config.defaults import MAX_FILE_NAME_BYTES
from ytdl_sub.script.script import Script
from ytdl_sub.utils.file_path import FilePathTruncater
from ytdl_sub.utils.subtitles import SUBTITLE_EXTENSIONS
from ytdl_sub.validators.file_path_validators import FilePathValidatorMixin
from ytdl_sub.validators.file_path_validators import StringFormatterFileNameValidator
@pytest.mark.usefixtures("register_custom_functions")
class TestStringFormatterFilePathValidator:
@pytest.mark.parametrize(
"ext",
@ -28,7 +30,9 @@ class TestStringFormatterFilePathValidator:
file_path = str(Path(temp_dir) / file_name)
formatter = StringFormatterFileNameValidator(name="test", value=str(file_path))
truncated_file_path = formatter.apply_formatter({})
truncated_file_path = (
Script({"file_name": formatter.format_string}).resolve().get_str("file_name")
)
assert truncated_file_path.count(".") == ext.count(".")
assert str(Path(temp_dir)) in truncated_file_path
@ -56,7 +60,9 @@ class TestStringFormatterFilePathValidator:
file_path = str(Path(temp_dir) / f"{base_file_name}{ext}")
formatter = StringFormatterFileNameValidator(name="test", value=str(file_path))
truncated_file_path = formatter.apply_formatter({})
truncated_file_path = (
Script({"file_name": formatter.format_string}).resolve().get_str("file_name")
)
assert truncated_file_path == str(
Path(temp_dir)
@ -74,16 +80,16 @@ class TestStringFormatterFilePathValidator:
@pytest.mark.parametrize(
"file_name_max_bytes, expected_max",
[
(50, 50 - FilePathValidatorMixin._EXTENSION_BYTES),
(50, 50 - FilePathTruncater._EXTENSION_BYTES),
(0, 16),
(10000, MAX_FILE_NAME_BYTES - FilePathValidatorMixin._EXTENSION_BYTES),
(10000, MAX_FILE_NAME_BYTES - FilePathTruncater._EXTENSION_BYTES),
],
)
def test_config_changes_max_file_name_bytes(self, file_name_max_bytes: int, expected_max: int):
# Ensure the default is set
assert (
FilePathValidatorMixin._MAX_BASE_FILE_NAME_BYTES
== FilePathValidatorMixin._DEFAULT_MAX_BASE_FILE_NAME_BYTES
FilePathTruncater._MAX_BASE_FILE_NAME_BYTES
== FilePathTruncater._DEFAULT_MAX_BASE_FILE_NAME_BYTES
)
try:
@ -98,8 +104,8 @@ class TestStringFormatterFilePathValidator:
}
)
assert FilePathValidatorMixin._MAX_BASE_FILE_NAME_BYTES == expected_max
assert FilePathTruncater._MAX_BASE_FILE_NAME_BYTES == expected_max
finally:
FilePathValidatorMixin._MAX_BASE_FILE_NAME_BYTES = (
FilePathValidatorMixin._DEFAULT_MAX_BASE_FILE_NAME_BYTES
FilePathTruncater._MAX_BASE_FILE_NAME_BYTES = (
FilePathTruncater._DEFAULT_MAX_BASE_FILE_NAME_BYTES
)

155
tests/unit/validators/test_string_formatter_validator.py
View file

@ -9,22 +9,6 @@ from ytdl_sub.validators.string_formatter_validators import OverridesStringForma
from ytdl_sub.validators.string_formatter_validators import StringFormatterValidator
@pytest.fixture
def error_message_unequal_brackets_str():
return (
"Brackets are reserved for {variable_names} and "
"should contain a single open and close bracket."
)
@pytest.fixture
def error_message_unequal_regex_matches_str():
return (
"{variable_names} must start with a lowercase letter, should only contain lowercase "
"letters, numbers, underscores, and have a single open and close bracket."
)
@pytest.mark.parametrize(
"string_formatter_class", [StringFormatterValidator, OverridesStringFormatterValidator]
)
@ -34,16 +18,6 @@ class TestStringFormatterValidator(object):
validator = string_formatter_class(name="test_format_variables", value=format_string)
assert validator.format_string == format_string
assert validator.format_variables == ["var_one1", "var_two"]
def test_format_variables(self, string_formatter_class):
format_string = "No vars 💩"
assert (
string_formatter_class(
name="test_format_variables_empty", value=format_string
).format_variables
== []
)
@pytest.mark.parametrize(
"format_string",
@ -54,12 +28,8 @@ class TestStringFormatterValidator(object):
"Try }var_one} and {var_one}",
],
)
def test_validate_fail_uneven_brackets(
self, string_formatter_class, format_string, error_message_unequal_brackets_str
):
expected_error_msg = f"Validation error in fail: {error_message_unequal_brackets_str}"
with pytest.raises(ValidationException, match=expected_error_msg):
def test_validate_fail_uneven_brackets(self, string_formatter_class, format_string):
with pytest.raises(ValidationException, match="Validation error in fail:"):
_ = string_formatter_class(name="fail", value=format_string)
@pytest.mark.parametrize(
@ -74,124 +44,10 @@ class TestStringFormatterValidator(object):
"Try {} empty",
],
)
def test_validate_fail_bad_variable(
self, string_formatter_class, format_string, error_message_unequal_regex_matches_str
):
expected_error_msg = f"Validation error in fail: {error_message_unequal_regex_matches_str}"
with pytest.raises(ValidationException, match=expected_error_msg):
def test_validate_fail_bad_variable(self, string_formatter_class, format_string):
with pytest.raises(ValidationException, match="Validation error in fail:"):
_ = string_formatter_class(name="fail", value=format_string)
@pytest.mark.parametrize(
"format_string, bad_variable",
[
("keyword {while}", "while"),
("{try} {valid_var}", "try"),
],
)
def test_validate_fail_variable_keyword_or_not_identifier(
self, string_formatter_class, format_string, bad_variable
):
expected_error_msg = (
f"Validation error in fail: "
f"'{bad_variable}' is a Python keyword and cannot be used as a variable."
)
with pytest.raises(ValidationException, match=expected_error_msg):
_ = string_formatter_class(name="fail", value=format_string)
def test_entry_formatter_fails_missing_field(self, string_formatter_class):
format_string = string_formatter_class(name="test", value=f"prefix {{bah_humbug}} suffix")
variable_dict = {"varb": "a", "vara": "b"}
expected_error_msg = (
f"Validation error in test: Format variable 'bah_humbug' does not exist. "
f"Available variables: {', '.join(sorted(variable_dict.keys()))}"
)
if string_formatter_class == OverridesStringFormatterValidator:
expected_error_msg = (
f"Validation error in test: Override variable 'bah_humbug' does not exist. "
f"For this field, ensure your override variable does not contain any source "
f"variables - it is a requirement that this be a static string. "
f"Available override variables: {', '.join(sorted(variable_dict.keys()))}"
)
with pytest.raises(StringFormattingException, match=expected_error_msg):
assert format_string.apply_formatter(variable_dict=variable_dict)
def test_string_formatter_single_field(self, string_formatter_class):
uid = "this uid"
format_string = string_formatter_class(name="test", value=f"prefix {{uid}} suffix")
expected_string = f"prefix {uid} suffix"
assert format_string.apply_formatter(variable_dict={"uid": uid}) == expected_string
def test_entry_formatter_duplicate_fields(self, string_formatter_class):
upload_year = "2022"
format_string = string_formatter_class(
name="test", value=f"prefix {{upload_year}} {{upload_year}} suffix"
)
expected_string = f"prefix {upload_year} {upload_year} suffix"
assert (
format_string.apply_formatter(variable_dict={"upload_year": upload_year})
== expected_string
)
def test_entry_formatter_override_recursive(self, string_formatter_class):
variable_dict = {
"level_a": "level a",
"level_b": "level b and {level_a}",
"level_c": "level c and {level_b}",
}
format_string = string_formatter_class(name="test", value="level d and {level_c}")
expected_string = "level d and level c and level b and level a"
assert format_string.apply_formatter(variable_dict=variable_dict) == expected_string
def test_entry_formatter_override_sanitized_recursive(self, string_formatter_class):
variable_dict = {
"level_a": "level a",
"level_b": "level b ? {level_a}",
"level_c": "level c and {level_b}",
}
format_string = string_formatter_class(name="test", value="level d and {level_c_sanitized}")
expected_string = "level d and " + sanitize_filename("level c and level b ? level a")
assert format_string.apply_formatter(variable_dict=variable_dict) == expected_string
def test_entry_formatter_override_sanitized_recursive_inner(self, string_formatter_class):
variable_dict = {
"level_a": "level a ?",
"level_b": "level b ? {level_a_sanitized}",
"level_c": "level c and {level_b_sanitized}",
}
format_string = string_formatter_class(name="test", value="level d and {level_c}")
expected_string = "level d and level c and " + sanitize_filename("level b ? level a ?")
assert format_string.apply_formatter(variable_dict=variable_dict) == expected_string
def test_entry_formatter_override_recursive_fail_cycle(self, string_formatter_class):
variable_dict = {
"level_a": "{level_b}",
"level_b": "{level_a}",
}
# Max depth is 3 so should go level_a -(0)-> level_b -(1)-> level_a -(2)-> level_b
expected_error_msg = (
"Validation error in test: Attempted to format but failed after reaching max recursion "
"depth of 3. Try to keep variables dependent on only one other variable at max. "
"Unresolved variables: level_b"
)
format_string = string_formatter_class(name="test", value="{level_a}")
format_string._max_format_recursion = 3
with pytest.raises(StringFormattingException, match=expected_error_msg):
_ = format_string.apply_formatter(variable_dict=variable_dict)
class TestDictFormatterValidator(object):
@pytest.mark.parametrize(
@ -215,9 +71,6 @@ class TestDictFormatterValidator(object):
assert validator.dict["key1"].format_string == key1_format_string
assert validator.dict["key2"].format_string == key2_format_string
assert validator.dict["key1"].format_variables == ["variable"]
assert validator.dict["key2"].format_variables == []
assert validator.dict_with_format_strings == {
"key1": key1_format_string,
"key2": key2_format_string,