This commit is contained in:
Jesse Bannon 2023-03-25 20:07:07 -07:00
parent 53f1d80285
commit 9093483722
5 changed files with 80 additions and 42 deletions

View file

@ -37,7 +37,9 @@ class ConfigFile(ConfigValidator):
ffprobe_path=self.config_options.ffprobe_path, ffprobe_path=self.config_options.ffprobe_path,
) )
FilePathValidatorMixin.MAX_FILE_NAME_BYTES = self.config_options.file_name_max_bytes FilePathValidatorMixin.set_max_file_name_bytes(
max_file_name_bytes=self.config_options.file_name_max_bytes
)
return self return self

View file

@ -1,4 +1,3 @@
import os
from typing import Any from typing import Any
from typing import Dict from typing import Dict
from typing import Optional from typing import Optional
@ -6,31 +5,19 @@ from typing import Optional
from mergedeep import mergedeep from mergedeep import mergedeep
from yt_dlp.utils import datetime_from_str from yt_dlp.utils import datetime_from_str
from ytdl_sub.config.defaults import DEFAULT_FFMPEG_PATH
from ytdl_sub.config.defaults import DEFAULT_FFPROBE_PATH
from ytdl_sub.config.defaults import DEFAULT_LOCK_DIRECTORY
from ytdl_sub.config.defaults import MAX_FILE_NAME_BYTES
from ytdl_sub.prebuilt_presets import PREBUILT_PRESETS from ytdl_sub.prebuilt_presets import PREBUILT_PRESETS
from ytdl_sub.utils.system import IS_WINDOWS
from ytdl_sub.validators.file_path_validators import FFmpegFileValidator from ytdl_sub.validators.file_path_validators import FFmpegFileValidator
from ytdl_sub.validators.file_path_validators import FFprobeFileValidator from ytdl_sub.validators.file_path_validators import FFprobeFileValidator
from ytdl_sub.validators.strict_dict_validator import StrictDictValidator from ytdl_sub.validators.strict_dict_validator import StrictDictValidator
from ytdl_sub.validators.validators import BoolValidator, IntValidator from ytdl_sub.validators.validators import BoolValidator
from ytdl_sub.validators.validators import IntValidator
from ytdl_sub.validators.validators import LiteralDictValidator from ytdl_sub.validators.validators import LiteralDictValidator
from ytdl_sub.validators.validators import StringValidator from ytdl_sub.validators.validators import StringValidator
if IS_WINDOWS:
_DEFAULT_LOCK_DIRECTORY = "" # Not supported in Windows
_DEFAULT_FFMPEG_PATH = ".\\ffmpeg.exe"
_DEFAULT_FFPROBE_PATH = ".\\ffprobe.exe"
_MAX_FILE_NAME_BYTES = 255
else:
_DEFAULT_LOCK_DIRECTORY = "/tmp"
_DEFAULT_FFMPEG_PATH = "/usr/bin/ffmpeg"
_DEFAULT_FFPROBE_PATH = "/usr/bin/ffprobe"
_MAX_FILE_NAME_BYTES = os.pathconf("/", "PC_NAME_MAX")
# Save file-name bytes for the -thumb.jpg portion
DEFAULT_MAX_FILE_NAME_BYTES = _MAX_FILE_NAME_BYTES - len("-thumb.jpg".encode("utf-8")) - 8
class ExperimentalValidator(StrictDictValidator): class ExperimentalValidator(StrictDictValidator):
_optional_keys = {"enable_update_with_info_json"} _optional_keys = {"enable_update_with_info_json"}
@ -138,19 +125,19 @@ class ConfigOptions(StrictDictValidator):
key="persist_logs", validator=PersistLogsValidator key="persist_logs", validator=PersistLogsValidator
) )
self._lock_directory = self._validate_key( self._lock_directory = self._validate_key(
key="lock_directory", validator=StringValidator, default=_DEFAULT_LOCK_DIRECTORY key="lock_directory", validator=StringValidator, default=DEFAULT_LOCK_DIRECTORY
) )
self._ffmpeg_path = self._validate_key( self._ffmpeg_path = self._validate_key(
key="ffmpeg_path", validator=FFmpegFileValidator, default=_DEFAULT_FFMPEG_PATH key="ffmpeg_path", validator=FFmpegFileValidator, default=DEFAULT_FFMPEG_PATH
) )
self._ffprobe_path = self._validate_key( self._ffprobe_path = self._validate_key(
key="ffprobe_path", validator=FFprobeFileValidator, default=_DEFAULT_FFPROBE_PATH key="ffprobe_path", validator=FFprobeFileValidator, default=DEFAULT_FFPROBE_PATH
) )
self._experimental = self._validate_key( self._experimental = self._validate_key(
key="experimental", validator=ExperimentalValidator, default={} key="experimental", validator=ExperimentalValidator, default={}
) )
self._file_name_max_bytes = self._validate_key( self._file_name_max_bytes = self._validate_key(
key="file_name_max_bytes", validator=IntValidator, default=DEFAULT_MAX_FILE_NAME_BYTES key="file_name_max_bytes", validator=IntValidator, default=MAX_FILE_NAME_BYTES
) )
@property @property

View file

@ -0,0 +1,16 @@
import os
from ytdl_sub.utils.system import IS_WINDOWS
if IS_WINDOWS:
DEFAULT_LOCK_DIRECTORY = "" # Not supported in Windows
DEFAULT_FFMPEG_PATH = ".\\ffmpeg.exe"
DEFAULT_FFPROBE_PATH = ".\\ffprobe.exe"
MAX_FILE_NAME_BYTES = 255
else:
DEFAULT_LOCK_DIRECTORY = "/tmp"
DEFAULT_FFMPEG_PATH = "/usr/bin/ffmpeg"
DEFAULT_FFPROBE_PATH = "/usr/bin/ffprobe"
MAX_FILE_NAME_BYTES = os.pathconf("/", "PC_NAME_MAX")

View file

@ -4,7 +4,7 @@ from typing import Any
from typing import Dict from typing import Dict
from typing import Tuple from typing import Tuple
from ytdl_sub.config.config_validator import DEFAULT_MAX_FILE_NAME_BYTES from ytdl_sub.config.defaults import MAX_FILE_NAME_BYTES
from ytdl_sub.utils.file_handler import get_file_extension from ytdl_sub.utils.file_handler import get_file_extension
from ytdl_sub.utils.subtitles import SUBTITLE_EXTENSIONS from ytdl_sub.utils.subtitles import SUBTITLE_EXTENSIONS
from ytdl_sub.validators.string_formatter_validators import OverridesStringFormatterValidator from ytdl_sub.validators.string_formatter_validators import OverridesStringFormatterValidator
@ -12,7 +12,6 @@ from ytdl_sub.validators.string_formatter_validators import StringFormatterValid
from ytdl_sub.validators.validators import StringValidator from ytdl_sub.validators.validators import StringValidator
class FFmpegFileValidator(StringValidator): class FFmpegFileValidator(StringValidator):
_expected_value_type_name = "ffmpeg dependency" _expected_value_type_name = "ffmpeg dependency"
_ffmpeg_dependency = "ffmpeg" _ffmpeg_dependency = "ffmpeg"
@ -37,11 +36,27 @@ class FFprobeFileValidator(FFmpegFileValidator):
class FilePathValidatorMixin: class FilePathValidatorMixin:
MAX_FILE_NAME_BYTES: int = DEFAULT_MAX_FILE_NAME_BYTES _EXTENSION_BYTES = len("-thumb.jpg".encode("utf-8")) + 8
_DEFAULT_MAX_BASE_FILE_NAME_BYTES: int = MAX_FILE_NAME_BYTES - _EXTENSION_BYTES
_MAX_BASE_FILE_NAME_BYTES: int = _DEFAULT_MAX_BASE_FILE_NAME_BYTES
@classmethod
def set_max_file_name_bytes(cls, max_file_name_bytes: int) -> None:
"""Actually sets the max _base_ file name in bytes (excludes extension)"""
max_base_file_name_bytes = max_file_name_bytes - cls._EXTENSION_BYTES
# bound between (extension_bytes + 20, MAX_FILE_NAME_BYTES)
max_base_file_name_bytes = max(max_base_file_name_bytes, 16)
max_base_file_name_bytes = min(
max_base_file_name_bytes, MAX_FILE_NAME_BYTES - cls._EXTENSION_BYTES
)
cls._MAX_BASE_FILE_NAME_BYTES = max_base_file_name_bytes
@classmethod @classmethod
def _is_file_name_too_long(cls, file_name: str) -> bool: def _is_file_name_too_long(cls, file_name: str) -> bool:
return len(file_name.encode("utf-8")) > cls.MAX_FILE_NAME_BYTES return len(file_name.encode("utf-8")) > cls._MAX_BASE_FILE_NAME_BYTES
@classmethod @classmethod
def _get_extension_split(cls, file_name: str) -> Tuple[str, str]: def _get_extension_split(cls, file_name: str) -> Tuple[str, str]:
@ -52,7 +67,7 @@ class FilePathValidatorMixin:
def _truncate_file_name(cls, file_name: str) -> str: def _truncate_file_name(cls, file_name: str) -> str:
file_sub_name, file_ext = cls._get_extension_split(file_name) file_sub_name, file_ext = cls._get_extension_split(file_name)
desired_size = cls.MAX_FILE_NAME_BYTES - len(file_ext.encode("utf-8")) - 1 desired_size = cls._MAX_BASE_FILE_NAME_BYTES - len(file_ext.encode("utf-8")) - 1
while len(file_sub_name.encode("utf-8")) > desired_size: while len(file_sub_name.encode("utf-8")) > desired_size:
file_sub_name = file_sub_name[:-1] file_sub_name = file_sub_name[:-1]
@ -84,7 +99,7 @@ class StringFormatterFileNameValidator(StringFormatterValidator, FilePathValidat
@classmethod @classmethod
def _is_file_name_too_long(cls, file_name: str) -> bool: def _is_file_name_too_long(cls, file_name: str) -> bool:
return len(file_name.encode("utf-8")) > cls.MAX_FILE_NAME_BYTES return len(file_name.encode("utf-8")) > cls._MAX_BASE_FILE_NAME_BYTES
@classmethod @classmethod
def _get_extension_split(cls, file_name: str) -> Tuple[str, str]: def _get_extension_split(cls, file_name: str) -> Tuple[str, str]:
@ -113,7 +128,7 @@ class StringFormatterFileNameValidator(StringFormatterValidator, FilePathValidat
def _truncate_file_name(cls, file_name: str) -> str: def _truncate_file_name(cls, file_name: str) -> str:
file_sub_name, file_ext = cls._get_extension_split(file_name) file_sub_name, file_ext = cls._get_extension_split(file_name)
while len(file_sub_name.encode("utf-8")) > cls.MAX_FILE_NAME_BYTES: while len(file_sub_name.encode("utf-8")) > cls._MAX_BASE_FILE_NAME_BYTES:
file_sub_name = file_sub_name[:-1] file_sub_name = file_sub_name[:-1]
return f"{file_sub_name}{file_ext}" return f"{file_sub_name}{file_ext}"

View file

@ -4,10 +4,10 @@ from pathlib import Path
import pytest import pytest
from ytdl_sub.config.config_file import ConfigFile from ytdl_sub.config.config_file import ConfigFile
from ytdl_sub.config.config_validator import DEFAULT_MAX_FILE_NAME_BYTES from ytdl_sub.config.defaults import MAX_FILE_NAME_BYTES
from ytdl_sub.utils.subtitles import SUBTITLE_EXTENSIONS from ytdl_sub.utils.subtitles import SUBTITLE_EXTENSIONS
from ytdl_sub.validators.file_path_validators import StringFormatterFileNameValidator, \ from ytdl_sub.validators.file_path_validators import FilePathValidatorMixin
FilePathValidatorMixin from ytdl_sub.validators.file_path_validators import StringFormatterFileNameValidator
class TestStringFormatterFilePathValidator: class TestStringFormatterFilePathValidator:
@ -71,17 +71,35 @@ class TestStringFormatterFilePathValidator:
assert len(dir_paths) == 1 assert len(dir_paths) == 1
assert Path(truncated_file_path) == dir_paths[0] assert Path(truncated_file_path) == dir_paths[0]
def test_config_changes_max_file_name_bytes(self): @pytest.mark.parametrize(
"file_name_max_bytes, expected_max",
[
(50, 50 - FilePathValidatorMixin._EXTENSION_BYTES),
(0, 16),
(10000, MAX_FILE_NAME_BYTES - FilePathValidatorMixin._EXTENSION_BYTES),
],
)
def test_config_changes_max_file_name_bytes(self, file_name_max_bytes: int, expected_max: int):
# Ensure the default is set # Ensure the default is set
assert FilePathValidatorMixin.MAX_FILE_NAME_BYTES == DEFAULT_MAX_FILE_NAME_BYTES assert (
FilePathValidatorMixin._MAX_BASE_FILE_NAME_BYTES
== FilePathValidatorMixin._DEFAULT_MAX_BASE_FILE_NAME_BYTES
)
try: try:
# Initializes the config values, setting the max to 10 bytes # Initializes the config values, setting the max to 10 bytes
_ = ConfigFile.from_dict({ _ = ConfigFile.from_dict(
"working_directory": ".", {
"file_name_max_bytes": 10, "configuration": {
}) "working_directory": ".",
"file_name_max_bytes": file_name_max_bytes,
},
"presets": {},
}
)
assert FilePathValidatorMixin.MAX_FILE_NAME_BYTES == 10 assert FilePathValidatorMixin._MAX_BASE_FILE_NAME_BYTES == expected_max
finally: finally:
FilePathValidatorMixin.MAX_FILE_NAME_BYTES = DEFAULT_MAX_FILE_NAME_BYTES FilePathValidatorMixin._MAX_BASE_FILE_NAME_BYTES = (
FilePathValidatorMixin._DEFAULT_MAX_BASE_FILE_NAME_BYTES
)