From 4b9709aeab152b7ac715efff5d37affc0de15d87 Mon Sep 17 00:00:00 2001 From: Jesse Bannon Date: Sat, 25 Mar 2023 20:17:38 -0700 Subject: [PATCH] [FEATURE] Configurable file_name_max_bytes (#562) * config for max file size, need to move to defaults file * wroking --- src/ytdl_sub/config/config_file.py | 7 +++- src/ytdl_sub/config/config_validator.py | 32 ++++++++++------- src/ytdl_sub/config/defaults.py | 16 +++++++++ .../validators/file_path_validators.py | 36 ++++++++++++------- src/ytdl_sub/validators/validators.py | 5 +++ .../validators/test_file_path_validators.py | 36 +++++++++++++++++++ 6 files changed, 105 insertions(+), 27 deletions(-) create mode 100644 src/ytdl_sub/config/defaults.py diff --git a/src/ytdl_sub/config/config_file.py b/src/ytdl_sub/config/config_file.py index d7874e28..6d6bcc66 100644 --- a/src/ytdl_sub/config/config_file.py +++ b/src/ytdl_sub/config/config_file.py @@ -6,6 +6,7 @@ from ytdl_sub.config.config_validator import ConfigValidator from ytdl_sub.config.preset import Preset from ytdl_sub.utils.ffmpeg import FFMPEG from ytdl_sub.utils.yaml import load_yaml +from ytdl_sub.validators.file_path_validators import FilePathValidatorMixin class ConfigFile(ConfigValidator): @@ -22,7 +23,7 @@ class ConfigFile(ConfigValidator): def _initialize(self): """ - Configures things (umask, pgid) prior to any downloading + Configures things (umask, pgid, etc) prior to any downloading Returns ------- @@ -36,6 +37,10 @@ class ConfigFile(ConfigValidator): ffprobe_path=self.config_options.ffprobe_path, ) + FilePathValidatorMixin.set_max_file_name_bytes( + max_file_name_bytes=self.config_options.file_name_max_bytes + ) + return self @classmethod diff --git a/src/ytdl_sub/config/config_validator.py b/src/ytdl_sub/config/config_validator.py index 4fb643c9..d2c353e6 100644 --- a/src/ytdl_sub/config/config_validator.py +++ b/src/ytdl_sub/config/config_validator.py @@ -5,24 +5,19 @@ from typing import Optional from mergedeep import mergedeep from yt_dlp.utils import datetime_from_str +from ytdl_sub.config.defaults import DEFAULT_FFMPEG_PATH +from ytdl_sub.config.defaults import DEFAULT_FFPROBE_PATH +from ytdl_sub.config.defaults import DEFAULT_LOCK_DIRECTORY +from ytdl_sub.config.defaults import MAX_FILE_NAME_BYTES from ytdl_sub.prebuilt_presets import PREBUILT_PRESETS -from ytdl_sub.utils.system import IS_WINDOWS from ytdl_sub.validators.file_path_validators import FFmpegFileValidator from ytdl_sub.validators.file_path_validators import FFprobeFileValidator from ytdl_sub.validators.strict_dict_validator import StrictDictValidator from ytdl_sub.validators.validators import BoolValidator +from ytdl_sub.validators.validators import IntValidator from ytdl_sub.validators.validators import LiteralDictValidator from ytdl_sub.validators.validators import StringValidator -if IS_WINDOWS: - _DEFAULT_LOCK_DIRECTORY = "" # Not supported in Windows - _DEFAULT_FFMPEG_PATH = ".\\ffmpeg.exe" - _DEFAULT_FFPROBE_PATH = ".\\ffprobe.exe" -else: - _DEFAULT_LOCK_DIRECTORY = "/tmp" - _DEFAULT_FFMPEG_PATH = "/usr/bin/ffmpeg" - _DEFAULT_FFPROBE_PATH = "/usr/bin/ffprobe" - class ExperimentalValidator(StrictDictValidator): _optional_keys = {"enable_update_with_info_json"} @@ -110,6 +105,7 @@ class ConfigOptions(StrictDictValidator): "lock_directory", "ffmpeg_path", "ffprobe_path", + "file_name_max_bytes", "experimental", } @@ -129,17 +125,20 @@ class ConfigOptions(StrictDictValidator): key="persist_logs", validator=PersistLogsValidator ) self._lock_directory = self._validate_key( - key="lock_directory", validator=StringValidator, default=_DEFAULT_LOCK_DIRECTORY + key="lock_directory", validator=StringValidator, default=DEFAULT_LOCK_DIRECTORY ) self._ffmpeg_path = self._validate_key( - key="ffmpeg_path", validator=FFmpegFileValidator, default=_DEFAULT_FFMPEG_PATH + key="ffmpeg_path", validator=FFmpegFileValidator, default=DEFAULT_FFMPEG_PATH ) self._ffprobe_path = self._validate_key( - key="ffprobe_path", validator=FFprobeFileValidator, default=_DEFAULT_FFPROBE_PATH + key="ffprobe_path", validator=FFprobeFileValidator, default=DEFAULT_FFPROBE_PATH ) self._experimental = self._validate_key( key="experimental", validator=ExperimentalValidator, default={} ) + self._file_name_max_bytes = self._validate_key( + key="file_name_max_bytes", validator=IntValidator, default=MAX_FILE_NAME_BYTES + ) @property def working_directory(self) -> str: @@ -193,6 +192,13 @@ class ConfigOptions(StrictDictValidator): """ return self._persist_logs + @property + def file_name_max_bytes(self) -> int: + """ + Optional. Max file name size in bytes. Most OS's typically default to 255 bytes. + """ + return self._file_name_max_bytes.value + @property def experimental(self) -> ExperimentalValidator: """ diff --git a/src/ytdl_sub/config/defaults.py b/src/ytdl_sub/config/defaults.py new file mode 100644 index 00000000..29f40102 --- /dev/null +++ b/src/ytdl_sub/config/defaults.py @@ -0,0 +1,16 @@ +import os + +from ytdl_sub.utils.system import IS_WINDOWS + +if IS_WINDOWS: + DEFAULT_LOCK_DIRECTORY = "" # Not supported in Windows + DEFAULT_FFMPEG_PATH = ".\\ffmpeg.exe" + DEFAULT_FFPROBE_PATH = ".\\ffprobe.exe" + + MAX_FILE_NAME_BYTES = 255 +else: + DEFAULT_LOCK_DIRECTORY = "/tmp" + DEFAULT_FFMPEG_PATH = "/usr/bin/ffmpeg" + DEFAULT_FFPROBE_PATH = "/usr/bin/ffprobe" + + MAX_FILE_NAME_BYTES = os.pathconf("/", "PC_NAME_MAX") diff --git a/src/ytdl_sub/validators/file_path_validators.py b/src/ytdl_sub/validators/file_path_validators.py index 617328a5..8d5afe11 100644 --- a/src/ytdl_sub/validators/file_path_validators.py +++ b/src/ytdl_sub/validators/file_path_validators.py @@ -4,21 +4,13 @@ from typing import Any from typing import Dict from typing import Tuple +from ytdl_sub.config.defaults import MAX_FILE_NAME_BYTES from ytdl_sub.utils.file_handler import get_file_extension from ytdl_sub.utils.subtitles import SUBTITLE_EXTENSIONS -from ytdl_sub.utils.system import IS_WINDOWS from ytdl_sub.validators.string_formatter_validators import OverridesStringFormatterValidator from ytdl_sub.validators.string_formatter_validators import StringFormatterValidator from ytdl_sub.validators.validators import StringValidator -if IS_WINDOWS: - _MAX_FILE_NAME_BYTES = 255 -else: - _MAX_FILE_NAME_BYTES = os.pathconf("/", "PC_NAME_MAX") - -# Save file-name bytes for the -thumb.jpg portion -_MAX_BASE_FILE_NAME_BYTES = _MAX_FILE_NAME_BYTES - len("-thumb.jpg".encode("utf-8")) - 8 - class FFmpegFileValidator(StringValidator): _expected_value_type_name = "ffmpeg dependency" @@ -44,9 +36,27 @@ class FFprobeFileValidator(FFmpegFileValidator): class FilePathValidatorMixin: + _EXTENSION_BYTES = len("-thumb.jpg".encode("utf-8")) + 8 + _DEFAULT_MAX_BASE_FILE_NAME_BYTES: int = MAX_FILE_NAME_BYTES - _EXTENSION_BYTES + + _MAX_BASE_FILE_NAME_BYTES: int = _DEFAULT_MAX_BASE_FILE_NAME_BYTES + + @classmethod + def set_max_file_name_bytes(cls, max_file_name_bytes: int) -> None: + """Actually sets the max _base_ file name in bytes (excludes extension)""" + max_base_file_name_bytes = max_file_name_bytes - cls._EXTENSION_BYTES + + # bound between (extension_bytes + 20, MAX_FILE_NAME_BYTES) + max_base_file_name_bytes = max(max_base_file_name_bytes, 16) + max_base_file_name_bytes = min( + max_base_file_name_bytes, MAX_FILE_NAME_BYTES - cls._EXTENSION_BYTES + ) + + cls._MAX_BASE_FILE_NAME_BYTES = max_base_file_name_bytes + @classmethod def _is_file_name_too_long(cls, file_name: str) -> bool: - return len(file_name.encode("utf-8")) > _MAX_FILE_NAME_BYTES + return len(file_name.encode("utf-8")) > cls._MAX_BASE_FILE_NAME_BYTES @classmethod def _get_extension_split(cls, file_name: str) -> Tuple[str, str]: @@ -57,7 +67,7 @@ class FilePathValidatorMixin: def _truncate_file_name(cls, file_name: str) -> str: file_sub_name, file_ext = cls._get_extension_split(file_name) - desired_size = _MAX_FILE_NAME_BYTES - len(file_ext.encode("utf-8")) - 1 + desired_size = cls._MAX_BASE_FILE_NAME_BYTES - len(file_ext.encode("utf-8")) - 1 while len(file_sub_name.encode("utf-8")) > desired_size: file_sub_name = file_sub_name[:-1] @@ -89,7 +99,7 @@ class StringFormatterFileNameValidator(StringFormatterValidator, FilePathValidat @classmethod def _is_file_name_too_long(cls, file_name: str) -> bool: - return len(file_name.encode("utf-8")) > _MAX_FILE_NAME_BYTES + return len(file_name.encode("utf-8")) > cls._MAX_BASE_FILE_NAME_BYTES @classmethod def _get_extension_split(cls, file_name: str) -> Tuple[str, str]: @@ -118,7 +128,7 @@ class StringFormatterFileNameValidator(StringFormatterValidator, FilePathValidat def _truncate_file_name(cls, file_name: str) -> str: file_sub_name, file_ext = cls._get_extension_split(file_name) - while len(file_sub_name.encode("utf-8")) > _MAX_BASE_FILE_NAME_BYTES: + while len(file_sub_name.encode("utf-8")) > cls._MAX_BASE_FILE_NAME_BYTES: file_sub_name = file_sub_name[:-1] return f"{file_sub_name}{file_ext}" diff --git a/src/ytdl_sub/validators/validators.py b/src/ytdl_sub/validators/validators.py index 51842d81..784442e2 100644 --- a/src/ytdl_sub/validators/validators.py +++ b/src/ytdl_sub/validators/validators.py @@ -124,6 +124,11 @@ class FloatValidator(ValueValidator[float]): _expected_value_type_name = "float" +class IntValidator(ValueValidator[int]): + _expected_value_type = int + _expected_value_type_name = "int" + + class ListValidator(Validator, ABC, Generic[ValidatorT]): """ Validates a list of objects to validate diff --git a/tests/unit/validators/test_file_path_validators.py b/tests/unit/validators/test_file_path_validators.py index a27314e6..7cf7b9f2 100644 --- a/tests/unit/validators/test_file_path_validators.py +++ b/tests/unit/validators/test_file_path_validators.py @@ -3,7 +3,10 @@ from pathlib import Path import pytest +from ytdl_sub.config.config_file import ConfigFile +from ytdl_sub.config.defaults import MAX_FILE_NAME_BYTES from ytdl_sub.utils.subtitles import SUBTITLE_EXTENSIONS +from ytdl_sub.validators.file_path_validators import FilePathValidatorMixin from ytdl_sub.validators.file_path_validators import StringFormatterFileNameValidator @@ -67,3 +70,36 @@ class TestStringFormatterFilePathValidator: assert len(dir_paths) == 1 assert Path(truncated_file_path) == dir_paths[0] + + @pytest.mark.parametrize( + "file_name_max_bytes, expected_max", + [ + (50, 50 - FilePathValidatorMixin._EXTENSION_BYTES), + (0, 16), + (10000, MAX_FILE_NAME_BYTES - FilePathValidatorMixin._EXTENSION_BYTES), + ], + ) + def test_config_changes_max_file_name_bytes(self, file_name_max_bytes: int, expected_max: int): + # Ensure the default is set + assert ( + FilePathValidatorMixin._MAX_BASE_FILE_NAME_BYTES + == FilePathValidatorMixin._DEFAULT_MAX_BASE_FILE_NAME_BYTES + ) + + try: + # Initializes the config values, setting the max to 10 bytes + _ = ConfigFile.from_dict( + { + "configuration": { + "working_directory": ".", + "file_name_max_bytes": file_name_max_bytes, + }, + "presets": {}, + } + ) + + assert FilePathValidatorMixin._MAX_BASE_FILE_NAME_BYTES == expected_max + finally: + FilePathValidatorMixin._MAX_BASE_FILE_NAME_BYTES = ( + FilePathValidatorMixin._DEFAULT_MAX_BASE_FILE_NAME_BYTES + )