From 53f1d80285a5c018bcd5c147abc1e4ee6ff5f9a4 Mon Sep 17 00:00:00 2001 From: Jesse Bannon Date: Sat, 25 Mar 2023 19:30:53 -0700 Subject: [PATCH] config for max file size, need to move to defaults file --- src/ytdl_sub/config/config_file.py | 5 ++++- src/ytdl_sub/config/config_validator.py | 21 ++++++++++++++++++- src/ytdl_sub/config/defaults.py | 0 .../validators/file_path_validators.py | 19 +++++++---------- src/ytdl_sub/validators/validators.py | 5 +++++ .../validators/test_file_path_validators.py | 20 +++++++++++++++++- 6 files changed, 55 insertions(+), 15 deletions(-) create mode 100644 src/ytdl_sub/config/defaults.py diff --git a/src/ytdl_sub/config/config_file.py b/src/ytdl_sub/config/config_file.py index d7874e28..fed1a2e4 100644 --- a/src/ytdl_sub/config/config_file.py +++ b/src/ytdl_sub/config/config_file.py @@ -6,6 +6,7 @@ from ytdl_sub.config.config_validator import ConfigValidator from ytdl_sub.config.preset import Preset from ytdl_sub.utils.ffmpeg import FFMPEG from ytdl_sub.utils.yaml import load_yaml +from ytdl_sub.validators.file_path_validators import FilePathValidatorMixin class ConfigFile(ConfigValidator): @@ -22,7 +23,7 @@ class ConfigFile(ConfigValidator): def _initialize(self): """ - Configures things (umask, pgid) prior to any downloading + Configures things (umask, pgid, etc) prior to any downloading Returns ------- @@ -36,6 +37,8 @@ class ConfigFile(ConfigValidator): ffprobe_path=self.config_options.ffprobe_path, ) + FilePathValidatorMixin.MAX_FILE_NAME_BYTES = self.config_options.file_name_max_bytes + return self @classmethod diff --git a/src/ytdl_sub/config/config_validator.py b/src/ytdl_sub/config/config_validator.py index 4fb643c9..0af30380 100644 --- a/src/ytdl_sub/config/config_validator.py +++ b/src/ytdl_sub/config/config_validator.py @@ -1,3 +1,4 @@ +import os from typing import Any from typing import Dict from typing import Optional @@ -10,7 +11,7 @@ from ytdl_sub.utils.system import IS_WINDOWS from ytdl_sub.validators.file_path_validators import FFmpegFileValidator from ytdl_sub.validators.file_path_validators import FFprobeFileValidator from ytdl_sub.validators.strict_dict_validator import StrictDictValidator -from ytdl_sub.validators.validators import BoolValidator +from ytdl_sub.validators.validators import BoolValidator, IntValidator from ytdl_sub.validators.validators import LiteralDictValidator from ytdl_sub.validators.validators import StringValidator @@ -18,11 +19,18 @@ if IS_WINDOWS: _DEFAULT_LOCK_DIRECTORY = "" # Not supported in Windows _DEFAULT_FFMPEG_PATH = ".\\ffmpeg.exe" _DEFAULT_FFPROBE_PATH = ".\\ffprobe.exe" + + _MAX_FILE_NAME_BYTES = 255 else: _DEFAULT_LOCK_DIRECTORY = "/tmp" _DEFAULT_FFMPEG_PATH = "/usr/bin/ffmpeg" _DEFAULT_FFPROBE_PATH = "/usr/bin/ffprobe" + _MAX_FILE_NAME_BYTES = os.pathconf("/", "PC_NAME_MAX") + +# Save file-name bytes for the -thumb.jpg portion +DEFAULT_MAX_FILE_NAME_BYTES = _MAX_FILE_NAME_BYTES - len("-thumb.jpg".encode("utf-8")) - 8 + class ExperimentalValidator(StrictDictValidator): _optional_keys = {"enable_update_with_info_json"} @@ -110,6 +118,7 @@ class ConfigOptions(StrictDictValidator): "lock_directory", "ffmpeg_path", "ffprobe_path", + "file_name_max_bytes", "experimental", } @@ -140,6 +149,9 @@ class ConfigOptions(StrictDictValidator): self._experimental = self._validate_key( key="experimental", validator=ExperimentalValidator, default={} ) + self._file_name_max_bytes = self._validate_key( + key="file_name_max_bytes", validator=IntValidator, default=DEFAULT_MAX_FILE_NAME_BYTES + ) @property def working_directory(self) -> str: @@ -193,6 +205,13 @@ class ConfigOptions(StrictDictValidator): """ return self._persist_logs + @property + def file_name_max_bytes(self) -> int: + """ + Optional. Max file name size in bytes. Most OS's typically default to 255 bytes. + """ + return self._file_name_max_bytes.value + @property def experimental(self) -> ExperimentalValidator: """ diff --git a/src/ytdl_sub/config/defaults.py b/src/ytdl_sub/config/defaults.py new file mode 100644 index 00000000..e69de29b diff --git a/src/ytdl_sub/validators/file_path_validators.py b/src/ytdl_sub/validators/file_path_validators.py index 617328a5..8ac3917d 100644 --- a/src/ytdl_sub/validators/file_path_validators.py +++ b/src/ytdl_sub/validators/file_path_validators.py @@ -4,20 +4,13 @@ from typing import Any from typing import Dict from typing import Tuple +from ytdl_sub.config.config_validator import DEFAULT_MAX_FILE_NAME_BYTES from ytdl_sub.utils.file_handler import get_file_extension from ytdl_sub.utils.subtitles import SUBTITLE_EXTENSIONS -from ytdl_sub.utils.system import IS_WINDOWS from ytdl_sub.validators.string_formatter_validators import OverridesStringFormatterValidator from ytdl_sub.validators.string_formatter_validators import StringFormatterValidator from ytdl_sub.validators.validators import StringValidator -if IS_WINDOWS: - _MAX_FILE_NAME_BYTES = 255 -else: - _MAX_FILE_NAME_BYTES = os.pathconf("/", "PC_NAME_MAX") - -# Save file-name bytes for the -thumb.jpg portion -_MAX_BASE_FILE_NAME_BYTES = _MAX_FILE_NAME_BYTES - len("-thumb.jpg".encode("utf-8")) - 8 class FFmpegFileValidator(StringValidator): @@ -44,9 +37,11 @@ class FFprobeFileValidator(FFmpegFileValidator): class FilePathValidatorMixin: + MAX_FILE_NAME_BYTES: int = DEFAULT_MAX_FILE_NAME_BYTES + @classmethod def _is_file_name_too_long(cls, file_name: str) -> bool: - return len(file_name.encode("utf-8")) > _MAX_FILE_NAME_BYTES + return len(file_name.encode("utf-8")) > cls.MAX_FILE_NAME_BYTES @classmethod def _get_extension_split(cls, file_name: str) -> Tuple[str, str]: @@ -57,7 +52,7 @@ class FilePathValidatorMixin: def _truncate_file_name(cls, file_name: str) -> str: file_sub_name, file_ext = cls._get_extension_split(file_name) - desired_size = _MAX_FILE_NAME_BYTES - len(file_ext.encode("utf-8")) - 1 + desired_size = cls.MAX_FILE_NAME_BYTES - len(file_ext.encode("utf-8")) - 1 while len(file_sub_name.encode("utf-8")) > desired_size: file_sub_name = file_sub_name[:-1] @@ -89,7 +84,7 @@ class StringFormatterFileNameValidator(StringFormatterValidator, FilePathValidat @classmethod def _is_file_name_too_long(cls, file_name: str) -> bool: - return len(file_name.encode("utf-8")) > _MAX_FILE_NAME_BYTES + return len(file_name.encode("utf-8")) > cls.MAX_FILE_NAME_BYTES @classmethod def _get_extension_split(cls, file_name: str) -> Tuple[str, str]: @@ -118,7 +113,7 @@ class StringFormatterFileNameValidator(StringFormatterValidator, FilePathValidat def _truncate_file_name(cls, file_name: str) -> str: file_sub_name, file_ext = cls._get_extension_split(file_name) - while len(file_sub_name.encode("utf-8")) > _MAX_BASE_FILE_NAME_BYTES: + while len(file_sub_name.encode("utf-8")) > cls.MAX_FILE_NAME_BYTES: file_sub_name = file_sub_name[:-1] return f"{file_sub_name}{file_ext}" diff --git a/src/ytdl_sub/validators/validators.py b/src/ytdl_sub/validators/validators.py index 51842d81..784442e2 100644 --- a/src/ytdl_sub/validators/validators.py +++ b/src/ytdl_sub/validators/validators.py @@ -124,6 +124,11 @@ class FloatValidator(ValueValidator[float]): _expected_value_type_name = "float" +class IntValidator(ValueValidator[int]): + _expected_value_type = int + _expected_value_type_name = "int" + + class ListValidator(Validator, ABC, Generic[ValidatorT]): """ Validates a list of objects to validate diff --git a/tests/unit/validators/test_file_path_validators.py b/tests/unit/validators/test_file_path_validators.py index a27314e6..929ddb6f 100644 --- a/tests/unit/validators/test_file_path_validators.py +++ b/tests/unit/validators/test_file_path_validators.py @@ -3,8 +3,11 @@ from pathlib import Path import pytest +from ytdl_sub.config.config_file import ConfigFile +from ytdl_sub.config.config_validator import DEFAULT_MAX_FILE_NAME_BYTES from ytdl_sub.utils.subtitles import SUBTITLE_EXTENSIONS -from ytdl_sub.validators.file_path_validators import StringFormatterFileNameValidator +from ytdl_sub.validators.file_path_validators import StringFormatterFileNameValidator, \ + FilePathValidatorMixin class TestStringFormatterFilePathValidator: @@ -67,3 +70,18 @@ class TestStringFormatterFilePathValidator: assert len(dir_paths) == 1 assert Path(truncated_file_path) == dir_paths[0] + + def test_config_changes_max_file_name_bytes(self): + # Ensure the default is set + assert FilePathValidatorMixin.MAX_FILE_NAME_BYTES == DEFAULT_MAX_FILE_NAME_BYTES + + try: + # Initializes the config values, setting the max to 10 bytes + _ = ConfigFile.from_dict({ + "working_directory": ".", + "file_name_max_bytes": 10, + }) + + assert FilePathValidatorMixin.MAX_FILE_NAME_BYTES == 10 + finally: + FilePathValidatorMixin.MAX_FILE_NAME_BYTES = DEFAULT_MAX_FILE_NAME_BYTES