From f9318b9d07cdce3a121247057441249b357eb813 Mon Sep 17 00:00:00 2001 From: Jesse Bannon Date: Fri, 10 Mar 2023 07:20:43 -0800 Subject: [PATCH] [BUGFIX] Handle truncating file names if too long (#524) * [BUGFIX] Handle truncating file names if too long * always import os * test fix * fix for windows * lint --- src/ytdl_sub/plugins/subtitles.py | 5 +- src/ytdl_sub/utils/subtitles.py | 3 ++ .../validators/file_path_validators.py | 46 ++++++++++++++++++- .../validators/test_file_path_validators.py | 42 +++++++++++++++++ 4 files changed, 91 insertions(+), 5 deletions(-) create mode 100644 src/ytdl_sub/utils/subtitles.py create mode 100644 tests/unit/validators/test_file_path_validators.py diff --git a/src/ytdl_sub/plugins/subtitles.py b/src/ytdl_sub/plugins/subtitles.py index c63d87b0..2b96ab01 100644 --- a/src/ytdl_sub/plugins/subtitles.py +++ b/src/ytdl_sub/plugins/subtitles.py @@ -2,7 +2,6 @@ from pathlib import Path from typing import Dict from typing import List from typing import Optional -from typing import Set from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder from ytdl_sub.entries.entry import Entry @@ -11,15 +10,13 @@ from ytdl_sub.plugins.plugin import PluginOptions from ytdl_sub.utils.file_handler import FileHandler from ytdl_sub.utils.file_handler import FileMetadata from ytdl_sub.utils.logger import Logger +from ytdl_sub.utils.subtitles import SUBTITLE_EXTENSIONS from ytdl_sub.validators.file_path_validators import StringFormatterFilePathValidator from ytdl_sub.validators.string_formatter_validators import StringFormatterValidator from ytdl_sub.validators.string_select_validator import StringSelectValidator from ytdl_sub.validators.validators import BoolValidator from ytdl_sub.validators.validators import StringListValidator -SUBTITLE_EXTENSIONS: Set[str] = {"srt", "vtt", "ass", "lrc"} - - logger = Logger.get(name="subtitles") diff --git a/src/ytdl_sub/utils/subtitles.py b/src/ytdl_sub/utils/subtitles.py new file mode 100644 index 00000000..9330ccbe --- /dev/null +++ b/src/ytdl_sub/utils/subtitles.py @@ -0,0 +1,3 @@ +from typing import Set + +SUBTITLE_EXTENSIONS: Set[str] = {"srt", "vtt", "ass", "lrc"} diff --git a/src/ytdl_sub/validators/file_path_validators.py b/src/ytdl_sub/validators/file_path_validators.py index 0a9cc41f..592b52b0 100644 --- a/src/ytdl_sub/validators/file_path_validators.py +++ b/src/ytdl_sub/validators/file_path_validators.py @@ -2,11 +2,19 @@ import os from pathlib import Path from typing import Any from typing import Dict +from typing import Tuple +from ytdl_sub.utils.subtitles import SUBTITLE_EXTENSIONS +from ytdl_sub.utils.system import IS_WINDOWS from ytdl_sub.validators.string_formatter_validators import OverridesStringFormatterValidator from ytdl_sub.validators.string_formatter_validators import StringFormatterValidator from ytdl_sub.validators.validators import StringValidator +if IS_WINDOWS: + _MAX_FILE_NAME_BYTES = 255 +else: + _MAX_FILE_NAME_BYTES = os.pathconf("/", "PC_NAME_MAX") + class FFmpegFileValidator(StringValidator): _expected_value_type_name = "ffmpeg dependency" @@ -34,9 +42,45 @@ class FFprobeFileValidator(FFmpegFileValidator): class StringFormatterFilePathValidator(StringFormatterValidator): _expected_value_type_name = "filepath" + @classmethod + def _is_file_name_too_long(cls, file_name: str) -> bool: + return len(file_name.encode("utf-8")) > _MAX_FILE_NAME_BYTES + + @classmethod + def _get_extension_split(cls, file_name: str) -> Tuple[str, str]: + if file_name.endswith(".info.json"): + ext = "info.json" + elif any(file_name.endswith(f".{subtitle_ext}") for subtitle_ext in SUBTITLE_EXTENSIONS): + file_name_split = file_name.split(".") + ext = file_name_split[-1] + + # Try to capture .lang.ext + if len(file_name_split) > 2 and len(file_name_split[-2]) < 6: + ext = f"{file_name_split[-2]}.{file_name_split[-1]}" + else: + ext = file_name.rsplit(".", maxsplit=1)[-1] + + return file_name[: -len(ext)], ext + + @classmethod + def _truncate_file_name(cls, file_name: str) -> str: + file_sub_name, file_ext = cls._get_extension_split(file_name) + + desired_size = _MAX_FILE_NAME_BYTES - len(file_ext.encode("utf-8")) - 1 + while len(file_sub_name.encode("utf-8")) > desired_size: + file_sub_name = file_sub_name[:-1] + + return f"{file_sub_name}.{file_ext}" + def apply_formatter(self, variable_dict: Dict[str, str]) -> str: """Turn into a Path, then a string, to get correct directory separators""" - return str(Path(super().apply_formatter(variable_dict))) + file_path = Path(super().apply_formatter(variable_dict)) + file_directory, file_name = os.path.split(Path(file_path)) + + if self._is_file_name_too_long(file_name): + return str(Path(file_directory) / self._truncate_file_name(file_name)) + + return str(file_path) class OverridesStringFormatterValidatorFilePathValidator(OverridesStringFormatterValidator): diff --git a/tests/unit/validators/test_file_path_validators.py b/tests/unit/validators/test_file_path_validators.py new file mode 100644 index 00000000..4a667c79 --- /dev/null +++ b/tests/unit/validators/test_file_path_validators.py @@ -0,0 +1,42 @@ +import tempfile +from pathlib import Path + +import pytest + +from ytdl_sub.utils.subtitles import SUBTITLE_EXTENSIONS +from ytdl_sub.validators.file_path_validators import StringFormatterFilePathValidator + + +class TestStringFormatterFilePathValidator: + @pytest.mark.parametrize( + "ext", + [ + "mp4", + "info.json", + ] + + [f"en-US.{ext}" for ext in SUBTITLE_EXTENSIONS], + ) + @pytest.mark.parametrize("file_name_char", ["a", "𒃀"]) + @pytest.mark.parametrize("file_name_len", [10, 10000]) + def test_truncates_file_name_successfully( + self, ext: str, file_name_char: str, file_name_len: int + ): + ext = f".{ext}" # pytest args with . in the beginning act weird + with tempfile.TemporaryDirectory() as temp_dir: + file_name = (file_name_char * file_name_len) + ext + file_path = str(Path(temp_dir) / file_name) + + formatter = StringFormatterFilePathValidator(name="test", value=str(file_path)) + truncated_file_path = formatter.apply_formatter({}) + + assert truncated_file_path.count(".") == ext.count(".") + assert str(Path(temp_dir)) in truncated_file_path + assert ext in truncated_file_path + + # Ensure it can actually open the file + with open(truncated_file_path, "w", encoding="utf-8"): + # Make sure the file is actually in the directory + dir_paths = list(Path(temp_dir).rglob("*")) + + assert len(dir_paths) == 1 + assert Path(truncated_file_path) == dir_paths[0]