From 759948361f10ec69433d03c2aceea84ec17e1345 Mon Sep 17 00:00:00 2001 From: Parker Moore <237985+parkr@users.noreply.github.com> Date: Tue, 17 Sep 2024 20:24:33 -0700 Subject: [PATCH] [BACKEND] get_file_md5_hash: read small blocks instead of the entire file (#1052) This prevents MemoryError when running on systems with less memory than the file being processed. Fixes #1051. Thanks @parkr --- src/ytdl_sub/utils/file_handler.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/ytdl_sub/utils/file_handler.py b/src/ytdl_sub/utils/file_handler.py index faa7d2dc..f7c6e44f 100644 --- a/src/ytdl_sub/utils/file_handler.py +++ b/src/ytdl_sub/utils/file_handler.py @@ -51,8 +51,12 @@ def get_file_md5_hash(full_file_path: Path | str) -> str: ------- md5 hash of its contents """ + md5hash = hashlib.md5() + block_size = 128 * 1000 # md5 uses 128-byte digest blocks with open(full_file_path, "rb") as file: - return hashlib.md5(file.read()).hexdigest() + while chunk := file.read(block_size): + md5hash.update(chunk) + return md5hash.hexdigest() def files_equal(full_file_path_a: Path | str, full_file_path_b: Path | str) -> bool: