[BACKEND] get_file_md5_hash: read small blocks instead of the entire file (#1052)
This prevents MemoryError when running on systems with less memory than the file being processed. Fixes #1051. Thanks @parkr
This commit is contained in:
parent
941fa43bb9
commit
759948361f
1 changed files with 5 additions and 1 deletions
|
|
@ -51,8 +51,12 @@ def get_file_md5_hash(full_file_path: Path | str) -> str:
|
|||
-------
|
||||
md5 hash of its contents
|
||||
"""
|
||||
md5hash = hashlib.md5()
|
||||
block_size = 128 * 1000 # md5 uses 128-byte digest blocks
|
||||
with open(full_file_path, "rb") as file:
|
||||
return hashlib.md5(file.read()).hexdigest()
|
||||
while chunk := file.read(block_size):
|
||||
md5hash.update(chunk)
|
||||
return md5hash.hexdigest()
|
||||
|
||||
|
||||
def files_equal(full_file_path_a: Path | str, full_file_path_b: Path | str) -> bool:
|
||||
|
|
|
|||
Loading…
Reference in a new issue