danvics/ytdl-sub
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

get_file_md5_hash: read small blocks instead of the entire file

Browse source 
This prevents MemoryError when running on systems with less memory than the file being processed.
This commit is contained in:
Parker Moore 2024-09-16 09:32:04 -07:00
parent 941fa43bb9
commit badafec313
No known key found for this signature in database
GPG key ID: 3FB953E3195E9F3D
1 changed files with 5 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
src/ytdl_sub/utils/file_handler.py
View file

@ -51,8 +51,12 @@ def get_file_md5_hash(full_file_path: Path | str) -> str:
------- -------
md5 hash of its contents md5 hash of its contents
""" """
md5hash = hashlib.md5()
block_size = 128 * 1000 # md5 uses 128-byte digest blocks
with open(full_file_path, "rb") as file: with open(full_file_path, "rb") as file:
return hashlib.md5(file.read()).hexdigest() while chunk := file.read(block_size):
md5hash.update(chunk)
return md5hash.hexdigest()
def files_equal(full_file_path_a: Path | str, full_file_path_b: Path | str) -> bool: def files_equal(full_file_path_a: Path | str, full_file_path_b: Path | str) -> bool: