From 9804c568fe0b1f1007fe53c415ef38e46402046b Mon Sep 17 00:00:00 2001 From: Jesse Bannon Date: Thu, 8 Sep 2022 09:54:00 -0700 Subject: [PATCH] [FEATURE] Have notion of modified files. Do not include files that have not changed --- src/ytdl_sub/utils/file_handler.py | 101 +++++++++++++++++++++++++---- tests/e2e/expected_download.py | 12 ++-- 2 files changed, 92 insertions(+), 21 deletions(-) diff --git a/src/ytdl_sub/utils/file_handler.py b/src/ytdl_sub/utils/file_handler.py index 7c339f1a..ad48d73f 100644 --- a/src/ytdl_sub/utils/file_handler.py +++ b/src/ytdl_sub/utils/file_handler.py @@ -1,3 +1,4 @@ +import hashlib import json import os import shutil @@ -10,6 +11,41 @@ from typing import Set from typing import Union +def get_file_md5_hash(full_file_path: Path | str) -> str: + """ + Parameters + ---------- + full_file_path + Path to the file + + Returns + ------- + md5 hash of its contents + """ + with open(full_file_path, "rb") as file: + return hashlib.md5(file.read()).hexdigest() + + +def files_equal(full_file_path_a: Path | str, full_file_path_b: Path | str) -> bool: + """ + Parameters + ---------- + full_file_path_a + full_file_path_b + + Returns + ------- + True if the files are equal in contents. False otherwise. + """ + if not (os.path.isfile(full_file_path_a) and os.path.isfile(full_file_path_b)): + return False + if os.path.getsize(full_file_path_a) != os.path.getsize(full_file_path_b): + return False + if get_file_md5_hash(full_file_path_a) != get_file_md5_hash(full_file_path_b): + return False + return True + + class FileMetadata: """ Stores pretty-printed information about a file. Each line in the metadata represents a newline @@ -120,6 +156,7 @@ class FileHandlerTransactionLog: def __init__(self): self.files_created: Dict[str, FileMetadata] = {} + self.files_modified: Dict[str, FileMetadata] = {} self.files_removed: Set[str] = set() @property @@ -129,7 +166,11 @@ class FileHandlerTransactionLog: ------- True if no transaction logs are recorded. False otherwise """ - return len(self.files_created) == 0 and len(self.files_removed) == 0 + return ( + len(self.files_created) == 0 + and len(self.files_removed) == 0 + and len(self.files_modified) == 0 + ) def log_created_file( self, file_name: str, file_metadata: Optional[FileMetadata] = None @@ -150,6 +191,25 @@ class FileHandlerTransactionLog: self.files_created[file_name] = file_metadata return self + def log_modified_file( + self, file_name: str, file_metadata: Optional[FileMetadata] = None + ) -> "FileHandlerTransactionLog": + """ + Adds a modified file to the transaction log + + Parameters + ---------- + file_name + Name of the file in the output directory + file_metadata + Optional. If the file has metadata, add it to the transaction log + """ + if not file_metadata: + file_metadata = FileMetadata() + + self.files_modified[file_name] = file_metadata + return self + def log_removed_file(self, file_name: str) -> "FileHandlerTransactionLog": """ Records a file removed from the output directory @@ -179,23 +239,31 @@ class FileHandlerTransactionLog: rstrip_line = line.rstrip() return f" {rstrip_line}" if rstrip_line else "" + line_dash = "-" * 40 + if self.files_created: created_line = f"Files created in '{output_directory}'" - created_line_dash = "-" * 40 - lines.extend([created_line, created_line_dash]) + lines.extend([created_line, line_dash]) for file_path, file_metadata in sorted(self.files_created.items()): lines.append(file_path) if file_metadata: lines.extend([_indent_metadata_line(line) for line in file_metadata.metadata]) + if self.files_modified: + modified_line = f"Files modified in '{output_directory}'" + lines.extend([modified_line, line_dash]) + for file_path, file_metadata in sorted(self.files_modified.items()): + lines.append(file_path) + if file_metadata: + lines.extend([_indent_metadata_line(line) for line in file_metadata.metadata]) + if self.files_removed: # Add a blank line to separate created/removed files if self.files_created: lines.append("") removed_line = f"Files removed from '{output_directory}'" - removed_line_dash = "-" * 40 - lines.extend([removed_line, removed_line_dash]) + lines.extend([removed_line, line_dash]) for file_path in sorted(self.files_removed): lines.append(file_path) @@ -274,17 +342,24 @@ class FileHandler: file_metadata Optional. Metadata to record to the transaction log for this file """ - self._file_handler_transaction_log.log_created_file( - file_name=output_file_name, file_metadata=file_metadata - ) + source_file_path = Path(self.working_directory) / file_name + output_file_path = Path(self.output_directory) / output_file_name + + # output file exists, see if we modify it + if os.path.isfile(output_file_path): + if not files_equal(source_file_path, output_file_path): + self._file_handler_transaction_log.log_modified_file( + file_name=output_file_name, file_metadata=file_metadata + ) + # output file does not already exist, creates a new file + else: + self._file_handler_transaction_log.log_created_file( + file_name=output_file_name, file_metadata=file_metadata + ) if not self.dry_run: - output_file_path = Path(self.output_directory) / output_file_name os.makedirs(os.path.dirname(output_file_path), exist_ok=True) - self.move( - src_file_path=Path(self.working_directory) / file_name, - dst_file_path=output_file_path, - ) + self.move(src_file_path=source_file_path, dst_file_path=output_file_path) def delete_file_from_output_directory(self, file_name: str): """ diff --git a/tests/e2e/expected_download.py b/tests/e2e/expected_download.py index 1c996dee..77ecb76c 100644 --- a/tests/e2e/expected_download.py +++ b/tests/e2e/expected_download.py @@ -1,4 +1,3 @@ -import hashlib import json import os.path from dataclasses import dataclass @@ -6,6 +5,8 @@ from pathlib import Path from typing import List from typing import Optional +from ytdl_sub.utils.file_handler import get_file_md5_hash + _EXPECTED_DOWNLOADS_SUMMARY_PATH = Path("tests/e2e/resources/expected_downloads_summaries") @@ -19,11 +20,6 @@ def _get_files_in_directory(relative_directory: Path | str) -> List[Path]: return relative_file_paths -def _get_file_md5_hash(full_file_path: Path | str) -> str: - with open(full_file_path, "rb") as file: - return hashlib.md5(file.read()).hexdigest() - - @dataclass class ExpectedDownloadFile: path: Path @@ -74,7 +70,7 @@ class ExpectedDownloads: if path in ignore_md5_hashes_for or path.endswith(".info.json"): continue - md5_hash = _get_file_md5_hash(full_file_path=full_path) + md5_hash = get_file_md5_hash(full_file_path=full_path) assert md5_hash in expected_download.md5, ( f"MD5 hash for {str(expected_download.path)} does not match: " f"{md5_hash} != {expected_download.md5}" @@ -98,7 +94,7 @@ class ExpectedDownloads: def from_directory(cls, directory_path: str | Path) -> "ExpectedDownloads": relative_file_paths = _get_files_in_directory(relative_directory=directory_path) expected_downloads_dict = { - str(file_path): _get_file_md5_hash(full_file_path=Path(directory_path) / file_path) + str(file_path): get_file_md5_hash(full_file_path=Path(directory_path) / file_path) for file_path in relative_file_paths } return cls.from_dict(expected_downloads_dict)