From 2c86d640a444b6ccde9493cd423b5909724d0f52 Mon Sep 17 00:00:00 2001 From: Jesse Bannon Date: Thu, 8 Sep 2022 12:24:46 -0700 Subject: [PATCH] [FEATURE] Have notion of modified files. Do not include files that have not changed (#225) --- src/ytdl_sub/utils/file_handler.py | 107 +++++++++++++++--- tests/e2e/expected_download.py | 12 +- tests/e2e/plugins/test_date_range.py | 28 ++--- .../plugins/date_range/no_downloads.txt | 11 +- .../test_channel_rolling_recent.txt | 10 +- ...ters_with_regex_no_chapters_video_drop.txt | 1 + 6 files changed, 116 insertions(+), 53 deletions(-) diff --git a/src/ytdl_sub/utils/file_handler.py b/src/ytdl_sub/utils/file_handler.py index 7c339f1a..a1fa97d5 100644 --- a/src/ytdl_sub/utils/file_handler.py +++ b/src/ytdl_sub/utils/file_handler.py @@ -1,3 +1,4 @@ +import hashlib import json import os import shutil @@ -10,6 +11,41 @@ from typing import Set from typing import Union +def get_file_md5_hash(full_file_path: Path | str) -> str: + """ + Parameters + ---------- + full_file_path + Path to the file + + Returns + ------- + md5 hash of its contents + """ + with open(full_file_path, "rb") as file: + return hashlib.md5(file.read()).hexdigest() + + +def files_equal(full_file_path_a: Path | str, full_file_path_b: Path | str) -> bool: + """ + Parameters + ---------- + full_file_path_a + full_file_path_b + + Returns + ------- + True if the files are equal in contents. False otherwise. + """ + if not (os.path.isfile(full_file_path_a) and os.path.isfile(full_file_path_b)): + return False + if os.path.getsize(full_file_path_a) != os.path.getsize(full_file_path_b): + return False + if get_file_md5_hash(full_file_path_a) != get_file_md5_hash(full_file_path_b): + return False + return True + + class FileMetadata: """ Stores pretty-printed information about a file. Each line in the metadata represents a newline @@ -120,6 +156,7 @@ class FileHandlerTransactionLog: def __init__(self): self.files_created: Dict[str, FileMetadata] = {} + self.files_modified: Dict[str, FileMetadata] = {} self.files_removed: Set[str] = set() @property @@ -129,7 +166,11 @@ class FileHandlerTransactionLog: ------- True if no transaction logs are recorded. False otherwise """ - return len(self.files_created) == 0 and len(self.files_removed) == 0 + return ( + len(self.files_created) == 0 + and len(self.files_removed) == 0 + and len(self.files_modified) == 0 + ) def log_created_file( self, file_name: str, file_metadata: Optional[FileMetadata] = None @@ -150,6 +191,25 @@ class FileHandlerTransactionLog: self.files_created[file_name] = file_metadata return self + def log_modified_file( + self, file_name: str, file_metadata: Optional[FileMetadata] = None + ) -> "FileHandlerTransactionLog": + """ + Adds a modified file to the transaction log + + Parameters + ---------- + file_name + Name of the file in the output directory + file_metadata + Optional. If the file has metadata, add it to the transaction log + """ + if not file_metadata: + file_metadata = FileMetadata() + + self.files_modified[file_name] = file_metadata + return self + def log_removed_file(self, file_name: str) -> "FileHandlerTransactionLog": """ Records a file removed from the output directory @@ -179,26 +239,37 @@ class FileHandlerTransactionLog: rstrip_line = line.rstrip() return f" {rstrip_line}" if rstrip_line else "" + line_dash = "-" * 40 + if self.files_created: created_line = f"Files created in '{output_directory}'" - created_line_dash = "-" * 40 - lines.extend([created_line, created_line_dash]) + lines.extend([created_line, line_dash]) for file_path, file_metadata in sorted(self.files_created.items()): lines.append(file_path) if file_metadata: lines.extend([_indent_metadata_line(line) for line in file_metadata.metadata]) + if self.files_modified: + modified_line = f"Files modified in '{output_directory}'" + lines.extend([modified_line, line_dash]) + for file_path, file_metadata in sorted(self.files_modified.items()): + lines.append(file_path) + if file_metadata: + lines.extend([_indent_metadata_line(line) for line in file_metadata.metadata]) + if self.files_removed: # Add a blank line to separate created/removed files if self.files_created: lines.append("") removed_line = f"Files removed from '{output_directory}'" - removed_line_dash = "-" * 40 - lines.extend([removed_line, removed_line_dash]) + lines.extend([removed_line, line_dash]) for file_path in sorted(self.files_removed): lines.append(file_path) + if self.is_empty: + lines.append(f"No new, modified, or removed files in '{output_directory}'") + return "\n".join(lines) @@ -274,17 +345,27 @@ class FileHandler: file_metadata Optional. Metadata to record to the transaction log for this file """ - self._file_handler_transaction_log.log_created_file( - file_name=output_file_name, file_metadata=file_metadata - ) + source_file_path = Path(self.working_directory) / file_name + output_file_path = Path(self.output_directory) / output_file_name + + # output file exists, and it's not marked as created already, see if we modify it + if ( + os.path.isfile(output_file_path) + and output_file_name not in self.file_handler_transaction_log.files_created + ): + if not files_equal(source_file_path, output_file_path): + self.file_handler_transaction_log.log_modified_file( + file_name=output_file_name, file_metadata=file_metadata + ) + # output file does not already exist, creates a new file + else: + self.file_handler_transaction_log.log_created_file( + file_name=output_file_name, file_metadata=file_metadata + ) if not self.dry_run: - output_file_path = Path(self.output_directory) / output_file_name os.makedirs(os.path.dirname(output_file_path), exist_ok=True) - self.move( - src_file_path=Path(self.working_directory) / file_name, - dst_file_path=output_file_path, - ) + self.move(src_file_path=source_file_path, dst_file_path=output_file_path) def delete_file_from_output_directory(self, file_name: str): """ diff --git a/tests/e2e/expected_download.py b/tests/e2e/expected_download.py index 1c996dee..77ecb76c 100644 --- a/tests/e2e/expected_download.py +++ b/tests/e2e/expected_download.py @@ -1,4 +1,3 @@ -import hashlib import json import os.path from dataclasses import dataclass @@ -6,6 +5,8 @@ from pathlib import Path from typing import List from typing import Optional +from ytdl_sub.utils.file_handler import get_file_md5_hash + _EXPECTED_DOWNLOADS_SUMMARY_PATH = Path("tests/e2e/resources/expected_downloads_summaries") @@ -19,11 +20,6 @@ def _get_files_in_directory(relative_directory: Path | str) -> List[Path]: return relative_file_paths -def _get_file_md5_hash(full_file_path: Path | str) -> str: - with open(full_file_path, "rb") as file: - return hashlib.md5(file.read()).hexdigest() - - @dataclass class ExpectedDownloadFile: path: Path @@ -74,7 +70,7 @@ class ExpectedDownloads: if path in ignore_md5_hashes_for or path.endswith(".info.json"): continue - md5_hash = _get_file_md5_hash(full_file_path=full_path) + md5_hash = get_file_md5_hash(full_file_path=full_path) assert md5_hash in expected_download.md5, ( f"MD5 hash for {str(expected_download.path)} does not match: " f"{md5_hash} != {expected_download.md5}" @@ -98,7 +94,7 @@ class ExpectedDownloads: def from_directory(cls, directory_path: str | Path) -> "ExpectedDownloads": relative_file_paths = _get_files_in_directory(relative_directory=directory_path) expected_downloads_dict = { - str(file_path): _get_file_md5_hash(full_file_path=Path(directory_path) / file_path) + str(file_path): get_file_md5_hash(full_file_path=Path(directory_path) / file_path) for file_path in relative_file_paths } return cls.from_dict(expected_downloads_dict) diff --git a/tests/e2e/plugins/test_date_range.py b/tests/e2e/plugins/test_date_range.py index d4080c4c..3b06c82b 100644 --- a/tests/e2e/plugins/test_date_range.py +++ b/tests/e2e/plugins/test_date_range.py @@ -102,19 +102,21 @@ class TestDateRange: preset_dict=recent_preset_dict, ) - # Run twice, ensure nothing changes between runsyoutube - for _ in range(2): - transaction_log = recent_channel_no_vids_in_range_subscription.download(dry_run=dry_run) - assert_transaction_log_matches( - output_directory=output_directory, - transaction_log=transaction_log, - transaction_log_summary_file_name="plugins/date_range/no_downloads.txt", - ) - assert_expected_downloads( - output_directory=output_directory, - dry_run=dry_run, - expected_download_summary_file_name="plugins/date_range/no_downloads.json", - ) + # run once to initialize all output directory files + _ = recent_channel_no_vids_in_range_subscription.download(dry_run=False) + + # Run again, ensure no downloads + transaction_log = recent_channel_no_vids_in_range_subscription.download(dry_run=dry_run) + assert_transaction_log_matches( + output_directory=output_directory, + transaction_log=transaction_log, + transaction_log_summary_file_name="plugins/date_range/no_downloads.txt", + ) + assert_expected_downloads( + output_directory=output_directory, + dry_run=False, + expected_download_summary_file_name="plugins/date_range/no_downloads.json", + ) @pytest.mark.parametrize("dry_run", [True, False]) def test_rolling_recent_channel_download( diff --git a/tests/e2e/resources/transaction_log_summaries/plugins/date_range/no_downloads.txt b/tests/e2e/resources/transaction_log_summaries/plugins/date_range/no_downloads.txt index 16888915..7192b6c7 100644 --- a/tests/e2e/resources/transaction_log_summaries/plugins/date_range/no_downloads.txt +++ b/tests/e2e/resources/transaction_log_summaries/plugins/date_range/no_downloads.txt @@ -1,10 +1 @@ -Files created in '{output_directory}' ----------------------------------------- -.ytdl-sub-recent-download-archive.json -fanart.jpg -poster.jpg -tvshow.nfo - NFO tags: - tvshow: - plot: Plugin and map updates for the server Project Zombie. - title: Project / Zombie \ No newline at end of file +No new, modified, or removed files in '{output_directory}' \ No newline at end of file diff --git a/tests/e2e/resources/transaction_log_summaries/plugins/date_range/test_channel_rolling_recent.txt b/tests/e2e/resources/transaction_log_summaries/plugins/date_range/test_channel_rolling_recent.txt index d9b74e8d..ee834a45 100644 --- a/tests/e2e/resources/transaction_log_summaries/plugins/date_range/test_channel_rolling_recent.txt +++ b/tests/e2e/resources/transaction_log_summaries/plugins/date_range/test_channel_rolling_recent.txt @@ -1,14 +1,6 @@ -Files created in '{output_directory}' +Files modified in '{output_directory}' ---------------------------------------- .ytdl-sub-recent-download-archive.json -fanart.jpg -poster.jpg -tvshow.nfo - NFO tags: - tvshow: - plot: Plugin and map updates for the server Project Zombie. - title: Project / Zombie - Files removed from '{output_directory}' ---------------------------------------- Season 2018/s2018.e1029 - Jesse's Minecraft Server ｜ Teaser Trailer-thumb.jpg diff --git a/tests/e2e/resources/transaction_log_summaries/plugins/split_by_chapters_with_regex_no_chapters_video_drop.txt b/tests/e2e/resources/transaction_log_summaries/plugins/split_by_chapters_with_regex_no_chapters_video_drop.txt index e69de29b..7192b6c7 100644 --- a/tests/e2e/resources/transaction_log_summaries/plugins/split_by_chapters_with_regex_no_chapters_video_drop.txt +++ b/tests/e2e/resources/transaction_log_summaries/plugins/split_by_chapters_with_regex_no_chapters_video_drop.txt @@ -0,0 +1 @@ +No new, modified, or removed files in '{output_directory}' \ No newline at end of file