[FEATURE] Have notion of modified files. Do not include files that have not changed (#225)

src/ytdl_sub/utils/file_handler.py

@@ -1,3 +1,4 @@
+import hashlib
 import json
 import os
 import shutil
@@ -10,6 +11,41 @@ from typing import Set
 from typing import Union
 
 
+def get_file_md5_hash(full_file_path: Path | str) -> str:
+    """
+    Parameters
+    ----------
+    full_file_path
+        Path to the file
+
+    Returns
+    -------
+    md5 hash of its contents
+    """
+    with open(full_file_path, "rb") as file:
+        return hashlib.md5(file.read()).hexdigest()
+
+
+def files_equal(full_file_path_a: Path | str, full_file_path_b: Path | str) -> bool:
+    """
+    Parameters
+    ----------
+    full_file_path_a
+    full_file_path_b
+
+    Returns
+    -------
+    True if the files are equal in contents. False otherwise.
+    """
+    if not (os.path.isfile(full_file_path_a) and os.path.isfile(full_file_path_b)):
+        return False
+    if os.path.getsize(full_file_path_a) != os.path.getsize(full_file_path_b):
+        return False
+    if get_file_md5_hash(full_file_path_a) != get_file_md5_hash(full_file_path_b):
+        return False
+    return True
+
+
 class FileMetadata:
     """
     Stores pretty-printed information about a file. Each line in the metadata represents a newline
@@ -120,6 +156,7 @@ class FileHandlerTransactionLog:
 
     def __init__(self):
         self.files_created: Dict[str, FileMetadata] = {}
+        self.files_modified: Dict[str, FileMetadata] = {}
         self.files_removed: Set[str] = set()
 
     @property
@@ -129,7 +166,11 @@ class FileHandlerTransactionLog:
         -------
         True if no transaction logs are recorded. False otherwise
         """
-        return len(self.files_created) == 0 and len(self.files_removed) == 0
+        return (
+            len(self.files_created) == 0
+            and len(self.files_removed) == 0
+            and len(self.files_modified) == 0
+        )
 
     def log_created_file(
         self, file_name: str, file_metadata: Optional[FileMetadata] = None
@@ -150,6 +191,25 @@ class FileHandlerTransactionLog:
         self.files_created[file_name] = file_metadata
         return self
 
+    def log_modified_file(
+        self, file_name: str, file_metadata: Optional[FileMetadata] = None
+    ) -> "FileHandlerTransactionLog":
+        """
+        Adds a modified file to the transaction log
+
+        Parameters
+        ----------
+        file_name
+            Name of the file in the output directory
+        file_metadata
+            Optional. If the file has metadata, add it to the transaction log
+        """
+        if not file_metadata:
+            file_metadata = FileMetadata()
+
+        self.files_modified[file_name] = file_metadata
+        return self
+
     def log_removed_file(self, file_name: str) -> "FileHandlerTransactionLog":
         """
         Records a file removed from the output directory
@@ -179,26 +239,37 @@ class FileHandlerTransactionLog:
             rstrip_line = line.rstrip()
             return f"  {rstrip_line}" if rstrip_line else ""
 
+        line_dash = "-" * 40
+
         if self.files_created:
             created_line = f"Files created in '{output_directory}'"
-            created_line_dash = "-" * 40
+            lines.extend([created_line, line_dash])
-            lines.extend([created_line, created_line_dash])
             for file_path, file_metadata in sorted(self.files_created.items()):
                 lines.append(file_path)
                 if file_metadata:
                     lines.extend([_indent_metadata_line(line) for line in file_metadata.metadata])
 
+        if self.files_modified:
+            modified_line = f"Files modified in '{output_directory}'"
+            lines.extend([modified_line, line_dash])
+            for file_path, file_metadata in sorted(self.files_modified.items()):
+                lines.append(file_path)
+                if file_metadata:
+                    lines.extend([_indent_metadata_line(line) for line in file_metadata.metadata])
+
         if self.files_removed:
             # Add a blank line to separate created/removed files
             if self.files_created:
                 lines.append("")
 
             removed_line = f"Files removed from '{output_directory}'"
-            removed_line_dash = "-" * 40
+            lines.extend([removed_line, line_dash])
-            lines.extend([removed_line, removed_line_dash])
             for file_path in sorted(self.files_removed):
                 lines.append(file_path)
 
+        if self.is_empty:
+            lines.append(f"No new, modified, or removed files in '{output_directory}'")
+
         return "\n".join(lines)
 
 
@@ -274,17 +345,27 @@ class FileHandler:
         file_metadata
             Optional. Metadata to record to the transaction log for this file
         """
-        self._file_handler_transaction_log.log_created_file(
+        source_file_path = Path(self.working_directory) / file_name
-            file_name=output_file_name, file_metadata=file_metadata
+        output_file_path = Path(self.output_directory) / output_file_name
-        )
+
+        # output file exists, and it's not marked as created already, see if we modify it
+        if (
+            os.path.isfile(output_file_path)
+            and output_file_name not in self.file_handler_transaction_log.files_created
+        ):
+            if not files_equal(source_file_path, output_file_path):
+                self.file_handler_transaction_log.log_modified_file(
+                    file_name=output_file_name, file_metadata=file_metadata
+                )
+        # output file does not already exist, creates a new file
+        else:
+            self.file_handler_transaction_log.log_created_file(
+                file_name=output_file_name, file_metadata=file_metadata
+            )
 
         if not self.dry_run:
-            output_file_path = Path(self.output_directory) / output_file_name
             os.makedirs(os.path.dirname(output_file_path), exist_ok=True)
-            self.move(
+            self.move(src_file_path=source_file_path, dst_file_path=output_file_path)
-                src_file_path=Path(self.working_directory) / file_name,
-                dst_file_path=output_file_path,
-            )
 
     def delete_file_from_output_directory(self, file_name: str):
         """

tests/e2e/expected_download.py

@@ -1,4 +1,3 @@
-import hashlib
 import json
 import os.path
 from dataclasses import dataclass
@@ -6,6 +5,8 @@ from pathlib import Path
 from typing import List
 from typing import Optional
 
+from ytdl_sub.utils.file_handler import get_file_md5_hash
+
 _EXPECTED_DOWNLOADS_SUMMARY_PATH = Path("tests/e2e/resources/expected_downloads_summaries")
 
 
@@ -19,11 +20,6 @@ def _get_files_in_directory(relative_directory: Path | str) -> List[Path]:
     return relative_file_paths
 
 
-def _get_file_md5_hash(full_file_path: Path | str) -> str:
-    with open(full_file_path, "rb") as file:
-        return hashlib.md5(file.read()).hexdigest()
-
-
 @dataclass
 class ExpectedDownloadFile:
     path: Path
@@ -74,7 +70,7 @@ class ExpectedDownloads:
             if path in ignore_md5_hashes_for or path.endswith(".info.json"):
                 continue
 
-            md5_hash = _get_file_md5_hash(full_file_path=full_path)
+            md5_hash = get_file_md5_hash(full_file_path=full_path)
             assert md5_hash in expected_download.md5, (
                 f"MD5  hash for {str(expected_download.path)} does not match: "
                 f"{md5_hash} != {expected_download.md5}"
@@ -98,7 +94,7 @@ class ExpectedDownloads:
     def from_directory(cls, directory_path: str | Path) -> "ExpectedDownloads":
         relative_file_paths = _get_files_in_directory(relative_directory=directory_path)
         expected_downloads_dict = {
-            str(file_path): _get_file_md5_hash(full_file_path=Path(directory_path) / file_path)
+            str(file_path): get_file_md5_hash(full_file_path=Path(directory_path) / file_path)
             for file_path in relative_file_paths
         }
         return cls.from_dict(expected_downloads_dict)

tests/e2e/plugins/test_date_range.py

@@ -102,19 +102,21 @@ class TestDateRange:
             preset_dict=recent_preset_dict,
         )
 
-        # Run twice, ensure nothing changes between runsyoutube
+        # run once to initialize all output directory files
-        for _ in range(2):
+        _ = recent_channel_no_vids_in_range_subscription.download(dry_run=False)
-            transaction_log = recent_channel_no_vids_in_range_subscription.download(dry_run=dry_run)
+
-            assert_transaction_log_matches(
+        # Run again, ensure no downloads
-                output_directory=output_directory,
+        transaction_log = recent_channel_no_vids_in_range_subscription.download(dry_run=dry_run)
-                transaction_log=transaction_log,
+        assert_transaction_log_matches(
-                transaction_log_summary_file_name="plugins/date_range/no_downloads.txt",
+            output_directory=output_directory,
-            )
+            transaction_log=transaction_log,
-            assert_expected_downloads(
+            transaction_log_summary_file_name="plugins/date_range/no_downloads.txt",
-                output_directory=output_directory,
+        )
-                dry_run=dry_run,
+        assert_expected_downloads(
-                expected_download_summary_file_name="plugins/date_range/no_downloads.json",
+            output_directory=output_directory,
-            )
+            dry_run=False,
+            expected_download_summary_file_name="plugins/date_range/no_downloads.json",
+        )
 
     @pytest.mark.parametrize("dry_run", [True, False])
     def test_rolling_recent_channel_download(

tests/e2e/resources/transaction_log_summaries/plugins/date_range/no_downloads.txt

@@ -1,10 +1 @@
-Files created in '{output_directory}'
+No new, modified, or removed files in '{output_directory}'
-----------------------------------------
-.ytdl-sub-recent-download-archive.json
-fanart.jpg
-poster.jpg
-tvshow.nfo
-  NFO tags:
-    tvshow:
-      plot: Plugin and map updates for the server Project Zombie.
-      title: Project / Zombie

tests/e2e/resources/transaction_log_summaries/plugins/date_range/test_channel_rolling_recent.txt

@@ -1,14 +1,6 @@
-Files created in '{output_directory}'
+Files modified in '{output_directory}'
 ----------------------------------------
 .ytdl-sub-recent-download-archive.json
-fanart.jpg
-poster.jpg
-tvshow.nfo
-  NFO tags:
-    tvshow:
-      plot: Plugin and map updates for the server Project Zombie.
-      title: Project / Zombie
-
 Files removed from '{output_directory}'
 ----------------------------------------
 Season 2018/s2018.e1029 - Jesse's Minecraft Server ｜ Teaser Trailer-thumb.jpg

tests/e2e/resources/transaction_log_summaries/plugins/split_by_chapters_with_regex_no_chapters_video_drop.txt

@@ -0,0 +1 @@
+No new, modified, or removed files in '{output_directory}'