danvics/ytdl-sub
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

[FEATURE] Have notion of modified files. Do not include files that have not changed (#225)

Browse source
This commit is contained in:
Jesse Bannon 2022-09-08 12:24:46 -07:00 committed by GitHub
parent 4b6c79caef
commit 2c86d640a4
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 116 additions and 53 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

107
src/ytdl_sub/utils/file_handler.py
View file

@ -1,3 +1,4 @@
import hashlib
import json
import os
import shutil
@ -10,6 +11,41 @@ from typing import Set
from typing import Union
def get_file_md5_hash(full_file_path: Path | str) -> str:
"""
Parameters
----------
full_file_path
Path to the file
Returns
-------
md5 hash of its contents
"""
with open(full_file_path, "rb") as file:
return hashlib.md5(file.read()).hexdigest()
def files_equal(full_file_path_a: Path | str, full_file_path_b: Path | str) -> bool:
"""
Parameters
----------
full_file_path_a
full_file_path_b
Returns
-------
True if the files are equal in contents. False otherwise.
"""
if not (os.path.isfile(full_file_path_a) and os.path.isfile(full_file_path_b)):
return False
if os.path.getsize(full_file_path_a) != os.path.getsize(full_file_path_b):
return False
if get_file_md5_hash(full_file_path_a) != get_file_md5_hash(full_file_path_b):
return False
return True
class FileMetadata:
"""
Stores pretty-printed information about a file. Each line in the metadata represents a newline
@ -120,6 +156,7 @@ class FileHandlerTransactionLog:
def __init__(self):
self.files_created: Dict[str, FileMetadata] = {}
self.files_modified: Dict[str, FileMetadata] = {}
self.files_removed: Set[str] = set()
@property
@ -129,7 +166,11 @@ class FileHandlerTransactionLog:
-------
True if no transaction logs are recorded. False otherwise
"""
return len(self.files_created) == 0 and len(self.files_removed) == 0
return (
len(self.files_created) == 0
and len(self.files_removed) == 0
and len(self.files_modified) == 0
)
def log_created_file(
self, file_name: str, file_metadata: Optional[FileMetadata] = None
@ -150,6 +191,25 @@ class FileHandlerTransactionLog:
self.files_created[file_name] = file_metadata
return self
def log_modified_file(
self, file_name: str, file_metadata: Optional[FileMetadata] = None
) -> "FileHandlerTransactionLog":
"""
Adds a modified file to the transaction log
Parameters
----------
file_name
Name of the file in the output directory
file_metadata
Optional. If the file has metadata, add it to the transaction log
"""
if not file_metadata:
file_metadata = FileMetadata()
self.files_modified[file_name] = file_metadata
return self
def log_removed_file(self, file_name: str) -> "FileHandlerTransactionLog":
"""
Records a file removed from the output directory
@ -179,26 +239,37 @@ class FileHandlerTransactionLog:
rstrip_line = line.rstrip()
return f" {rstrip_line}" if rstrip_line else ""
line_dash = "-" * 40
if self.files_created:
created_line = f"Files created in '{output_directory}'"
created_line_dash = "-" * 40
lines.extend([created_line, created_line_dash])
lines.extend([created_line, line_dash])
for file_path, file_metadata in sorted(self.files_created.items()):
lines.append(file_path)
if file_metadata:
lines.extend([_indent_metadata_line(line) for line in file_metadata.metadata])
if self.files_modified:
modified_line = f"Files modified in '{output_directory}'"
lines.extend([modified_line, line_dash])
for file_path, file_metadata in sorted(self.files_modified.items()):
lines.append(file_path)
if file_metadata:
lines.extend([_indent_metadata_line(line) for line in file_metadata.metadata])
if self.files_removed:
# Add a blank line to separate created/removed files
if self.files_created:
lines.append("")
removed_line = f"Files removed from '{output_directory}'"
removed_line_dash = "-" * 40
lines.extend([removed_line, removed_line_dash])
lines.extend([removed_line, line_dash])
for file_path in sorted(self.files_removed):
lines.append(file_path)
if self.is_empty:
lines.append(f"No new, modified, or removed files in '{output_directory}'")
return "\n".join(lines)
@ -274,17 +345,27 @@ class FileHandler:
file_metadata
Optional. Metadata to record to the transaction log for this file
"""
self._file_handler_transaction_log.log_created_file(
file_name=output_file_name, file_metadata=file_metadata
)
source_file_path = Path(self.working_directory) / file_name
output_file_path = Path(self.output_directory) / output_file_name
# output file exists, and it's not marked as created already, see if we modify it
if (
os.path.isfile(output_file_path)
and output_file_name not in self.file_handler_transaction_log.files_created
):
if not files_equal(source_file_path, output_file_path):
self.file_handler_transaction_log.log_modified_file(
file_name=output_file_name, file_metadata=file_metadata
)
# output file does not already exist, creates a new file
else:
self.file_handler_transaction_log.log_created_file(
file_name=output_file_name, file_metadata=file_metadata
)
if not self.dry_run:
output_file_path = Path(self.output_directory) / output_file_name
os.makedirs(os.path.dirname(output_file_path), exist_ok=True)
self.move(
src_file_path=Path(self.working_directory) / file_name,
dst_file_path=output_file_path,
)
self.move(src_file_path=source_file_path, dst_file_path=output_file_path)
def delete_file_from_output_directory(self, file_name: str):
"""

12
tests/e2e/expected_download.py
View file

@ -1,4 +1,3 @@
import hashlib
import json
import os.path
from dataclasses import dataclass
@ -6,6 +5,8 @@ from pathlib import Path
from typing import List
from typing import Optional
from ytdl_sub.utils.file_handler import get_file_md5_hash
_EXPECTED_DOWNLOADS_SUMMARY_PATH = Path("tests/e2e/resources/expected_downloads_summaries")
@ -19,11 +20,6 @@ def _get_files_in_directory(relative_directory: Path | str) -> List[Path]:
return relative_file_paths
def _get_file_md5_hash(full_file_path: Path | str) -> str:
with open(full_file_path, "rb") as file:
return hashlib.md5(file.read()).hexdigest()
@dataclass
class ExpectedDownloadFile:
path: Path
@ -74,7 +70,7 @@ class ExpectedDownloads:
if path in ignore_md5_hashes_for or path.endswith(".info.json"):
continue
md5_hash = _get_file_md5_hash(full_file_path=full_path)
md5_hash = get_file_md5_hash(full_file_path=full_path)
assert md5_hash in expected_download.md5, (
f"MD5 hash for {str(expected_download.path)} does not match: "
f"{md5_hash} != {expected_download.md5}"
@ -98,7 +94,7 @@ class ExpectedDownloads:
def from_directory(cls, directory_path: str | Path) -> "ExpectedDownloads":
relative_file_paths = _get_files_in_directory(relative_directory=directory_path)
expected_downloads_dict = {
str(file_path): _get_file_md5_hash(full_file_path=Path(directory_path) / file_path)
str(file_path): get_file_md5_hash(full_file_path=Path(directory_path) / file_path)
for file_path in relative_file_paths
}
return cls.from_dict(expected_downloads_dict)

28
tests/e2e/plugins/test_date_range.py
View file

@ -102,19 +102,21 @@ class TestDateRange:
preset_dict=recent_preset_dict,
)
# Run twice, ensure nothing changes between runsyoutube
for _ in range(2):
transaction_log = recent_channel_no_vids_in_range_subscription.download(dry_run=dry_run)
assert_transaction_log_matches(
output_directory=output_directory,
transaction_log=transaction_log,
transaction_log_summary_file_name="plugins/date_range/no_downloads.txt",
)
assert_expected_downloads(
output_directory=output_directory,
dry_run=dry_run,
expected_download_summary_file_name="plugins/date_range/no_downloads.json",
)
# run once to initialize all output directory files
_ = recent_channel_no_vids_in_range_subscription.download(dry_run=False)
# Run again, ensure no downloads
transaction_log = recent_channel_no_vids_in_range_subscription.download(dry_run=dry_run)
assert_transaction_log_matches(
output_directory=output_directory,
transaction_log=transaction_log,
transaction_log_summary_file_name="plugins/date_range/no_downloads.txt",
)
assert_expected_downloads(
output_directory=output_directory,
dry_run=False,
expected_download_summary_file_name="plugins/date_range/no_downloads.json",
)
@pytest.mark.parametrize("dry_run", [True, False])
def test_rolling_recent_channel_download(

11
tests/e2e/resources/transaction_log_summaries/plugins/date_range/no_downloads.txt
View file

@ -1,10 +1 @@
Files created in '{output_directory}'
----------------------------------------
.ytdl-sub-recent-download-archive.json
fanart.jpg
poster.jpg
tvshow.nfo
NFO tags:
tvshow:
plot: Plugin and map updates for the server Project Zombie.
title: Project / Zombie
No new, modified, or removed files in '{output_directory}'

10
tests/e2e/resources/transaction_log_summaries/plugins/date_range/test_channel_rolling_recent.txt
View file

@ -1,14 +1,6 @@
Files created in '{output_directory}'
Files modified in '{output_directory}'
----------------------------------------
.ytdl-sub-recent-download-archive.json
fanart.jpg
poster.jpg
tvshow.nfo
NFO tags:
tvshow:
plot: Plugin and map updates for the server Project Zombie.
title: Project / Zombie
Files removed from '{output_directory}'
----------------------------------------
Season 2018/s2018.e1029 - Jesse's Minecraft Server ｜ Teaser Trailer-thumb.jpg

1
tests/e2e/resources/transaction_log_summaries/plugins/split_by_chapters_with_regex_no_chapters_video_drop.txt
View file

@ -0,0 +1 @@
No new, modified, or removed files in '{output_directory}'