From e464235ac0b9912acfe105e92835a8b9226509a9 Mon Sep 17 00:00:00 2001 From: Jesse Bannon Date: Thu, 19 Oct 2023 23:24:51 -0700 Subject: [PATCH] lint --- src/ytdl_sub/cli/entrypoint.py | 72 +++++++++---------- src/ytdl_sub/cli/output_summary.py | 14 ++-- src/ytdl_sub/cli/output_transaction_log.py | 14 ++-- .../subscriptions/base_subscription.py | 21 ++++++ .../subscriptions/subscription_download.py | 17 +++++ 5 files changed, 86 insertions(+), 52 deletions(-) diff --git a/src/ytdl_sub/cli/entrypoint.py b/src/ytdl_sub/cli/entrypoint.py index c685fb6f..d91d184d 100644 --- a/src/ytdl_sub/cli/entrypoint.py +++ b/src/ytdl_sub/cli/entrypoint.py @@ -5,7 +5,6 @@ from datetime import datetime from pathlib import Path from typing import List from typing import Optional -from typing import Tuple from yt_dlp.utils import sanitize_filename @@ -20,7 +19,6 @@ from ytdl_sub.subscriptions.subscription import Subscription from ytdl_sub.utils.exceptions import ExperimentalFeatureNotEnabled from ytdl_sub.utils.exceptions import ValidationException from ytdl_sub.utils.file_handler import FileHandler -from ytdl_sub.utils.file_handler import FileHandlerTransactionLog from ytdl_sub.utils.file_lock import working_directory_lock from ytdl_sub.utils.logger import Logger @@ -67,7 +65,7 @@ def _maybe_write_subscription_log_file( def _download_subscriptions_from_yaml_files( config: ConfigFile, subscription_paths: List[str], update_with_info_json: bool, dry_run: bool -) -> List[Tuple[Subscription, FileHandlerTransactionLog]]: +) -> List[Subscription]: """ Downloads all subscriptions from one or many subscription yaml files. @@ -84,7 +82,7 @@ def _download_subscriptions_from_yaml_files( Returns ------- - List of (subscription, transaction_log) + List of subscriptions processed Raises ------ @@ -92,7 +90,7 @@ def _download_subscriptions_from_yaml_files( Any exception during download """ subscriptions: List[Subscription] = [] - output: List[Tuple[Subscription, FileHandlerTransactionLog]] = [] + has_exception = False # Load all the subscriptions first to perform all validation before downloading for path in subscription_paths: @@ -106,31 +104,31 @@ def _download_subscriptions_from_yaml_files( ) logger.debug("Subscription full yaml:\n%s", subscription.as_yaml()) - try: + with subscription.exception_handling(): if update_with_info_json: - transaction_log = subscription.update_with_info_json(dry_run=dry_run) + subscription.update_with_info_json(dry_run=dry_run) else: - transaction_log = subscription.download(dry_run=dry_run) - except Exception as exc: # pylint: disable=broad-except - _maybe_write_subscription_log_file( - config=config, subscription=subscription, dry_run=dry_run, exception=exc - ) - raise - else: - output.append((subscription, transaction_log)) - _maybe_write_subscription_log_file( - config=config, subscription=subscription, dry_run=dry_run - ) - Logger.cleanup() # Cleanup logger after each successful subscription download - finally: - gc.collect() # Garbage collect after each subscription download + subscription.download(dry_run=dry_run) - return output + has_exception = has_exception or subscription.exception is not None + _maybe_write_subscription_log_file( + config=config, + subscription=subscription, + dry_run=dry_run, + exception=subscription.exception, + ) + + if not has_exception: + Logger.cleanup() # Cleanup logger if no exceptions occurred + + gc.collect() # Garbage collect after each subscription download + + return subscriptions def _download_subscription_from_cli( config: ConfigFile, dry_run: bool, extra_args: List[str] -) -> Tuple[Subscription, FileHandlerTransactionLog]: +) -> Subscription: """ Downloads a one-off subscription using the CLI @@ -158,12 +156,12 @@ def _download_subscription_from_cli( ) logger.info("Beginning CLI %s", ("dry run" if dry_run else "download")) - return subscription, subscription.download(dry_run=dry_run) + subscription.download(dry_run=dry_run) + + return subscription -def _view_url_from_cli( - config: ConfigFile, url: str, split_chapters: bool -) -> Tuple[Subscription, FileHandlerTransactionLog]: +def _view_url_from_cli(config: ConfigFile, url: str, split_chapters: bool) -> Subscription: """ `ytdl-sub view` dry-runs a URL to print its source variables. Use the pre-built `_view` preset, inject the URL argument, and dry-run. @@ -180,10 +178,12 @@ def _view_url_from_cli( url, " with split chapters" if split_chapters else "", ) - return subscription, subscription.download(dry_run=True) + subscription.download(dry_run=True) + + return subscription -def main() -> List[Tuple[Subscription, FileHandlerTransactionLog]]: +def main() -> List[Subscription]: """ Entrypoint for ytdl-sub, without the error handling """ @@ -203,7 +203,7 @@ def main() -> List[Tuple[Subscription, FileHandlerTransactionLog]]: logger.info("No config specified, using defaults.") config = ConfigFile(name="default_config", value={}) - transaction_logs: List[Tuple[Subscription, FileHandlerTransactionLog]] = [] + subscriptions: List[Subscription] = [] # If transaction log file is specified, make sure we can open it _maybe_validate_transaction_log_file(transaction_log_file_path=args.transaction_log) @@ -222,7 +222,7 @@ def main() -> List[Tuple[Subscription, FileHandlerTransactionLog]]: "full backup before usage. You have been warned!", ) - transaction_logs = _download_subscriptions_from_yaml_files( + subscriptions = _download_subscriptions_from_yaml_files( config=config, subscription_paths=args.subscription_paths, update_with_info_json=args.update_with_info_json, @@ -231,13 +231,13 @@ def main() -> List[Tuple[Subscription, FileHandlerTransactionLog]]: # One-off download elif args.subparser == "dl": - transaction_logs.append( + subscriptions.append( _download_subscription_from_cli( config=config, dry_run=args.dry_run, extra_args=extra_args ) ) elif args.subparser == "view": - transaction_logs.append( + subscriptions.append( _view_url_from_cli(config=config, url=args.url, split_chapters=args.split_chapters) ) else: @@ -245,10 +245,10 @@ def main() -> List[Tuple[Subscription, FileHandlerTransactionLog]]: if not args.suppress_transaction_log: output_transaction_log( - transaction_logs=transaction_logs, + subscriptions=subscriptions, transaction_log_file_path=args.transaction_log, ) - output_summary(transaction_logs) + output_summary(subscriptions) - return transaction_logs + return subscriptions diff --git a/src/ytdl_sub/cli/output_summary.py b/src/ytdl_sub/cli/output_summary.py index 607f0bf6..d08ff1bf 100644 --- a/src/ytdl_sub/cli/output_summary.py +++ b/src/ytdl_sub/cli/output_summary.py @@ -1,10 +1,8 @@ from typing import List -from typing import Tuple from colorama import Fore from ytdl_sub.subscriptions.subscription import Subscription -from ytdl_sub.utils.file_handler import FileHandlerTransactionLog from ytdl_sub.utils.logger import Logger logger = Logger.get() @@ -37,12 +35,12 @@ def _color_int(value: int) -> str: return _no_color(str_int) -def output_summary(transaction_logs: List[Tuple[Subscription, FileHandlerTransactionLog]]) -> str: +def output_summary(subscriptions: List[Subscription]) -> str: """ Parameters ---------- - transaction_logs - Transaction logs from downloaded subscriptions + subscriptions + Processed subscriptions Returns ------- @@ -58,7 +56,7 @@ def output_summary(transaction_logs: List[Tuple[Subscription, FileHandlerTransac width_num_entries: int = 0 # Calculate min width needed - for subscription, _ in transaction_logs: + for subscription in subscriptions: width_sub_name = max(width_sub_name, len(subscription.name)) width_num_entries_added = max( width_num_entries_added, len(_color_int(subscription.num_entries_added)) @@ -76,12 +74,12 @@ def output_summary(transaction_logs: List[Tuple[Subscription, FileHandlerTransac width_num_entries += 4 # Build the summary - for subscription, _ in transaction_logs: + for subscription in subscriptions: num_entries_added = _color_int(subscription.num_entries_added) num_entries_modified = _color_int(subscription.num_entries_modified) num_entries_removed = _color_int(subscription.num_entries_removed * -1) num_entries = str(subscription.num_entries) - status = _green("success") + status = _red("error") if subscription.exception else _green("success") summary.append( f"{subscription.name:<{width_sub_name}} " diff --git a/src/ytdl_sub/cli/output_transaction_log.py b/src/ytdl_sub/cli/output_transaction_log.py index 6bf5e07b..ce46de0c 100644 --- a/src/ytdl_sub/cli/output_transaction_log.py +++ b/src/ytdl_sub/cli/output_transaction_log.py @@ -1,10 +1,8 @@ from typing import List from typing import Optional -from typing import Tuple from ytdl_sub.subscriptions.subscription import Subscription from ytdl_sub.utils.exceptions import ValidationException -from ytdl_sub.utils.file_handler import FileHandlerTransactionLog from ytdl_sub.utils.logger import Logger logger = Logger.get() @@ -23,7 +21,7 @@ def _maybe_validate_transaction_log_file(transaction_log_file_path: Optional[str def output_transaction_log( - transaction_logs: List[Tuple[Subscription, FileHandlerTransactionLog]], + subscriptions: List[Subscription], transaction_log_file_path: Optional[str], ) -> None: """ @@ -31,19 +29,19 @@ def output_transaction_log( Parameters ---------- - transaction_logs - The transaction logs from downloaded subscriptions + subscriptions + Processed subscriptions transaction_log_file_path Optional file path to write to """ transaction_log_file_contents = "" - for subscription, transaction_log in transaction_logs: - if transaction_log.is_empty: + for subscription in subscriptions: + if subscription.transaction_log.is_empty: transaction_log_contents = f"\nNo files changed for {subscription.name}" else: transaction_log_contents = ( f"Transaction log for {subscription.name}:\n" - f"{transaction_log.to_output_message(subscription.output_directory)}" + f"{subscription.transaction_log.to_output_message(subscription.output_directory)}" ) if transaction_log_file_path: diff --git a/src/ytdl_sub/subscriptions/base_subscription.py b/src/ytdl_sub/subscriptions/base_subscription.py index 28f9caa6..0c61f7e1 100644 --- a/src/ytdl_sub/subscriptions/base_subscription.py +++ b/src/ytdl_sub/subscriptions/base_subscription.py @@ -9,6 +9,7 @@ from ytdl_sub.config.preset_options import OutputOptions from ytdl_sub.config.preset_options import Overrides from ytdl_sub.config.preset_options import YTDLOptions from ytdl_sub.downloaders.url.validators import MultiUrlValidator +from ytdl_sub.utils.file_handler import FileHandlerTransactionLog from ytdl_sub.utils.logger import Logger from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive @@ -59,6 +60,8 @@ class BaseSubscription(ABC): migrated_file_name=migrated_file_name, ) + self._exception: Optional[Exception] = None + @property def downloader_options(self) -> MultiUrlValidator: """ @@ -167,6 +170,24 @@ class BaseSubscription(ABC): """ return self._enhanced_download_archive.num_entries + @property + def transaction_log(self) -> FileHandlerTransactionLog: + """ + Returns + ------- + Transaction log from the subscription + """ + return self._enhanced_download_archive.get_file_handler_transaction_log() + + @property + def exception(self) -> Optional[Exception]: + """ + Returns + ------- + An exception if one occurred while processing the subscription + """ + return self._exception + def as_yaml(self) -> str: """ Returns diff --git a/src/ytdl_sub/subscriptions/subscription_download.py b/src/ytdl_sub/subscriptions/subscription_download.py index 4e6eebcc..366b8a6e 100644 --- a/src/ytdl_sub/subscriptions/subscription_download.py +++ b/src/ytdl_sub/subscriptions/subscription_download.py @@ -326,7 +326,9 @@ class SubscriptionDownload(BaseSubscription, ABC): If true, do not download any video/audio files or move anything to the output directory. """ + self._exception = None self._enhanced_download_archive.reinitialize(dry_run=dry_run) + plugins = self._initialize_plugins() subscription_ytdl_options = SubscriptionYTDLOptions( @@ -353,6 +355,19 @@ class SubscriptionDownload(BaseSubscription, ABC): dry_run=dry_run, ) + @contextlib.contextmanager + def exception_handling(self) -> None: + """ + Try to perform something on the subscription. + Store the error if one occurs. + """ + try: + yield + except Exception as exc: # pylint: disable=broad-except + self._exception = exc + + return self.transaction_log + def update_with_info_json(self, dry_run: bool = False) -> FileHandlerTransactionLog: """ Performs the subscription update using local info json files. @@ -362,7 +377,9 @@ class SubscriptionDownload(BaseSubscription, ABC): dry_run If true, do not modify any video/audio files or move anything to the output directory. """ + self._exception = None self._enhanced_download_archive.reinitialize(dry_run=dry_run) + plugins = self._initialize_plugins() subscription_ytdl_options = SubscriptionYTDLOptions(