From 9382be5c42e6725957d8e839e2c81d9994f71db1 Mon Sep 17 00:00:00 2001 From: Jesse Bannon Date: Mon, 8 Jan 2024 13:37:07 -0800 Subject: [PATCH] [FEATURE] `--match` flag to only run subset of subscriptions (#881) Implements https://github.com/jmbannon/ytdl-sub/issues/880 and maybe fixes https://github.com/jmbannon/ytdl-sub/issues/827 Can now do `ytdl-sub sub --match SubA SubB` or `ytdl-sub sub --match SubA --match SubB`, which will only run subscriptions that contain `SubA` or `SubB` in their names --- src/ytdl_sub/cli/entrypoint.py | 17 ++++++++-- src/ytdl_sub/cli/parsers/main.py | 15 +++++++++ src/ytdl_sub/main.py | 2 +- src/ytdl_sub/utils/logger.py | 26 ++++++++------ tests/unit/cli/test_entrypoint.py | 16 ++++++--- tests/unit/main/test_main.py | 56 +++++++++++++++++++++++++++++-- tests/unit/utils/test_logger.py | 10 +++--- 7 files changed, 117 insertions(+), 25 deletions(-) diff --git a/src/ytdl_sub/cli/entrypoint.py b/src/ytdl_sub/cli/entrypoint.py index bc7fc04e..c877e0f1 100644 --- a/src/ytdl_sub/cli/entrypoint.py +++ b/src/ytdl_sub/cli/entrypoint.py @@ -67,7 +67,11 @@ def _maybe_write_subscription_log_file( def _download_subscriptions_from_yaml_files( - config: ConfigFile, subscription_paths: List[str], update_with_info_json: bool, dry_run: bool + config: ConfigFile, + subscription_paths: List[str], + subscription_matches: List[str], + update_with_info_json: bool, + dry_run: bool, ) -> List[Subscription]: """ Downloads all subscriptions from one or many subscription yaml files. @@ -78,6 +82,8 @@ def _download_subscriptions_from_yaml_files( Configuration file subscription_paths Path to subscription files to download + subscription_matches + Optional list of substrings to match subscription names to (only run if matched) update_with_info_json Whether to actually download or update using existing info json dry_run @@ -98,6 +104,12 @@ def _download_subscriptions_from_yaml_files( for path in subscription_paths: subscriptions += Subscription.from_file_path(config=config, subscription_path=path) + if subscriptions and subscription_matches: + logger.info("Filtering subscriptions by name based on --match arguments") + subscriptions = [ + sub for sub in subscriptions if any(match in sub.name for match in subscription_matches) + ] + for subscription in subscriptions: with subscription.exception_handling(): logger.info( @@ -119,7 +131,7 @@ def _download_subscriptions_from_yaml_files( exception=subscription.exception, ) - Logger.cleanup(cleanup_error_log=False) + Logger.cleanup(has_error=False) gc.collect() # Garbage collect after each subscription download return subscriptions @@ -225,6 +237,7 @@ def main() -> List[Subscription]: subscriptions = _download_subscriptions_from_yaml_files( config=config, subscription_paths=args.subscription_paths, + subscription_matches=args.match, update_with_info_json=args.update_with_info_json, dry_run=args.dry_run, ) diff --git a/src/ytdl_sub/cli/parsers/main.py b/src/ytdl_sub/cli/parsers/main.py index ccb50734..8d3630bc 100644 --- a/src/ytdl_sub/cli/parsers/main.py +++ b/src/ytdl_sub/cli/parsers/main.py @@ -40,6 +40,10 @@ class MainArguments: long="--suppress-transaction-log", is_positional=True, ) + MATCH = CLIArgument( + short="-m", + long="--match", + ) @classmethod def all(cls) -> List[CLIArgument]: @@ -54,6 +58,7 @@ class MainArguments: cls.LOG_LEVEL, cls.TRANSACTION_LOG, cls.SUPPRESS_TRANSACTION_LOG, + cls.MATCH, ] @classmethod @@ -124,6 +129,16 @@ def _add_shared_arguments(arg_parser: argparse.ArgumentParser, suppress_defaults help="do not output transaction logs to console or file", default=argparse.SUPPRESS if suppress_defaults else False, ) + arg_parser.add_argument( + MainArguments.MATCH.short, + MainArguments.MATCH.long, + dest="match", + nargs="+", + action="extend", + type=str, + help="match subscription names to one or more substrings, and only run those subscriptions", + default=argparse.SUPPRESS if suppress_defaults else [], + ) ################################################################################################### diff --git a/src/ytdl_sub/main.py b/src/ytdl_sub/main.py index b4e7e776..44258a5c 100644 --- a/src/ytdl_sub/main.py +++ b/src/ytdl_sub/main.py @@ -27,7 +27,7 @@ def main(): """ try: return_code = _main() - Logger.cleanup(cleanup_error_log=return_code == 0) + Logger.cleanup(has_error=return_code != 0) sys.exit(return_code) except Exception as exc: # pylint: disable=broad-except Logger.log_exception(exception=exc) diff --git a/src/ytdl_sub/utils/logger.py b/src/ytdl_sub/utils/logger.py index 0b531a32..62e40ba1 100644 --- a/src/ytdl_sub/utils/logger.py +++ b/src/ytdl_sub/utils/logger.py @@ -210,6 +210,14 @@ class Logger: finally: redirect_stream.flush() + @classmethod + def _append_to_error_log(cls): + # Any time an exception occurs, dump all debug logs into the error log + with open(cls.debug_log_filename(), mode="r", encoding="utf-8") as debug_logs, open( + cls.error_log_filename(), mode="a", encoding="utf-8" + ) as error_logs: + error_logs.writelines(debug_logs.readlines()) + @classmethod def log_exception(cls, exception: Exception, log_filepath: Optional[Path] = None): """ @@ -248,14 +256,10 @@ class Logger: log_filepath if log_filepath else Logger.error_log_filename(), ) - # Any time an exception occurs, dump all debug logs into the error log - with open(cls.debug_log_filename(), mode="r", encoding="utf-8") as debug_logs, open( - cls.error_log_filename(), mode="a", encoding="utf-8" - ) as error_logs: - error_logs.writelines(debug_logs.readlines()) + cls._append_to_error_log() @classmethod - def cleanup(cls, cleanup_error_log: bool = False): + def cleanup(cls, has_error: bool = False): """ Cleans up debug log file left behind """ @@ -263,9 +267,11 @@ class Logger: for handler in logger.handlers: handler.close() - cls._DEBUG_LOGGER_FILE.close() - FileHandler.delete(cls.debug_log_filename()) - - if cleanup_error_log: + if has_error: + cls._append_to_error_log() + else: cls._ERROR_LOG_FILE.close() FileHandler.delete(cls.error_log_filename()) + + cls._DEBUG_LOGGER_FILE.close() + FileHandler.delete(cls.debug_log_filename()) diff --git a/tests/unit/cli/test_entrypoint.py b/tests/unit/cli/test_entrypoint.py index 7a4c66e7..f9be7dc3 100644 --- a/tests/unit/cli/test_entrypoint.py +++ b/tests/unit/cli/test_entrypoint.py @@ -2,6 +2,7 @@ import re import sys from pathlib import Path from typing import Callable +from typing import List from unittest.mock import patch import pytest @@ -22,6 +23,7 @@ from ytdl_sub.utils.exceptions import ExperimentalFeatureNotEnabled @pytest.mark.parametrize("dry_run", [True, False]) @pytest.mark.parametrize("mock_success_output", [True, False]) @pytest.mark.parametrize("keep_successful_logs", [True, False]) +@pytest.mark.parametrize("match", [[], ["Rick", "Michael"]]) def test_subscription_logs_write_to_file( persist_logs_directory: str, persist_logs_config_factory: Callable, @@ -30,8 +32,11 @@ def test_subscription_logs_write_to_file( dry_run: bool, mock_success_output: bool, keep_successful_logs: bool, + match: List[str], ): - subscripton_names = ["Rick Astley", "Michael Jackson", "Eric Clapton"] + subscription_names = ["Rick Astley", "Michael Jackson", "Eric Clapton"] + if match: + subscription_names = ["Rick Astley", "Michael Jackson"] num_runs = 2 config = persist_logs_config_factory(keep_successful_logs=keep_successful_logs) @@ -47,6 +52,7 @@ def test_subscription_logs_write_to_file( _download_subscriptions_from_yaml_files( config=config, subscription_paths=subscription_paths, + subscription_matches=match, update_with_info_json=False, dry_run=dry_run, ) @@ -61,8 +67,8 @@ def test_subscription_logs_write_to_file( return # If not success, expect 2 log files for both sub errors elif not mock_success_output: - assert len(log_directory_files) == (num_runs * len(subscripton_names)) - for log_path, subscription_name in zip(log_directory_files, subscripton_names): + assert len(log_directory_files) == (num_runs * len(subscription_names)) + for log_path, subscription_name in zip(log_directory_files, subscription_names): subscription_log_file_name = subscription_name.lower().replace(" ", "_") assert bool(re.match(rf"\d\.{subscription_log_file_name}\.error\.log", log_path.name)) @@ -74,9 +80,9 @@ def test_subscription_logs_write_to_file( ) # If success and success logging, expect 3 log files else: - assert len(log_directory_files) == (num_runs * len(subscripton_names)) + assert len(log_directory_files) == (num_runs * len(subscription_names)) for log_file_path, subscription_name in zip( - log_directory_files, subscripton_names * num_runs + log_directory_files, subscription_names * num_runs ): subscription_log_file_name = subscription_name.lower().replace(" ", "_") diff --git a/tests/unit/main/test_main.py b/tests/unit/main/test_main.py index a42e21a5..a88faa13 100644 --- a/tests/unit/main/test_main.py +++ b/tests/unit/main/test_main.py @@ -47,8 +47,8 @@ def test_main_exit_code(mock_sys_exit, return_code: int): main() assert mock_logger_cleanup.call_count == 1 - assert mock_logger_cleanup.call_args.kwargs["cleanup_error_log"] == ( - True if return_code == 0 else False + assert mock_logger_cleanup.call_args.kwargs["has_error"] == ( + True if return_code != 0 else False ) @@ -107,6 +107,58 @@ def test_args_after_sub_work(mock_sys_exit, tv_show_config_path): assert mock_sub.call_count == 1 assert mock_sub.call_args.kwargs["subscription_paths"] == ["subscriptions.yaml"] assert mock_sub.call_args.kwargs["config"]._name == tv_show_config_path + assert mock_sub.call_args.kwargs["subscription_matches"] == [] + assert Logger._LOGGER_LEVEL == LoggerLevels.VERBOSE + + +def test_sub_match_arguments_before(mock_sys_exit, tv_show_config_path): + with mock_sys_exit(expected_exit_code=0), patch.object( + sys, + "argv", + [ + "ytdl-sub", + "--match", + "testA", + "testB", + "-c", + tv_show_config_path, + "sub", + "--log-level", + "verbose", + ], + ), patch("ytdl_sub.cli.entrypoint._download_subscriptions_from_yaml_files") as mock_sub: + main() + + assert mock_sub.call_count == 1 + assert mock_sub.call_args.kwargs["subscription_paths"] == ["subscriptions.yaml"] + assert mock_sub.call_args.kwargs["config"]._name == tv_show_config_path + assert mock_sub.call_args.kwargs["subscription_matches"] == ["testA", "testB"] + assert Logger._LOGGER_LEVEL == LoggerLevels.VERBOSE + + +def test_sub_match_arguments_after_many(mock_sys_exit, tv_show_config_path): + with mock_sys_exit(expected_exit_code=0), patch.object( + sys, + "argv", + [ + "ytdl-sub", + "-c", + tv_show_config_path, + "sub", + "--log-level", + "verbose", + "--match", + "testA", + "--match", + "testB", + ], + ), patch("ytdl_sub.cli.entrypoint._download_subscriptions_from_yaml_files") as mock_sub: + main() + + assert mock_sub.call_count == 1 + assert mock_sub.call_args.kwargs["subscription_paths"] == ["subscriptions.yaml"] + assert mock_sub.call_args.kwargs["config"]._name == tv_show_config_path + assert mock_sub.call_args.kwargs["subscription_matches"] == ["testA", "testB"] assert Logger._LOGGER_LEVEL == LoggerLevels.VERBOSE diff --git a/tests/unit/utils/test_logger.py b/tests/unit/utils/test_logger.py index 5a75e5c3..97ce6bbd 100644 --- a/tests/unit/utils/test_logger.py +++ b/tests/unit/utils/test_logger.py @@ -111,8 +111,8 @@ class TestLogger: Logger.cleanup() assert not os.path.isfile(Logger._DEBUG_LOGGER_FILE.name) - @pytest.mark.parametrize("clean_error_log", [True, False]) - def test_logger_can_be_cleaned_during_execution(self, clean_error_log: bool): + @pytest.mark.parametrize("has_error", [True, False]) + def test_logger_can_be_cleaned_during_execution(self, has_error: bool): Logger._LOGGER_LEVEL = LoggerLevels.INFO logger = Logger.get(name="name_test") @@ -133,11 +133,11 @@ class TestLogger: except ValueError as exc: Logger.log_exception(exception=exc) - Logger.cleanup(cleanup_error_log=clean_error_log) + Logger.cleanup(has_error=has_error) assert not os.path.isfile(Logger.debug_log_filename()) - assert clean_error_log == (not os.path.isfile(Logger.error_log_filename())) - if not clean_error_log: + assert not has_error == (not os.path.isfile(Logger.error_log_filename())) + if has_error: with open(Logger.error_log_filename(), mode="r", encoding="utf-8") as err_file: err_logs = err_file.readlines() expected = [