danvics/ytdl-sub
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

lint

Browse source
This commit is contained in:
Jesse Bannon 2023-10-19 23:24:51 -07:00
parent 82c9beb00a
commit e464235ac0
5 changed files with 86 additions and 52 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

72
src/ytdl_sub/cli/entrypoint.py
View file

@ -5,7 +5,6 @@ from datetime import datetime
from pathlib import Path
from typing import List
from typing import Optional
from typing import Tuple
from yt_dlp.utils import sanitize_filename
@ -20,7 +19,6 @@ from ytdl_sub.subscriptions.subscription import Subscription
from ytdl_sub.utils.exceptions import ExperimentalFeatureNotEnabled
from ytdl_sub.utils.exceptions import ValidationException
from ytdl_sub.utils.file_handler import FileHandler
from ytdl_sub.utils.file_handler import FileHandlerTransactionLog
from ytdl_sub.utils.file_lock import working_directory_lock
from ytdl_sub.utils.logger import Logger
@ -67,7 +65,7 @@ def _maybe_write_subscription_log_file(
def _download_subscriptions_from_yaml_files(
config: ConfigFile, subscription_paths: List[str], update_with_info_json: bool, dry_run: bool
) -> List[Tuple[Subscription, FileHandlerTransactionLog]]:
) -> List[Subscription]:
"""
Downloads all subscriptions from one or many subscription yaml files.
@ -84,7 +82,7 @@ def _download_subscriptions_from_yaml_files(
Returns
-------
List of (subscription, transaction_log)
List of subscriptions processed
Raises
------
@ -92,7 +90,7 @@ def _download_subscriptions_from_yaml_files(
Any exception during download
"""
subscriptions: List[Subscription] = []
output: List[Tuple[Subscription, FileHandlerTransactionLog]] = []
has_exception = False
# Load all the subscriptions first to perform all validation before downloading
for path in subscription_paths:
@ -106,31 +104,31 @@ def _download_subscriptions_from_yaml_files(
)
logger.debug("Subscription full yaml:\n%s", subscription.as_yaml())
try:
with subscription.exception_handling():
if update_with_info_json:
transaction_log = subscription.update_with_info_json(dry_run=dry_run)
subscription.update_with_info_json(dry_run=dry_run)
else:
transaction_log = subscription.download(dry_run=dry_run)
except Exception as exc: # pylint: disable=broad-except
_maybe_write_subscription_log_file(
config=config, subscription=subscription, dry_run=dry_run, exception=exc
)
raise
else:
output.append((subscription, transaction_log))
_maybe_write_subscription_log_file(
config=config, subscription=subscription, dry_run=dry_run
)
Logger.cleanup() # Cleanup logger after each successful subscription download
finally:
gc.collect() # Garbage collect after each subscription download
subscription.download(dry_run=dry_run)
return output
has_exception = has_exception or subscription.exception is not None
_maybe_write_subscription_log_file(
config=config,
subscription=subscription,
dry_run=dry_run,
exception=subscription.exception,
)
if not has_exception:
Logger.cleanup() # Cleanup logger if no exceptions occurred
gc.collect() # Garbage collect after each subscription download
return subscriptions
def _download_subscription_from_cli(
config: ConfigFile, dry_run: bool, extra_args: List[str]
) -> Tuple[Subscription, FileHandlerTransactionLog]:
) -> Subscription:
"""
Downloads a one-off subscription using the CLI
@ -158,12 +156,12 @@ def _download_subscription_from_cli(
)
logger.info("Beginning CLI %s", ("dry run" if dry_run else "download"))
return subscription, subscription.download(dry_run=dry_run)
subscription.download(dry_run=dry_run)
return subscription
def _view_url_from_cli(
config: ConfigFile, url: str, split_chapters: bool
) -> Tuple[Subscription, FileHandlerTransactionLog]:
def _view_url_from_cli(config: ConfigFile, url: str, split_chapters: bool) -> Subscription:
"""
`ytdl-sub view` dry-runs a URL to print its source variables. Use the pre-built `_view` preset,
inject the URL argument, and dry-run.
@ -180,10 +178,12 @@ def _view_url_from_cli(
url,
" with split chapters" if split_chapters else "",
)
return subscription, subscription.download(dry_run=True)
subscription.download(dry_run=True)
return subscription
def main() -> List[Tuple[Subscription, FileHandlerTransactionLog]]:
def main() -> List[Subscription]:
"""
Entrypoint for ytdl-sub, without the error handling
"""
@ -203,7 +203,7 @@ def main() -> List[Tuple[Subscription, FileHandlerTransactionLog]]:
logger.info("No config specified, using defaults.")
config = ConfigFile(name="default_config", value={})
transaction_logs: List[Tuple[Subscription, FileHandlerTransactionLog]] = []
subscriptions: List[Subscription] = []
# If transaction log file is specified, make sure we can open it
_maybe_validate_transaction_log_file(transaction_log_file_path=args.transaction_log)
@ -222,7 +222,7 @@ def main() -> List[Tuple[Subscription, FileHandlerTransactionLog]]:
"full backup before usage. You have been warned!",
)
transaction_logs = _download_subscriptions_from_yaml_files(
subscriptions = _download_subscriptions_from_yaml_files(
config=config,
subscription_paths=args.subscription_paths,
update_with_info_json=args.update_with_info_json,
@ -231,13 +231,13 @@ def main() -> List[Tuple[Subscription, FileHandlerTransactionLog]]:
# One-off download
elif args.subparser == "dl":
transaction_logs.append(
subscriptions.append(
_download_subscription_from_cli(
config=config, dry_run=args.dry_run, extra_args=extra_args
)
)
elif args.subparser == "view":
transaction_logs.append(
subscriptions.append(
_view_url_from_cli(config=config, url=args.url, split_chapters=args.split_chapters)
)
else:
@ -245,10 +245,10 @@ def main() -> List[Tuple[Subscription, FileHandlerTransactionLog]]:
if not args.suppress_transaction_log:
output_transaction_log(
transaction_logs=transaction_logs,
subscriptions=subscriptions,
transaction_log_file_path=args.transaction_log,
)
output_summary(transaction_logs)
output_summary(subscriptions)
return transaction_logs
return subscriptions

14
src/ytdl_sub/cli/output_summary.py
View file

@ -1,10 +1,8 @@
from typing import List
from typing import Tuple
from colorama import Fore
from ytdl_sub.subscriptions.subscription import Subscription
from ytdl_sub.utils.file_handler import FileHandlerTransactionLog
from ytdl_sub.utils.logger import Logger
logger = Logger.get()
@ -37,12 +35,12 @@ def _color_int(value: int) -> str:
return _no_color(str_int)
def output_summary(transaction_logs: List[Tuple[Subscription, FileHandlerTransactionLog]]) -> str:
def output_summary(subscriptions: List[Subscription]) -> str:
"""
Parameters
----------
transaction_logs
Transaction logs from downloaded subscriptions
subscriptions
Processed subscriptions
Returns
-------
@ -58,7 +56,7 @@ def output_summary(transaction_logs: List[Tuple[Subscription, FileHandlerTransac
width_num_entries: int = 0
# Calculate min width needed
for subscription, _ in transaction_logs:
for subscription in subscriptions:
width_sub_name = max(width_sub_name, len(subscription.name))
width_num_entries_added = max(
width_num_entries_added, len(_color_int(subscription.num_entries_added))
@ -76,12 +74,12 @@ def output_summary(transaction_logs: List[Tuple[Subscription, FileHandlerTransac
width_num_entries += 4
# Build the summary
for subscription, _ in transaction_logs:
for subscription in subscriptions:
num_entries_added = _color_int(subscription.num_entries_added)
num_entries_modified = _color_int(subscription.num_entries_modified)
num_entries_removed = _color_int(subscription.num_entries_removed * -1)
num_entries = str(subscription.num_entries)
status = _green("success")
status = _red("error") if subscription.exception else _green("success")
summary.append(
f"{subscription.name:<{width_sub_name}} "

14
src/ytdl_sub/cli/output_transaction_log.py
View file

@ -1,10 +1,8 @@
from typing import List
from typing import Optional
from typing import Tuple
from ytdl_sub.subscriptions.subscription import Subscription
from ytdl_sub.utils.exceptions import ValidationException
from ytdl_sub.utils.file_handler import FileHandlerTransactionLog
from ytdl_sub.utils.logger import Logger
logger = Logger.get()
@ -23,7 +21,7 @@ def _maybe_validate_transaction_log_file(transaction_log_file_path: Optional[str
def output_transaction_log(
transaction_logs: List[Tuple[Subscription, FileHandlerTransactionLog]],
subscriptions: List[Subscription],
transaction_log_file_path: Optional[str],
) -> None:
"""
@ -31,19 +29,19 @@ def output_transaction_log(
Parameters
----------
transaction_logs
The transaction logs from downloaded subscriptions
subscriptions
Processed subscriptions
transaction_log_file_path
Optional file path to write to
"""
transaction_log_file_contents = ""
for subscription, transaction_log in transaction_logs:
if transaction_log.is_empty:
for subscription in subscriptions:
if subscription.transaction_log.is_empty:
transaction_log_contents = f"\nNo files changed for {subscription.name}"
else:
transaction_log_contents = (
f"Transaction log for {subscription.name}:\n"
f"{transaction_log.to_output_message(subscription.output_directory)}"
f"{subscription.transaction_log.to_output_message(subscription.output_directory)}"
)
if transaction_log_file_path:

21
src/ytdl_sub/subscriptions/base_subscription.py
View file

@ -9,6 +9,7 @@ from ytdl_sub.config.preset_options import OutputOptions
from ytdl_sub.config.preset_options import Overrides
from ytdl_sub.config.preset_options import YTDLOptions
from ytdl_sub.downloaders.url.validators import MultiUrlValidator
from ytdl_sub.utils.file_handler import FileHandlerTransactionLog
from ytdl_sub.utils.logger import Logger
from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive
@ -59,6 +60,8 @@ class BaseSubscription(ABC):
migrated_file_name=migrated_file_name,
)
self._exception: Optional[Exception] = None
@property
def downloader_options(self) -> MultiUrlValidator:
"""
@ -167,6 +170,24 @@ class BaseSubscription(ABC):
"""
return self._enhanced_download_archive.num_entries
@property
def transaction_log(self) -> FileHandlerTransactionLog:
"""
Returns
-------
Transaction log from the subscription
"""
return self._enhanced_download_archive.get_file_handler_transaction_log()
@property
def exception(self) -> Optional[Exception]:
"""
Returns
-------
An exception if one occurred while processing the subscription
"""
return self._exception
def as_yaml(self) -> str:
"""
Returns

17
src/ytdl_sub/subscriptions/subscription_download.py
View file

@ -326,7 +326,9 @@ class SubscriptionDownload(BaseSubscription, ABC):
If true, do not download any video/audio files or move anything to the output
directory.
"""
self._exception = None
self._enhanced_download_archive.reinitialize(dry_run=dry_run)
plugins = self._initialize_plugins()
subscription_ytdl_options = SubscriptionYTDLOptions(
@ -353,6 +355,19 @@ class SubscriptionDownload(BaseSubscription, ABC):
dry_run=dry_run,
)
@contextlib.contextmanager
def exception_handling(self) -> None:
"""
Try to perform something on the subscription.
Store the error if one occurs.
"""
try:
yield
except Exception as exc: # pylint: disable=broad-except
self._exception = exc
return self.transaction_log
def update_with_info_json(self, dry_run: bool = False) -> FileHandlerTransactionLog:
"""
Performs the subscription update using local info json files.
@ -362,7 +377,9 @@ class SubscriptionDownload(BaseSubscription, ABC):
dry_run
If true, do not modify any video/audio files or move anything to the output directory.
"""
self._exception = None
self._enhanced_download_archive.reinitialize(dry_run=dry_run)
plugins = self._initialize_plugins()
subscription_ytdl_options = SubscriptionYTDLOptions(