From 0545ada1f87f33cfbe0af3f7232334c3813787a8 Mon Sep 17 00:00:00 2001 From: jbannon Date: Tue, 12 Apr 2022 05:35:56 +0000 Subject: [PATCH] pylint spree --- .pylintrc | 3 +- ytdl_subscribe/downloaders/downloader.py | 10 ++--- .../downloaders/youtube_downloader.py | 10 ++--- ytdl_subscribe/entries/entry.py | 12 ++++-- ytdl_subscribe/entries/soundcloud.py | 2 +- ytdl_subscribe/subscriptions/soundcloud.py | 8 ++++ ytdl_subscribe/subscriptions/subscription.py | 37 ++++++++++++++++--- ytdl_subscribe/subscriptions/youtube.py | 16 ++++++++ .../validators/base/string_datetime.py | 4 +- .../base/string_formatter_validators.py | 13 ++++++- .../metadata_options_validator.py | 6 ++- .../config/metadata_options/nfo_validator.py | 4 ++ .../validators/config/preset_validator.py | 6 ++- .../ytdl_options/ytdl_options_validator.py | 2 - .../enhanced_download_archive.py | 8 ++-- 15 files changed, 106 insertions(+), 35 deletions(-) diff --git a/.pylintrc b/.pylintrc index 1b1b8b8c..4a1bade4 100644 --- a/.pylintrc +++ b/.pylintrc @@ -1,4 +1,5 @@ [MASTER] disable= C0114, # missing-module-docstring - R0903, # too-few-public-methods \ No newline at end of file + R0903, # too-few-public-methods + R0801, # similar lines \ No newline at end of file diff --git a/ytdl_subscribe/downloaders/downloader.py b/ytdl_subscribe/downloaders/downloader.py index f881122f..7639ff93 100644 --- a/ytdl_subscribe/downloaders/downloader.py +++ b/ytdl_subscribe/downloaders/downloader.py @@ -1,4 +1,3 @@ -import tempfile from contextlib import contextmanager from pathlib import Path from typing import Dict @@ -50,17 +49,14 @@ class Downloader: def __init__( self, - output_directory: str, + working_directory: str, ytdl_options: Optional[Dict] = None, download_archive_file_name: Optional[str] = None, ): - self.output_directory = output_directory - if self.output_directory is None: - self.output_directory = tempfile.TemporaryDirectory().name - + self.working_directory = working_directory self.ytdl_options = Downloader._configure_ytdl_options( ytdl_options=ytdl_options, - working_directory=self.output_directory, + working_directory=self.working_directory, download_archive_file_name=download_archive_file_name, ) diff --git a/ytdl_subscribe/downloaders/youtube_downloader.py b/ytdl_subscribe/downloaders/youtube_downloader.py index cbcee617..2b6a1337 100644 --- a/ytdl_subscribe/downloaders/youtube_downloader.py +++ b/ytdl_subscribe/downloaders/youtube_downloader.py @@ -62,9 +62,9 @@ class YoutubeDownloader(Downloader): entries: List[YoutubeVideo] = [] # Load the entries from info.json, ignore the playlist entry - for file_name in os.listdir(self.output_directory): + for file_name in os.listdir(self.working_directory): if file_name.endswith(".info.json") and not file_name.startswith(playlist_id): - with open(Path(self.output_directory) / file_name, "r", encoding="utf-8") as file: + with open(Path(self.working_directory) / file_name, "r", encoding="utf-8") as file: entries.append(YoutubeVideo(**json.load(file))) return entries @@ -72,8 +72,6 @@ class YoutubeDownloader(Downloader): def download_channel(self, channel_id: str) -> List[YoutubeVideo]: """ Downloads all videos from a channel - TODO: Add caching via ids in the metadata. Scrape output directory for any vid ids and - TODO: include it in the archive """ self._download_with_metadata(url=self.channel_url(channel_id)) @@ -82,9 +80,9 @@ class YoutubeDownloader(Downloader): # Load the entries from info.json # TODO dupe code between this and playlist - for file_name in os.listdir(self.output_directory): + for file_name in os.listdir(self.working_directory): if file_name.endswith(".info.json") and not file_name.startswith(channel_id): - with open(Path(self.output_directory) / file_name, "r", encoding="utf-8") as file: + with open(Path(self.working_directory) / file_name, "r", encoding="utf-8") as file: entries.append(YoutubeVideo(**json.load(file))) return entries diff --git a/ytdl_subscribe/entries/entry.py b/ytdl_subscribe/entries/entry.py index 8676afc8..0943126a 100644 --- a/ytdl_subscribe/entries/entry.py +++ b/ytdl_subscribe/entries/entry.py @@ -1,24 +1,28 @@ from abc import ABC from dataclasses import dataclass -from pathlib import Path from typing import Any from typing import Dict from typing import Optional from sanitize_filename import sanitize -from ytdl_subscribe.validators.base.string_formatter_validators import StringFormatterValidator -from ytdl_subscribe.validators.config.overrides.overrides_validator import OverridesValidator - @dataclass class PlaylistMetadata: + """ + Metadata for storing playlist information. + """ + playlist_index: int playlist_id: str playlist_extractor: str class BaseEntry(ABC): + """ + Abstract entry object to represent anything download from ytdl (playlist metadata, media, etc). + """ + def __init__(self, **kwargs): """ Initialize the entry using ytdl metadata diff --git a/ytdl_subscribe/entries/soundcloud.py b/ytdl_subscribe/entries/soundcloud.py index 8faed23f..e3345464 100644 --- a/ytdl_subscribe/entries/soundcloud.py +++ b/ytdl_subscribe/entries/soundcloud.py @@ -107,7 +107,7 @@ class SoundcloudAlbumTrack(SoundcloudTrack): album=album, album_year=album_year, playlist_metadata=playlist_metadata, - **soundcloud_track._kwargs, + **soundcloud_track._kwargs, # pylint: disable=protected-access ) diff --git a/ytdl_subscribe/subscriptions/soundcloud.py b/ytdl_subscribe/subscriptions/soundcloud.py index 589bb20a..aa208f52 100644 --- a/ytdl_subscribe/subscriptions/soundcloud.py +++ b/ytdl_subscribe/subscriptions/soundcloud.py @@ -16,6 +16,10 @@ from ytdl_subscribe.validators.config.source_options.soundcloud_validators impor class SoundcloudSubscription(Subscription[SourceT], ABC): + """ + Abstract class for all Soundcloud-based subscriptions. Sets entry type to SoundcloudTrack + """ + def __init__( self, name: str, @@ -33,6 +37,10 @@ class SoundcloudSubscription(Subscription[SourceT], ABC): class SoundcloudAlbumsAndSinglesSubscription( SoundcloudSubscription[SoundcloudAlbumsAndSinglesSourceValidator] ): + """ + Soundcloud subscription to download albums and tracks as singles. + """ + def _extract_info(self) -> List[SoundcloudTrack]: tracks: List[SoundcloudTrack] = [] downloader = self.get_downloader(SoundcloudDownloader) diff --git a/ytdl_subscribe/subscriptions/subscription.py b/ytdl_subscribe/subscriptions/subscription.py index 8d100e2e..15f1aecc 100644 --- a/ytdl_subscribe/subscriptions/subscription.py +++ b/ytdl_subscribe/subscriptions/subscription.py @@ -40,6 +40,19 @@ DownloaderT = TypeVar("DownloaderT", bound=Downloader) class Subscription(Generic[SourceT], ABC): + """ + Subscription classes are the 'controllers' that perform... + + - Downloading via ytdlp + - Adding metadata + - Placing files in the output directory + + while configuring each step with provided configs. Child classes are expected to + provide SourceValidator (SourceT), which defines the source and its configurable options. + In addition, they should provide in the init an Entry type (EntryT), which is the entry that + will be returned after downloading. + """ + def __init__( self, name: str, @@ -68,6 +81,9 @@ class Subscription(Generic[SourceT], ABC): @property def entry_type(self) -> EntryT: + """ + :return: The Entry type this subscription uses to represent downloaded media + """ return self.__entry_type @property @@ -96,10 +112,19 @@ class Subscription(Generic[SourceT], ABC): return str(Path(self.__config_options.working_directory.value) / Path(self.name)) @property - def output_directory(self): + def output_directory(self) -> str: + """ + :return: The formatted output directory + """ return self._apply_formatter(formatter=self.output_options.output_directory) def _archive_entry_file_name(self, entry: Optional[Entry], relative_file_name: str) -> None: + """ + Adds an entry and a file name that belongs to it into the archive mapping. + + :param entry: Optional. The entry the file belongs to + :param relative_file_name: The name of the file + """ if entry: self._enhanced_download_archive.mapping.add_entry( entry=entry, entry_file_path=relative_file_name @@ -181,16 +206,18 @@ class Subscription(Generic[SourceT], ABC): # Bug that mismatches webp and jpg extensions. Try to hotfix here if not os.path.isfile(source_thumbnail_path): + to_replace = f".{entry.thumbnail_ext}" actual_thumbnail_ext = ".webp" if entry.thumbnail_ext == "webp": actual_thumbnail_ext = ".jpg" - source_thumbnail_path = source_thumbnail_path.replace( - f".{entry.thumbnail_ext}", actual_thumbnail_ext + source_thumbnail_name = entry.download_thumbnail_name.replace( + to_replace, actual_thumbnail_ext ) + source_thumbnail_path = Path(self.working_directory) / source_thumbnail_name + if not os.path.isfile(source_thumbnail_path): - # TODO: make more formal - raise ValueError("Youtube thumbnails are a lie") + raise ValueError("Hotfix for getting thumbnail file extension failed") output_thumbnail_name = self._apply_formatter( formatter=self.output_options.thumbnail_name, entry=entry diff --git a/ytdl_subscribe/subscriptions/youtube.py b/ytdl_subscribe/subscriptions/youtube.py index 03b4d4e5..b0138d3b 100644 --- a/ytdl_subscribe/subscriptions/youtube.py +++ b/ytdl_subscribe/subscriptions/youtube.py @@ -21,6 +21,10 @@ from ytdl_subscribe.validators.config.source_options.youtube_validators import ( class YoutubeSubscription(Subscription[SourceT], ABC): + """ + Abstract class for all Youtube-based subscriptions. Sets entry type to YoutubeVideo + """ + def __init__( self, name: str, @@ -36,6 +40,10 @@ class YoutubeSubscription(Subscription[SourceT], ABC): class YoutubePlaylistSubscription(YoutubeSubscription[YoutubePlaylistSourceValidator]): + """ + Youtube subscription to download videos from a playlist + """ + def _extract_info(self) -> List[YoutubeVideo]: return self.get_downloader(YoutubeDownloader).download_playlist( playlist_id=self.source_options.playlist_id.value @@ -43,6 +51,10 @@ class YoutubePlaylistSubscription(YoutubeSubscription[YoutubePlaylistSourceValid class YoutubeChannelSubscription(YoutubeSubscription[YoutubeChannelSourceValidator]): + """ + Youtube subscription to download videos from a channel + """ + def _extract_info(self) -> List[YoutubeVideo]: source_ytdl_options = {} source_date_range = self.source_options.get_date_range() @@ -54,6 +66,10 @@ class YoutubeChannelSubscription(YoutubeSubscription[YoutubeChannelSourceValidat class YoutubeVideoSubscription(YoutubeSubscription[YoutubeVideoSourceValidator]): + """ + Youtube subscription to download a single video + """ + def _extract_info(self) -> List[YoutubeVideo]: entry = self.get_downloader(YoutubeDownloader).download_video( video_id=self.source_options.video_id.value diff --git a/ytdl_subscribe/validators/base/string_datetime.py b/ytdl_subscribe/validators/base/string_datetime.py index 3b034d45..56b26004 100644 --- a/ytdl_subscribe/validators/base/string_datetime.py +++ b/ytdl_subscribe/validators/base/string_datetime.py @@ -17,8 +17,8 @@ class StringDatetimeValidator(Validator): try: _ = datetime_from_str(self._value) - except Exception as e: - raise ValidationException(e) + except Exception as exc: + raise ValidationException(exc) from exc @property def datetime_str(self) -> str: diff --git a/ytdl_subscribe/validators/base/string_formatter_validators.py b/ytdl_subscribe/validators/base/string_formatter_validators.py index 49349a8e..034c74ab 100644 --- a/ytdl_subscribe/validators/base/string_formatter_validators.py +++ b/ytdl_subscribe/validators/base/string_formatter_validators.py @@ -131,11 +131,13 @@ class DictFormatterValidator(LiteralDictValidator): Validates a dictionary made up of key: string_formatters """ + _key_validator = StringFormatterValidator + def __init__(self, name, value): super().__init__(name, value) for key in self._keys: - self._value[key] = self._validate_key(key=key, validator=StringFormatterValidator) + self._value[key] = self._validate_key(key=key, validator=self._key_validator) @property def dict(self) -> Dict[str, StringFormatterValidator]: @@ -146,3 +148,12 @@ class DictFormatterValidator(LiteralDictValidator): def dict_with_format_strings(self) -> Dict[str, str]: """Returns dict with the format strings themselves""" return {key: string_formatter.format_string for key, string_formatter in self.dict.items()} + + +class OverridesDictFormatterValidator(DictFormatterValidator): + """ + Validates a dictionary made up of key: string_formatters, that must be resolved by overrides + only. + """ + + _key_validator = OverridesStringFormatterValidator diff --git a/ytdl_subscribe/validators/config/metadata_options/metadata_options_validator.py b/ytdl_subscribe/validators/config/metadata_options/metadata_options_validator.py index bcf1c107..fe1a0122 100644 --- a/ytdl_subscribe/validators/config/metadata_options/metadata_options_validator.py +++ b/ytdl_subscribe/validators/config/metadata_options/metadata_options_validator.py @@ -1,6 +1,9 @@ from ytdl_subscribe.validators.base.strict_dict_validator import StrictDictValidator from ytdl_subscribe.validators.config.metadata_options.id3_validator import Id3Validator from ytdl_subscribe.validators.config.metadata_options.nfo_validator import NFOValidator +from ytdl_subscribe.validators.config.metadata_options.nfo_validator import ( + OutputDirectoryNFOValidator, +) class MetadataOptionsValidator(StrictDictValidator): @@ -12,7 +15,6 @@ class MetadataOptionsValidator(StrictDictValidator): self.id3 = self._validate_key_if_present(key="id3", validator=Id3Validator) self.nfo = self._validate_key_if_present(key="nfo", validator=NFOValidator) - # TODO: Ensure this does not depend on entry variables, only overrides self.output_directory_nfo = self._validate_key_if_present( - key="output_directory_nfo", validator=NFOValidator + key="output_directory_nfo", validator=OutputDirectoryNFOValidator ) diff --git a/ytdl_subscribe/validators/config/metadata_options/nfo_validator.py b/ytdl_subscribe/validators/config/metadata_options/nfo_validator.py index e4706569..9a0a912d 100644 --- a/ytdl_subscribe/validators/config/metadata_options/nfo_validator.py +++ b/ytdl_subscribe/validators/config/metadata_options/nfo_validator.py @@ -23,3 +23,7 @@ class OutputDirectoryNFOValidator(NFOValidator): self.nfo_name = self._validate_key( key="nfo_name", validator=OverridesStringFormatterValidator ) + self.nfo_root = self._validate_key( + key="nfo_root", validator=OverridesStringFormatterValidator + ) + self.tags = self._validate_key(key="tags", validator=DictFormatterValidator) diff --git a/ytdl_subscribe/validators/config/preset_validator.py b/ytdl_subscribe/validators/config/preset_validator.py index 3267b87a..9117ab2e 100644 --- a/ytdl_subscribe/validators/config/preset_validator.py +++ b/ytdl_subscribe/validators/config/preset_validator.py @@ -110,9 +110,13 @@ class PresetValidator(StrictDictValidator): if validator_dict is None: validator_dict = self._validator_dict - for name, validator in validator_dict.items(): + for validator in validator_dict.values(): if isinstance(validator, DictValidator): + # Usage of protected variables in other validators is fine. The reason to keep them + # protected is for readability when using them in subscriptions. + # pylint: disable=protected-access self.__recursive_preset_validate(validator._validator_dict) + # pylint: enable=protected-access if isinstance(validator, OverridesStringFormatterValidator): self.__validate_override_string_formatter_validator(validator) diff --git a/ytdl_subscribe/validators/config/ytdl_options/ytdl_options_validator.py b/ytdl_subscribe/validators/config/ytdl_options/ytdl_options_validator.py index 96e05eb6..1b76d0cc 100644 --- a/ytdl_subscribe/validators/config/ytdl_options/ytdl_options_validator.py +++ b/ytdl_subscribe/validators/config/ytdl_options/ytdl_options_validator.py @@ -3,5 +3,3 @@ from ytdl_subscribe.validators.base.validators import LiteralDictValidator class YTDLOptionsValidator(LiteralDictValidator): """Ensures `ytdl_options` is a dict""" - - pass diff --git a/ytdl_subscribe/ytdl_additions/enhanced_download_archive.py b/ytdl_subscribe/ytdl_additions/enhanced_download_archive.py index 7f206e95..a331c216 100644 --- a/ytdl_subscribe/ytdl_additions/enhanced_download_archive.py +++ b/ytdl_subscribe/ytdl_additions/enhanced_download_archive.py @@ -66,8 +66,8 @@ class DownloadArchive: @classmethod def from_file(cls, file_path: str) -> "DownloadArchive": - lines = open(file_path, "r", encoding="utf8").readlines() - return cls.from_lines(lines=lines) + with open(file_path, "r", encoding="utf8") as file: + return cls.from_lines(lines=file.readlines()) def to_file(self, file_path: str) -> "DownloadArchive": with open(file_path, "w", encoding="utf8") as file: @@ -93,7 +93,9 @@ class DownloadMappings: @classmethod def from_file(cls, json_file_path: str) -> "DownloadMappings": - entry_mappings_json = json.load(open(json_file_path, "r", encoding="utf8")) + with open(json_file_path, "r", encoding="utf8") as json_file: + entry_mappings_json = json.load(json_file) + for uid in entry_mappings_json.keys(): entry_mappings_json[uid] = DownloadMapping.from_dict( mapping_dict=entry_mappings_json[uid]