From ba86733e5e02d280dfc9a72cf666ec07b6eb0d1b Mon Sep 17 00:00:00 2001 From: jbannon Date: Fri, 29 Apr 2022 06:28:54 +0000 Subject: [PATCH] 10/10 --- pyproject.toml | 19 ++ src/ytdl_sub/config/preset_class_mappings.py | 2 +- src/ytdl_sub/downloaders/downloader.py | 12 + src/ytdl_sub/entries/base_entry.py | 2 +- src/ytdl_sub/entries/soundcloud.py | 41 +++- src/ytdl_sub/subscriptions/subscription.py | 2 +- .../validators/string_formatter_validators.py | 2 +- .../enhanced_download_archive.py | 224 ++++++++++++++++-- tools/linter | 1 + 9 files changed, 277 insertions(+), 28 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 85058f6e..0b532b09 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,7 +12,26 @@ disable = [ "C0114", # missing-module-docstring "R0903", # too-few-public-methods "R0801", # similar lines + "R0913", # Too many arguments "W0511", # TODO ] load-plugins = "pylint.extensions.docparams" + +[tool.pydocstyle] +inherit = false +match = "[^test_].*\\.py" +ignore = [ + "D100", # docstring in public module + "D101", # Missing docstring in public class (covered by pylint) + "D104", # docstring in public package + "D107", # docstring in init + "D200", # One-line should fit on one line + "D203", # 1 blank line before class docstring + "D205", # 1 blank line between summary and description + "D212", # Multi-line should start at first line + "D400", # Should end with a period + "D401", # Return vs Returns + "D413", # Missing blank line after last section + "D415", # Should end with a period +] diff --git a/src/ytdl_sub/config/preset_class_mappings.py b/src/ytdl_sub/config/preset_class_mappings.py index e912f764..958fa957 100644 --- a/src/ytdl_sub/config/preset_class_mappings.py +++ b/src/ytdl_sub/config/preset_class_mappings.py @@ -129,7 +129,7 @@ class PluginMapping: The plugin class Raises - ------- + ------ ValueError Raised if the plugin does not exist """ diff --git a/src/ytdl_sub/downloaders/downloader.py b/src/ytdl_sub/downloaders/downloader.py index 4f61a753..c9f12e4a 100644 --- a/src/ytdl_sub/downloaders/downloader.py +++ b/src/ytdl_sub/downloaders/downloader.py @@ -77,6 +77,18 @@ class Downloader(Generic[DownloaderOptionsT, DownloaderEntryT], ABC): ytdl_options: Optional[Dict] = None, download_archive_file_name: Optional[str] = None, ): + """ + Parameters + ---------- + working_directory + Path to the working directory + download_options + Options validator for this downloader + ytdl_options + YTDL options validator + download_archive_file_name + Optional. Name of the download archive file that should reside in the working directory + """ self.working_directory = working_directory self.download_options = download_options self.ytdl_options = Downloader._configure_ytdl_options( diff --git a/src/ytdl_sub/entries/base_entry.py b/src/ytdl_sub/entries/base_entry.py index 4981601a..76dfb2a1 100644 --- a/src/ytdl_sub/entries/base_entry.py +++ b/src/ytdl_sub/entries/base_entry.py @@ -52,7 +52,7 @@ class BaseEntry(ABC): The working directory Raises - ------- + ------ ValueError The working directory was never defined in the init """ diff --git a/src/ytdl_sub/entries/soundcloud.py b/src/ytdl_sub/entries/soundcloud.py index 5d5fdcb8..15c75530 100644 --- a/src/ytdl_sub/entries/soundcloud.py +++ b/src/ytdl_sub/entries/soundcloud.py @@ -64,6 +64,22 @@ class SoundcloudAlbumTrack(SoundcloudTrack): soundcloud_track: SoundcloudTrack, playlist_metadata: PlaylistMetadata, ) -> "SoundcloudAlbumTrack": + """ + Parameters + ---------- + album: + Album name + album_year: + Album year + soundcloud_track: + Track to convert to an album track + playlist_metadata: + Metadata for playlist ordering + + Returns + ------- + SoundcloudTrack converted to a SoundcloudAlbumTrack + """ return SoundcloudAlbumTrack( entry_dict=soundcloud_track._kwargs, # pylint: disable=protected-access working_directory=soundcloud_track.working_directory(), @@ -91,7 +107,9 @@ class SoundcloudAlbum(Entry): def album_tracks(self, skip_premiere_tracks: bool = True) -> List[SoundcloudAlbumTrack]: """ - Returns all tracks in the album represented as album-tracks. They will share the + Returns + ------- + All tracks in the album represented as album-tracks. They will share the same album name, have ordered track numbers, and a shared album year. """ tracks = [ @@ -118,17 +136,26 @@ class SoundcloudAlbum(Entry): @property def album_year(self) -> int: - """Returns the album's year, computed by the max upload year amongst all album tracks""" + """ + Returns + ------- + The album's year, computed by the max upload year amongst all album tracks + """ return max(track.upload_year for track in self._single_tracks) @property def track_count(self) -> int: + """ + Returns + ------- + Number of tracks in the album (technically a playlist) + """ return self.kwargs("playlist_count") - @property - def downloaded_track_count(self) -> int: - return len(self.kwargs("entries")) - def contains(self, track: SoundcloudTrack) -> bool: - """Returns whether the album contains a track""" + """ + Returns + ------- + True if this album contains this track. False otherwise. + """ return any(track.uid == t.uid for t in self._single_tracks) diff --git a/src/ytdl_sub/subscriptions/subscription.py b/src/ytdl_sub/subscriptions/subscription.py index f851c429..54dbe718 100644 --- a/src/ytdl_sub/subscriptions/subscription.py +++ b/src/ytdl_sub/subscriptions/subscription.py @@ -207,7 +207,7 @@ class Subscription: date_range=self.output_options.delete_stale_files.get_date_range() ) - self._enhanced_download_archive.save_download_archive() + self._enhanced_download_archive.save_download_mappings() def _initialize_plugins(self) -> List[Plugin]: """ diff --git a/src/ytdl_sub/validators/string_formatter_validators.py b/src/ytdl_sub/validators/string_formatter_validators.py index ae5e0678..e1b7df97 100644 --- a/src/ytdl_sub/validators/string_formatter_validators.py +++ b/src/ytdl_sub/validators/string_formatter_validators.py @@ -31,7 +31,7 @@ class StringFormatterValidator(Validator): List of format variables in the format string Raises - ------- + ------ ValidationException If the format string contains invalid variable formatting """ diff --git a/src/ytdl_sub/ytdl_additions/enhanced_download_archive.py b/src/ytdl_sub/ytdl_additions/enhanced_download_archive.py index be4471aa..7db054c7 100644 --- a/src/ytdl_sub/ytdl_additions/enhanced_download_archive.py +++ b/src/ytdl_sub/ytdl_additions/enhanced_download_archive.py @@ -34,6 +34,16 @@ class DownloadMapping: @classmethod def from_dict(cls, mapping_dict: dict) -> "DownloadMapping": + """ + Parameters + ---------- + mapping_dict + Download mapping in dict format + + Returns + ------- + Instantiated DownloadMapping class + """ return DownloadMapping( upload_date=mapping_dict["upload_date"], extractor=mapping_dict["extractor"], @@ -42,6 +52,16 @@ class DownloadMapping: @classmethod def from_entry(cls, entry: Entry) -> "DownloadMapping": + """ + Parameters + ---------- + entry + Entry to create a download mapping for + + Returns + ------- + DownloadMapping for the entry + """ return DownloadMapping( upload_date=entry.upload_date_standardized, extractor=entry.extractor, @@ -55,30 +75,70 @@ class DownloadArchive: possible in case of future changes. """ - def __init__(self): - self._download_archive_lines: List[str] = [] - - @classmethod - def from_lines(cls, lines: List[str]) -> "DownloadArchive": - download_archive = DownloadArchive() - download_archive._download_archive_lines = lines - return download_archive + def __init__(self, download_archive_lines: List[str]): + """ + Parameters + ---------- + download_archive_lines + Lines found in a YTDL download archive file, i.e. youtube id-32342343423 + """ + self._download_archive_lines = download_archive_lines @classmethod def from_file(cls, file_path: str) -> "DownloadArchive": + """ + Parameters + ---------- + file_path + Path to a download archive file + + Returns + ------- + Instantiated DownloadArchive class + """ with open(file_path, "r", encoding="utf8") as file: - return cls.from_lines(lines=file.readlines()) + return cls(download_archive_lines=file.readlines()) def to_file(self, file_path: str) -> "DownloadArchive": + """ + Parameters + ---------- + file_path + File path to store this download archive to + + Returns + ------- + self + """ with open(file_path, "w", encoding="utf8") as file: for line in self._download_archive_lines: file.write(f"{line}\n") return self def contains(self, entry_id: str) -> bool: + """ + Parameters + ---------- + entry_id + Id of the entry + + Returns + ------- + True if the entry id is within this download archive. False otherwise. + """ return any(entry_id in line for line in self._download_archive_lines) def remove_entry(self, entry_id: str) -> "DownloadArchive": + """ + Parameters + ---------- + entry_id + Entry ID to remove if it exists in this download archive + + Returns + ------- + self + """ self._download_archive_lines = [ line for line in self._download_archive_lines if entry_id not in line ] @@ -89,10 +149,23 @@ class DownloadMappings: _strptime_format = "%Y-%m-%d" def __init__(self): + """ + Initializes an empty mapping + """ self._entry_mappings: Dict[str, DownloadMapping] = {} @classmethod def from_file(cls, json_file_path: str) -> "DownloadMappings": + """ + Parameters + ---------- + json_file_path + Path to a json file that contains download mappings + + Returns + ------- + Instantiated DownloadMappings class + """ with open(json_file_path, "r", encoding="utf8") as json_file: entry_mappings_json = json.load(json_file) @@ -107,13 +180,37 @@ class DownloadMappings: @property def entry_ids(self) -> List[str]: + """ + Returns + ------- + List of entry ids in the mapping + """ return list(self._entry_mappings.keys()) @property def is_empty(self) -> bool: + """ + Returns + ------- + True if there are no entry mappings. False otherwise. + """ return len(self._entry_mappings) == 0 def add_entry(self, entry: Entry, entry_file_path: str) -> "DownloadMappings": + """ + Adds a file path for the entry. An entry can map to multiple file paths. + + Parameters + ---------- + entry + Entry that this file belongs to + entry_file_path + Relative path to the file that belongs to the entry + + Returns + ------- + self + """ if entry.uid not in self.entry_ids: self._entry_mappings[entry.uid] = DownloadMapping.from_entry(entry=entry) @@ -121,14 +218,30 @@ class DownloadMappings: return self def remove_entry(self, entry_id: str) -> "DownloadMappings": + """ + Parameters + ---------- + entry_id + Id of the entry to remove + + Returns + ------- + self + """ if entry_id in self.entry_ids: del self._entry_mappings[entry_id] return self def get_entries_out_of_range(self, date_range: DateRange) -> Dict[str, DownloadMapping]: """ - :param date_range: range of dates that entries' upload dates must be within - :return: dict of entry_id: mapping if the upload date is not in the date range + Parameters + ---------- + date_range + Range of dates that entries' upload dates must be within + + Returns + ------- + Dict of entry_id: mapping if the upload date is not in the date range """ out_of_range_entry_mappings = copy.deepcopy(self._entry_mappings) for uid in list(out_of_range_entry_mappings.keys()): @@ -143,7 +256,16 @@ class DownloadMappings: return out_of_range_entry_mappings def to_file(self, output_json_file: str) -> "DownloadMappings": + """ + Parameters + ---------- + output_json_file + Output json file path to write the download mappings to + Returns + ------- + self + """ # Create json string first to ensure it is valid before writing anything to file json_str = json.dumps( obj={ @@ -163,11 +285,17 @@ class DownloadMappings: return self def to_download_archive(self) -> DownloadArchive: + """ + Returns + ------- + A DownloadArchive created from the DownloadMappings' ids and extractors. YTDL will use this + to avoid redownloading entries already downloaded. + """ lines: List[str] = [] for entry_id, metadata in self._entry_mappings.items(): lines.append(f"{metadata.extractor} {entry_id}") - return DownloadArchive.from_lines(lines) + return DownloadArchive(download_archive_lines=lines) class EnhancedDownloadArchive: @@ -211,39 +339,64 @@ class EnhancedDownloadArchive: @property def archive_file_name(self) -> str: """ - :return: The download archive's file name (no path) + Returns + ------- + The download archive's file name (no path) """ return f".ytdl-subscribe-{self.subscription_name}-download-archive.txt" @property def _mapping_file_name(self) -> str: """ - :return: The download mapping's file name (no path) + Returns + ------- + The download mapping's file name (no path) """ return f".ytdl-subscribe-{self.subscription_name}-download-mapping.json" @property def _mapping_output_file_path(self): """ - :return: The download mapping's file path in the output directory. + Returns + ------- + The download mapping's file path in the output directory. """ return str(Path(self.output_directory) / self._mapping_file_name) @property def _archive_working_file_path(self) -> str: """ - :return: The download archive's file path in the working directory. + Returns + ------- + The download archive's file path in the working directory. """ return str(Path(self.working_directory) / self.archive_file_name) @property def mapping(self) -> DownloadMappings: + """ + Returns + ------- + Loaded DownloadMappings + + Raises + ------ + ValueError + If the download mappings was not loaded + """ if self._download_mapping is None: raise ValueError("Tried to use download mapping before it was loaded") return self._download_mapping def _load(self) -> "EnhancedDownloadArchive": + """ + Tries to load download mappings if they are present in the output directory. + If they are not, initialize an empty mapping. + Returns + ------- + self + """ # If a mapping file exists in the output directory, load it up. if os.path.isfile(self._mapping_output_file_path): self._download_mapping = DownloadMappings.from_file( @@ -257,6 +410,14 @@ class EnhancedDownloadArchive: return self def _copy_to_working_directory(self) -> "EnhancedDownloadArchive": + """ + If the mapping is not empty, create a download archive from it and save it into the + working directory. This will tell YTDL to not redownload already downloaded entries. + + Returns + ------- + self + """ # If the download mapping is empty, do nothing since the ytdl downloader will create a new # download archive file if self.mapping.is_empty: @@ -268,11 +429,32 @@ class EnhancedDownloadArchive: return self def prepare_download_archive(self) -> "EnhancedDownloadArchive": + """ + Helper function to load mappings and create a YTDL download archive file in the + working directory if mappings exist. + + Returns + ------- + self + """ self._load() self._copy_to_working_directory() return self def remove_stale_files(self, date_range: DateRange) -> "EnhancedDownloadArchive": + """ + Checks all entries within the mappings. If any entries' upload dates are not within the + provided date range, delete them. + + Parameters + ---------- + date_range + Date range the upload date must be in to not get deleted + + Returns + ------- + self + """ stale_mappings: Dict[str, DownloadMapping] = self.mapping.get_entries_out_of_range( date_range=date_range ) @@ -289,7 +471,15 @@ class EnhancedDownloadArchive: return self - def save_download_archive(self) -> "EnhancedDownloadArchive": + def save_download_mappings(self) -> "EnhancedDownloadArchive": + """ + Saves the updated download mappings to the output directory + + Returns + ------- + self + """ + # TODO: Make sure this logic is actually right... # Load the download archive from the working directory, which should contain any past # and new entries downloaded in this session self._download_archive = DownloadArchive.from_file(self._archive_working_file_path) diff --git a/tools/linter b/tools/linter index 9e9bc054..480ce4ac 100755 --- a/tools/linter +++ b/tools/linter @@ -2,3 +2,4 @@ isort . black . pylint src/ +pydocstyle src/*