diff --git a/config.yaml b/config.yaml index a03a7b6d..d6f16c20 100644 --- a/config.yaml +++ b/config.yaml @@ -53,6 +53,7 @@ presets: file_name: "{output_file_name}.{ext}" convert_thumbnail: "jpeg" thumbnail_name: "{output_file_name}.jpeg" + maintain_download_archive: True metadata_options: nfo: nfo_name: "{output_file_name}.nfo" diff --git a/ytdl_subscribe/downloaders/downloader.py b/ytdl_subscribe/downloaders/downloader.py index a67e6b79..f881122f 100644 --- a/ytdl_subscribe/downloaders/downloader.py +++ b/ytdl_subscribe/downloaders/downloader.py @@ -24,7 +24,12 @@ class Downloader: return {} @classmethod - def _configure_ytdl_options(cls, ytdl_options: Optional[Dict], working_directory: str) -> Dict: + def _configure_ytdl_options( + cls, + working_directory: str, + ytdl_options: Optional[Dict], + download_archive_file_name: Optional[str], + ) -> Dict: """Configure the ytdl options for the downloader""" if ytdl_options is None: ytdl_options = {} @@ -35,14 +40,19 @@ class Downloader: # Overwrite defaults + input with global options ytdl_options = dict(ytdl_options, **cls.ytdl_option_overrides()) - # Finally overwrite the output location with the specified working directory + # Overwrite the output location with the specified working directory ytdl_options["outtmpl"] = str(Path(working_directory) / "%(id)s.%(ext)s") + + # If a download archive file name is provided, set it to that + ytdl_options["download_archive"] = str(Path(working_directory) / download_archive_file_name) + return ytdl_options def __init__( self, output_directory: str, ytdl_options: Optional[Dict] = None, + download_archive_file_name: Optional[str] = None, ): self.output_directory = output_directory if self.output_directory is None: @@ -51,6 +61,7 @@ class Downloader: self.ytdl_options = Downloader._configure_ytdl_options( ytdl_options=ytdl_options, working_directory=self.output_directory, + download_archive_file_name=download_archive_file_name, ) @contextmanager diff --git a/ytdl_subscribe/downloaders/youtube_downloader.py b/ytdl_subscribe/downloaders/youtube_downloader.py index eae37fa7..cbcee617 100644 --- a/ytdl_subscribe/downloaders/youtube_downloader.py +++ b/ytdl_subscribe/downloaders/youtube_downloader.py @@ -37,7 +37,6 @@ class YoutubeDownloader(Downloader): not fetch the metadata (maybe there is a way??) """ ytdl_metadata_override = { - "download_archive": str(Path(self.output_directory) / "ytdl-download-archive.txt"), "writeinfojson": True, } diff --git a/ytdl_subscribe/entries/entry.py b/ytdl_subscribe/entries/entry.py index 67102e68..10219a3c 100644 --- a/ytdl_subscribe/entries/entry.py +++ b/ytdl_subscribe/entries/entry.py @@ -1,4 +1,3 @@ -from collections import OrderedDict from pathlib import Path from typing import Any from typing import Dict @@ -8,7 +7,6 @@ from sanitize_filename import sanitize from ytdl_subscribe.validators.base.string_formatter_validators import StringFormatterValidator from ytdl_subscribe.validators.config.overrides.overrides_validator import OverridesValidator -from ytdl_subscribe.validators.exceptions import ValidationException class Entry: diff --git a/ytdl_subscribe/main.py b/ytdl_subscribe/main.py index 4a5cdbb8..1aadf90b 100644 --- a/ytdl_subscribe/main.py +++ b/ytdl_subscribe/main.py @@ -51,7 +51,7 @@ if __name__ == "__main__": ) for subscription in subscriptions: - subscription.to_subscription().extract_info() + subscription.to_subscription().download() print("hi") # for subscription_path in subscription_paths: diff --git a/ytdl_subscribe/subscriptions/soundcloud.py b/ytdl_subscribe/subscriptions/soundcloud.py index 76b74385..7ae75cc4 100644 --- a/ytdl_subscribe/subscriptions/soundcloud.py +++ b/ytdl_subscribe/subscriptions/soundcloud.py @@ -12,7 +12,7 @@ from ytdl_subscribe.validators.config.source_options.soundcloud_validators impor class SoundcloudAlbumsAndSinglesSubscription( Subscription[SoundcloudAlbumsAndSinglesSourceValidator] ): - def extract_info(self): + def _extract_info(self): tracks: List[SoundcloudTrack] = [] downloader = self.get_downloader(SoundcloudDownloader) diff --git a/ytdl_subscribe/subscriptions/subscription.py b/ytdl_subscribe/subscriptions/subscription.py index 894d27e6..06d624b1 100644 --- a/ytdl_subscribe/subscriptions/subscription.py +++ b/ytdl_subscribe/subscriptions/subscription.py @@ -1,4 +1,5 @@ import os +import shutil from abc import ABC from pathlib import Path from shutil import copyfile @@ -61,13 +62,16 @@ class Subscription(Generic[S], ABC): self, downloader_type: Type[D], source_ytdl_options: Optional[Dict] = None ) -> D: """Returns the downloader that will be used to download media for this subscription""" - # if source_ytdl_options are present, override the ytdl_options with them ytdl_options = self.__preset_options.ytdl_options.dict if source_ytdl_options: ytdl_options = dict(ytdl_options, **source_ytdl_options) - return downloader_type(output_directory=self.working_directory, ytdl_options=ytdl_options) + return downloader_type( + output_directory=self.working_directory, + ytdl_options=ytdl_options, + download_archive_file_name=self.download_archive_file_name, + ) @property def output_options(self) -> OutputOptionsValidator: @@ -89,6 +93,14 @@ class Subscription(Generic[S], ABC): """Returns the directory that the downloader saves files to""" return str(Path(self.__config_options.working_directory.value) / Path(self.name)) + @property + def output_directory(self): + return self._apply_formatter(formatter=self.output_options.output_directory) + + @property + def download_archive_file_name(self): + return f".ytdl-subscribe-{self.name}-download-archive.txt" + def _apply_formatter( self, formatter: StringFormatterValidator, entry: Optional[Entry] = None ) -> str: @@ -139,7 +151,23 @@ class Subscription(Generic[S], ABC): with open(nfo_file_path, "wb") as nfo_file: nfo_file.write(xml) - def extract_info(self): + def download(self): + """ + Performs the subscription download. + """ + existing_download_archive = Path(self.output_directory) / self.download_archive_file_name + working_directory_archive = Path(self.working_directory) / self.download_archive_file_name + + if self.output_options.maintain_download_archive: + if os.path.exists(existing_download_archive): + shutil.copy(existing_download_archive, working_directory_archive) + + self._extract_info() + + if self.output_options.maintain_download_archive: + shutil.copy(working_directory_archive, existing_download_archive) + + def _extract_info(self): """ Extracts only the info of the source, does not download it """ diff --git a/ytdl_subscribe/subscriptions/youtube.py b/ytdl_subscribe/subscriptions/youtube.py index d45cfeac..e33df6c0 100644 --- a/ytdl_subscribe/subscriptions/youtube.py +++ b/ytdl_subscribe/subscriptions/youtube.py @@ -14,7 +14,7 @@ from ytdl_subscribe.validators.config.source_options.youtube_validators import ( class YoutubePlaylistSubscription(Subscription[YoutubePlaylistSourceValidator]): - def extract_info(self): + def _extract_info(self): entries = self.get_downloader(YoutubeDownloader).download_playlist( playlist_id=self.source_options.playlist_id.value ) @@ -24,7 +24,7 @@ class YoutubePlaylistSubscription(Subscription[YoutubePlaylistSourceValidator]): class YoutubeChannelSubscription(Subscription[YoutubeChannelSourceValidator]): - def extract_info(self): + def _extract_info(self): source_ytdl_options = {} if self.source_options.before or self.source_options.after: source_ytdl_options["daterange"] = DateRange( @@ -40,7 +40,7 @@ class YoutubeChannelSubscription(Subscription[YoutubeChannelSourceValidator]): class YoutubeVideoSubscription(Subscription[YoutubeVideoSourceValidator]): - def extract_info(self): + def _extract_info(self): entry = self.get_downloader(YoutubeDownloader).download_video( video_id=self.source_options.video_id.value ) diff --git a/ytdl_subscribe/validators/config/output_options/output_options_validator.py b/ytdl_subscribe/validators/config/output_options/output_options_validator.py index 8e356f6e..2a845e2d 100644 --- a/ytdl_subscribe/validators/config/output_options/output_options_validator.py +++ b/ytdl_subscribe/validators/config/output_options/output_options_validator.py @@ -4,6 +4,7 @@ from ytdl_subscribe.validators.base.string_formatter_validators import ( ) from ytdl_subscribe.validators.base.string_formatter_validators import StringFormatterValidator from ytdl_subscribe.validators.base.string_select_validator import StringSelectValidator +from ytdl_subscribe.validators.base.validators import BoolValidator class ConvertThumbnailValidator(StringSelectValidator): @@ -16,7 +17,7 @@ class OutputOptionsValidator(StrictDictValidator): """Where to output the final files and thumbnails""" _required_keys = {"output_directory", "file_name"} - _optional_keys = {"convert_thumbnail", "thumbnail_name"} + _optional_keys = {"convert_thumbnail", "thumbnail_name", "maintain_download_archive"} def __init__(self, name, value): super().__init__(name, value) @@ -37,3 +38,7 @@ class OutputOptionsValidator(StrictDictValidator): self.thumbnail_name = self._validate_key_if_present( key="thumbnail_name", validator=StringFormatterValidator ) + + self.maintain_download_archive = self._validate_key_if_present( + key="maintain_download_archive", validator=BoolValidator + )