maintain download archive

config.yaml

@@ -53,6 +53,7 @@ presets:
       file_name: "{output_file_name}.{ext}"
       convert_thumbnail: "jpeg"
       thumbnail_name: "{output_file_name}.jpeg"
+      maintain_download_archive: True
     metadata_options:
       nfo:
         nfo_name: "{output_file_name}.nfo"

ytdl_subscribe/downloaders/downloader.py

@@ -24,7 +24,12 @@ class Downloader:
         return {}
 
     @classmethod
-    def _configure_ytdl_options(cls, ytdl_options: Optional[Dict], working_directory: str) -> Dict:
+    def _configure_ytdl_options(
+        cls,
+        working_directory: str,
+        ytdl_options: Optional[Dict],
+        download_archive_file_name: Optional[str],
+    ) -> Dict:
         """Configure the ytdl options for the downloader"""
         if ytdl_options is None:
             ytdl_options = {}
@@ -35,14 +40,19 @@ class Downloader:
         # Overwrite defaults + input with global options
         ytdl_options = dict(ytdl_options, **cls.ytdl_option_overrides())
 
-        # Finally overwrite the output location with the specified working directory
+        # Overwrite the output location with the specified working directory
         ytdl_options["outtmpl"] = str(Path(working_directory) / "%(id)s.%(ext)s")
+
+        # If a download archive file name is provided, set it to that
+        ytdl_options["download_archive"] = str(Path(working_directory) / download_archive_file_name)
+
         return ytdl_options
 
     def __init__(
         self,
         output_directory: str,
         ytdl_options: Optional[Dict] = None,
+        download_archive_file_name: Optional[str] = None,
     ):
         self.output_directory = output_directory
         if self.output_directory is None:
@@ -51,6 +61,7 @@ class Downloader:
         self.ytdl_options = Downloader._configure_ytdl_options(
             ytdl_options=ytdl_options,
             working_directory=self.output_directory,
+            download_archive_file_name=download_archive_file_name,
         )
 
     @contextmanager

ytdl_subscribe/downloaders/youtube_downloader.py

@@ -37,7 +37,6 @@ class YoutubeDownloader(Downloader):
         not fetch the metadata (maybe there is a way??)
         """
         ytdl_metadata_override = {
-            "download_archive": str(Path(self.output_directory) / "ytdl-download-archive.txt"),
             "writeinfojson": True,
         }
 

ytdl_subscribe/entries/entry.py

@@ -1,4 +1,3 @@
-from collections import OrderedDict
 from pathlib import Path
 from typing import Any
 from typing import Dict
@@ -8,7 +7,6 @@ from sanitize_filename import sanitize
 
 from ytdl_subscribe.validators.base.string_formatter_validators import StringFormatterValidator
 from ytdl_subscribe.validators.config.overrides.overrides_validator import OverridesValidator
-from ytdl_subscribe.validators.exceptions import ValidationException
 
 
 class Entry:

ytdl_subscribe/main.py

@@ -51,7 +51,7 @@ if __name__ == "__main__":
             )
 
         for subscription in subscriptions:
-            subscription.to_subscription().extract_info()
+            subscription.to_subscription().download()
 
         print("hi")
         # for subscription_path in subscription_paths:

ytdl_subscribe/subscriptions/soundcloud.py

@@ -12,7 +12,7 @@ from ytdl_subscribe.validators.config.source_options.soundcloud_validators impor
 class SoundcloudAlbumsAndSinglesSubscription(
     Subscription[SoundcloudAlbumsAndSinglesSourceValidator]
 ):
-    def extract_info(self):
+    def _extract_info(self):
         tracks: List[SoundcloudTrack] = []
         downloader = self.get_downloader(SoundcloudDownloader)
 

ytdl_subscribe/subscriptions/subscription.py

@@ -1,4 +1,5 @@
 import os
+import shutil
 from abc import ABC
 from pathlib import Path
 from shutil import copyfile
@@ -61,13 +62,16 @@ class Subscription(Generic[S], ABC):
         self, downloader_type: Type[D], source_ytdl_options: Optional[Dict] = None
     ) -> D:
         """Returns the downloader that will be used to download media for this subscription"""
-
         # if source_ytdl_options are present, override the ytdl_options with them
         ytdl_options = self.__preset_options.ytdl_options.dict
         if source_ytdl_options:
             ytdl_options = dict(ytdl_options, **source_ytdl_options)
 
-        return downloader_type(output_directory=self.working_directory, ytdl_options=ytdl_options)
+        return downloader_type(
+            output_directory=self.working_directory,
+            ytdl_options=ytdl_options,
+            download_archive_file_name=self.download_archive_file_name,
+        )
 
     @property
     def output_options(self) -> OutputOptionsValidator:
@@ -89,6 +93,14 @@ class Subscription(Generic[S], ABC):
         """Returns the directory that the downloader saves files to"""
         return str(Path(self.__config_options.working_directory.value) / Path(self.name))
 
+    @property
+    def output_directory(self):
+        return self._apply_formatter(formatter=self.output_options.output_directory)
+
+    @property
+    def download_archive_file_name(self):
+        return f".ytdl-subscribe-{self.name}-download-archive.txt"
+
     def _apply_formatter(
         self, formatter: StringFormatterValidator, entry: Optional[Entry] = None
     ) -> str:
@@ -139,7 +151,23 @@ class Subscription(Generic[S], ABC):
         with open(nfo_file_path, "wb") as nfo_file:
             nfo_file.write(xml)
 
-    def extract_info(self):
+    def download(self):
+        """
+        Performs the subscription download.
+        """
+        existing_download_archive = Path(self.output_directory) / self.download_archive_file_name
+        working_directory_archive = Path(self.working_directory) / self.download_archive_file_name
+
+        if self.output_options.maintain_download_archive:
+            if os.path.exists(existing_download_archive):
+                shutil.copy(existing_download_archive, working_directory_archive)
+
+        self._extract_info()
+
+        if self.output_options.maintain_download_archive:
+            shutil.copy(working_directory_archive, existing_download_archive)
+
+    def _extract_info(self):
         """
         Extracts only the info of the source, does not download it
         """

ytdl_subscribe/subscriptions/youtube.py

@@ -14,7 +14,7 @@ from ytdl_subscribe.validators.config.source_options.youtube_validators import (
 
 
 class YoutubePlaylistSubscription(Subscription[YoutubePlaylistSourceValidator]):
-    def extract_info(self):
+    def _extract_info(self):
         entries = self.get_downloader(YoutubeDownloader).download_playlist(
             playlist_id=self.source_options.playlist_id.value
         )
@@ -24,7 +24,7 @@ class YoutubePlaylistSubscription(Subscription[YoutubePlaylistSourceValidator]):
 
 
 class YoutubeChannelSubscription(Subscription[YoutubeChannelSourceValidator]):
-    def extract_info(self):
+    def _extract_info(self):
         source_ytdl_options = {}
         if self.source_options.before or self.source_options.after:
             source_ytdl_options["daterange"] = DateRange(
@@ -40,7 +40,7 @@ class YoutubeChannelSubscription(Subscription[YoutubeChannelSourceValidator]):
 
 
 class YoutubeVideoSubscription(Subscription[YoutubeVideoSourceValidator]):
-    def extract_info(self):
+    def _extract_info(self):
         entry = self.get_downloader(YoutubeDownloader).download_video(
             video_id=self.source_options.video_id.value
         )

ytdl_subscribe/validators/config/output_options/output_options_validator.py

@@ -4,6 +4,7 @@ from ytdl_subscribe.validators.base.string_formatter_validators import (
 )
 from ytdl_subscribe.validators.base.string_formatter_validators import StringFormatterValidator
 from ytdl_subscribe.validators.base.string_select_validator import StringSelectValidator
+from ytdl_subscribe.validators.base.validators import BoolValidator
 
 
 class ConvertThumbnailValidator(StringSelectValidator):
@@ -16,7 +17,7 @@ class OutputOptionsValidator(StrictDictValidator):
     """Where to output the final files and thumbnails"""
 
     _required_keys = {"output_directory", "file_name"}
-    _optional_keys = {"convert_thumbnail", "thumbnail_name"}
+    _optional_keys = {"convert_thumbnail", "thumbnail_name", "maintain_download_archive"}
 
     def __init__(self, name, value):
         super().__init__(name, value)
@@ -37,3 +38,7 @@ class OutputOptionsValidator(StrictDictValidator):
         self.thumbnail_name = self._validate_key_if_present(
             key="thumbnail_name", validator=StringFormatterValidator
         )
+
+        self.maintain_download_archive = self._validate_key_if_present(
+            key="maintain_download_archive", validator=BoolValidator
+        )