working

src/ytdl_sub/config/overrides.py

@@ -7,8 +7,8 @@ import mergedeep
 
 from ytdl_sub.entries.entry import Entry
 from ytdl_sub.entries.script.variable_definitions import VARIABLES
+from ytdl_sub.entries.variables.override_variables import REQUIRED_OVERRIDE_VARIABLE_NAMES
 from ytdl_sub.entries.variables.override_variables import OverrideHelpers
-from ytdl_sub.entries.variables.override_variables import SubscriptionVariables
 from ytdl_sub.script.parser import parse
 from ytdl_sub.script.script import Script
 from ytdl_sub.script.utils.exceptions import ScriptVariableNotResolved
@@ -62,6 +62,7 @@ class Overrides(DictFormatterValidator, Scriptable):
             self.ensure_variable_name_valid(key)
 
         self.unresolvable.add(VARIABLES.entry_metadata.variable_name)
+        self.unresolvable.update(REQUIRED_OVERRIDE_VARIABLE_NAMES)
 
     def ensure_added_plugin_variable_valid(self, added_variable: str) -> bool:
         """

src/ytdl_sub/config/validators/variable_validation.py

@@ -13,7 +13,6 @@ from ytdl_sub.config.plugin.preset_plugins import PresetPlugins
 from ytdl_sub.config.preset_options import OutputOptions
 from ytdl_sub.config.validators.options import OptionsValidator
 from ytdl_sub.downloaders.url.validators import MultiUrlValidator
-from ytdl_sub.entries.variables.override_variables import REQUIRED_OVERRIDE_VARIABLE_DEFINITIONS
 from ytdl_sub.entries.variables.override_variables import REQUIRED_OVERRIDE_VARIABLE_NAMES
 from ytdl_sub.script.script import Script
 from ytdl_sub.script.script import _is_function
@@ -106,9 +105,7 @@ class VariableValidation:
         """
         Do some gymnastics to initialize the Overrides script.
         """
-        override_variables = (
+        override_variables = set(list(overrides.initial_variables().keys()))
-            set(list(overrides.initial_variables().keys())) | REQUIRED_OVERRIDE_VARIABLE_NAMES
-        )
 
         # Set resolved variables as all entry + override variables
         # at this point to generate every possible added/modified variable

src/ytdl_sub/entries/variables/override_variables.py

@@ -8,7 +8,6 @@ from ytdl_sub.entries.script.variable_types import MapVariable
 from ytdl_sub.entries.script.variable_types import StringVariable
 from ytdl_sub.entries.script.variable_types import Variable
 from ytdl_sub.script.functions import Functions
-from ytdl_sub.script.types.syntax_tree import SyntaxTree
 from ytdl_sub.script.utils.name_validation import is_valid_name
 
 # TODO: use this

src/ytdl_sub/subscriptions/base_subscription.py

@@ -47,21 +47,19 @@ class BaseSubscription(ABC):
         self.name = name
         self._config_options = config_options
         self._preset_options = preset_options
-
+        self._enhanced_download_archive: Optional[EnhancedDownloadArchive] = None
-        migrated_file_name: Optional[str] = None
-        if migrated_file_name_option := self.output_options.migrated_download_archive_name:
-            migrated_file_name = self.overrides.apply_formatter(migrated_file_name_option)
-
-        # TODO: Do not include this as part of the subscription
-        self._enhanced_download_archive = EnhancedDownloadArchive(
-            file_name=self.overrides.apply_formatter(self.output_options.download_archive_name),
-            working_directory=self.working_directory,
-            output_directory=self.output_directory,
-            migrated_file_name=migrated_file_name,
-        )
-
         self._exception: Optional[Exception] = None
 
+    @property
+    def download_archive(self) -> EnhancedDownloadArchive:
+        """
+        Returns
+        -------
+        Initialized download archive
+        """
+        assert self._enhanced_download_archive is not None
+        return self._enhanced_download_archive
+
     @property
     def downloader_options(self) -> MultiUrlValidator:
         """
@@ -141,7 +139,7 @@ class BaseSubscription(ABC):
         -------
         Number of entries added
         """
-        return self._enhanced_download_archive.num_entries_added
+        return self.download_archive.num_entries_added
 
     @property
     def num_entries_modified(self) -> int:
@@ -150,7 +148,7 @@ class BaseSubscription(ABC):
         -------
         Number of entries modified
         """
-        return self._enhanced_download_archive.num_entries_modified
+        return self.download_archive.num_entries_modified
 
     @property
     def num_entries_removed(self) -> int:
@@ -159,7 +157,7 @@ class BaseSubscription(ABC):
         -------
         Number of entries removed
         """
-        return self._enhanced_download_archive.num_entries_removed
+        return self.download_archive.num_entries_removed
 
     @property
     def num_entries(self) -> int:
@@ -168,7 +166,7 @@ class BaseSubscription(ABC):
         -------
         The number of entries
         """
-        return self._enhanced_download_archive.num_entries
+        return self.download_archive.num_entries
 
     @property
     def transaction_log(self) -> FileHandlerTransactionLog:
@@ -177,7 +175,7 @@ class BaseSubscription(ABC):
         -------
         Transaction log from the subscription
         """
-        return self._enhanced_download_archive.get_file_handler_transaction_log()
+        return self.download_archive.get_file_handler_transaction_log()
 
     @property
     def exception(self) -> Optional[Exception]:

src/ytdl_sub/subscriptions/subscription_download.py

@@ -26,6 +26,7 @@ from ytdl_sub.utils.file_handler import FileHandler
 from ytdl_sub.utils.file_handler import FileHandlerTransactionLog
 from ytdl_sub.utils.file_handler import FileMetadata
 from ytdl_sub.utils.logger import Logger
+from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive
 
 logger: logging.Logger = Logger.get()
 
@@ -68,7 +69,7 @@ class SubscriptionDownload(BaseSubscription, ABC):
         output_file_name = self.overrides.apply_formatter(
             formatter=self.output_options.file_name, entry=entry
         )
-        self._enhanced_download_archive.save_file_to_output_directory(
+        self.download_archive.save_file_to_output_directory(
             file_name=entry.get_download_file_name(),
             file_metadata=entry_metadata,
             output_file_name=output_file_name,
@@ -82,7 +83,7 @@ class SubscriptionDownload(BaseSubscription, ABC):
             )
 
             # Copy the thumbnails since they could be used later for other things
-            self._enhanced_download_archive.save_file_to_output_directory(
+            self.download_archive.save_file_to_output_directory(
                 file_name=entry.get_download_thumbnail_name(),
                 output_file_name=output_thumbnail_name,
                 entry=entry,
@@ -102,7 +103,7 @@ class SubscriptionDownload(BaseSubscription, ABC):
             if not dry_run:
                 entry.write_info_json()
 
-            self._enhanced_download_archive.save_file_to_output_directory(
+            self.download_archive.save_file_to_output_directory(
                 file_name=entry.get_download_info_json_name(),
                 output_file_name=output_info_json_name,
                 entry=entry,
@@ -136,7 +137,7 @@ class SubscriptionDownload(BaseSubscription, ABC):
         Context manager to initialize the enhanced download archive
         """
         if self.maintain_download_archive:
-            self._enhanced_download_archive.prepare_download_archive()
+            self.download_archive.prepare_download_archive()
 
         yield
 
@@ -157,19 +158,19 @@ class SubscriptionDownload(BaseSubscription, ABC):
                 )
 
             if date_range_to_keep or self.output_options.keep_max_files is not None:
-                self._enhanced_download_archive.remove_stale_files(
+                self.download_archive.remove_stale_files(
                     date_range=date_range_to_keep, keep_max_files=keep_max_files
                 )
 
-            self._enhanced_download_archive.save_download_mappings()
+            self.download_archive.save_download_mappings()
-            FileHandler.delete(self._enhanced_download_archive.working_file_path)
+            FileHandler.delete(self.download_archive.working_file_path)
 
     @contextlib.contextmanager
     def _remove_empty_directories_in_output_directory(self):
         try:
             yield
         finally:
-            if not self._enhanced_download_archive.is_dry_run:
+            if not self.download_archive.is_dry_run:
                 for root, dir_names, _ in os.walk(Path(self.output_directory), topdown=False):
                     for dir_name in dir_names:
                         dir_path = Path(root) / dir_name
@@ -195,7 +196,7 @@ class SubscriptionDownload(BaseSubscription, ABC):
             plugin_type(
                 options=plugin_options,
                 overrides=self.overrides,
-                enhanced_download_archive=self._enhanced_download_archive,
+                enhanced_download_archive=self.download_archive,
             )
             for plugin_type, plugin_options in self.plugins.zipped()
         ]
@@ -234,7 +235,7 @@ class SubscriptionDownload(BaseSubscription, ABC):
 
         # Re-save the download archive after each entry is moved to the output directory
         if self.maintain_download_archive:
-            self._enhanced_download_archive.save_download_mappings()
+            self.download_archive.save_download_mappings()
 
     def _process_entry(
         self, plugins: List[Plugin], dry_run: bool, entry: Entry, entry_metadata: FileMetadata
@@ -324,17 +325,35 @@ class SubscriptionDownload(BaseSubscription, ABC):
         for plugin in plugins:
             plugin.post_process_subscription()
 
-        return self._enhanced_download_archive.get_file_handler_transaction_log()
+        return self.download_archive.get_file_handler_transaction_log()
 
-    def _initialize_subscription_overrides(self):
+    def _initialize_subscription_overrides_pre_archive(self) -> None:
-        required_overrides = {
+        self.overrides.add(
-            SubscriptionVariables.subscription_name(): self.name,
+            {
-            SubscriptionVariables.subscription_has_download_archive(): f"""{{
+                SubscriptionVariables.subscription_name(): self.name,
-                %bool({self._enhanced_download_archive.num_entries > 0})
+            }
+        )
+
+    def _initialize_subscription_overrides_post_archive(self) -> None:
+        self.overrides.add(
+            {
+                SubscriptionVariables.subscription_has_download_archive(): f"""{{
+                %bool({self.download_archive.num_entries > 0})
             }}""",
-        }
+            }
-        assert SubscriptionVariables
+        )
-        self.overrides.add(required_overrides)
+
+    def _initialize_download_archive(self, dry_run: bool) -> None:
+        migrated_file_name: Optional[str] = None
+        if migrated_file_name_option := self.output_options.migrated_download_archive_name:
+            migrated_file_name = self.overrides.apply_formatter(migrated_file_name_option)
+
+        self._enhanced_download_archive = EnhancedDownloadArchive(
+            file_name=self.overrides.apply_formatter(self.output_options.download_archive_name),
+            working_directory=self.working_directory,
+            output_directory=self.output_directory,
+            migrated_file_name=migrated_file_name,
+        ).reinitialize(dry_run=dry_run)
 
     def download(self, dry_run: bool = False) -> FileHandlerTransactionLog:
         """
@@ -347,15 +366,17 @@ class SubscriptionDownload(BaseSubscription, ABC):
             directory.
         """
         self._exception = None
-        self._enhanced_download_archive.reinitialize(dry_run=dry_run)
 
-        self._initialize_subscription_overrides()
+        self._initialize_subscription_overrides_pre_archive()
+        self._initialize_download_archive(dry_run=dry_run)
+        self._initialize_subscription_overrides_post_archive()
+
         plugins = self._initialize_plugins()
 
         subscription_ytdl_options = SubscriptionYTDLOptions(
             preset=self._preset_options,
             plugins=plugins,
-            enhanced_download_archive=self._enhanced_download_archive,
+            enhanced_download_archive=self.download_archive,
             overrides=self.overrides,
             working_directory=self.working_directory,
             dry_run=dry_run,
@@ -363,7 +384,7 @@ class SubscriptionDownload(BaseSubscription, ABC):
 
         downloader = MultiUrlDownloader(
             options=self.downloader_options,
-            enhanced_download_archive=self._enhanced_download_archive,
+            enhanced_download_archive=self.download_archive,
             download_ytdl_options=subscription_ytdl_options.download_builder(),
             metadata_ytdl_options=subscription_ytdl_options.metadata_builder(),
             overrides=self.overrides,
@@ -401,14 +422,17 @@ class SubscriptionDownload(BaseSubscription, ABC):
             If true, do not modify any video/audio files or move anything to the output directory.
         """
         self._exception = None
-        self._enhanced_download_archive.reinitialize(dry_run=dry_run)
+
+        self._initialize_subscription_overrides_pre_archive()
+        self._initialize_download_archive(dry_run=dry_run)
+        self._initialize_subscription_overrides_post_archive()
 
         plugins = self._initialize_plugins()
 
         subscription_ytdl_options = SubscriptionYTDLOptions(
             preset=self._preset_options,
             plugins=plugins,
-            enhanced_download_archive=self._enhanced_download_archive,
+            enhanced_download_archive=self.download_archive,
             overrides=self.overrides,
             working_directory=self.working_directory,
             dry_run=dry_run,
@@ -418,7 +442,7 @@ class SubscriptionDownload(BaseSubscription, ABC):
         plugins.extend(
             MultiUrlDownloader(
                 options=self.downloader_options,
-                enhanced_download_archive=self._enhanced_download_archive,
+                enhanced_download_archive=self.download_archive,
                 download_ytdl_options=subscription_ytdl_options.download_builder(),
                 metadata_ytdl_options=subscription_ytdl_options.metadata_builder(),
                 overrides=self.overrides,
@@ -427,7 +451,7 @@ class SubscriptionDownload(BaseSubscription, ABC):
 
         downloader = InfoJsonDownloader(
             options=InfoJsonDownloaderOptions(name="no-op", value={}),
-            enhanced_download_archive=self._enhanced_download_archive,
+            enhanced_download_archive=self.download_archive,
             download_ytdl_options=YTDLOptionsBuilder(),
             metadata_ytdl_options=YTDLOptionsBuilder(),
             overrides=self.overrides,