diff --git a/src/ytdl_sub/config/overrides.py b/src/ytdl_sub/config/overrides.py index 9ad6ca56..76092735 100644 --- a/src/ytdl_sub/config/overrides.py +++ b/src/ytdl_sub/config/overrides.py @@ -7,8 +7,8 @@ import mergedeep from ytdl_sub.entries.entry import Entry from ytdl_sub.entries.script.variable_definitions import VARIABLES +from ytdl_sub.entries.variables.override_variables import REQUIRED_OVERRIDE_VARIABLE_NAMES from ytdl_sub.entries.variables.override_variables import OverrideHelpers -from ytdl_sub.entries.variables.override_variables import SubscriptionVariables from ytdl_sub.script.parser import parse from ytdl_sub.script.script import Script from ytdl_sub.script.utils.exceptions import ScriptVariableNotResolved @@ -62,6 +62,7 @@ class Overrides(DictFormatterValidator, Scriptable): self.ensure_variable_name_valid(key) self.unresolvable.add(VARIABLES.entry_metadata.variable_name) + self.unresolvable.update(REQUIRED_OVERRIDE_VARIABLE_NAMES) def ensure_added_plugin_variable_valid(self, added_variable: str) -> bool: """ diff --git a/src/ytdl_sub/config/validators/variable_validation.py b/src/ytdl_sub/config/validators/variable_validation.py index b838c714..466762d9 100644 --- a/src/ytdl_sub/config/validators/variable_validation.py +++ b/src/ytdl_sub/config/validators/variable_validation.py @@ -13,7 +13,6 @@ from ytdl_sub.config.plugin.preset_plugins import PresetPlugins from ytdl_sub.config.preset_options import OutputOptions from ytdl_sub.config.validators.options import OptionsValidator from ytdl_sub.downloaders.url.validators import MultiUrlValidator -from ytdl_sub.entries.variables.override_variables import REQUIRED_OVERRIDE_VARIABLE_DEFINITIONS from ytdl_sub.entries.variables.override_variables import REQUIRED_OVERRIDE_VARIABLE_NAMES from ytdl_sub.script.script import Script from ytdl_sub.script.script import _is_function @@ -106,9 +105,7 @@ class VariableValidation: """ Do some gymnastics to initialize the Overrides script. """ - override_variables = ( - set(list(overrides.initial_variables().keys())) | REQUIRED_OVERRIDE_VARIABLE_NAMES - ) + override_variables = set(list(overrides.initial_variables().keys())) # Set resolved variables as all entry + override variables # at this point to generate every possible added/modified variable diff --git a/src/ytdl_sub/entries/variables/override_variables.py b/src/ytdl_sub/entries/variables/override_variables.py index 8eee70f1..68ec72d4 100644 --- a/src/ytdl_sub/entries/variables/override_variables.py +++ b/src/ytdl_sub/entries/variables/override_variables.py @@ -8,7 +8,6 @@ from ytdl_sub.entries.script.variable_types import MapVariable from ytdl_sub.entries.script.variable_types import StringVariable from ytdl_sub.entries.script.variable_types import Variable from ytdl_sub.script.functions import Functions -from ytdl_sub.script.types.syntax_tree import SyntaxTree from ytdl_sub.script.utils.name_validation import is_valid_name # TODO: use this diff --git a/src/ytdl_sub/subscriptions/base_subscription.py b/src/ytdl_sub/subscriptions/base_subscription.py index c7872e8a..28d590db 100644 --- a/src/ytdl_sub/subscriptions/base_subscription.py +++ b/src/ytdl_sub/subscriptions/base_subscription.py @@ -47,21 +47,19 @@ class BaseSubscription(ABC): self.name = name self._config_options = config_options self._preset_options = preset_options - - migrated_file_name: Optional[str] = None - if migrated_file_name_option := self.output_options.migrated_download_archive_name: - migrated_file_name = self.overrides.apply_formatter(migrated_file_name_option) - - # TODO: Do not include this as part of the subscription - self._enhanced_download_archive = EnhancedDownloadArchive( - file_name=self.overrides.apply_formatter(self.output_options.download_archive_name), - working_directory=self.working_directory, - output_directory=self.output_directory, - migrated_file_name=migrated_file_name, - ) - + self._enhanced_download_archive: Optional[EnhancedDownloadArchive] = None self._exception: Optional[Exception] = None + @property + def download_archive(self) -> EnhancedDownloadArchive: + """ + Returns + ------- + Initialized download archive + """ + assert self._enhanced_download_archive is not None + return self._enhanced_download_archive + @property def downloader_options(self) -> MultiUrlValidator: """ @@ -141,7 +139,7 @@ class BaseSubscription(ABC): ------- Number of entries added """ - return self._enhanced_download_archive.num_entries_added + return self.download_archive.num_entries_added @property def num_entries_modified(self) -> int: @@ -150,7 +148,7 @@ class BaseSubscription(ABC): ------- Number of entries modified """ - return self._enhanced_download_archive.num_entries_modified + return self.download_archive.num_entries_modified @property def num_entries_removed(self) -> int: @@ -159,7 +157,7 @@ class BaseSubscription(ABC): ------- Number of entries removed """ - return self._enhanced_download_archive.num_entries_removed + return self.download_archive.num_entries_removed @property def num_entries(self) -> int: @@ -168,7 +166,7 @@ class BaseSubscription(ABC): ------- The number of entries """ - return self._enhanced_download_archive.num_entries + return self.download_archive.num_entries @property def transaction_log(self) -> FileHandlerTransactionLog: @@ -177,7 +175,7 @@ class BaseSubscription(ABC): ------- Transaction log from the subscription """ - return self._enhanced_download_archive.get_file_handler_transaction_log() + return self.download_archive.get_file_handler_transaction_log() @property def exception(self) -> Optional[Exception]: diff --git a/src/ytdl_sub/subscriptions/subscription_download.py b/src/ytdl_sub/subscriptions/subscription_download.py index 076096b7..382e25ce 100644 --- a/src/ytdl_sub/subscriptions/subscription_download.py +++ b/src/ytdl_sub/subscriptions/subscription_download.py @@ -26,6 +26,7 @@ from ytdl_sub.utils.file_handler import FileHandler from ytdl_sub.utils.file_handler import FileHandlerTransactionLog from ytdl_sub.utils.file_handler import FileMetadata from ytdl_sub.utils.logger import Logger +from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive logger: logging.Logger = Logger.get() @@ -68,7 +69,7 @@ class SubscriptionDownload(BaseSubscription, ABC): output_file_name = self.overrides.apply_formatter( formatter=self.output_options.file_name, entry=entry ) - self._enhanced_download_archive.save_file_to_output_directory( + self.download_archive.save_file_to_output_directory( file_name=entry.get_download_file_name(), file_metadata=entry_metadata, output_file_name=output_file_name, @@ -82,7 +83,7 @@ class SubscriptionDownload(BaseSubscription, ABC): ) # Copy the thumbnails since they could be used later for other things - self._enhanced_download_archive.save_file_to_output_directory( + self.download_archive.save_file_to_output_directory( file_name=entry.get_download_thumbnail_name(), output_file_name=output_thumbnail_name, entry=entry, @@ -102,7 +103,7 @@ class SubscriptionDownload(BaseSubscription, ABC): if not dry_run: entry.write_info_json() - self._enhanced_download_archive.save_file_to_output_directory( + self.download_archive.save_file_to_output_directory( file_name=entry.get_download_info_json_name(), output_file_name=output_info_json_name, entry=entry, @@ -136,7 +137,7 @@ class SubscriptionDownload(BaseSubscription, ABC): Context manager to initialize the enhanced download archive """ if self.maintain_download_archive: - self._enhanced_download_archive.prepare_download_archive() + self.download_archive.prepare_download_archive() yield @@ -157,19 +158,19 @@ class SubscriptionDownload(BaseSubscription, ABC): ) if date_range_to_keep or self.output_options.keep_max_files is not None: - self._enhanced_download_archive.remove_stale_files( + self.download_archive.remove_stale_files( date_range=date_range_to_keep, keep_max_files=keep_max_files ) - self._enhanced_download_archive.save_download_mappings() - FileHandler.delete(self._enhanced_download_archive.working_file_path) + self.download_archive.save_download_mappings() + FileHandler.delete(self.download_archive.working_file_path) @contextlib.contextmanager def _remove_empty_directories_in_output_directory(self): try: yield finally: - if not self._enhanced_download_archive.is_dry_run: + if not self.download_archive.is_dry_run: for root, dir_names, _ in os.walk(Path(self.output_directory), topdown=False): for dir_name in dir_names: dir_path = Path(root) / dir_name @@ -195,7 +196,7 @@ class SubscriptionDownload(BaseSubscription, ABC): plugin_type( options=plugin_options, overrides=self.overrides, - enhanced_download_archive=self._enhanced_download_archive, + enhanced_download_archive=self.download_archive, ) for plugin_type, plugin_options in self.plugins.zipped() ] @@ -234,7 +235,7 @@ class SubscriptionDownload(BaseSubscription, ABC): # Re-save the download archive after each entry is moved to the output directory if self.maintain_download_archive: - self._enhanced_download_archive.save_download_mappings() + self.download_archive.save_download_mappings() def _process_entry( self, plugins: List[Plugin], dry_run: bool, entry: Entry, entry_metadata: FileMetadata @@ -324,17 +325,35 @@ class SubscriptionDownload(BaseSubscription, ABC): for plugin in plugins: plugin.post_process_subscription() - return self._enhanced_download_archive.get_file_handler_transaction_log() + return self.download_archive.get_file_handler_transaction_log() - def _initialize_subscription_overrides(self): - required_overrides = { - SubscriptionVariables.subscription_name(): self.name, - SubscriptionVariables.subscription_has_download_archive(): f"""{{ - %bool({self._enhanced_download_archive.num_entries > 0}) + def _initialize_subscription_overrides_pre_archive(self) -> None: + self.overrides.add( + { + SubscriptionVariables.subscription_name(): self.name, + } + ) + + def _initialize_subscription_overrides_post_archive(self) -> None: + self.overrides.add( + { + SubscriptionVariables.subscription_has_download_archive(): f"""{{ + %bool({self.download_archive.num_entries > 0}) }}""", - } - assert SubscriptionVariables - self.overrides.add(required_overrides) + } + ) + + def _initialize_download_archive(self, dry_run: bool) -> None: + migrated_file_name: Optional[str] = None + if migrated_file_name_option := self.output_options.migrated_download_archive_name: + migrated_file_name = self.overrides.apply_formatter(migrated_file_name_option) + + self._enhanced_download_archive = EnhancedDownloadArchive( + file_name=self.overrides.apply_formatter(self.output_options.download_archive_name), + working_directory=self.working_directory, + output_directory=self.output_directory, + migrated_file_name=migrated_file_name, + ).reinitialize(dry_run=dry_run) def download(self, dry_run: bool = False) -> FileHandlerTransactionLog: """ @@ -347,15 +366,17 @@ class SubscriptionDownload(BaseSubscription, ABC): directory. """ self._exception = None - self._enhanced_download_archive.reinitialize(dry_run=dry_run) - self._initialize_subscription_overrides() + self._initialize_subscription_overrides_pre_archive() + self._initialize_download_archive(dry_run=dry_run) + self._initialize_subscription_overrides_post_archive() + plugins = self._initialize_plugins() subscription_ytdl_options = SubscriptionYTDLOptions( preset=self._preset_options, plugins=plugins, - enhanced_download_archive=self._enhanced_download_archive, + enhanced_download_archive=self.download_archive, overrides=self.overrides, working_directory=self.working_directory, dry_run=dry_run, @@ -363,7 +384,7 @@ class SubscriptionDownload(BaseSubscription, ABC): downloader = MultiUrlDownloader( options=self.downloader_options, - enhanced_download_archive=self._enhanced_download_archive, + enhanced_download_archive=self.download_archive, download_ytdl_options=subscription_ytdl_options.download_builder(), metadata_ytdl_options=subscription_ytdl_options.metadata_builder(), overrides=self.overrides, @@ -401,14 +422,17 @@ class SubscriptionDownload(BaseSubscription, ABC): If true, do not modify any video/audio files or move anything to the output directory. """ self._exception = None - self._enhanced_download_archive.reinitialize(dry_run=dry_run) + + self._initialize_subscription_overrides_pre_archive() + self._initialize_download_archive(dry_run=dry_run) + self._initialize_subscription_overrides_post_archive() plugins = self._initialize_plugins() subscription_ytdl_options = SubscriptionYTDLOptions( preset=self._preset_options, plugins=plugins, - enhanced_download_archive=self._enhanced_download_archive, + enhanced_download_archive=self.download_archive, overrides=self.overrides, working_directory=self.working_directory, dry_run=dry_run, @@ -418,7 +442,7 @@ class SubscriptionDownload(BaseSubscription, ABC): plugins.extend( MultiUrlDownloader( options=self.downloader_options, - enhanced_download_archive=self._enhanced_download_archive, + enhanced_download_archive=self.download_archive, download_ytdl_options=subscription_ytdl_options.download_builder(), metadata_ytdl_options=subscription_ytdl_options.metadata_builder(), overrides=self.overrides, @@ -427,7 +451,7 @@ class SubscriptionDownload(BaseSubscription, ABC): downloader = InfoJsonDownloader( options=InfoJsonDownloaderOptions(name="no-op", value={}), - enhanced_download_archive=self._enhanced_download_archive, + enhanced_download_archive=self.download_archive, download_ytdl_options=YTDLOptionsBuilder(), metadata_ytdl_options=YTDLOptionsBuilder(), overrides=self.overrides,