danvics/ytdl-sub
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

working

Browse source
This commit is contained in:
Jesse Bannon 2024-03-30 15:59:27 -07:00
parent 7d6666a8f2
commit 3e4e2c0e28
5 changed files with 70 additions and 51 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
src/ytdl_sub/config/overrides.py
View file

@ -7,8 +7,8 @@ import mergedeep
from ytdl_sub.entries.entry import Entry
from ytdl_sub.entries.script.variable_definitions import VARIABLES
from ytdl_sub.entries.variables.override_variables import REQUIRED_OVERRIDE_VARIABLE_NAMES
from ytdl_sub.entries.variables.override_variables import OverrideHelpers
from ytdl_sub.entries.variables.override_variables import SubscriptionVariables
from ytdl_sub.script.parser import parse
from ytdl_sub.script.script import Script
from ytdl_sub.script.utils.exceptions import ScriptVariableNotResolved
@ -62,6 +62,7 @@ class Overrides(DictFormatterValidator, Scriptable):
self.ensure_variable_name_valid(key)
self.unresolvable.add(VARIABLES.entry_metadata.variable_name)
self.unresolvable.update(REQUIRED_OVERRIDE_VARIABLE_NAMES)
def ensure_added_plugin_variable_valid(self, added_variable: str) -> bool:
"""

5
src/ytdl_sub/config/validators/variable_validation.py
View file

@ -13,7 +13,6 @@ from ytdl_sub.config.plugin.preset_plugins import PresetPlugins
from ytdl_sub.config.preset_options import OutputOptions
from ytdl_sub.config.validators.options import OptionsValidator
from ytdl_sub.downloaders.url.validators import MultiUrlValidator
from ytdl_sub.entries.variables.override_variables import REQUIRED_OVERRIDE_VARIABLE_DEFINITIONS
from ytdl_sub.entries.variables.override_variables import REQUIRED_OVERRIDE_VARIABLE_NAMES
from ytdl_sub.script.script import Script
from ytdl_sub.script.script import _is_function
@ -106,9 +105,7 @@ class VariableValidation:
"""
Do some gymnastics to initialize the Overrides script.
"""
override_variables = (
set(list(overrides.initial_variables().keys())) | REQUIRED_OVERRIDE_VARIABLE_NAMES
)
override_variables = set(list(overrides.initial_variables().keys()))
# Set resolved variables as all entry + override variables
# at this point to generate every possible added/modified variable

1
src/ytdl_sub/entries/variables/override_variables.py
View file

@ -8,7 +8,6 @@ from ytdl_sub.entries.script.variable_types import MapVariable
from ytdl_sub.entries.script.variable_types import StringVariable
from ytdl_sub.entries.script.variable_types import Variable
from ytdl_sub.script.functions import Functions
from ytdl_sub.script.types.syntax_tree import SyntaxTree
from ytdl_sub.script.utils.name_validation import is_valid_name
# TODO: use this

34
src/ytdl_sub/subscriptions/base_subscription.py
View file

@ -47,21 +47,19 @@ class BaseSubscription(ABC):
self.name = name
self._config_options = config_options
self._preset_options = preset_options
migrated_file_name: Optional[str] = None
if migrated_file_name_option := self.output_options.migrated_download_archive_name:
migrated_file_name = self.overrides.apply_formatter(migrated_file_name_option)
# TODO: Do not include this as part of the subscription
self._enhanced_download_archive = EnhancedDownloadArchive(
file_name=self.overrides.apply_formatter(self.output_options.download_archive_name),
working_directory=self.working_directory,
output_directory=self.output_directory,
migrated_file_name=migrated_file_name,
)
self._enhanced_download_archive: Optional[EnhancedDownloadArchive] = None
self._exception: Optional[Exception] = None
@property
def download_archive(self) -> EnhancedDownloadArchive:
"""
Returns
-------
Initialized download archive
"""
assert self._enhanced_download_archive is not None
return self._enhanced_download_archive
@property
def downloader_options(self) -> MultiUrlValidator:
"""
@ -141,7 +139,7 @@ class BaseSubscription(ABC):
-------
Number of entries added
"""
return self._enhanced_download_archive.num_entries_added
return self.download_archive.num_entries_added
@property
def num_entries_modified(self) -> int:
@ -150,7 +148,7 @@ class BaseSubscription(ABC):
-------
Number of entries modified
"""
return self._enhanced_download_archive.num_entries_modified
return self.download_archive.num_entries_modified
@property
def num_entries_removed(self) -> int:
@ -159,7 +157,7 @@ class BaseSubscription(ABC):
-------
Number of entries removed
"""
return self._enhanced_download_archive.num_entries_removed
return self.download_archive.num_entries_removed
@property
def num_entries(self) -> int:
@ -168,7 +166,7 @@ class BaseSubscription(ABC):
-------
The number of entries
"""
return self._enhanced_download_archive.num_entries
return self.download_archive.num_entries
@property
def transaction_log(self) -> FileHandlerTransactionLog:
@ -177,7 +175,7 @@ class BaseSubscription(ABC):
-------
Transaction log from the subscription
"""
return self._enhanced_download_archive.get_file_handler_transaction_log()
return self.download_archive.get_file_handler_transaction_log()
@property
def exception(self) -> Optional[Exception]:

78
src/ytdl_sub/subscriptions/subscription_download.py
View file

@ -26,6 +26,7 @@ from ytdl_sub.utils.file_handler import FileHandler
from ytdl_sub.utils.file_handler import FileHandlerTransactionLog
from ytdl_sub.utils.file_handler import FileMetadata
from ytdl_sub.utils.logger import Logger
from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive
logger: logging.Logger = Logger.get()
@ -68,7 +69,7 @@ class SubscriptionDownload(BaseSubscription, ABC):
output_file_name = self.overrides.apply_formatter(
formatter=self.output_options.file_name, entry=entry
)
self._enhanced_download_archive.save_file_to_output_directory(
self.download_archive.save_file_to_output_directory(
file_name=entry.get_download_file_name(),
file_metadata=entry_metadata,
output_file_name=output_file_name,
@ -82,7 +83,7 @@ class SubscriptionDownload(BaseSubscription, ABC):
)
# Copy the thumbnails since they could be used later for other things
self._enhanced_download_archive.save_file_to_output_directory(
self.download_archive.save_file_to_output_directory(
file_name=entry.get_download_thumbnail_name(),
output_file_name=output_thumbnail_name,
entry=entry,
@ -102,7 +103,7 @@ class SubscriptionDownload(BaseSubscription, ABC):
if not dry_run:
entry.write_info_json()
self._enhanced_download_archive.save_file_to_output_directory(
self.download_archive.save_file_to_output_directory(
file_name=entry.get_download_info_json_name(),
output_file_name=output_info_json_name,
entry=entry,
@ -136,7 +137,7 @@ class SubscriptionDownload(BaseSubscription, ABC):
Context manager to initialize the enhanced download archive
"""
if self.maintain_download_archive:
self._enhanced_download_archive.prepare_download_archive()
self.download_archive.prepare_download_archive()
yield
@ -157,19 +158,19 @@ class SubscriptionDownload(BaseSubscription, ABC):
)
if date_range_to_keep or self.output_options.keep_max_files is not None:
self._enhanced_download_archive.remove_stale_files(
self.download_archive.remove_stale_files(
date_range=date_range_to_keep, keep_max_files=keep_max_files
)
self._enhanced_download_archive.save_download_mappings()
FileHandler.delete(self._enhanced_download_archive.working_file_path)
self.download_archive.save_download_mappings()
FileHandler.delete(self.download_archive.working_file_path)
@contextlib.contextmanager
def _remove_empty_directories_in_output_directory(self):
try:
yield
finally:
if not self._enhanced_download_archive.is_dry_run:
if not self.download_archive.is_dry_run:
for root, dir_names, _ in os.walk(Path(self.output_directory), topdown=False):
for dir_name in dir_names:
dir_path = Path(root) / dir_name
@ -195,7 +196,7 @@ class SubscriptionDownload(BaseSubscription, ABC):
plugin_type(
options=plugin_options,
overrides=self.overrides,
enhanced_download_archive=self._enhanced_download_archive,
enhanced_download_archive=self.download_archive,
)
for plugin_type, plugin_options in self.plugins.zipped()
]
@ -234,7 +235,7 @@ class SubscriptionDownload(BaseSubscription, ABC):
# Re-save the download archive after each entry is moved to the output directory
if self.maintain_download_archive:
self._enhanced_download_archive.save_download_mappings()
self.download_archive.save_download_mappings()
def _process_entry(
self, plugins: List[Plugin], dry_run: bool, entry: Entry, entry_metadata: FileMetadata
@ -324,17 +325,35 @@ class SubscriptionDownload(BaseSubscription, ABC):
for plugin in plugins:
plugin.post_process_subscription()
return self._enhanced_download_archive.get_file_handler_transaction_log()
return self.download_archive.get_file_handler_transaction_log()
def _initialize_subscription_overrides(self):
required_overrides = {
SubscriptionVariables.subscription_name(): self.name,
SubscriptionVariables.subscription_has_download_archive(): f"""{{
%bool({self._enhanced_download_archive.num_entries > 0})
def _initialize_subscription_overrides_pre_archive(self) -> None:
self.overrides.add(
{
SubscriptionVariables.subscription_name(): self.name,
}
)
def _initialize_subscription_overrides_post_archive(self) -> None:
self.overrides.add(
{
SubscriptionVariables.subscription_has_download_archive(): f"""{{
%bool({self.download_archive.num_entries > 0})
}}""",
}
assert SubscriptionVariables
self.overrides.add(required_overrides)
}
)
def _initialize_download_archive(self, dry_run: bool) -> None:
migrated_file_name: Optional[str] = None
if migrated_file_name_option := self.output_options.migrated_download_archive_name:
migrated_file_name = self.overrides.apply_formatter(migrated_file_name_option)
self._enhanced_download_archive = EnhancedDownloadArchive(
file_name=self.overrides.apply_formatter(self.output_options.download_archive_name),
working_directory=self.working_directory,
output_directory=self.output_directory,
migrated_file_name=migrated_file_name,
).reinitialize(dry_run=dry_run)
def download(self, dry_run: bool = False) -> FileHandlerTransactionLog:
"""
@ -347,15 +366,17 @@ class SubscriptionDownload(BaseSubscription, ABC):
directory.
"""
self._exception = None
self._enhanced_download_archive.reinitialize(dry_run=dry_run)
self._initialize_subscription_overrides()
self._initialize_subscription_overrides_pre_archive()
self._initialize_download_archive(dry_run=dry_run)
self._initialize_subscription_overrides_post_archive()
plugins = self._initialize_plugins()
subscription_ytdl_options = SubscriptionYTDLOptions(
preset=self._preset_options,
plugins=plugins,
enhanced_download_archive=self._enhanced_download_archive,
enhanced_download_archive=self.download_archive,
overrides=self.overrides,
working_directory=self.working_directory,
dry_run=dry_run,
@ -363,7 +384,7 @@ class SubscriptionDownload(BaseSubscription, ABC):
downloader = MultiUrlDownloader(
options=self.downloader_options,
enhanced_download_archive=self._enhanced_download_archive,
enhanced_download_archive=self.download_archive,
download_ytdl_options=subscription_ytdl_options.download_builder(),
metadata_ytdl_options=subscription_ytdl_options.metadata_builder(),
overrides=self.overrides,
@ -401,14 +422,17 @@ class SubscriptionDownload(BaseSubscription, ABC):
If true, do not modify any video/audio files or move anything to the output directory.
"""
self._exception = None
self._enhanced_download_archive.reinitialize(dry_run=dry_run)
self._initialize_subscription_overrides_pre_archive()
self._initialize_download_archive(dry_run=dry_run)
self._initialize_subscription_overrides_post_archive()
plugins = self._initialize_plugins()
subscription_ytdl_options = SubscriptionYTDLOptions(
preset=self._preset_options,
plugins=plugins,
enhanced_download_archive=self._enhanced_download_archive,
enhanced_download_archive=self.download_archive,
overrides=self.overrides,
working_directory=self.working_directory,
dry_run=dry_run,
@ -418,7 +442,7 @@ class SubscriptionDownload(BaseSubscription, ABC):
plugins.extend(
MultiUrlDownloader(
options=self.downloader_options,
enhanced_download_archive=self._enhanced_download_archive,
enhanced_download_archive=self.download_archive,
download_ytdl_options=subscription_ytdl_options.download_builder(),
metadata_ytdl_options=subscription_ytdl_options.metadata_builder(),
overrides=self.overrides,
@ -427,7 +451,7 @@ class SubscriptionDownload(BaseSubscription, ABC):
downloader = InfoJsonDownloader(
options=InfoJsonDownloaderOptions(name="no-op", value={}),
enhanced_download_archive=self._enhanced_download_archive,
enhanced_download_archive=self.download_archive,
download_ytdl_options=YTDLOptionsBuilder(),
metadata_ytdl_options=YTDLOptionsBuilder(),
overrides=self.overrides,