danvics/ytdl-sub
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

stale file support - more sunday night yolos without testing

Browse source
This commit is contained in:
jbannon 2022-04-11 07:24:30 +00:00
parent b88e8e8166
commit bf97c93cbc
7 changed files with 101 additions and 38 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
config.yaml
View file

@ -54,6 +54,7 @@ presets:
convert_thumbnail: "jpeg"
thumbnail_name: "{output_file_name}.jpeg"
maintain_download_archive: True
maintain_stale_file_deletion: True
metadata_options:
nfo:
nfo_name: "{output_file_name}.nfo"

70
ytdl_subscribe/subscriptions/subscription.py
View file

@ -29,7 +29,9 @@ from ytdl_subscribe.validators.config.output_options.output_options_validator im
)
from ytdl_subscribe.validators.config.overrides.overrides_validator import OverridesValidator
from ytdl_subscribe.validators.config.preset_validator import PresetValidator
from ytdl_subscribe.validators.config.source_options.mixins import DownloadDateRangeSource
from ytdl_subscribe.validators.config.source_options.source_validators import SourceValidator
from ytdl_subscribe.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive
S = TypeVar("S", bound=SourceValidator)
D = TypeVar("D", bound=Downloader)
@ -54,26 +56,17 @@ class Subscription(Generic[S], ABC):
self.__config_options = config_options
self.__preset_options = preset_options
self._enhanced_download_archive = EnhancedDownloadArchive(
subscription_name=name,
working_directory=self.working_directory,
output_directory=self.output_directory,
)
@property
def source_options(self) -> S:
"""Returns the source options defined for this subscription"""
return self.__preset_options.subscription_source
def get_downloader(
self, downloader_type: Type[D], source_ytdl_options: Optional[Dict] = None
) -> D:
"""Returns the downloader that will be used to download media for this subscription"""
# if source_ytdl_options are present, override the ytdl_options with them
ytdl_options = self.__preset_options.ytdl_options.dict
if source_ytdl_options:
ytdl_options = dict(ytdl_options, **source_ytdl_options)
return downloader_type(
output_directory=self.working_directory,
ytdl_options=ytdl_options,
download_archive_file_name=self.download_archive_file_name,
)
@property
def output_options(self) -> OutputOptionsValidator:
"""Returns the output options defined for this subscription"""
@ -98,9 +91,26 @@ class Subscription(Generic[S], ABC):
def output_directory(self):
return self._apply_formatter(formatter=self.output_options.output_directory)
@property
def download_archive_file_name(self):
return f".ytdl-subscribe-{self.name}-download-archive.txt"
def _archive_entry_file_name(self, entry: Optional[Entry], relative_file_name: str) -> None:
if entry:
self._enhanced_download_archive.mapping.add_entry(
entry=entry, entry_file_path=relative_file_name
)
def get_downloader(
self, downloader_type: Type[D], source_ytdl_options: Optional[Dict] = None
) -> D:
"""Returns the downloader that will be used to download media for this subscription"""
# if source_ytdl_options are present, override the ytdl_options with them
ytdl_options = self.__preset_options.ytdl_options.dict
if source_ytdl_options:
ytdl_options = dict(ytdl_options, **source_ytdl_options)
return downloader_type(
output_directory=self.working_directory,
ytdl_options=ytdl_options,
download_archive_file_name=self._enhanced_download_archive.archive_file_name,
)
def _apply_formatter(
self, formatter: StringFormatterValidator, entry: Optional[Entry] = None
@ -156,20 +166,26 @@ class Subscription(Generic[S], ABC):
with open(nfo_file_path, "wb") as nfo_file:
nfo_file.write(xml)
# Archive the nfo's file name
self._archive_entry_file_name(entry=entry, relative_file_name=nfo_file_name)
@contextlib.contextmanager
def _maintain_archive_file(self):
if not self.output_options.maintain_download_archive:
return
existing_download_archive = Path(self.output_directory) / self.download_archive_file_name
working_directory_archive = Path(self.working_directory) / self.download_archive_file_name
if os.path.exists(existing_download_archive):
shutil.copy(existing_download_archive, working_directory_archive)
self._enhanced_download_archive.prepare_download_archive()
yield
shutil.copy(working_directory_archive, existing_download_archive)
date_range = None
if isinstance(self.source_options, DownloadDateRangeSource):
date_range = self.source_options.get_date_range()
if date_range and self.output_options.maintain_stale_file_deletion:
self._enhanced_download_archive.remove_stale_files(date_range=date_range)
self._enhanced_download_archive.save_download_archive()
def download(self):
"""
@ -203,6 +219,9 @@ class Subscription(Generic[S], ABC):
os.makedirs(os.path.dirname(entry_destination_file_path), exist_ok=True)
copyfile(entry_source_file_path, entry_destination_file_path)
self._archive_entry_file_name(entry=entry, relative_file_name=output_file_name)
# TODO: move thumbnail to separate function
# Download the thumbnail if its present
if self.output_options.thumbnail_name:
source_thumbnail_path = entry.thumbnail_path(relative_directory=self.working_directory)
@ -239,6 +258,9 @@ class Subscription(Generic[S], ABC):
else:
copyfile(source_thumbnail_path, output_thumbnail_path)
# Archive the thumbnail file name
self._archive_entry_file_name(entry=entry, relative_file_name=output_thumbnail_name)
if self.metadata_options.nfo:
self._post_process_nfo(nfo_options=self.metadata_options.nfo, entry=entry)

8
ytdl_subscribe/subscriptions/youtube.py
View file

@ -26,11 +26,9 @@ class YoutubePlaylistSubscription(Subscription[YoutubePlaylistSourceValidator]):
class YoutubeChannelSubscription(Subscription[YoutubeChannelSourceValidator]):
def _extract_info(self):
source_ytdl_options = {}
if self.source_options.before or self.source_options.after:
source_ytdl_options["daterange"] = DateRange(
start=self.source_options.after.datetime_str if self.source_options.after else None,
end=self.source_options.before.datetime_str if self.source_options.before else None,
)
source_date_range = self.source_options.get_date_range()
if source_date_range:
source_ytdl_options["daterange"] = source_ytdl_options
downloader = self.get_downloader(YoutubeDownloader, source_ytdl_options=source_ytdl_options)
entries = downloader.download_channel(channel_id=self.source_options.channel_id.value)

17
ytdl_subscribe/validators/config/output_options/output_options_validator.py
View file

@ -17,7 +17,12 @@ class OutputOptionsValidator(StrictDictValidator):
"""Where to output the final files and thumbnails"""
_required_keys = {"output_directory", "file_name"}
_optional_keys = {"convert_thumbnail", "thumbnail_name", "maintain_download_archive"}
_optional_keys = {
"convert_thumbnail",
"thumbnail_name",
"maintain_download_archive",
"maintain_stale_file_deletion",
}
def __init__(self, name, value):
super().__init__(name, value)
@ -40,5 +45,13 @@ class OutputOptionsValidator(StrictDictValidator):
)
self.maintain_download_archive = self._validate_key_if_present(
key="maintain_download_archive", validator=BoolValidator
key="maintain_download_archive", validator=BoolValidator, default=False
)
self.maintain_stale_file_deletion = self._validate_key_if_present(
key="maintain_stale_file_deletion", validator=BoolValidator, default=False
)
if self.maintain_stale_file_deletion.value and not self.maintain_download_archive.value:
raise self._validation_exception(
"maintain_stale_file_deletion requires maintain_download_archive set to True"
)

24
ytdl_subscribe/validators/config/source_options/mixins.py Normal file
View file

@ -0,0 +1,24 @@
from abc import ABC
from typing import Optional
from yt_dlp import DateRange
from ytdl_subscribe.validators.base.strict_dict_validator import StrictDictValidator
from ytdl_subscribe.validators.base.string_datetime import StringDatetimeValidator
class DownloadDateRangeSource(StrictDictValidator, ABC):
optional_keys = {"before", "after"}
def __init__(self, name, value):
super().__init__(name, value)
self.before = self._validate_key_if_present("before", StringDatetimeValidator)
self.after = self._validate_key_if_present("after", StringDatetimeValidator)
def get_date_range(self) -> Optional[DateRange]:
if self.before or self.after:
return DateRange(
start=self.after.datetime_str if self.after else None,
end=self.before.datetime_str if self.before else None,
)
return None

8
ytdl_subscribe/validators/config/source_options/youtube_validators.py
View file

@ -1,5 +1,6 @@
from ytdl_subscribe.validators.base.string_datetime import StringDatetimeValidator
from ytdl_subscribe.validators.base.validators import StringValidator
from ytdl_subscribe.validators.config.source_options.mixins import DownloadDateRangeSource
from ytdl_subscribe.validators.config.source_options.source_validators import YoutubeSourceValidator
@ -19,12 +20,11 @@ class YoutubeVideoSourceValidator(YoutubeSourceValidator):
self.video_id = self._validate_key("video_id", StringValidator)
class YoutubeChannelSourceValidator(YoutubeSourceValidator):
class YoutubeChannelSourceValidator(YoutubeSourceValidator, DownloadDateRangeSource):
_required_keys = {"channel_id"}
_optional_keys = {"before", "after"}
def __init__(self, name, value):
super().__init__(name, value)
YoutubeSourceValidator.__init__(self, name, value)
DownloadDateRangeSource.__init__(self, name, value)
self.channel_id = self._validate_key("channel_id", StringValidator)
self.before = self._validate_key_if_present("before", StringDatetimeValidator)
self.after = self._validate_key_if_present("after", StringDatetimeValidator)

11
ytdl_subscribe/ytdl_additions/enhanced_download_archive.py
View file

@ -163,7 +163,7 @@ class EnhancedDownloadArchive:
self._download_mapping: Optional[DownloadMappings] = None
@property
def _archive_file_name(self) -> str:
def archive_file_name(self) -> str:
"""
:return: The download archive's file name (no path)
"""
@ -210,7 +210,7 @@ class EnhancedDownloadArchive:
raise ValueError("Tried to use download mapping before it was loaded")
return self._download_mapping
def load(self) -> "EnhancedDownloadArchive":
def _load(self) -> "EnhancedDownloadArchive":
download_archive_is_file = os.path.isfile(self._mapping_output_file_path)
download_mapping_is_file = os.path.isfile(self._archive_output_file_path)
@ -243,7 +243,7 @@ class EnhancedDownloadArchive:
return self
def copy_to_working_directory(self) -> "EnhancedDownloadArchive":
def _copy_to_working_directory(self) -> "EnhancedDownloadArchive":
# If no download archive was found, then there is nothing to copy
if not self._download_archive:
return self
@ -253,6 +253,11 @@ class EnhancedDownloadArchive:
return self
def prepare_download_archive(self) -> "EnhancedDownloadArchive":
self._load()
self._copy_to_working_directory()
return self
def remove_stale_files(self, date_range: DateRange) -> "EnhancedDownloadArchive":
stale_mappings: Dict[str, DownloadMapping] = self.mapping.get_entries_out_of_range(
date_range=date_range