From d3736f011a083992b854fcedc0a12720ab5c5317 Mon Sep 17 00:00:00 2001 From: Jesse Bannon Date: Wed, 15 Mar 2023 00:24:16 -0700 Subject: [PATCH] info json downloader --- .../downloaders/info_json/__init__.py | 0 .../info_json/info_json_downloader.py | 104 ++++++++++++++++++ .../subscriptions/subscription_download.py | 93 ++++------------ 3 files changed, 123 insertions(+), 74 deletions(-) create mode 100644 src/ytdl_sub/downloaders/info_json/__init__.py create mode 100644 src/ytdl_sub/downloaders/info_json/info_json_downloader.py diff --git a/src/ytdl_sub/downloaders/info_json/__init__.py b/src/ytdl_sub/downloaders/info_json/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/ytdl_sub/downloaders/info_json/info_json_downloader.py b/src/ytdl_sub/downloaders/info_json/info_json_downloader.py new file mode 100644 index 00000000..f523f916 --- /dev/null +++ b/src/ytdl_sub/downloaders/info_json/info_json_downloader.py @@ -0,0 +1,104 @@ +import json +import os +from collections import defaultdict +from pathlib import Path +from typing import Dict +from typing import Iterable +from typing import List + +from ytdl_sub.downloaders.base_downloader import BaseDownloader +from ytdl_sub.downloaders.base_downloader import BaseDownloaderValidator +from ytdl_sub.entries.entry import Entry +from ytdl_sub.utils.exceptions import ValidationException +from ytdl_sub.utils.file_handler import FileHandler +from ytdl_sub.utils.file_handler import get_file_extension +from ytdl_sub.ytdl_additions.enhanced_download_archive import DownloadMapping +from ytdl_sub.ytdl_additions.enhanced_download_archive import DownloadMappings + + +class InfoJsonDownloaderOptions(BaseDownloaderValidator): + _optional_keys = {"no-op"} + + +class InfoJsonDownloader(BaseDownloader[InfoJsonDownloaderOptions]): + downloader_options_type = InfoJsonDownloaderOptions + + @property + def output_directory(self) -> str: + return self._enhanced_download_archive._file_handler.output_directory + + @property + def download_mappings(self) -> DownloadMappings: + return self._enhanced_download_archive.mapping + + def _get_entry_from_download_mapping(self, download_mapping: DownloadMapping): + """ + Try to load an entry from a download mapping's info json + """ + for file_name in download_mapping.file_names: + if file_name.endswith(".info.json"): + try: + with open( + Path(self.output_directory) / file_name, "r", encoding="utf-8" + ) as maybe_info_json: + entry_dict = json.load(maybe_info_json) + except Exception as exc: + raise ValidationException( + "info.json file cannot be loaded - subscription cannot be reformatted" + ) from exc + + return Entry( + entry_dict=entry_dict, + working_directory=self.working_directory, + ) + + raise ValidationException( + "info.json file could not be found - subscription cannot be reformatted" + ) + + def download_metadata(self) -> Iterable[Entry]: + """ + Loads all entries via their info.json files first (to ensure they are all valid), then + iterates them + """ + # Track to see if files were modified + file_names_mtime: Dict[str, Dict[str, float]] = defaultdict(dict) + entries: List[Entry] = [] + + for download_mapping in self.download_mappings._entry_mappings.values(): + entry = self._get_entry_from_download_mapping(download_mapping) + entries.append(entry) + + for file_name in download_mapping.file_names: + file_path = Path(self.output_directory) / file_name + file_names_mtime[entry.ytdl_uid()][file_name] = os.path.getmtime(file_path) + + for entry in entries: + yield entry + + for file_name, mtime in file_names_mtime[entry.ytdl_uid()].items(): + # If the entry file_path is unchanged, then delete it since it was not part of the + # reformat output + if os.path.getmtime(Path(self.output_directory) / file_name) == mtime: + self._enhanced_download_archive._file_handler.delete_file_from_output_directory( + file_name + ) + + def download(self, entry: Entry) -> Entry: + entry_file_names = self.download_mappings._entry_mappings.get(entry.uid).file_names + for file_name in entry_file_names: + ext = get_file_extension(file_name) + file_path = Path(self.output_directory) / file_name + working_directory_file_path = Path(self.working_directory) / f"{entry.uid}.{ext}" + + # NFO files will always get rewritten, so ignore + if ext == "nfo": + continue + + if not self.is_dry_run: + FileHandler.copy( + src_file_path=file_path, + dst_file_path=working_directory_file_path, + ) + + return entry diff --git a/src/ytdl_sub/subscriptions/subscription_download.py b/src/ytdl_sub/subscriptions/subscription_download.py index fa482ae6..0b8c6c18 100644 --- a/src/ytdl_sub/subscriptions/subscription_download.py +++ b/src/ytdl_sub/subscriptions/subscription_download.py @@ -1,17 +1,15 @@ import contextlib -import json import os import shutil from abc import ABC from pathlib import Path -from typing import Dict -from typing import Iterable from typing import List from typing import Optional -from typing import Set -from typing import Tuple -from ytdl_sub.downloaders.downloader import BaseDownloader +from ytdl_sub.downloaders.base_downloader import BaseDownloader +from ytdl_sub.downloaders.info_json.info_json_downloader import InfoJsonDownloader +from ytdl_sub.downloaders.info_json.info_json_downloader import InfoJsonDownloaderOptions +from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder from ytdl_sub.entries.entry import Entry from ytdl_sub.plugins.plugin import Plugin from ytdl_sub.subscriptions.base_subscription import BaseSubscription @@ -355,75 +353,22 @@ class SubscriptionDownload(BaseSubscription, ABC): dry_run=dry_run, ) - def _get_entries_for_reformat( - self, download_mappings: DownloadMappings, dry_run: bool - ) -> Iterable[Entry]: - entry_mapping: List[Tuple[Entry, Set[str]]] = [] - for download_mapping in download_mappings._entry_mappings.values(): - maybe_entry: Optional[Entry] = None - for file_name in download_mapping.file_names: - if file_name.endswith(".info.json"): - try: - with open( - Path(self.output_directory) / file_name, "r", encoding="utf-8" - ) as maybe_info_json: - maybe_entry = Entry( - entry_dict=json.load(maybe_info_json), - working_directory=self.working_directory, - ) - except Exception as exc: - raise ValidationException( - "info.json file cannot be loaded - subscription cannot be reformatted" - ) from exc - - if not maybe_entry: - raise ValidationException( - ".info.json file could not be found - subscription cannot be reformatted" - ) - - entry_mapping.append((maybe_entry, download_mapping.file_names)) - - for entry, file_names in entry_mapping: - file_names_mtime: Dict[str, float] = {} - for file_name in file_names: - ext = get_file_extension(file_name) - - file_path = Path(self.output_directory) / file_name - working_directory_file_path = Path(self.working_directory) / f"{entry.uid}.{ext}" - - file_names_mtime[file_name] = os.path.getmtime(file_path) - - # NFO files will always get rewritten, so ignore - if ext == "nfo": - continue - - if not dry_run: - FileHandler.copy( - src_file_path=file_path, - dst_file_path=working_directory_file_path, - ) - - yield entry - - for file_name, mtime in file_names_mtime.items(): - # If the entry file_path is unchanged, then delete it since it was not part of the - # reformat output - if os.path.getmtime(Path(self.output_directory) / file_name) == mtime: - self._enhanced_download_archive._file_handler.delete_file_from_output_directory( - file_name - ) - def update_with_info_json(self, dry_run: bool = False) -> FileHandlerTransactionLog: + self._enhanced_download_archive.reinitialize(dry_run=dry_run) plugins = self._initialize_plugins() - with self._subscription_download_context_managers(): - download_mappings = self._enhanced_download_archive.mapping - self._enhanced_download_archive.reinitialize(dry_run=dry_run) + downloader = InfoJsonDownloader( + download_options=InfoJsonDownloaderOptions(name="no-op", value={}), + enhanced_download_archive=self._enhanced_download_archive, + download_ytdl_options=YTDLOptionsBuilder(), + metadata_ytdl_options=YTDLOptionsBuilder(), + overrides=self.overrides, + ) + # This could be cleaned up.... + plugins.extend(downloader.added_plugins()) - return self._process_subscription( - plugins=plugins, - entries=self._get_entries_for_reformat( - download_mappings=download_mappings, dry_run=dry_run - ), - dry_run=dry_run, - ) + return self._process_subscription( + plugins=plugins, + downloader=downloader, + dry_run=dry_run, + )