info json downloader

src/ytdl_sub/downloaders/info_json/info_json_downloader.py

@@ -0,0 +1,104 @@
+import json
+import os
+from collections import defaultdict
+from pathlib import Path
+from typing import Dict
+from typing import Iterable
+from typing import List
+
+from ytdl_sub.downloaders.base_downloader import BaseDownloader
+from ytdl_sub.downloaders.base_downloader import BaseDownloaderValidator
+from ytdl_sub.entries.entry import Entry
+from ytdl_sub.utils.exceptions import ValidationException
+from ytdl_sub.utils.file_handler import FileHandler
+from ytdl_sub.utils.file_handler import get_file_extension
+from ytdl_sub.ytdl_additions.enhanced_download_archive import DownloadMapping
+from ytdl_sub.ytdl_additions.enhanced_download_archive import DownloadMappings
+
+
+class InfoJsonDownloaderOptions(BaseDownloaderValidator):
+    _optional_keys = {"no-op"}
+
+
+class InfoJsonDownloader(BaseDownloader[InfoJsonDownloaderOptions]):
+    downloader_options_type = InfoJsonDownloaderOptions
+
+    @property
+    def output_directory(self) -> str:
+        return self._enhanced_download_archive._file_handler.output_directory
+
+    @property
+    def download_mappings(self) -> DownloadMappings:
+        return self._enhanced_download_archive.mapping
+
+    def _get_entry_from_download_mapping(self, download_mapping: DownloadMapping):
+        """
+        Try to load an entry from a download mapping's info json
+        """
+        for file_name in download_mapping.file_names:
+            if file_name.endswith(".info.json"):
+                try:
+                    with open(
+                        Path(self.output_directory) / file_name, "r", encoding="utf-8"
+                    ) as maybe_info_json:
+                        entry_dict = json.load(maybe_info_json)
+                except Exception as exc:
+                    raise ValidationException(
+                        "info.json file cannot be loaded - subscription cannot be reformatted"
+                    ) from exc
+
+                return Entry(
+                    entry_dict=entry_dict,
+                    working_directory=self.working_directory,
+                )
+
+        raise ValidationException(
+            "info.json file could not be found - subscription cannot be reformatted"
+        )
+
+    def download_metadata(self) -> Iterable[Entry]:
+        """
+        Loads all entries via their info.json files first (to ensure they are all valid), then
+        iterates them
+        """
+        # Track to see if files were modified
+        file_names_mtime: Dict[str, Dict[str, float]] = defaultdict(dict)
+        entries: List[Entry] = []
+
+        for download_mapping in self.download_mappings._entry_mappings.values():
+            entry = self._get_entry_from_download_mapping(download_mapping)
+            entries.append(entry)
+
+            for file_name in download_mapping.file_names:
+                file_path = Path(self.output_directory) / file_name
+                file_names_mtime[entry.ytdl_uid()][file_name] = os.path.getmtime(file_path)
+
+        for entry in entries:
+            yield entry
+
+            for file_name, mtime in file_names_mtime[entry.ytdl_uid()].items():
+                # If the entry file_path is unchanged, then delete it since it was not part of the
+                # reformat output
+                if os.path.getmtime(Path(self.output_directory) / file_name) == mtime:
+                    self._enhanced_download_archive._file_handler.delete_file_from_output_directory(
+                        file_name
+                    )
+
+    def download(self, entry: Entry) -> Entry:
+        entry_file_names = self.download_mappings._entry_mappings.get(entry.uid).file_names
+        for file_name in entry_file_names:
+            ext = get_file_extension(file_name)
+            file_path = Path(self.output_directory) / file_name
+            working_directory_file_path = Path(self.working_directory) / f"{entry.uid}.{ext}"
+
+            # NFO files will always get rewritten, so ignore
+            if ext == "nfo":
+                continue
+
+            if not self.is_dry_run:
+                FileHandler.copy(
+                    src_file_path=file_path,
+                    dst_file_path=working_directory_file_path,
+                )
+
+        return entry

src/ytdl_sub/subscriptions/subscription_download.py

@@ -1,17 +1,15 @@
 import contextlib
-import json
 import os
 import shutil
 from abc import ABC
 from pathlib import Path
-from typing import Dict
-from typing import Iterable
 from typing import List
 from typing import Optional
-from typing import Set
-from typing import Tuple
 
-from ytdl_sub.downloaders.downloader import BaseDownloader
+from ytdl_sub.downloaders.base_downloader import BaseDownloader
+from ytdl_sub.downloaders.info_json.info_json_downloader import InfoJsonDownloader
+from ytdl_sub.downloaders.info_json.info_json_downloader import InfoJsonDownloaderOptions
+from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder
 from ytdl_sub.entries.entry import Entry
 from ytdl_sub.plugins.plugin import Plugin
 from ytdl_sub.subscriptions.base_subscription import BaseSubscription
@@ -355,75 +353,22 @@ class SubscriptionDownload(BaseSubscription, ABC):
             dry_run=dry_run,
         )
 
-    def _get_entries_for_reformat(
-        self, download_mappings: DownloadMappings, dry_run: bool
-    ) -> Iterable[Entry]:
-        entry_mapping: List[Tuple[Entry, Set[str]]] = []
-        for download_mapping in download_mappings._entry_mappings.values():
-            maybe_entry: Optional[Entry] = None
-            for file_name in download_mapping.file_names:
-                if file_name.endswith(".info.json"):
-                    try:
-                        with open(
-                            Path(self.output_directory) / file_name, "r", encoding="utf-8"
-                        ) as maybe_info_json:
-                            maybe_entry = Entry(
-                                entry_dict=json.load(maybe_info_json),
-                                working_directory=self.working_directory,
-                            )
-                    except Exception as exc:
-                        raise ValidationException(
-                            "info.json file cannot be loaded - subscription cannot be reformatted"
-                        ) from exc
-
-            if not maybe_entry:
-                raise ValidationException(
-                    ".info.json file could not be found - subscription cannot be reformatted"
-                )
-
-            entry_mapping.append((maybe_entry, download_mapping.file_names))
-
-        for entry, file_names in entry_mapping:
-            file_names_mtime: Dict[str, float] = {}
-            for file_name in file_names:
-                ext = get_file_extension(file_name)
-
-                file_path = Path(self.output_directory) / file_name
-                working_directory_file_path = Path(self.working_directory) / f"{entry.uid}.{ext}"
-
-                file_names_mtime[file_name] = os.path.getmtime(file_path)
-
-                # NFO files will always get rewritten, so ignore
-                if ext == "nfo":
-                    continue
-
-                if not dry_run:
-                    FileHandler.copy(
-                        src_file_path=file_path,
-                        dst_file_path=working_directory_file_path,
-                    )
-
-            yield entry
-
-            for file_name, mtime in file_names_mtime.items():
-                # If the entry file_path is unchanged, then delete it since it was not part of the
-                # reformat output
-                if os.path.getmtime(Path(self.output_directory) / file_name) == mtime:
-                    self._enhanced_download_archive._file_handler.delete_file_from_output_directory(
-                        file_name
-                    )
-
     def update_with_info_json(self, dry_run: bool = False) -> FileHandlerTransactionLog:
+        self._enhanced_download_archive.reinitialize(dry_run=dry_run)
         plugins = self._initialize_plugins()
 
-        with self._subscription_download_context_managers():
+        downloader = InfoJsonDownloader(
-            download_mappings = self._enhanced_download_archive.mapping
+            download_options=InfoJsonDownloaderOptions(name="no-op", value={}),
-            self._enhanced_download_archive.reinitialize(dry_run=dry_run)
+            enhanced_download_archive=self._enhanced_download_archive,
+            download_ytdl_options=YTDLOptionsBuilder(),
+            metadata_ytdl_options=YTDLOptionsBuilder(),
+            overrides=self.overrides,
+        )
+        # This could be cleaned up....
+        plugins.extend(downloader.added_plugins())
 
-            return self._process_subscription(
+        return self._process_subscription(
-                plugins=plugins,
+            plugins=plugins,
-                entries=self._get_entries_for_reformat(
+            downloader=downloader,
-                    download_mappings=download_mappings, dry_run=dry_run
+            dry_run=dry_run,
-                ),
+        )
-                dry_run=dry_run,
-            )