[BACKEND] Reduce memory footprint

src/ytdl_sub/cli/main.py

@@ -1,6 +1,7 @@
 import argparse
 import errno
 import fcntl
+import gc
 import os
 import sys
 import tempfile
@@ -59,6 +60,7 @@ def _download_subscriptions_from_yaml_files(
         transaction_log = subscription.download(dry_run=args.dry_run)
 
         output.append((subscription, transaction_log))
+        gc.collect()  # Garbage collect after each subscription download
 
     return output
 

src/ytdl_sub/downloaders/downloader.py

@@ -30,7 +30,6 @@ from ytdl_sub.downloaders.generic.validators import MultiUrlValidator
 from ytdl_sub.downloaders.generic.validators import UrlThumbnailListValidator
 from ytdl_sub.downloaders.generic.validators import UrlValidator
 from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder
-from ytdl_sub.entries.base_entry import BaseEntry
 from ytdl_sub.entries.entry import Entry
 from ytdl_sub.entries.entry_parent import EntryParent
 from ytdl_sub.entries.variables.kwargs import COMMENTS
@@ -56,10 +55,6 @@ from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadAr
 download_logger = Logger.get(name="downloader")
 
 
-def _entry_key(entry: BaseEntry) -> str:
-    return entry.extractor + entry.uid
-
-
 class DownloaderValidator(StrictDictValidator, AddsVariablesMixin, ABC):
     """
     Placeholder class to define downloader options
@@ -101,8 +96,6 @@ class URLDownloadState:
     def __init__(self, entries_total: int):
         self.entries_total = entries_total
         self.entries_downloaded = 0
-
-        self.entries: List[Entry] = []
         self.thumbnails_downloaded: Set[str] = set()
 
 
@@ -158,7 +151,7 @@ class Downloader(DownloadArchiver, Generic[DownloaderOptionsT], ABC):
         self.overrides = overrides
         self._download_ytdl_options_builder = download_ytdl_options
         self._metadata_ytdl_options_builder = metadata_ytdl_options
-        self.downloaded_entries: Dict[str, Entry] = {}
+        self._downloaded_entries: Set[str] = set()
 
         self._url_state: Optional[URLDownloadState] = None
 
@@ -386,10 +379,10 @@ class Downloader(DownloadArchiver, Generic[DownloaderOptionsT], ABC):
     # DOWNLOAD FUNCTIONS
 
     def _is_downloaded(self, entry: Entry) -> bool:
-        return _entry_key(entry) in self.downloaded_entries
+        return entry.ytdl_uid() in self._downloaded_entries
 
     def _mark_downloaded(self, entry: Entry) -> None:
-        self.downloaded_entries[_entry_key(entry)] = entry
+        self._downloaded_entries.add(entry.ytdl_uid())
 
     @property
     def collection(self) -> MultiUrlValidator:
@@ -586,10 +579,8 @@ class Downloader(DownloadArchiver, Generic[DownloaderOptionsT], ABC):
             )
             for entry in self._download(parents=parents, orphans=orphan_entries):
                 yield entry
-                # Add entry to URL state
-                self._url_state.entries.append(entry)
                 # Update thumbnails in case of last_entry
-                self._download_url_thumbnails(collection_url=collection_url)
+                self._download_url_thumbnails(collection_url=collection_url, entry=entry)
 
     @classmethod
     def _download_thumbnail(
@@ -659,27 +650,26 @@ class Downloader(DownloadArchiver, Generic[DownloaderOptionsT], ABC):
             else:
                 download_logger.warning("Failed to download thumbnail id '%s'", thumbnail_id)
 
-    def _download_url_thumbnails(self, collection_url: UrlValidator):
+    def _download_url_thumbnails(self, collection_url: UrlValidator, entry: Entry):
         """
         After all media entries have been downloaded, post processed, and moved to the output
         directory, run this function. This lets the downloader add any extra files directly to the
         output directory, for things like YT channel image, banner.
         """
-        for entry in self._url_state.entries:
+        if entry.kwargs_contains(PLAYLIST_ENTRY):
-            if entry.kwargs_contains(PLAYLIST_ENTRY):
+            self._download_parent_thumbnails(
-                self._download_parent_thumbnails(
+                thumbnail_list_info=collection_url.playlist_thumbnails,
-                    thumbnail_list_info=collection_url.playlist_thumbnails,
+                entry=entry,
-                    entry=entry,
+                parent=EntryParent(
-                    parent=EntryParent(
+                    entry.kwargs(PLAYLIST_ENTRY), working_directory=self.working_directory
-                        entry.kwargs(PLAYLIST_ENTRY), working_directory=self.working_directory
+                ),
-                    ),
+            )
-                )
 
-            if entry.kwargs_contains(SOURCE_ENTRY):
+        if entry.kwargs_contains(SOURCE_ENTRY):
-                self._download_parent_thumbnails(
+            self._download_parent_thumbnails(
-                    thumbnail_list_info=collection_url.source_thumbnails,
+                thumbnail_list_info=collection_url.source_thumbnails,
-                    entry=entry,
+                entry=entry,
-                    parent=EntryParent(
+                parent=EntryParent(
-                        entry.kwargs(SOURCE_ENTRY), working_directory=self.working_directory
+                    entry.kwargs(SOURCE_ENTRY), working_directory=self.working_directory
-                    ),
+                ),
-                )
+            )

src/ytdl_sub/entries/base_entry.py

@@ -358,3 +358,11 @@ class BaseEntry(BaseEntryVariables, ABC):
             entry_ext = entry_dict.get("ext")
 
         return entry_ext is not None
+
+    def ytdl_uid(self) -> str:
+        """
+        Returns
+        -------
+        extractor + uid, making this a unique hash for any entry
+        """
+        return self.extractor + self.uid

src/ytdl_sub/entries/entry_parent.py

@@ -1,4 +1,3 @@
-import functools
 import math
 from typing import Dict
 from typing import List
@@ -55,19 +54,16 @@ class EntryParent(BaseEntry):
         super().__init__(entry_dict=entry_dict, working_directory=working_directory)
         self.child_entries: List["EntryParent"] = []
 
-    @functools.cache
     def parent_children(self) -> List["EntryParent"]:
         """This parent's children that are also parents"""
         return _sort_entries([child for child in self.child_entries if self.is_entry_parent(child)])
 
-    @functools.cache
     def entry_children(self) -> List[Entry]:
         """This parent's children that are entries"""
         return _sort_entries(
             [child.to_type(Entry) for child in self.child_entries if self.is_entry(child)]
         )
 
-    @functools.cache
     def num_children(self) -> int:
         """
         Returns