diff --git a/src/ytdl_sub/cli/main.py b/src/ytdl_sub/cli/main.py index 1aae4069..e5fe9e38 100644 --- a/src/ytdl_sub/cli/main.py +++ b/src/ytdl_sub/cli/main.py @@ -1,6 +1,7 @@ import argparse import errno import fcntl +import gc import os import sys import tempfile @@ -59,6 +60,7 @@ def _download_subscriptions_from_yaml_files( transaction_log = subscription.download(dry_run=args.dry_run) output.append((subscription, transaction_log)) + gc.collect() # Garbage collect after each subscription download return output diff --git a/src/ytdl_sub/downloaders/downloader.py b/src/ytdl_sub/downloaders/downloader.py index 98c39d59..eca70886 100644 --- a/src/ytdl_sub/downloaders/downloader.py +++ b/src/ytdl_sub/downloaders/downloader.py @@ -30,7 +30,6 @@ from ytdl_sub.downloaders.generic.validators import MultiUrlValidator from ytdl_sub.downloaders.generic.validators import UrlThumbnailListValidator from ytdl_sub.downloaders.generic.validators import UrlValidator from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder -from ytdl_sub.entries.base_entry import BaseEntry from ytdl_sub.entries.entry import Entry from ytdl_sub.entries.entry_parent import EntryParent from ytdl_sub.entries.variables.kwargs import COMMENTS @@ -56,10 +55,6 @@ from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadAr download_logger = Logger.get(name="downloader") -def _entry_key(entry: BaseEntry) -> str: - return entry.extractor + entry.uid - - class DownloaderValidator(StrictDictValidator, AddsVariablesMixin, ABC): """ Placeholder class to define downloader options @@ -101,8 +96,6 @@ class URLDownloadState: def __init__(self, entries_total: int): self.entries_total = entries_total self.entries_downloaded = 0 - - self.entries: List[Entry] = [] self.thumbnails_downloaded: Set[str] = set() @@ -158,7 +151,7 @@ class Downloader(DownloadArchiver, Generic[DownloaderOptionsT], ABC): self.overrides = overrides self._download_ytdl_options_builder = download_ytdl_options self._metadata_ytdl_options_builder = metadata_ytdl_options - self.downloaded_entries: Dict[str, Entry] = {} + self._downloaded_entries: Set[str] = set() self._url_state: Optional[URLDownloadState] = None @@ -386,10 +379,10 @@ class Downloader(DownloadArchiver, Generic[DownloaderOptionsT], ABC): # DOWNLOAD FUNCTIONS def _is_downloaded(self, entry: Entry) -> bool: - return _entry_key(entry) in self.downloaded_entries + return entry.ytdl_uid() in self._downloaded_entries def _mark_downloaded(self, entry: Entry) -> None: - self.downloaded_entries[_entry_key(entry)] = entry + self._downloaded_entries.add(entry.ytdl_uid()) @property def collection(self) -> MultiUrlValidator: @@ -586,10 +579,8 @@ class Downloader(DownloadArchiver, Generic[DownloaderOptionsT], ABC): ) for entry in self._download(parents=parents, orphans=orphan_entries): yield entry - # Add entry to URL state - self._url_state.entries.append(entry) # Update thumbnails in case of last_entry - self._download_url_thumbnails(collection_url=collection_url) + self._download_url_thumbnails(collection_url=collection_url, entry=entry) @classmethod def _download_thumbnail( @@ -659,27 +650,26 @@ class Downloader(DownloadArchiver, Generic[DownloaderOptionsT], ABC): else: download_logger.warning("Failed to download thumbnail id '%s'", thumbnail_id) - def _download_url_thumbnails(self, collection_url: UrlValidator): + def _download_url_thumbnails(self, collection_url: UrlValidator, entry: Entry): """ After all media entries have been downloaded, post processed, and moved to the output directory, run this function. This lets the downloader add any extra files directly to the output directory, for things like YT channel image, banner. """ - for entry in self._url_state.entries: - if entry.kwargs_contains(PLAYLIST_ENTRY): - self._download_parent_thumbnails( - thumbnail_list_info=collection_url.playlist_thumbnails, - entry=entry, - parent=EntryParent( - entry.kwargs(PLAYLIST_ENTRY), working_directory=self.working_directory - ), - ) + if entry.kwargs_contains(PLAYLIST_ENTRY): + self._download_parent_thumbnails( + thumbnail_list_info=collection_url.playlist_thumbnails, + entry=entry, + parent=EntryParent( + entry.kwargs(PLAYLIST_ENTRY), working_directory=self.working_directory + ), + ) - if entry.kwargs_contains(SOURCE_ENTRY): - self._download_parent_thumbnails( - thumbnail_list_info=collection_url.source_thumbnails, - entry=entry, - parent=EntryParent( - entry.kwargs(SOURCE_ENTRY), working_directory=self.working_directory - ), - ) + if entry.kwargs_contains(SOURCE_ENTRY): + self._download_parent_thumbnails( + thumbnail_list_info=collection_url.source_thumbnails, + entry=entry, + parent=EntryParent( + entry.kwargs(SOURCE_ENTRY), working_directory=self.working_directory + ), + ) diff --git a/src/ytdl_sub/entries/base_entry.py b/src/ytdl_sub/entries/base_entry.py index 78a13aaf..7a714961 100644 --- a/src/ytdl_sub/entries/base_entry.py +++ b/src/ytdl_sub/entries/base_entry.py @@ -358,3 +358,11 @@ class BaseEntry(BaseEntryVariables, ABC): entry_ext = entry_dict.get("ext") return entry_ext is not None + + def ytdl_uid(self) -> str: + """ + Returns + ------- + extractor + uid, making this a unique hash for any entry + """ + return self.extractor + self.uid diff --git a/src/ytdl_sub/entries/entry_parent.py b/src/ytdl_sub/entries/entry_parent.py index 482fab43..88e70346 100644 --- a/src/ytdl_sub/entries/entry_parent.py +++ b/src/ytdl_sub/entries/entry_parent.py @@ -1,4 +1,3 @@ -import functools import math from typing import Dict from typing import List @@ -55,19 +54,16 @@ class EntryParent(BaseEntry): super().__init__(entry_dict=entry_dict, working_directory=working_directory) self.child_entries: List["EntryParent"] = [] - @functools.cache def parent_children(self) -> List["EntryParent"]: """This parent's children that are also parents""" return _sort_entries([child for child in self.child_entries if self.is_entry_parent(child)]) - @functools.cache def entry_children(self) -> List[Entry]: """This parent's children that are entries""" return _sort_entries( [child.to_type(Entry) for child in self.child_entries if self.is_entry(child)] ) - @functools.cache def num_children(self) -> int: """ Returns