good spot

This commit is contained in:
Jesse Bannon 2022-09-16 16:00:22 -07:00
parent 05e69d4d3b
commit 97df43a625
5 changed files with 104 additions and 41 deletions

View file

@ -47,20 +47,6 @@ def _entry_key(entry: BaseEntry) -> str:
return entry.extractor + entry.uid return entry.extractor + entry.uid
def _get_parent_entry_variables(parent: EntryParent) -> Dict[str, str | int]:
"""
Adds source variables to the child entry derived from the parent entry.
"""
if not parent.child_entries:
return {}
return {
"playlist_max_upload_year": max(
child_entry.to_type(Entry).upload_year for child_entry in parent.child_entries
)
}
class DownloaderValidator(StrictDictValidator, AddsVariablesMixin, ABC): class DownloaderValidator(StrictDictValidator, AddsVariablesMixin, ABC):
""" """
Placeholder class to define downloader options Placeholder class to define downloader options
@ -439,24 +425,22 @@ class Downloader(DownloadArchiver, Generic[DownloaderOptionsT, DownloaderEntryT]
) )
return self.parents, orphans return self.parents, orphans
def _download_url( def _download(
self, self,
collection_url: CollectionUrlValidator, parents: Optional[List[EntryParent]] = None,
parents: List[EntryParent], orphans: Optional[List[Entry]] = None,
orphans: List[Entry] = None,
) -> Generator[Entry, None, None]: ) -> Generator[Entry, None, None]:
""" """
Downloads the leaf entries from EntryParent trees Downloads the leaf entries from EntryParent trees
""" """
if parents is None:
parents = []
if orphans is None: if orphans is None:
orphans = [] orphans = []
with self._separate_download_archives(): with self._separate_download_archives():
for parent in parents: for parent in parents:
for entry_child in self._download_parent_entry(parent=parent): for entry_child in self._download_parent_entry(parent=parent):
entry_child.add_variables(
dict(_get_parent_entry_variables(parent), **collection_url.variables)
)
yield entry_child yield entry_child
for orphan in orphans: for orphan in orphans:
@ -469,9 +453,7 @@ class Downloader(DownloadArchiver, Generic[DownloaderOptionsT, DownloaderEntryT]
# download the bottom-most urls first since they are top-priority # download the bottom-most urls first since they are top-priority
for collection_url in reversed(self.collection.collection_urls.list): for collection_url in reversed(self.collection.collection_urls.list):
parents, orphan_entries = self._download_url_metadata(collection_url=collection_url) parents, orphan_entries = self._download_url_metadata(collection_url=collection_url)
for entry in self._download_url( for entry in self._download(parents=parents, orphans=orphan_entries):
collection_url=collection_url, parents=parents, orphans=orphan_entries
):
yield entry yield entry
def post_download(self): def post_download(self):

View file

@ -148,7 +148,7 @@ class YoutubeChannelDownloader(YoutubeDownloader[YoutubeChannelDownloaderOptions
} }
) )
for entry in super()._download_url(collection_url=collection_url, parents=self.parents): for entry in super()._download(parents=self.parents):
yield entry.to_type(YoutubeVideo) yield entry.to_type(YoutubeVideo)
def _download_thumbnail( def _download_thumbnail(

View file

@ -113,5 +113,5 @@ class YoutubePlaylistDownloader(
} }
) )
for entry in super()._download_url(collection_url=collection_url, parents=self.parents): for entry in super()._download(parents=self.parents):
yield entry.to_type(YoutubePlaylistVideo) yield entry.to_type(YoutubePlaylistVideo)

View file

@ -82,6 +82,20 @@ class BaseEntryVariables:
""" """
return "info.json" return "info.json"
def base_variable_dict(self) -> Dict[str, str]:
"""
Returns
-------
BaseEntry variables that can be nested for playlist, source, etc
"""
return {
"uid": self.uid,
"extractor": self.extractor,
"title": self.title,
"title_sanitized": self.title_sanitized,
"webpage_url": self.webpage_url,
}
# pylint: enable=no-member # pylint: enable=no-member
@ -141,6 +155,23 @@ class BaseEntry(BaseEntryVariables, ABC):
""" """
return self._working_directory return self._working_directory
def add_kwargs(self, variables_to_add: Dict[str, Any]) -> "BaseEntry":
"""
Adds variables to kwargs. Use with caution since yt-dlp data can be overwritten.
Plugins should use ``add_variables``.
Parameters
----------
variables_to_add
Variables to add to kwargs
Returns
-------
self
"""
self._kwargs = dict(self._kwargs, **variables_to_add)
return self
def add_variables(self, variables_to_add: Dict[str, str]) -> "BaseEntry": def add_variables(self, variables_to_add: Dict[str, str]) -> "BaseEntry":
""" """
Parameters Parameters

View file

@ -1,12 +1,26 @@
import math
import os import os
from typing import Dict from typing import Dict
from typing import List from typing import List
from typing import Optional from typing import Optional
import mergedeep
from ytdl_sub.entries.base_entry import BaseEntry from ytdl_sub.entries.base_entry import BaseEntry
from ytdl_sub.entries.base_entry import TBaseEntry
from ytdl_sub.entries.entry import Entry from ytdl_sub.entries.entry import Entry
class ParentType:
PLAYLIST = "playlist"
SOURCE = "source"
def _sort_entries(entries: List[TBaseEntry]) -> List[TBaseEntry]:
"""Try sorting by playlist_id first, then fall back to uid"""
return sorted(entries, key=lambda ent: (ent.kwargs_get("playlist_id", math.inf), ent.uid))
class EntryParent(BaseEntry): class EntryParent(BaseEntry):
def __init__(self, entry_dict: Dict, working_directory: str): def __init__(self, entry_dict: Dict, working_directory: str):
super().__init__(entry_dict=entry_dict, working_directory=working_directory) super().__init__(entry_dict=entry_dict, working_directory=working_directory)
@ -14,26 +28,57 @@ class EntryParent(BaseEntry):
def parent_children(self) -> List["EntryParent"]: def parent_children(self) -> List["EntryParent"]:
"""This parent's children that are also parents""" """This parent's children that are also parents"""
return [child for child in self.child_entries if self.is_entry_parent(child)] return _sort_entries([child for child in self.child_entries if self.is_entry_parent(child)])
def entry_children(self) -> List[Entry]: def entry_children(self) -> List[Entry]:
"""This parent's children that are entries""" """This parent's children that are entries"""
return [child.to_type(Entry) for child in self.child_entries if self.is_entry(child)] return _sort_entries(
[child.to_type(Entry) for child in self.child_entries if self.is_entry(child)]
)
def read_children_from_entry_dicts(self, entry_dicts: List[Dict]) -> "EntryParent": def _parent_variables(self, parent_type: str) -> Dict:
return dict(
{f"{parent_type}_entry": self._kwargs},
**{f"{parent_type}_{key}": value for key, value in self.base_variable_dict().items()},
)
# pylint: disable=protected-access
def _set_child_variables(self, parents: Optional[List["EntryParent"]] = None) -> "EntryParent":
if parents is None:
parents = [self]
kwargs_to_add: Dict = {}
if len(parents) >= 1:
mergedeep.merge(kwargs_to_add, parents[-1]._parent_variables(ParentType.PLAYLIST))
if len(parents) >= 2:
mergedeep.merge(kwargs_to_add, parents[-2]._parent_variables(ParentType.SOURCE))
if len(parents) >= 3:
raise ValueError(
"ytdl-sub currently does support more than 3 layers of playlists/entries. "
"If you encounter this error, please file a ticket with the URLs used."
)
for entry_child in self.entry_children():
entry_child.add_kwargs(kwargs_to_add)
for parent_child in self.parent_children():
parent_child._set_child_variables(parents=parents + [parent_child])
return self
def _read_children_from_entry_dicts(self, entry_dicts: List[Dict]) -> "EntryParent":
""" """
Populates a tree of EntryParents that belong to this instance Populates a tree of EntryParents that belong to this instance
""" """
child_entries: List["EntryParent"] = [] child_entries = [
EntryParent(
for entry_dict in entry_dicts: entry_dict=entry_dict,
if entry_dict.get("playlist_id") == self.uid: working_directory=self.working_directory(),
child_entries.append( )._read_children_from_entry_dicts(entry_dicts)
self.__class__( for entry_dict in entry_dicts
entry_dict=entry_dict, if entry_dict in self
working_directory=self.working_directory(), ]
).read_children_from_entry_dicts(entry_dicts)
)
self.child_entries = sorted(child_entries, key=lambda entry: entry.kwargs("playlist_index")) self.child_entries = sorted(child_entries, key=lambda entry: entry.kwargs("playlist_index"))
return self return self
@ -80,7 +125,7 @@ class EntryParent(BaseEntry):
parents = [ parents = [
EntryParent( EntryParent(
entry_dict=entry_dict, working_directory=working_directory entry_dict=entry_dict, working_directory=working_directory
).read_children_from_entry_dicts(entry_dicts) )._read_children_from_entry_dicts(entry_dicts)
for entry_dict in entry_dicts for entry_dict in entry_dicts
if cls.is_entry_parent(entry_dict) if cls.is_entry_parent(entry_dict)
] ]
@ -96,10 +141,15 @@ class EntryParent(BaseEntry):
parents.remove(first_parent) parents.remove(first_parent)
first_parent.child_entries = parents first_parent.child_entries = parents
return [first_parent] parents = [first_parent]
for parent in parents:
parent._set_child_variables()
return parents return parents
# pylint: enable=protected-access
@classmethod @classmethod
def from_entry_dicts_with_no_parents( def from_entry_dicts_with_no_parents(
cls, parents: List["EntryParent"], entry_dicts: List[Dict], working_directory: str cls, parents: List["EntryParent"], entry_dicts: List[Dict], working_directory: str