things passing maybe?
This commit is contained in:
parent
b359b2367f
commit
7e02e6a5d5
2 changed files with 58 additions and 14 deletions
|
|
@ -403,6 +403,18 @@ class Downloader(DownloadArchiver, Generic[DownloaderOptionsT, DownloaderEntryT]
|
||||||
for entry_child in self._download_parent_entry(parent=parent_child):
|
for entry_child in self._download_parent_entry(parent=parent_child):
|
||||||
yield entry_child
|
yield entry_child
|
||||||
|
|
||||||
|
def _set_collection_variables(
|
||||||
|
self, collection_url: CollectionUrlValidator, entry: Entry | EntryParent
|
||||||
|
):
|
||||||
|
if isinstance(entry, EntryParent):
|
||||||
|
for child in entry.parent_children():
|
||||||
|
self._set_collection_variables(collection_url, child)
|
||||||
|
for child in entry.entry_children():
|
||||||
|
child.add_variables(variables_to_add=collection_url.variables)
|
||||||
|
|
||||||
|
elif isinstance(entry, Entry):
|
||||||
|
entry.add_variables(variables_to_add=collection_url.variables)
|
||||||
|
|
||||||
def _download_url_metadata(
|
def _download_url_metadata(
|
||||||
self, collection_url: CollectionUrlValidator
|
self, collection_url: CollectionUrlValidator
|
||||||
) -> Tuple[List[EntryParent], List[Entry]]:
|
) -> Tuple[List[EntryParent], List[Entry]]:
|
||||||
|
|
@ -417,12 +429,19 @@ class Downloader(DownloadArchiver, Generic[DownloaderOptionsT, DownloaderEntryT]
|
||||||
)
|
)
|
||||||
|
|
||||||
self.parents = EntryParent.from_entry_dicts(
|
self.parents = EntryParent.from_entry_dicts(
|
||||||
entry_dicts=entry_dicts, working_directory=self.working_directory
|
url=collection_url.url,
|
||||||
|
entry_dicts=entry_dicts,
|
||||||
|
working_directory=self.working_directory,
|
||||||
)
|
)
|
||||||
|
|
||||||
orphans = EntryParent.from_entry_dicts_with_no_parents(
|
orphans = EntryParent.from_entry_dicts_with_no_parents(
|
||||||
parents=self.parents, entry_dicts=entry_dicts, working_directory=self.working_directory
|
parents=self.parents, entry_dicts=entry_dicts, working_directory=self.working_directory
|
||||||
)
|
)
|
||||||
|
|
||||||
|
for parent_entry in self.parents:
|
||||||
|
self._set_collection_variables(collection_url, parent_entry)
|
||||||
|
for entry in orphans:
|
||||||
|
self._set_collection_variables(collection_url, entry)
|
||||||
|
|
||||||
return self.parents, orphans
|
return self.parents, orphans
|
||||||
|
|
||||||
def _download(
|
def _download(
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,5 @@
|
||||||
import functools
|
import functools
|
||||||
import math
|
import math
|
||||||
import os
|
|
||||||
from typing import Dict
|
from typing import Dict
|
||||||
from typing import List
|
from typing import List
|
||||||
from typing import Optional
|
from typing import Optional
|
||||||
|
|
@ -59,7 +58,7 @@ class EntryParent(BaseEntry):
|
||||||
out = dict(
|
out = dict(
|
||||||
out,
|
out,
|
||||||
**{
|
**{
|
||||||
"playlist_index": idx,
|
"playlist_index": idx + 1,
|
||||||
"playlist_count": len(children),
|
"playlist_count": len(children),
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
|
|
@ -169,9 +168,39 @@ class EntryParent(BaseEntry):
|
||||||
child.__contains__(item) for child in self.child_entries
|
child.__contains__(item) for child in self.child_entries
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def _get_disconnected_root_parent(
|
||||||
|
cls, url: str, parents: List["EntryParent"]
|
||||||
|
) -> Optional["EntryParent"]:
|
||||||
|
"""
|
||||||
|
Sometimes the root-level parent is disconnected via playlist_ids Find it if it exists.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def _url_matches(webpage_url: str):
|
||||||
|
return webpage_url in url or url in webpage_url
|
||||||
|
|
||||||
|
top_level_parents = [
|
||||||
|
parent
|
||||||
|
for parent in parents
|
||||||
|
if not parent.child_entries and _url_matches(parent.webpage_url)
|
||||||
|
]
|
||||||
|
if len(top_level_parents) == 0:
|
||||||
|
return None
|
||||||
|
|
||||||
|
match len(top_level_parents):
|
||||||
|
case 0:
|
||||||
|
return None
|
||||||
|
case 1:
|
||||||
|
return top_level_parents[0]
|
||||||
|
case _:
|
||||||
|
raise ValueError(
|
||||||
|
"Detected multiple top-level parents. "
|
||||||
|
"Please file an issue on GitHub with the URLs used to produce this error"
|
||||||
|
)
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def from_entry_dicts(
|
def from_entry_dicts(
|
||||||
cls, entry_dicts: List[Dict], working_directory: str
|
cls, url: str, entry_dicts: List[Dict], working_directory: str
|
||||||
) -> List["EntryParent"]:
|
) -> List["EntryParent"]:
|
||||||
"""
|
"""
|
||||||
Reads all entry dicts and builds a tree of EntryParents
|
Reads all entry dicts and builds a tree of EntryParents
|
||||||
|
|
@ -187,15 +216,11 @@ class EntryParent(BaseEntry):
|
||||||
if not parents:
|
if not parents:
|
||||||
return []
|
return []
|
||||||
|
|
||||||
# find disconnected root parent if one exists
|
# If a disconnected root parent exists, connect it here
|
||||||
first_parent = min(
|
if (root_parent := cls._get_disconnected_root_parent(url, parents)) is not None:
|
||||||
parents, key=lambda x: os.stat(x.get_download_info_json_path()).st_ctime_ns
|
parents.remove(root_parent)
|
||||||
)
|
root_parent.child_entries = parents
|
||||||
if len(first_parent.child_entries) == 0:
|
parents = [root_parent]
|
||||||
parents.remove(first_parent)
|
|
||||||
first_parent.child_entries = parents
|
|
||||||
|
|
||||||
parents = [first_parent]
|
|
||||||
|
|
||||||
for parent in parents:
|
for parent in parents:
|
||||||
parent._set_child_variables()
|
parent._set_child_variables()
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue