157 lines
5.5 KiB
Python
157 lines
5.5 KiB
Python
"""Playlist processing."""
|
|
|
|
import asyncio
|
|
import logging
|
|
from typing import TYPE_CHECKING, Any
|
|
|
|
from app.library.Utils import merge_dict, ytdlp_reject
|
|
|
|
from .utils import handle_task_exception
|
|
|
|
if TYPE_CHECKING:
|
|
from app.library.ItemDTO import Item
|
|
|
|
from .queue_manager import DownloadQueue
|
|
|
|
LOG: logging.Logger = logging.getLogger("downloads.playlist")
|
|
|
|
|
|
async def process_playlist(
|
|
queue: "DownloadQueue", entry: dict, item: "Item", already=None, yt_params: dict | None = None
|
|
) -> dict[str, str]:
|
|
"""
|
|
Process a playlist entry.
|
|
|
|
Args:
|
|
queue: DownloadQueue instance for accessing queue/config
|
|
entry: Playlist entry dict from yt-dlp
|
|
item: Item configuration
|
|
already: Set of already processed IDs
|
|
yt_params: yt-dlp parameters
|
|
|
|
Returns:
|
|
dict: Status dict with "status" and optional "msg" keys
|
|
|
|
"""
|
|
if not yt_params:
|
|
yt_params = {}
|
|
|
|
entries = entry.get("entries", [])
|
|
|
|
playlist_name: str = f"{entry.get('id')}: {entry.get('title')}"
|
|
|
|
LOG.info(f"Processing '{playlist_name} ({len(entries)})' Playlist.")
|
|
|
|
playlistCount = entry.get("playlist_count")
|
|
playlistCount: int = int(playlistCount) if playlistCount else len(entries)
|
|
|
|
playlist_keys: dict[str, Any] = {
|
|
"playlist_count": playlistCount,
|
|
"playlist": entry.get("title") or entry.get("id"),
|
|
"playlist_id": entry.get("id"),
|
|
"playlist_title": entry.get("title"),
|
|
"playlist_uploader": entry.get("uploader"),
|
|
"playlist_uploader_id": entry.get("uploader_id"),
|
|
"playlist_channel": entry.get("channel"),
|
|
"playlist_channel_id": entry.get("channel_id"),
|
|
"playlist_webpage_url": entry.get("webpage_url"),
|
|
"__last_playlist_index": playlistCount - 1,
|
|
"n_entries": len(entries),
|
|
}
|
|
|
|
async def playlist_processor(i: int, etr: dict):
|
|
acquired = False
|
|
try:
|
|
item_name: str = (
|
|
f"'{entry.get('title')}: {i}/{playlist_keys['n_entries']}' - '{etr.get('id')}: {etr.get('title')}'"
|
|
)
|
|
LOG.debug(f"Waiting to acquire lock for {item_name}")
|
|
await queue.processors.acquire()
|
|
acquired = True
|
|
LOG.debug(f"Acquired lock for {item_name}")
|
|
|
|
LOG.info(f"Processing '{item_name}'.")
|
|
|
|
_status, _msg = ytdlp_reject(entry=etr, yt_params=yt_params)
|
|
if not _status:
|
|
return {"status": "error", "msg": _msg}
|
|
|
|
extras: dict[str, Any] = {
|
|
**playlist_keys,
|
|
"playlist_index": i,
|
|
"playlist_index_number": i,
|
|
"playlist_autonumber": i,
|
|
}
|
|
|
|
for property in ("id", "title", "uploader", "uploader_id"):
|
|
if property in entry:
|
|
extras[f"playlist_{property}"] = entry.get(property)
|
|
|
|
extractor_key = entry.get("ie_key") or entry.get("extractor_key") or entry.get("extractor") or ""
|
|
if "thumbnail" not in etr and "youtube" in str(extractor_key).lower():
|
|
extras["thumbnail"] = "https://img.youtube.com/vi/{id}/maxresdefault.jpg".format(**etr)
|
|
|
|
newItem: Item = item.new_with(url=etr.get("url") or etr.get("webpage_url"), extras=extras)
|
|
|
|
if ("video" == etr.get("_type") and etr.get("url")) or (
|
|
"formats" in etr and isinstance(etr["formats"], list) and len(etr["formats"]) > 0
|
|
):
|
|
dct = merge_dict(merge_dict({"_type": "video"}, etr), entry)
|
|
dct.pop("entries", None)
|
|
return await queue.add(item=newItem, entry=dct, already=already)
|
|
|
|
return await queue.add(item=newItem, already=already)
|
|
finally:
|
|
if acquired:
|
|
queue.processors.release()
|
|
|
|
max_downloads: int = -1
|
|
ytdlp_opts: dict[str, Any] = item.get_ytdlp_opts().get_all()
|
|
if ytdlp_opts.get("max_downloads") and isinstance(ytdlp_opts.get("max_downloads"), int):
|
|
max_downloads: int = ytdlp_opts.get("max_downloads")
|
|
|
|
batch_size: int = max(50, int(queue.config.playlist_items_concurrency) * 10)
|
|
|
|
async def run_batch(batch: list[tuple[int, dict]]) -> list[dict]:
|
|
tasks: list[asyncio.Task] = []
|
|
for i, etr in batch:
|
|
task = asyncio.create_task(
|
|
playlist_processor(i, etr),
|
|
name=f"playlist_processor_{etr.get('id')}_{i}",
|
|
)
|
|
task.add_done_callback(lambda t: handle_task_exception(t, LOG))
|
|
tasks.append(task)
|
|
|
|
batch_results: list[dict] = await asyncio.gather(*tasks)
|
|
await asyncio.sleep(0)
|
|
return batch_results
|
|
|
|
results: list[dict] = []
|
|
batch: list[tuple[int, dict]] = []
|
|
|
|
for i, etr in enumerate(entries, start=1):
|
|
if max_downloads > 0 and i > max_downloads:
|
|
break
|
|
|
|
batch.append((i, etr))
|
|
if len(batch) >= batch_size:
|
|
results.extend(await run_batch(batch))
|
|
batch.clear()
|
|
|
|
if batch:
|
|
results.extend(await run_batch(batch))
|
|
|
|
log_msg: str = f"Playlist '{playlist_name}' processing completed with '{len(results)}' entries."
|
|
if max_downloads > 0 and len(entries) > max_downloads:
|
|
skipped: int = len(entries) - max_downloads
|
|
log_msg += f" Limited to '{max_downloads}' items, skipped '{skipped}' remaining items."
|
|
|
|
LOG.info(log_msg)
|
|
|
|
if any("error" == res["status"] for res in results):
|
|
return {
|
|
"status": "error",
|
|
"msg": ", ".join(res["msg"] for res in results if "error" == res["status"] and "msg" in res),
|
|
}
|
|
|
|
return {"status": "ok"}
|