520 lines
20 KiB
Python
520 lines
20 KiB
Python
import asyncio
|
|
import logging
|
|
import time
|
|
import uuid
|
|
from numbers import Number
|
|
from pathlib import Path
|
|
from typing import TYPE_CHECKING, Any
|
|
|
|
import yt_dlp.utils
|
|
|
|
from app.features.conditions.service import Conditions
|
|
from app.features.presets.service import Presets
|
|
from app.features.ytdlp.extractor import fetch_info
|
|
from app.features.ytdlp.utils import archive_add, archive_read, arg_converter, get_extras, ytdlp_reject
|
|
from app.library.Events import Events
|
|
from app.library.ItemDTO import ItemDTO
|
|
from app.library.log import get_logger
|
|
from app.library.Utils import create_cookies_file, merge_dict
|
|
|
|
from .core import Download
|
|
from .playlist_processor import process_playlist
|
|
from .video_processor import add_video
|
|
|
|
if TYPE_CHECKING:
|
|
from app.features.presets.schemas import Preset
|
|
from app.library.ItemDTO import Item
|
|
|
|
from .queue_manager import DownloadQueue
|
|
|
|
LOG = get_logger()
|
|
|
|
|
|
def _get_ignored_conditions(extras: dict | None) -> list[str]:
|
|
if not extras or not isinstance(extras, dict):
|
|
return []
|
|
|
|
ignore_conditions = extras.get("ignore_conditions")
|
|
if not isinstance(ignore_conditions, list):
|
|
return []
|
|
|
|
ignored: list[str] = []
|
|
for value in ignore_conditions:
|
|
if isinstance(value, bool) or not isinstance(value, (str, Number)):
|
|
continue
|
|
|
|
identifier = str(value).strip()
|
|
if not identifier:
|
|
continue
|
|
|
|
ignored.append(identifier)
|
|
|
|
return ignored
|
|
|
|
|
|
def _task_ignored_logs(item: "Item") -> list[str] | None:
|
|
extras = item.extras if isinstance(item.extras, dict) else {}
|
|
if not extras.get("source_handler"):
|
|
return None
|
|
|
|
return ["has already been recorded in the archive"]
|
|
|
|
|
|
async def add_item(
|
|
queue: "DownloadQueue",
|
|
entry: dict,
|
|
item: "Item",
|
|
already=None,
|
|
logs: list | None = None,
|
|
yt_params: dict | None = None,
|
|
) -> dict[str, Any]:
|
|
"""
|
|
Route an entry to the appropriate processor based on type.
|
|
|
|
Args:
|
|
queue: DownloadQueue instance
|
|
entry: Entry dict from yt-dlp
|
|
item: Item configuration
|
|
already: Set of already processed URLs
|
|
logs: yt-dlp logs
|
|
yt_params: yt-dlp parameters
|
|
|
|
Returns:
|
|
dict: Status dict with "status" and optional "msg" keys
|
|
|
|
"""
|
|
if not entry:
|
|
return {"status": "error", "msg": "Invalid/empty data was given."}
|
|
|
|
event_type = entry.get("_type", "video")
|
|
|
|
if event_type.startswith("playlist"):
|
|
return await process_playlist(queue=queue, entry=entry, item=item, already=already, yt_params=yt_params)
|
|
|
|
if event_type.startswith("url"):
|
|
return await add(queue=queue, item=item.new_with(url=entry.get("url")), already=already)
|
|
|
|
if event_type.startswith("video"):
|
|
return await add_video(queue=queue, entry=entry, item=item, logs=logs)
|
|
|
|
return {"status": "error", "msg": f'Unsupported event type "{event_type}".'}
|
|
|
|
|
|
async def add(
|
|
queue: "DownloadQueue", item: "Item", already: set | None = None, entry: dict | None = None
|
|
) -> dict[str, Any]:
|
|
"""
|
|
Add an item to the download queue.
|
|
|
|
Args:
|
|
queue: DownloadQueue instance
|
|
item: Item to be added to the queue
|
|
already: Set of already downloaded items
|
|
entry: Entry associated with the item (if already extracted)
|
|
|
|
Returns:
|
|
dict[str, str]: Status dict with "status" and optional "msg" keys
|
|
|
|
"""
|
|
_preset: Preset | None = Presets.get_instance().get(item.preset)
|
|
logs = []
|
|
|
|
if item.has_cli():
|
|
try:
|
|
arg_converter(args=item.cli, level=True)
|
|
except Exception as e:
|
|
LOG.error(
|
|
"Invalid yt-dlp command options for '%s'.",
|
|
item.url,
|
|
extra={"url": item.url, "preset": item.preset, "exception_type": type(e).__name__},
|
|
)
|
|
return {"status": "error", "msg": f"Invalid command options for yt-dlp '{item.cli}'. {e!s}"}
|
|
|
|
if _preset:
|
|
if _preset.folder and not item.folder:
|
|
item.folder = _preset.folder
|
|
|
|
if _preset.template and not item.template:
|
|
item.template = _preset.template
|
|
|
|
yt_conf = {}
|
|
cookie_file: Path = Path(queue.config.temp_path) / f"c_{uuid.uuid4().hex}.txt"
|
|
|
|
LOG.info(
|
|
f"Adding '{item.url}' to downloads.",
|
|
extra={
|
|
"download": {
|
|
"url": item.url,
|
|
"preset": item.preset,
|
|
"folder": item.folder,
|
|
"has_cookies": bool(item.cookies),
|
|
"auto_start": item.auto_start,
|
|
}
|
|
},
|
|
)
|
|
|
|
already = set() if already is None else already
|
|
|
|
if item.url in already:
|
|
LOG.warning(
|
|
"Skipping recursive download URL '%s'.",
|
|
item.url,
|
|
extra={"url": item.url, "preset": item.preset},
|
|
)
|
|
return {"status": "ok"}
|
|
|
|
already.add(item.url)
|
|
|
|
try:
|
|
yt_conf: dict = item.get_ytdlp_opts().get_all()
|
|
|
|
if yt_conf.get("external_downloader"):
|
|
LOG.warning(
|
|
"Using external downloader '%s' for '%s'.",
|
|
yt_conf.get("external_downloader"),
|
|
item.url,
|
|
extra={
|
|
"url": item.url,
|
|
"preset": item.preset,
|
|
"external_downloader": yt_conf.get("external_downloader"),
|
|
},
|
|
)
|
|
item.extras.update({"external_downloader": True})
|
|
|
|
archive_id: str | None = item.get_archive_id()
|
|
|
|
# Early archive check to avoid unnecessary extraction calls
|
|
# This sometimes can be different from the final extracted ID, so we need to verify again after extraction.
|
|
if archive_id and item.is_archived():
|
|
store_type, dlInfo = await queue.get_item(archive_id=archive_id)
|
|
if not store_type and not queue.config.ignore_archived_items:
|
|
dlInfo = Download(
|
|
info=ItemDTO(
|
|
id=archive_id.split()[1],
|
|
title=archive_id,
|
|
url=item.url,
|
|
preset=item.preset,
|
|
folder=item.folder,
|
|
status="skip",
|
|
cookies=item.cookies,
|
|
template=item.template,
|
|
msg="URL is already downloaded.",
|
|
extras=item.extras,
|
|
)
|
|
)
|
|
|
|
if archive_file := dlInfo.info.get_ytdlp_opts().get_all().get("download_archive"):
|
|
dlInfo.info.msg = (dlInfo.info.msg or "") + f" Found in archive '{archive_file}'."
|
|
|
|
await queue.done.put(dlInfo)
|
|
|
|
queue._notify.emit(
|
|
Events.ITEM_MOVED,
|
|
data={"to": "history", "preset": dlInfo.info.preset, "item": dlInfo.info},
|
|
title="Download History Update",
|
|
message=f"Download history updated with '{item.url}'.",
|
|
)
|
|
return {"status": "ok"}
|
|
|
|
message: str = f"The URL '{item.url}':'{archive_id}' is already downloaded and recorded in archive."
|
|
LOG.warning(message)
|
|
queue._notify.emit(
|
|
Events.LOG_INFO, data={"preset": item.preset}, title="Already Downloaded", message=message
|
|
)
|
|
return {"status": "error", "msg": message, "hidden": True}
|
|
|
|
started: float = time.perf_counter()
|
|
|
|
if item.cookies:
|
|
try:
|
|
yt_conf["cookiefile"] = str(create_cookies_file(item.cookies, cookie_file))
|
|
except Exception as e:
|
|
msg = f"Failed to create cookie file for '{item.url}'. '{e!s}'."
|
|
LOG.exception(
|
|
msg,
|
|
extra={
|
|
"download": {
|
|
"url": item.url,
|
|
"preset": item.preset,
|
|
"path": str(cookie_file),
|
|
"has_cookies": True,
|
|
"exception_type": type(e).__name__,
|
|
}
|
|
},
|
|
)
|
|
return {"status": "error", "msg": msg}
|
|
|
|
if entry:
|
|
LOG.info(
|
|
f"Processing pre-extracted info for '{item.url}'.",
|
|
extra={
|
|
"download": {
|
|
"url": item.url,
|
|
"preset": item.preset,
|
|
"has_cookies": bool(yt_conf.get("cookiefile")),
|
|
"pre_extracted": True,
|
|
}
|
|
},
|
|
)
|
|
|
|
if not entry:
|
|
LOG.info(
|
|
f"Extracting info for '{item.url}'.",
|
|
extra={
|
|
"download": {
|
|
"url": item.url,
|
|
"preset": item.preset,
|
|
"has_cookies": bool(yt_conf.get("cookiefile")),
|
|
"pre_extracted": False,
|
|
}
|
|
},
|
|
)
|
|
(entry, logs) = await fetch_info(
|
|
config=yt_conf,
|
|
url=item.url,
|
|
debug=bool(queue.config.ytdlp_debug),
|
|
no_archive=False,
|
|
follow_redirect=True,
|
|
capture_logs=logging.WARNING,
|
|
budget_sleep=True,
|
|
suppress_logs=_task_ignored_logs(item),
|
|
)
|
|
|
|
if not entry:
|
|
LOG.error(
|
|
f"Unable to extract info for '{item.url}'.",
|
|
extra={
|
|
"download": {
|
|
"url": item.url,
|
|
"preset": item.preset,
|
|
"has_cookies": bool(yt_conf.get("cookiefile")),
|
|
"yt_dlp_logs": logs,
|
|
}
|
|
},
|
|
)
|
|
return {"status": "error", "msg": "Unable to extract info." + "\n".join(logs)}
|
|
|
|
# Sometimes playlists or extractor returns different ID than what we get from the make_archive_id()
|
|
# So, we need to re-check the archive after extraction to be sure the item was not downloaded.
|
|
# This also apply to old archive IDs that might have been used before.
|
|
if _archive_file := item.get_archive_file():
|
|
extra_ids: list[str] = []
|
|
|
|
if entry.get("_old_archive_ids") and isinstance(entry.get("_old_archive_ids"), list):
|
|
extra_ids.extend(entry.get("_old_archive_ids", []))
|
|
|
|
new_archive_id: str | None = None
|
|
|
|
if entry.get("extractor_key") and entry.get("id"):
|
|
_extractor_key = entry.get("extractor_key")
|
|
_entry_id = entry.get("id")
|
|
if isinstance(_extractor_key, str) and _entry_id is not None:
|
|
new_archive_id = f"{_extractor_key.lower()} {_entry_id!s}"
|
|
if new_archive_id != archive_id:
|
|
extra_ids.append(new_archive_id)
|
|
|
|
if len(extra_ids) > 0:
|
|
archive_ids: list[str] = archive_read(_archive_file, extra_ids)
|
|
if len(archive_ids) > 0:
|
|
store_type = None
|
|
for n in archive_ids:
|
|
store_type, dlInfo = await queue.get_item(archive_id=n)
|
|
if store_type:
|
|
break
|
|
|
|
if not store_type and not queue.config.ignore_archived_items:
|
|
new_archive_id = new_archive_id or extra_ids.pop(0)
|
|
dlInfo = Download(
|
|
info=ItemDTO(
|
|
id=new_archive_id.split()[1],
|
|
title=new_archive_id,
|
|
url=entry.get("url") or entry.get("webpage_url") or item.url,
|
|
preset=item.preset,
|
|
folder=item.folder,
|
|
status="skip",
|
|
cookies=item.cookies,
|
|
template=item.template,
|
|
msg="URL is already downloaded.",
|
|
extras=merge_dict(item.extras, get_extras(entry)),
|
|
)
|
|
)
|
|
|
|
dlInfo.info.msg = (dlInfo.info.msg or "") + f" Found in archive '{_archive_file}'."
|
|
await queue.done.put(dlInfo)
|
|
|
|
queue._notify.emit(
|
|
Events.ITEM_MOVED,
|
|
data={"to": "history", "preset": dlInfo.info.preset, "item": dlInfo.info},
|
|
title="Download History Update",
|
|
message=f"Download history updated with '{item.url}'.",
|
|
)
|
|
return {"status": "ok"}
|
|
|
|
message: str = (
|
|
f"The URL '{item.url}':'{archive_ids.pop(0)}' is already downloaded and recorded in archive."
|
|
)
|
|
LOG.warning(message)
|
|
queue._notify.emit(
|
|
Events.LOG_INFO, data={"preset": item.preset}, title="Already Downloaded", message=message
|
|
)
|
|
return {"status": "error", "msg": message, "hidden": True}
|
|
|
|
ignored_conditions = _get_ignored_conditions(item.extras)
|
|
|
|
if not item.requeued and (
|
|
condition := await Conditions.get_instance().match(info=entry, ignore_conditions=ignored_conditions)
|
|
):
|
|
already.pop()
|
|
|
|
display = str(entry.get("title") or entry.get("webpage_url") or entry.get("url") or item.url)
|
|
|
|
message = f"Condition '{condition.name}' matched for '{display}'."
|
|
|
|
if condition.cli:
|
|
message += f" Re-queuing with '{condition.cli}'."
|
|
|
|
LOG.info(message)
|
|
|
|
if condition.extras.get("ignore_download", False):
|
|
extra_msg: str = ""
|
|
if _archive_file and not condition.extras.get("no_archive", False) and archive_id:
|
|
archive_add(_archive_file, [archive_id])
|
|
extra_msg = f" and added to archive file '{_archive_file}'"
|
|
|
|
_name = str(entry.get("title") or entry.get("id") or item.url)
|
|
log_message = f"Ignoring download of '{_name}' as per condition '{condition.name}'{extra_msg}."
|
|
|
|
store_type, dlInfo = await queue.get_item(archive_id=archive_id)
|
|
if not store_type:
|
|
dlInfo = Download(
|
|
info=ItemDTO(
|
|
id=str(entry.get("id") or item.url),
|
|
title=_name,
|
|
url=item.url,
|
|
preset=item.preset,
|
|
folder=item.folder,
|
|
status="skip",
|
|
cookies=item.cookies,
|
|
template=item.template,
|
|
msg=log_message,
|
|
extras=merge_dict(item.extras, get_extras(entry)),
|
|
)
|
|
)
|
|
await queue.done.put(dlInfo)
|
|
|
|
LOG.info(log_message)
|
|
queue._notify.emit(Events.LOG_INFO, data={}, title="Ignored Download", message=log_message)
|
|
if dlInfo is None:
|
|
msg = "Download info is not available."
|
|
raise RuntimeError(msg)
|
|
queue._notify.emit(
|
|
Events.ITEM_MOVED,
|
|
data={"to": "history", "preset": dlInfo.info.preset, "item": dlInfo.info},
|
|
title="Download History Update",
|
|
message=f"Download history updated with '{item.url}'.",
|
|
)
|
|
return {"status": "ok"}
|
|
|
|
if condition.extras.get("set_preset") and (target_preset := condition.extras.get("set_preset")):
|
|
if Presets.get_instance().has(target_preset):
|
|
log_message: str = (
|
|
f"Switching preset from '{item.preset}' to '{target_preset}' for '{display}' "
|
|
f"as per condition '{condition.name}'."
|
|
)
|
|
LOG.info(log_message)
|
|
queue._notify.emit(Events.LOG_INFO, data={}, title="Preset Switched", message=log_message)
|
|
item = item.new_with(preset=target_preset)
|
|
else:
|
|
LOG.warning(
|
|
f"Preset '{target_preset}' specified in condition '{condition.name}' does not exist. Ignoring set_preset."
|
|
)
|
|
|
|
return await add(queue=queue, item=item.new_with(requeued=True, cli=condition.cli), already=already)
|
|
|
|
_status, _msg = ytdlp_reject(entry=entry, yt_params=yt_conf)
|
|
if not _status:
|
|
LOG.debug(_msg)
|
|
return {"status": "error", "msg": _msg}
|
|
|
|
end_time = time.perf_counter() - started
|
|
LOG.debug(
|
|
f"Extracted info for '{item.url}' in '{end_time:.3f}' seconds.",
|
|
extra={
|
|
"download": {
|
|
"url": item.url,
|
|
"preset": item.preset,
|
|
"has_cookies": bool(yt_conf.get("cookiefile")),
|
|
"elapsed_seconds": round(end_time, 3),
|
|
"entry_keys": len(entry),
|
|
}
|
|
},
|
|
)
|
|
except yt_dlp.utils.ExistingVideoReached as exc:
|
|
LOG.error(
|
|
"Video '%s' is already recorded in the download archive.",
|
|
item.url,
|
|
extra={
|
|
"download": {
|
|
"url": item.url,
|
|
"preset": item.preset,
|
|
"status": "skip",
|
|
"exception_type": type(exc).__name__,
|
|
}
|
|
},
|
|
)
|
|
return {"status": "error", "msg": "Video has been downloaded already and recorded in archive.log file."}
|
|
except yt_dlp.utils.YoutubeDLError as exc:
|
|
LOG.exception(
|
|
"Failed to extract media info for '%s'.",
|
|
item.url,
|
|
extra={
|
|
"download": {
|
|
"url": item.url,
|
|
"preset": item.preset,
|
|
"has_cookies": bool(yt_conf.get("cookiefile")),
|
|
"exception_type": type(exc).__name__,
|
|
}
|
|
},
|
|
)
|
|
return {"status": "error", "msg": str(exc)}
|
|
except asyncio.exceptions.TimeoutError as exc:
|
|
LOG.exception(
|
|
"Timed out extracting media info for '%s' after %s second(s).",
|
|
item.url,
|
|
queue.config.extract_info_timeout,
|
|
extra={
|
|
"download": {
|
|
"url": item.url,
|
|
"preset": item.preset,
|
|
"has_cookies": bool(yt_conf.get("cookiefile")),
|
|
"timeout_seconds": queue.config.extract_info_timeout,
|
|
"exception_type": type(exc).__name__,
|
|
}
|
|
},
|
|
)
|
|
return {
|
|
"status": "error",
|
|
"msg": f"TimeoutError: {queue.config.extract_info_timeout}s reached Unable to extract info.",
|
|
}
|
|
finally:
|
|
if cookie_file and cookie_file.exists():
|
|
try:
|
|
cookie_file.unlink(missing_ok=True)
|
|
yt_conf.pop("cookiefile", None)
|
|
except Exception as e:
|
|
LOG.exception(
|
|
"Failed to remove cookie file for '%s' at '%s'.",
|
|
item.url,
|
|
yt_conf["cookiefile"],
|
|
extra={
|
|
"download": {
|
|
"url": item.url,
|
|
"preset": item.preset,
|
|
"path": yt_conf["cookiefile"],
|
|
"has_cookies": True,
|
|
"exception_type": type(e).__name__,
|
|
}
|
|
},
|
|
)
|
|
|
|
return await add_item(queue=queue, entry=entry, item=item, already=already, logs=logs, yt_params=yt_conf)
|