357 lines
14 KiB
Python
357 lines
14 KiB
Python
import asyncio
|
|
import logging
|
|
import time
|
|
import uuid
|
|
from pathlib import Path
|
|
from typing import TYPE_CHECKING
|
|
|
|
import yt_dlp.utils
|
|
|
|
from app.features.conditions.service import Conditions
|
|
from app.library.Events import Events
|
|
from app.library.ItemDTO import ItemDTO
|
|
from app.library.Presets import Presets
|
|
from app.library.Utils import (
|
|
archive_add,
|
|
archive_read,
|
|
arg_converter,
|
|
create_cookies_file,
|
|
fetch_info,
|
|
get_extras,
|
|
merge_dict,
|
|
ytdlp_reject,
|
|
)
|
|
|
|
from .core import Download
|
|
from .playlist_processor import process_playlist
|
|
from .video_processor import add_video
|
|
|
|
if TYPE_CHECKING:
|
|
from app.library.ItemDTO import Item
|
|
from app.library.Presets import Preset
|
|
|
|
from .queue_manager import DownloadQueue
|
|
|
|
LOG: logging.Logger = logging.getLogger("downloads.add")
|
|
|
|
|
|
async def add_item(
|
|
queue: "DownloadQueue",
|
|
entry: dict,
|
|
item: "Item",
|
|
already=None,
|
|
logs: list | None = None,
|
|
yt_params: dict | None = None,
|
|
) -> dict[str, str]:
|
|
"""
|
|
Route an entry to the appropriate processor based on type.
|
|
|
|
Args:
|
|
queue: DownloadQueue instance
|
|
entry: Entry dict from yt-dlp
|
|
item: Item configuration
|
|
already: Set of already processed URLs
|
|
logs: yt-dlp logs
|
|
yt_params: yt-dlp parameters
|
|
|
|
Returns:
|
|
dict: Status dict with "status" and optional "msg" keys
|
|
|
|
"""
|
|
if not entry:
|
|
return {"status": "error", "msg": "Invalid/empty data was given."}
|
|
|
|
event_type = entry.get("_type", "video")
|
|
|
|
if event_type.startswith("playlist"):
|
|
return await process_playlist(queue=queue, entry=entry, item=item, already=already, yt_params=yt_params)
|
|
|
|
if event_type.startswith("url"):
|
|
return await add(queue=queue, item=item.new_with(url=entry.get("url")), already=already)
|
|
|
|
if event_type.startswith("video"):
|
|
return await add_video(queue=queue, entry=entry, item=item, logs=logs)
|
|
|
|
return {"status": "error", "msg": f'Unsupported event type "{event_type}".'}
|
|
|
|
|
|
async def add(
|
|
queue: "DownloadQueue", item: "Item", already: set | None = None, entry: dict | None = None
|
|
) -> dict[str, str]:
|
|
"""
|
|
Add an item to the download queue.
|
|
|
|
Args:
|
|
queue: DownloadQueue instance
|
|
item: Item to be added to the queue
|
|
already: Set of already downloaded items
|
|
entry: Entry associated with the item (if already extracted)
|
|
|
|
Returns:
|
|
dict[str, str]: Status dict with "status" and optional "msg" keys
|
|
|
|
"""
|
|
_preset: Preset | None = Presets.get_instance().get(item.preset)
|
|
|
|
if item.has_cli():
|
|
try:
|
|
arg_converter(args=item.cli, level=True)
|
|
except Exception as e:
|
|
LOG.error(f"Invalid command options for yt-dlp '{item.cli}'. {e!s}")
|
|
return {"status": "error", "msg": f"Invalid command options for yt-dlp '{item.cli}'. {e!s}"}
|
|
|
|
if _preset:
|
|
if _preset.folder and not item.folder:
|
|
item.folder = _preset.folder
|
|
|
|
if _preset.template and not item.template:
|
|
item.template = _preset.template
|
|
|
|
yt_conf = {}
|
|
cookie_file: Path = Path(queue.config.temp_path) / f"c_{uuid.uuid4().hex}.txt"
|
|
|
|
LOG.info(f"Adding '{item!r}'.")
|
|
|
|
already = set() if already is None else already
|
|
|
|
if item.url in already:
|
|
LOG.warning(f"Recursion detected with url '{item.url}' skipping.")
|
|
return {"status": "ok"}
|
|
|
|
already.add(item.url)
|
|
|
|
try:
|
|
logs: list = []
|
|
|
|
yt_conf: dict = {
|
|
"callback": {
|
|
"func": lambda _, msg: logs.append(msg),
|
|
"level": logging.WARNING,
|
|
"name": "callback-logger",
|
|
},
|
|
**item.get_ytdlp_opts().get_all(),
|
|
}
|
|
|
|
if yt_conf.get("external_downloader"):
|
|
LOG.warning(f"Using external downloader '{yt_conf.get('external_downloader')}' for '{item.url}'.")
|
|
item.extras.update({"external_downloader": True})
|
|
|
|
archive_id: str | None = item.get_archive_id()
|
|
|
|
# Early archive check to avoid unnecessary extraction calls
|
|
# This sometimes can be different from the final extracted ID, so we need to verify again after extraction.
|
|
if archive_id and item.is_archived():
|
|
store_type, dlInfo = await queue.get_item(archive_id=archive_id)
|
|
if not store_type and not queue.config.ignore_archived_items:
|
|
dlInfo = Download(
|
|
info=ItemDTO(
|
|
id=archive_id.split()[1],
|
|
title=archive_id,
|
|
url=item.url,
|
|
preset=item.preset,
|
|
folder=item.folder,
|
|
status="skip",
|
|
cookies=item.cookies,
|
|
template=item.template,
|
|
msg="URL is already downloaded.",
|
|
extras=item.extras,
|
|
)
|
|
)
|
|
|
|
if archive_file := dlInfo.info.get_ytdlp_opts().get_all().get("download_archive"):
|
|
dlInfo.info.msg += f" Found in archive '{archive_file}'."
|
|
|
|
await queue.done.put(dlInfo)
|
|
|
|
queue._notify.emit(
|
|
Events.ITEM_MOVED,
|
|
data={"to": "history", "preset": dlInfo.info.preset, "item": dlInfo.info},
|
|
title="Download History Update",
|
|
message=f"Download history updated with '{item.url}'.",
|
|
)
|
|
return {"status": "ok"}
|
|
|
|
message: str = f"The URL '{item.url}':'{archive_id}' is already downloaded and recorded in archive."
|
|
LOG.warning(message)
|
|
queue._notify.emit(
|
|
Events.LOG_INFO, data={"preset": item.preset}, title="Already Downloaded", message=message
|
|
)
|
|
return {"status": "error", "msg": message, "hidden": True}
|
|
|
|
started: float = time.perf_counter()
|
|
|
|
if item.cookies:
|
|
try:
|
|
yt_conf["cookiefile"] = str(create_cookies_file(item.cookies, cookie_file))
|
|
except Exception as e:
|
|
msg = f"Failed to create cookie file for '{item.url}'. '{e!s}'."
|
|
LOG.error(msg)
|
|
return {"status": "error", "msg": msg}
|
|
|
|
if entry:
|
|
LOG.info(f"[P] Extracting '{item.url}'{' with cookies' if yt_conf.get('cookiefile') else ''}.")
|
|
|
|
if not entry:
|
|
LOG.info(f"Extracting '{item.url}'{' with cookies' if yt_conf.get('cookiefile') else ''}.")
|
|
entry: dict | None = await fetch_info(
|
|
config=yt_conf,
|
|
url=item.url,
|
|
debug=bool(queue.config.ytdlp_debug),
|
|
no_archive=False,
|
|
follow_redirect=True,
|
|
)
|
|
|
|
if not entry:
|
|
LOG.error(f"Unable to extract info for '{item.url}'. Logs: {logs}")
|
|
return {"status": "error", "msg": "Unable to extract info." + "\n".join(logs)}
|
|
|
|
# Sometimes playlists or extractor returns different ID than what we get from the make_archive_id()
|
|
# So, we need to re-check the archive after extraction to be sure the item was not downloaded.
|
|
# This also apply to old archive IDs that might have been used before.
|
|
if _archive_file := item.get_archive_file():
|
|
extra_ids: list[str] = []
|
|
|
|
if entry.get("_old_archive_ids") and isinstance(entry.get("_old_archive_ids"), list):
|
|
extra_ids.extend(entry.get("_old_archive_ids", []))
|
|
|
|
new_archive_id: str | None = None
|
|
|
|
if entry.get("extractor_key") and entry.get("id"):
|
|
new_archive_id: str = f"{entry.get('extractor_key').lower()} {entry.get('id')}"
|
|
if new_archive_id != archive_id:
|
|
extra_ids.append(new_archive_id)
|
|
|
|
if len(extra_ids) > 0:
|
|
archive_ids: list[str] = archive_read(_archive_file, extra_ids)
|
|
if len(archive_ids) > 0:
|
|
store_type = None
|
|
for n in archive_ids:
|
|
store_type, dlInfo = await queue.get_item(archive_id=n)
|
|
if store_type:
|
|
break
|
|
|
|
if not store_type and not queue.config.ignore_archived_items:
|
|
new_archive_id = new_archive_id or extra_ids.pop(0)
|
|
dlInfo = Download(
|
|
info=ItemDTO(
|
|
id=new_archive_id.split()[1],
|
|
title=new_archive_id,
|
|
url=entry.get("url") or entry.get("webpage_url") or item.url,
|
|
preset=item.preset,
|
|
folder=item.folder,
|
|
status="skip",
|
|
cookies=item.cookies,
|
|
template=item.template,
|
|
msg="URL is already downloaded.",
|
|
extras=merge_dict(item.extras, get_extras(entry)),
|
|
)
|
|
)
|
|
|
|
dlInfo.info.msg += f" Found in archive '{_archive_file}'."
|
|
await queue.done.put(dlInfo)
|
|
|
|
queue._notify.emit(
|
|
Events.ITEM_MOVED,
|
|
data={"to": "history", "preset": dlInfo.info.preset, "item": dlInfo.info},
|
|
title="Download History Update",
|
|
message=f"Download history updated with '{item.url}'.",
|
|
)
|
|
return {"status": "ok"}
|
|
|
|
message: str = (
|
|
f"The URL '{item.url}':'{archive_ids.pop(0)}' is already downloaded and recorded in archive."
|
|
)
|
|
LOG.warning(message)
|
|
queue._notify.emit(
|
|
Events.LOG_INFO, data={"preset": item.preset}, title="Already Downloaded", message=message
|
|
)
|
|
return {"status": "error", "msg": message, "hidden": True}
|
|
|
|
if not item.requeued and (condition := await Conditions.get_instance().match(info=entry)):
|
|
already.pop()
|
|
|
|
message = f"Condition '{condition.name}' matched for '{item!r}'."
|
|
|
|
if condition.cli:
|
|
message += f" Re-queuing with '{condition.cli}'."
|
|
|
|
LOG.info(message)
|
|
|
|
if condition.extras.get("ignore_download", False):
|
|
extra_msg: str = ""
|
|
if _archive_file and not condition.extras.get("no_archive", False):
|
|
archive_add(_archive_file, [archive_id])
|
|
extra_msg = f" and added to archive file '{_archive_file}'"
|
|
|
|
_name = entry.get("title", entry.get("id"))
|
|
log_message = f"Ignoring download of '{_name!r}' as per condition '{condition.name}'{extra_msg}."
|
|
|
|
store_type, dlInfo = await queue.get_item(archive_id=archive_id)
|
|
if not store_type:
|
|
dlInfo = Download(
|
|
info=ItemDTO(
|
|
id=entry.get("id"),
|
|
title=_name,
|
|
url=item.url,
|
|
preset=item.preset,
|
|
folder=item.folder,
|
|
status="skip",
|
|
cookies=item.cookies,
|
|
template=item.template,
|
|
msg=log_message,
|
|
extras=merge_dict(item.extras, get_extras(entry)),
|
|
)
|
|
)
|
|
await queue.done.put(dlInfo)
|
|
|
|
LOG.info(log_message)
|
|
queue._notify.emit(Events.LOG_INFO, data={}, title="Ignored Download", message=log_message)
|
|
queue._notify.emit(
|
|
Events.ITEM_MOVED,
|
|
data={"to": "history", "preset": dlInfo.info.preset, "item": dlInfo.info},
|
|
title="Download History Update",
|
|
message=f"Download history updated with '{item.url}'.",
|
|
)
|
|
return {"status": "ok"}
|
|
|
|
if condition.extras.get("set_preset") and (target_preset := condition.extras.get("set_preset")):
|
|
if Presets.get_instance().has(target_preset):
|
|
log_message: str = f"Switching preset from '{item.preset}' to '{target_preset}' for '{item!r}' as per condition '{condition.name}'."
|
|
LOG.info(log_message)
|
|
queue._notify.emit(Events.LOG_INFO, data={}, title="Preset Switched", message=log_message)
|
|
item = item.new_with(preset=target_preset)
|
|
else:
|
|
LOG.warning(
|
|
f"Preset '{target_preset}' specified in condition '{condition.name}' does not exist. Ignoring set_preset."
|
|
)
|
|
|
|
return await add(queue=queue, item=item.new_with(requeued=True, cli=condition.cli), already=already)
|
|
|
|
_status, _msg = ytdlp_reject(entry=entry, yt_params=yt_conf)
|
|
if not _status:
|
|
LOG.debug(_msg)
|
|
return {"status": "error", "msg": _msg}
|
|
|
|
end_time = time.perf_counter() - started
|
|
LOG.debug(f"extract_info: for 'URL: {item.url}' is done in '{end_time:.3f}'. Length: '{len(entry)}/keys'.")
|
|
except yt_dlp.utils.ExistingVideoReached as exc:
|
|
LOG.error(f"Video has been downloaded already and recorded in archive.log file. '{exc!s}'.")
|
|
return {"status": "error", "msg": "Video has been downloaded already and recorded in archive.log file."}
|
|
except yt_dlp.utils.YoutubeDLError as exc:
|
|
LOG.error(f"YoutubeDLError: Unable to extract info. '{exc!s}'.")
|
|
return {"status": "error", "msg": str(exc)}
|
|
except asyncio.exceptions.TimeoutError as exc:
|
|
LOG.error(f"TimeoutError: Unable to extract info. '{exc!s}'.")
|
|
return {
|
|
"status": "error",
|
|
"msg": f"TimeoutError: {queue.config.extract_info_timeout}s reached Unable to extract info.",
|
|
}
|
|
finally:
|
|
if cookie_file and cookie_file.exists():
|
|
try:
|
|
cookie_file.unlink(missing_ok=True)
|
|
yt_conf.pop("cookiefile", None)
|
|
except Exception as e:
|
|
LOG.error(f"Failed to remove cookie file '{yt_conf['cookiefile']}'. {e!s}")
|
|
|
|
return await add_item(queue=queue, entry=entry, item=item, already=already, logs=logs, yt_params=yt_conf)
|