diff --git a/FAQ.md b/FAQ.md index 5e2ae000..fd034fc6 100644 --- a/FAQ.md +++ b/FAQ.md @@ -57,6 +57,7 @@ or the `environment:` section in `compose.yaml` file. | YTP_TASK_HANDLER_RANDOM_DELAY | The maximum random delay in seconds before starting a task handler. | `60` | | YTP_IGNORE_ARCHIVED_ITEMS | Don't report archived items in the download history. | `false` | | YTP_CHECK_FOR_UPDATES | Whether to check for application updates. | `true` | +| YTP_EXTRACT_INFO_CONCURRENCY | The number of concurrent extract info operations. | `4` | > [!NOTE] > To raise the maximum workers for specific extractor, you need to add a ENV variable that follows the pattern `YTP_MAX_WORKERS_FOR_`. diff --git a/app/library/DataStore.py b/app/library/DataStore.py index 33cb44d5..28302d62 100644 --- a/app/library/DataStore.py +++ b/app/library/DataStore.py @@ -4,7 +4,7 @@ from collections import OrderedDict from collections.abc import Iterable from enum import Enum -from .Download import Download +from .downloads import Download from .ItemDTO import ItemDTO from .operations import matches_condition from .sqlite_store import SqliteStore diff --git a/app/library/Download.py b/app/library/Download.py deleted file mode 100644 index 9308b466..00000000 --- a/app/library/Download.py +++ /dev/null @@ -1,742 +0,0 @@ -import asyncio -import hashlib -import logging -import multiprocessing -import os -import re -import signal -import time -from datetime import UTC, datetime -from email.utils import formatdate -from pathlib import Path -from typing import Any - -import yt_dlp.utils - -from .config import Config -from .Events import EventBus, Events -from .ffprobe import ffprobe -from .ItemDTO import ItemDTO -from .Utils import create_cookies_file, delete_dir, extract_info, extract_ytdlp_logs -from .ytdlp import YTDLP - -GENERIC_EXTRACTORS = ("HTML5MediaEmbed", "generic") -"Generic extractors identifiers." - - -class Terminator: - pass - - -class NestedLogger: - debug_messages: list[str] = ["[debug] ", "[download] "] - - def __init__(self, logger: logging.Logger) -> None: - self.logger: logging.Logger = logger - - def debug(self, msg: str) -> None: - levelno: int = logging.DEBUG if any(msg.startswith(x) for x in self.debug_messages) else logging.INFO - self.logger.log(level=levelno, msg=re.sub(r"^\[(debug|info)\] ", "", msg, flags=re.IGNORECASE)) - - def error(self, msg) -> None: - self.logger.error(msg) - - def warning(self, msg) -> None: - self.logger.warning(msg) - - -class Download: - """ - Download task. - """ - - temp_path: str = None - update_task = None - cancel_in_progress: bool = False - final_update = False - - bad_live_options: list = [ - "concurrent_fragment_downloads", - "fragment_retries", - "skip_unavailable_fragments", - ] - "Bad yt-dlp options which are known to cause issues with live stream." - - _ytdlp_fields: tuple = ( - "tmpfilename", - "filename", - "status", - "msg", - "total_bytes", - "total_bytes_estimate", - "downloaded_bytes", - "speed", - "eta", - ) - "Fields to be extracted from yt-dlp progress hook." - - def __init__(self, info: ItemDTO, info_dict: dict = None, logs: list | None = None): - """ - Initialize download task. - - Args: - info (ItemDTO): ItemDTO object. - info_dict (dict): yt-dlp metadata dict. - logs (list): Logs from yt-dlp. - - """ - config: Config = Config.get_instance() - - self.download_dir: str = info.download_dir - "Download directory." - self.temp_dir: str | None = info.temp_dir - "Temporary directory." - self.template: str | None = info.template - "Filename template." - self.template_chapter: str | None = info.template_chapter - "Chapter filename template." - self.download_info_expires = int(config.download_info_expires) - "Time in seconds before the download info is considered expired." - self.info: ItemDTO = info - "ItemDTO object." - self.id: str = info._id - "Download ID." - self.debug = bool(config.debug) - "Debug mode." - self.debug_ytdl = bool(config.ytdlp_debug) - "Debug mode for yt-dlp." - self.cancelled = False - "Download cancelled." - self.tmpfilename = None - "Temporary filename." - self.status_queue = None - "Status queue." - self.proc = None - "yt-dlp process." - self._notify: EventBus = EventBus.get_instance() - "Event bus instance." - self.max_workers = int(config.max_workers) - "Maximum number of concurrent downloads." - self.temp_keep = bool(config.temp_keep) - "Keep temp folder after download." - self.temp_disabled = bool(config.temp_disabled) - "Disable the temporary files feature." - self.is_live: bool = bool(info.is_live) or info.live_in is not None - "Is the download a live stream." - self.info_dict: dict = info_dict - "yt-dlp metadata dict." - self.logger: logging.Logger = logging.getLogger(f"Download.{info.id if info.id else info._id}") - "Logger for the download task." - self.started_time = 0 - "Time when the download started." - self.queue_time: datetime = datetime.now(tz=UTC) - "Time when the download was queued." - self.logs: list[str] = logs if logs else [] - "Logs from yt-dlp." - - def _progress_hook(self, data: dict) -> None: - if self.debug: - try: - d_safe: dict = { - "status": data.get("status"), - "filename": data.get("filename"), - "info_dict": { - k: v - for k, v in data.get("info_dict", {}).items() - if k not in ["formats", "thumbnails", "description", "tags", "_format_sort_fields"] - and not isinstance(v, (type(None).__bases__[0], type(lambda: None))) # Skip non-serializable - }, - } - self.logger.debug(f"PG Hook: {d_safe}") - except Exception as e: - self.logger.debug(f"PG Hook: Error creating debug info: {e}") - - self.status_queue.put( - { - "id": self.id, - "action": "progress", - **{k: v for k, v in data.items() if k in self._ytdlp_fields}, - } - ) - - def _postprocessor_hook(self, data: dict) -> None: - if self.debug: - try: - d_safe: dict = { - "postprocessor": data.get("postprocessor"), - "status": data.get("status"), - "info_dict": { - k: v - for k, v in data.get("info_dict", {}).items() - if k not in ["formats", "thumbnails", "description", "tags", "_format_sort_fields"] - and not isinstance(v, (type(None).__bases__[0], type(lambda: None))) # Skip non-serializable - }, - } - self.logger.debug(f"PP Hook: {d_safe}") - except Exception as e: - self.logger.debug(f"PP Hook: Error creating debug info: {e}") - - if "MoveFiles" == data.get("postprocessor") and "finished" == data.get("status"): - if "__finaldir" in data.get("info_dict", {}) and "filepath" in data.get("info_dict", {}): - filename = str(Path(data["info_dict"]["__finaldir"]) / Path(data["info_dict"]["filepath"]).name) - else: - filename: str | None = data.get("info_dict", {}).get("filepath", data.get("filename")) - - self.logger.debug(f"Final filename: '{filename}'.") - self.status_queue.put({"id": self.id, "action": "moved", "status": "finished", "final_name": filename}) - return - - dataDict = {k: v for k, v in data.items() if k in self._ytdlp_fields} - self.status_queue.put({"id": self.id, "action": "postprocessing", **dataDict, "status": "postprocessing"}) - - def _post_hooks(self, filename: str | None = None) -> None: - if not filename: - return - - self.status_queue.put({"id": self.id, "filename": filename}) - - def _download(self) -> None: - if not self._notify: - self._notify = EventBus.get_instance() - - cookie_file = None - - try: - params: dict = ( - self.info.get_ytdlp_opts() - .add( - config={ - "color": "no_color", - "paths": { - "home": str(self.download_dir), - "temp": str(self.temp_path) if self.temp_path else self.download_dir, - }, - "outtmpl": { - "default": self.template, - "chapter": self.template_chapter, - }, - "noprogress": True, - "ignoreerrors": False, - }, - from_user=False, - ) - .get_all() - ) - - params.update( - { - "progress_hooks": [self._progress_hook], - "postprocessor_hooks": [self._postprocessor_hook], - "post_hooks": [self._post_hooks], - } - ) - - if self.debug_ytdl: - params["verbose"] = True - params["noprogress"] = False - - if self.info.cookies: - try: - cookie_file: Path = ( - Path(self.temp_path if self.temp_path else self.temp_dir) / f"cookie_{self.info._id}.txt" - ) - self.logger.debug( - f"Creating cookie file for '{self.info.id}: {self.info.title}' - '{cookie_file}'." - ) - params["cookiefile"] = str(create_cookies_file(self.info.cookies, cookie_file)) - except Exception as e: - err_msg: str = f"Failed to create cookie file for '{self.info.id}: {self.info.title}'. '{e!s}'." - self.logger.error(err_msg) - raise ValueError(err_msg) from e - - if self.info_dict and isinstance(self.info_dict, dict): - # If info extractor_key is generic, we need to remove download_archive param from default preset. - if self.info_dict.get("extractor_key") in GENERIC_EXTRACTORS and self.info.get_preset().default: - self.logger.debug(f"Removing 'download_archive' for generic extractor. {self.info.url=}") - params.pop("download_archive", None) - - # Safe-guard in-case downloading take too long and the info expires. - if self.download_info_expires > 0: - _ts: int | None = self.info_dict.get("epoch", self.info_dict.get("timestamp", None)) - _ts = datetime.fromtimestamp(_ts, tz=UTC) if _ts else None - if not _ts or (datetime.now(tz=UTC) - _ts).total_seconds() > self.download_info_expires: - self.info_dict = None - self.logger.warning(f"Info for '{self.info.url}' has expired, re-extracting info.") - - if not self.info_dict or not isinstance(self.info_dict, dict): - self.logger.info(f"Extracting info for '{self.info.url}'.") - self.logs = [] - ie_params: dict = params.copy() - ie_params["callback"] = { - "func": lambda _, msg: self.logs.append(msg), - "level": logging.WARNING, - "name": "callback-logger", - } - - info: dict = extract_info( - config=params, - url=self.info.url, - debug=self.debug, - no_archive=not params.get("download_archive", False), - follow_redirect=True, - ) - - if info: - self.info_dict = info - - if self.is_live: - deletedOpts: list = [] - for opt in self.bad_live_options: - if opt in params: - params.pop(opt, None) - deletedOpts.append(opt) - - if len(deletedOpts) > 0: - self.logger.warning( - f"Live stream detected for '{self.info.title}', The following opts '{deletedOpts=}' have been deleted." - ) - - if isinstance(self.info_dict, dict) and not ( - len(self.info_dict.get("formats", [])) > 0 or self.info_dict.get("url") - ): - msg: str = f"Failed to extract any formats for '{self.info.url}'." - if filtered_logs := extract_ytdlp_logs(self.logs): - msg += " " + ", ".join(filtered_logs) - - raise ValueError(msg) # noqa: TRY301 - - self.logger.info( - f'Download {self.info.name()}, preset="{self.info.preset}", cookies="{bool(params.get("cookiefile"))}" started.' - ) - - if self.debug: - self.logger.debug(f"Params before passing to yt-dlp. {params}") - - params["logger"] = NestedLogger(self.logger) - - if "continuedl" in params and params["continuedl"] is False: - self.delete_temp(by_pass=True) - - cls = YTDLP(params=params) - - self.started_time = int(time.time()) - - if "posix" == os.name: - - def mark_cancelled(*_) -> None: - cls._interrupted = True - cls.to_screen("[info] Interrupt received, exiting cleanly...") - raise SystemExit(130) # noqa: TRY301 - - signal.signal(signal.SIGUSR1, mark_cancelled) - - self.status_queue.put({"id": self.id, "status": "downloading"}) - - if isinstance(self.info_dict, dict) and len(self.info_dict) > 1: - self.logger.debug(f"Downloading '{self.info.url}' using pre-info.") - _dct: dict = self.info_dict.copy() - if isinstance(self.info.extras, dict) and len(self.info.extras) > 0: - _dct.update( - { - k: v - for k, v in self.info.extras.items() - if k not in _dct and v is not None and k not in ("is_live",) - } - ) - - cls.process_ie_result(ie_result=_dct, download=True) - ret: int = cls._download_retcode - else: - self.logger.debug(f"Downloading using url: {self.info.url}") - ret: int = cls.download(url_list=[self.info.url]) - - self.status_queue.put({"id": self.id, "status": "finished" if ret == 0 else "error"}) - except yt_dlp.utils.ExistingVideoReached as exc: - self.logger.error(exc) - self.status_queue.put({"id": self.id, "status": "skip", "msg": "Item has already been downloaded."}) - except Exception as exc: - self.logger.exception(exc) - self.logger.error(exc) - self.status_queue.put({"id": self.id, "status": "error", "msg": str(exc), "error": str(exc)}) - finally: - self.status_queue.put(Terminator()) - if cookie_file and cookie_file.exists(): - try: - cookie_file.unlink() - self.logger.debug(f"Deleted cookie file: {cookie_file}") - except Exception as e: - self.logger.error(f"Failed to delete cookie file: {cookie_file}. {e}") - - self.logger.info( - f'Task {self.info.name()} preset="{self.info.preset}" cookies="{bool(params.get("cookiefile"))}" completed.' - ) - - async def start(self) -> None: - self.status_queue: multiprocessing.Queue[Any] = Config.get_manager().Queue() - - # Create temp dir for each download. - if not self.temp_disabled: - self.temp_path = Path(self.temp_dir) / hashlib.shake_256(f"D-{self.info.id}".encode()).hexdigest(5) - - if not self.temp_path.exists(): - self.temp_path.mkdir(parents=True, exist_ok=True) - - self.info.temp_path = str(self.temp_path) - - self.proc = multiprocessing.Process(name=f"download-{self.id}", target=self._download) - self.proc.start() - self.info.status = "preparing" - - self._notify.emit(Events.ITEM_UPDATED, data=self.info) - asyncio.create_task(self.progress_update(), name=f"update-{self.id}") - - ret = await asyncio.get_running_loop().run_in_executor(None, self.proc.join) - - if self.final_update or self.cancelled: - if self.cancelled: - self.info.status = "cancelled" - return ret - - self.status_queue.put(Terminator()) - self.logger.debug("Draining status queue.") - try: - drain_count: int = 50 + (self.status_queue.qsize() if hasattr(self.status_queue, "qsize") else 5) - except Exception: - drain_count = 55 - - for i in range(drain_count): - try: - if self.debug: - self.logger.debug(f"(50/{i}) Draining the status queue...") - if self.final_update: - if self.debug: - self.logger.debug("(50/{i}) Draining stopped. Final update received.") - break - next_status = self.status_queue.get(timeout=0.1) - if next_status is None or isinstance(next_status, Terminator): - continue - await self._process_status_update(next_status) - except Exception: # noqa: S112 - continue - - return ret - - def started(self) -> bool: - return self.proc is not None - - def cancel(self) -> bool: - """ - Cancel the download task. - For live streams and unresponsive processes, this will force-kill the process. - """ - if not self.started(): - return False - - self.cancelled = True - - # Actively kill the process instead of just setting a flag - return self.kill() - - async def close(self) -> bool: - """ - Close download process. - This method MUST be called to clear resources. - """ - if not self.started() or self.cancel_in_progress: - return False - - self.cancel_in_progress = True - procId: int | None = self.proc.ident - - if not procId: - if self.proc: - self.proc.close() - self.proc = None - self.logger.warning("Attempted to close download process, but it is not running.") - return False - - self.logger.info(f"Closing PID='{procId}' download process.") - - try: - try: - if "update_task" in self.__dict__ and self.update_task.done() is False: - self.update_task.cancel() - except Exception as e: - self.logger.error(f"Failed to close status queue: '{procId}'. {e}") - - self.kill() - - loop = asyncio.get_running_loop() - - if self.proc.is_alive(): - self.logger.debug(f"Waiting for PID='{procId}' to close.") - await loop.run_in_executor(None, self.proc.join) - self.logger.debug(f"PID='{procId}' closed.") - - if self.status_queue: - self.logger.debug(f"Closing status queue for PID='{procId}'.") - self.status_queue.put(Terminator()) - self.logger.debug(f"Closed status queue for PID='{procId}'.") - - if self.proc: - self.proc.close() - self.proc = None - - self.delete_temp() - - self.logger.debug(f"Closed PID='{procId}' download process.") - - return True - except Exception as e: - self.logger.error(f"Failed to close process: '{procId}'. {e}") - self.logger.exception(e) - - return False - - def running(self) -> bool: - try: - return self.proc is not None and self.proc.is_alive() - except ValueError: - return False - - def is_cancelled(self) -> bool: - return self.cancelled - - def kill(self) -> bool: - """ - Kill the download process. - Attempts graceful termination via signal first, then force-kills if necessary. - For live streams that may not respond to signals, uses force termination. - """ - if not self.running(): - return False - - procId: int | None = self.proc.ident - try: - self.logger.info(f"Killing download process: PID={self.proc.pid}, ident={procId}.") - - # First, try graceful termination with SIGUSR1 signal (if on POSIX) - if self.proc.pid and "posix" == os.name: - try: - self.logger.debug(f"Sending SIGUSR1 signal to PID={self.proc.pid}.") - os.kill(self.proc.pid, signal.SIGUSR1) - - # Wait briefly for graceful shutdown (1 second timeout for live streams) - start_time = time.time() - timeout = 2 if self.is_live else 5 # Live streams get shorter timeout - while self.proc.is_alive() and (time.time() - start_time) < timeout: - time.sleep(0.1) - - if self.proc.is_alive(): - self.logger.warning( - f"Process PID={self.proc.pid} did not respond to SIGUSR1 " - f"({'live stream' if self.is_live else 'regular download'}), " - f"forcing termination." - ) - else: - self.logger.debug(f"Process PID={self.proc.pid} terminated gracefully.") - return True - - except (OSError, AttributeError) as e: - self.logger.debug(f"Failed to send SIGUSR1 signal: {e}") - - # Force terminate if still running (for live streams or unresponsive processes) - if self.proc.is_alive(): - self.logger.info(f"Force-terminating process PID={self.proc.pid}.") - self.proc.terminate() - - # Give it a moment to terminate - start_time = time.time() - timeout = 1 if self.is_live else 2 - while self.proc.is_alive() and (time.time() - start_time) < timeout: - time.sleep(0.05) - - # If still alive, use hard kill - if self.proc.is_alive(): - self.logger.warning( - f"Process PID={self.proc.pid} did not respond to terminate(), killing forcefully." - ) - self.proc.kill() - # Give it final moment - start_time = time.time() - while self.proc.is_alive() and (time.time() - start_time) < 1: - time.sleep(0.05) - - self.logger.info(f"Process PID={self.proc.pid} killed.") - return True - - except Exception as e: - self.logger.error(f"Failed to kill process PID={self.proc.pid}, ident={procId}. {e}") - self.logger.exception(e) - - return False - - def delete_temp(self, by_pass: bool = False) -> None: - if self.temp_disabled or self.temp_keep is True or not self.temp_path: - return - - if ( - not by_pass - and "finished" != self.info.status - and self.info.downloaded_bytes - and self.info.downloaded_bytes > 0 - ): - self.logger.warning(f"Keeping temp folder '{self.temp_path}'. {self.info.status=}.") - return - - tmp_dir = Path(self.temp_path) - - if not tmp_dir.exists(): - return - - if str(tmp_dir) == str(self.temp_dir): - self.logger.warning(f"Refusing to delete video temp folder '{self.temp_path}' as it's temp root.") - return - - status: bool = delete_dir(tmp_dir) - if by_pass: - tmp_dir.mkdir(parents=True, exist_ok=True) - self.logger.info(f"Temp folder '{self.temp_path}' emptied.") - else: - self.logger.info(f"Temp folder '{self.temp_path}' deletion is {'success' if status else 'failed'}.") - - async def _process_status_update(self, status) -> None: - if status.get("id") != self.id or len(status) < 2: - self.logger.warning(f"Received invalid status update. {status}") - return - - if self.debug: - self.logger.debug(f"Status Update: {self.info._id=} {status=}") - - if isinstance(status, str): - self._notify.emit(Events.ITEM_UPDATED, data=self.info) - return - - self.tmpfilename: str | None = status.get("tmpfilename") - - fl = None - if "final_name" in status: - fl = Path(status.get("final_name")) - try: - self.info.filename = str(fl.relative_to(Path(self.download_dir))) - except ValueError as ve: - self.logger.debug(f"Failed to get relative path for '{fl}' from '{self.download_dir}'. {ve}") - if self.temp_path: - self.info.filename = str(fl.relative_to(Path(self.temp_path))) - else: - self.info.filename = str(fl) - - if fl.is_file() and fl.exists(): - self.final_update = True - self.logger.debug(f"Final file name: '{fl}'.") - - try: - self.info.file_size = fl.stat().st_size - except FileNotFoundError: - self.info.file_size = 0 - - self.info.status = status.get("status", self.info.status) - self.info.msg = status.get("msg") - - if "error" == self.info.status and "error" in status: - self.info.error = status.get("error") - self._notify.emit( - Events.LOG_ERROR, - data=self.info, - title="Download Error", - message=f"'{self.info.title}' failed to download. {self.info.error}", - ) - - if "downloaded_bytes" in status and status.get("downloaded_bytes", 0) > 0: - self.info.downloaded_bytes = status.get("downloaded_bytes") - total: float | None = status.get("total_bytes") or status.get("total_bytes_estimate") - if total: - try: - self.info.percent = status["downloaded_bytes"] / total * 100 - except ZeroDivisionError: - self.info.percent = 0 - self.info.total_bytes = total - - self.info.speed = status.get("speed") - self.info.eta = status.get("eta") - - if "finished" == self.info.status and fl and fl.is_file() and fl.exists(): - self.info.file_size = fl.stat().st_size - self.info.datetime = str(formatdate(time.time())) - - try: - ff = await ffprobe(str(fl)) - self.info.extras["is_video"] = ff.has_video() - self.info.extras["is_audio"] = ff.has_audio() - if (ff.has_video() or ff.has_audio()) and not self.info.extras.get("duration"): - self.info.extras["duration"] = int(float(ff.metadata.get("duration", 0.0))) - except Exception as e: - self.info.extras["is_video"] = True - self.info.extras["is_audio"] = True - self.logger.exception(e) - self.logger.error(f"Failed to run ffprobe. {status.get}. {e}") - - if not self.final_update or fl: - self._notify.emit(Events.ITEM_UPDATED, data=self.info) - - async def progress_update(self) -> None: - """ - Update status of download task and notify the client. - """ - while True: - try: - self.update_task = asyncio.get_running_loop().run_in_executor(None, self.status_queue.get) - status = await self.update_task - if status is None or isinstance(status, Terminator): - return - await self._process_status_update(status) - except (asyncio.CancelledError, OSError, FileNotFoundError): - return - - def is_stale(self) -> bool: - """ - Check if the download task is stale. - A download task is considered stale if it has not been updated for a certain period of time. - - Returns: - bool: True if the download task is stale, False otherwise. - - """ - if not self.info.auto_start: - return False - - if self.started_time < 1: - self.logger.debug(f"Download task '{self.info.name()}' not started yet.") - return False - - if int(time.time()) - self.started_time < 300: - self.logger.debug( - f"Download task '{self.info.title}: {self.info.id}' started for '{int(time.time()) - self.started_time}' seconds." - ) - return False - - if not self.running(): - self.logger.warning( - f"Download task '{self.info.title}: {self.info.id}' not started running for '{int(time.time()) - self.started_time}' seconds." - ) - return True - - if self.info.status not in ["finished", "error", "cancelled", "downloading", "postprocessing"]: - status: str = self.info.status if self.info.status else "unknown" - self.logger.warning( - f"Download task '{self.info.title}: {self.info.id}' has been stuck in '{status}' state for '{int(time.time()) - self.started_time}' seconds." - ) - return True - - return False - - def __getstate__(self) -> dict[str, Any]: - state: dict[str, Any] = self.__dict__.copy() - - # Exclude (unpickleable) keys during pickling, this issue arise mostly on Windows. - excluded_keys: tuple[str] = ("_notify",) - for key in excluded_keys: - if key in state: - state[key] = None - - return state diff --git a/app/library/DownloadQueue.py b/app/library/DownloadQueue.py deleted file mode 100644 index 08c6045e..00000000 --- a/app/library/DownloadQueue.py +++ /dev/null @@ -1,1425 +0,0 @@ -import asyncio -import functools -import glob -import logging -import os -import time -import traceback -import uuid -from datetime import UTC, datetime, timedelta -from email.utils import formatdate -from pathlib import Path -from typing import TYPE_CHECKING, Any - -import yt_dlp.utils -from aiohttp import web - -from .ag_utils import ag -from .conditions import Conditions -from .config import Config -from .DataStore import DataStore, StoreType -from .Download import Download -from .Events import EventBus, Events -from .ItemDTO import Item, ItemDTO -from .Presets import Presets -from .Scheduler import Scheduler -from .Services import Services -from .Singleton import Singleton -from .sqlite_store import SqliteStore -from .Utils import ( - archive_add, - archive_read, - arg_converter, - calc_download_path, - create_cookies_file, - dt_delta, - extract_info, - extract_ytdlp_logs, - get_extras, - merge_dict, - str_to_dt, - ytdlp_reject, -) - -if TYPE_CHECKING: - from app.library.Presets import Preset - -LOG: logging.Logger = logging.getLogger("DownloadQueue") - - -class DownloadQueue(metaclass=Singleton): - """ - DownloadQueue class is a singleton class that manages the download queue and the download history. - """ - - def __init__(self, config: Config | None = None): - self.config: Config = config or Config.get_instance() - "Configuration instance." - self._notify: EventBus = EventBus.get_instance() - "Event bus instance." - self.done = DataStore(type=StoreType.HISTORY, connection=SqliteStore.get_instance()) - "DataStore for the completed downloads." - self.queue = DataStore(type=StoreType.QUEUE, connection=SqliteStore.get_instance()) - "DataStore for the download queue." - self.workers = asyncio.Semaphore(self.config.max_workers) - "Semaphore to limit the number of concurrent downloads." - self.processors = asyncio.Semaphore(self.config.playlist_items_concurrency) - "Semaphore to limit the number of concurrent processors." - self.limits: dict[str, asyncio.Semaphore] = {} - "Per-extractor semaphores to limit concurrent downloads per extractor." - self.paused = asyncio.Event() - "Event to pause the download queue." - self.event = asyncio.Event() - "Event to signal the download queue to start downloading." - self._active: dict[str, Download] = {} - """Dictionary of active downloads.""" - - self.paused.set() - - @staticmethod - def get_instance(config: Config | None = None) -> "DownloadQueue": - """ - Get the instance of the DownloadQueue. - - Returns: - DownloadQueue: The instance of the DownloadQueue - - """ - return DownloadQueue(config=config) - - def _get_limit(self, extractor: str) -> asyncio.Semaphore: - """ - Get or create a semaphore for the given extractor. - - Args: - extractor (str): The extractor name. - - Returns: - asyncio.Semaphore: The semaphore for the extractor. - - """ - if extractor not in self.limits: - env_limit: str | None = os.environ.get(f"YTP_MAX_WORKERS_FOR_{extractor.upper()}") - - # Determine effective limit - if env_limit and env_limit.isdigit() and 1 <= int(env_limit): - limit: int = min(int(env_limit), self.config.max_workers) - else: - if env_limit: - LOG.warning(f"Invalid extractor limit '{env_limit}' for '{extractor}', using default limit.") - limit = self.config.max_workers_per_extractor - - limit = min(limit, self.config.max_workers) - - self.limits[extractor] = asyncio.Semaphore(limit) - LOG.info(f"Created limits container for extractor '{extractor}': {limit}") - - return self.limits[extractor] - - def attach(self, _: web.Application) -> None: - """ - Attach the download queue to the application. - - Args: - _ (web.Application): The application to attach the download queue to. - - """ - Services.get_instance().add("queue", self) - - async def event_handler(_, __): - await self.initialize() - - self._notify.subscribe(Events.STARTED, event_handler, f"{__class__.__name__}.{__class__.initialize.__name__}") - - Scheduler.get_instance().add( - timer="* * * * *", - func=self._check_for_stale, - id=f"{__class__.__name__}.{__class__._check_for_stale.__name__}", - ) - - Scheduler.get_instance().add( - timer="* * * * *", - func=self._check_live, - id=f"{__class__.__name__}.{__class__._check_live.__name__}", - ) - - if self.config.auto_clear_history_days > 0: - Scheduler.get_instance().add( - # timer="8 */1 * * *", - timer="* * * * *", - func=self._delete_old_history, - id=f"{__class__.__name__}.{__class__._delete_old_history.__name__}", - ) - - # app.on_shutdown.append(self.on_shutdown) - - async def test(self) -> bool: - """ - Test the datastore connection to the database. - - Returns: - bool: True if the test is successful, False otherwise. - - """ - await self.done.test() - return True - - async def initialize(self) -> None: - """ - Initialize the download queue. - """ - await self.queue.load() - LOG.info( - f"Using '{self.config.max_workers}' workers for downloading and '{self.config.max_workers_per_extractor}' per extractor." - ) - asyncio.create_task(self._download_pool(), name="download_pool") - - async def start_items(self, ids: list[str]) -> dict[str, str]: - """ - Start one or more queued downloads that were added with auto_started=False. - - Args: - ids (list[str]): List of item IDs to start. - - Returns: - dict[str, str]: Dictionary of per-ID results and overall status. - - """ - status: dict[str, str] = {"status": "ok"} - started = False - - for item_id in ids: - try: - item: Download = await self.queue.get(key=item_id) - except KeyError as e: - status[item_id] = f"not found: {e!s}" - status["status"] = "error" - LOG.warning(f"Start requested for non-existent item {item_id=}.") - continue - - if item.info.auto_start: - status[item_id] = "already started" - continue - - item.info.auto_start = True - updated: Download = await self.queue.put(item) - self._notify.emit(Events.ITEM_UPDATED, data=updated.info) - self._notify.emit( - Events.ITEM_RESUMED, - data=item.info, - title="Download Resumed", - message=f"Download '{item.info.title}' has been resumed.", - ) - status[item_id] = "started" - started = True - - if started: - self.event.set() - - return status - - async def pause_items(self, ids: list[str]) -> dict[str, str]: - """ - Pause one or more queued downloads that were added with auto_started=True. - - Args: - ids (list[str]): List of item IDs to pause. - - Returns: - dict[str, str]: Dictionary of per-ID results and overall status. - - """ - status: dict[str, str] = {"status": "ok"} - - for item_id in ids: - try: - item: Download = await self.queue.get(key=item_id) - except KeyError as e: - status[item_id] = f"not found: {e!s}" - status["status"] = "error" - LOG.warning(f"Start requested for non-existent item {item_id=}.") - continue - - if item.started() or item.is_cancelled(): - status[item_id] = "already started" - continue - - if item.info.auto_start is False: - status[item_id] = "not started" - continue - - item.info.auto_start = False - updated: Download = await self.queue.put(item) - self._notify.emit(Events.ITEM_UPDATED, data=updated.info) - self._notify.emit( - Events.ITEM_PAUSED, - data=item.info, - title="Download Paused", - message=f"Download '{item.info.title}' has been paused.", - ) - status[item_id] = "paused" - - return status - - def pause(self, shutdown: bool = False) -> bool: - """ - Pause the download queue. - - Returns: - bool: True if the download is paused, False otherwise - - """ - if self.paused.is_set(): - self.paused.clear() - if not shutdown: - LOG.warning(f"Download paused at. {datetime.now(tz=UTC).isoformat()}") - return True - - return False - - def resume(self) -> bool: - """ - Resume the download queue. - - Returns: - bool: True if the download is resumed, False otherwise - - """ - if not self.paused.is_set(): - self.paused.set() - LOG.warning(f"Downloading resumed at. {datetime.now(tz=UTC).isoformat()}") - return True - - return False - - def is_paused(self) -> bool: - """ - Check if the download queue is paused. - - Returns: - bool: True if the download queue is paused, False otherwise - - """ - return not self.paused.is_set() - - async def on_shutdown(self, _: web.Application): - LOG.debug("Canceling all active downloads.") - if self._active: - self.pause() - try: - await self.cancel(list(self._active.keys())) - except Exception as e: - LOG.error(f"Failed to cancel downloads. {e!s}") - - async def _process_playlist(self, entry: dict, item: Item, already=None, yt_params: dict | None = None): - if not yt_params: - yt_params = {} - - entries = entry.get("entries", []) - - playlist_name: str = f"{entry.get('id')}: {entry.get('title')}" - - LOG.info(f"Processing '{playlist_name} ({len(entries)})' Playlist.") - - playlistCount = entry.get("playlist_count") - playlistCount: int = int(playlistCount) if playlistCount else len(entries) - - playlist_keys: dict[str, Any] = { - "playlist_count": playlistCount, - "playlist": entry.get("title") or entry.get("id"), - "playlist_id": entry.get("id"), - "playlist_title": entry.get("title"), - "playlist_uploader": entry.get("uploader"), - "playlist_uploader_id": entry.get("uploader_id"), - "playlist_channel": entry.get("channel"), - "playlist_channel_id": entry.get("channel_id"), - "playlist_webpage_url": entry.get("webpage_url"), - "__last_playlist_index": playlistCount - 1, - "n_entries": len(entries), - } - - async def playlist_processor(i: int, etr: dict): - acquired = False - try: - item_name: str = ( - f"'{entry.get('title')}: {i}/{playlist_keys['n_entries']}' - '{etr.get('id')}: {etr.get('title')}'" - ) - LOG.debug(f"Waiting to acquire lock for {item_name}") - await self.processors.acquire() - acquired = True - LOG.debug(f"Acquired lock for {item_name}") - - LOG.info(f"Processing '{item_name}'.") - - _status, _msg = ytdlp_reject(entry=etr, yt_params=yt_params) - if not _status: - return {"status": "error", "msg": _msg} - - extras: dict[str, Any] = { - **playlist_keys, - "playlist_index": i, - "playlist_index_number": i, - "playlist_autonumber": i, - } - - for property in ("id", "title", "uploader", "uploader_id"): - if property in entry: - extras[f"playlist_{property}"] = entry.get(property) - - extractor_key = entry.get("ie_key") or entry.get("extractor_key") or entry.get("extractor") or "" - if "thumbnail" not in etr and "youtube" in str(extractor_key).lower(): - extras["thumbnail"] = "https://img.youtube.com/vi/{id}/maxresdefault.jpg".format(**etr) - - newItem: Item = item.new_with(url=etr.get("url") or etr.get("webpage_url"), extras=extras) - - if ("video" == etr.get("_type") and etr.get("url")) or ( - "formats" in etr and isinstance(etr["formats"], list) and len(etr["formats"]) > 0 - ): - dct = merge_dict(merge_dict({"_type": "video"}, etr), entry) - dct.pop("entries", None) - return await self.add(item=newItem, entry=dct, already=already) - - return await self.add(item=newItem, already=already) - finally: - if acquired: - self.processors.release() - - max_downloads: int = -1 - ytdlp_opts: dict[str, Any] = item.get_ytdlp_opts().get_all() - if ytdlp_opts.get("max_downloads") and isinstance(ytdlp_opts.get("max_downloads"), int): - max_downloads: int = ytdlp_opts.get("max_downloads") - - batch_size: int = max(50, int(self.config.playlist_items_concurrency) * 10) - - async def run_batch(batch: list[tuple[int, dict]]) -> list[dict]: - tasks: list[asyncio.Task] = [] - for i, etr in batch: - task = asyncio.create_task( - playlist_processor(i, etr), - name=f"playlist_processor_{etr.get('id')}_{i}", - ) - task.add_done_callback(self._handle_task_exception) - tasks.append(task) - - batch_results: list[dict] = await asyncio.gather(*tasks) - await asyncio.sleep(0) - return batch_results - - results: list[dict] = [] - batch: list[tuple[int, dict]] = [] - - for i, etr in enumerate(entries, start=1): - if max_downloads > 0 and i > max_downloads: - break - - batch.append((i, etr)) - if len(batch) >= batch_size: - results.extend(await run_batch(batch)) - batch.clear() - - if batch: - results.extend(await run_batch(batch)) - - log_msg: str = f"Playlist '{playlist_name}' processing completed with '{len(results)}' entries." - if max_downloads > 0 and len(entries) > max_downloads: - skipped: int = len(entries) - max_downloads - log_msg += f" Limited to '{max_downloads}' items, skipped '{skipped}' remaining items." - - LOG.info(log_msg) - - if any("error" == res["status"] for res in results): - return { - "status": "error", - "msg": ", ".join(res["msg"] for res in results if "error" == res["status"] and "msg" in res), - } - - return {"status": "ok"} - - async def _add_video(self, entry: dict, item: Item, logs: list[str] | None = None): - if not logs: - logs: list[str] = [] - - options: dict = {} - error: str | None = None - live_in: str | None = None - is_premiere: bool = bool(entry.get("is_premiere", False)) - - release_in: str | None = None - if entry.get("release_timestamp"): - release_in = formatdate(entry.get("release_timestamp"), usegmt=True) - item.extras["release_in"] = release_in - - # check if the video is live stream. - if "is_upcoming" == entry.get("live_status"): - if release_in: - live_in = release_in - item.extras["live_in"] = live_in - else: - error = f"No start time is set for {'premiere' if is_premiere else 'live stream'}." - else: - error = entry.get("msg") - - LOG.debug(f"Entry id '{entry.get('id')}' url '{entry.get('webpage_url')} - {entry.get('url')}'.") - - try: - _item: Download = await self.done.get(key=entry.get("id"), url=entry.get("webpage_url") or entry.get("url")) - err_msg: str = f"Removing {_item.info.name()} from history list." - LOG.warning(err_msg) - await self.clear([_item.info._id], remove_file=False) - except KeyError: - pass - - try: - _item: Download = await self.queue.get( - key=str(entry.get("id")), url=str(entry.get("webpage_url") or entry.get("url")) - ) - err_msg: str = f"Item {_item.info.name()} is already in download queue." - LOG.info(err_msg) - return {"status": "error", "msg": err_msg} - except KeyError: - pass - - live_status: list = ["is_live", "is_upcoming"] - is_live = bool(entry.get("is_live") or live_in or entry.get("live_status") in live_status) - - try: - download_dir: str = calc_download_path(base_path=self.config.download_path, folder=item.folder) - except Exception as e: - LOG.exception(e) - return {"status": "error", "msg": str(e)} - - for field in ("uploader", "channel", "thumbnail"): - if entry.get(field): - item.extras[field] = entry.get(field) - - for key in entry: - if isinstance(key, str) and key.startswith("playlist") and entry.get(key): - item.extras[key] = entry.get(key) - - item.extras["duration"] = entry.get("duration", item.extras.get("duration")) - - if not item.extras.get("live_in") and live_in: - item.extras["live_in"] = live_in - - if not item.extras.get("is_premiere") and is_premiere: - item.extras["is_premiere"] = is_premiere - - dl = ItemDTO( - id=str(entry.get("id")), - title=str(entry.get("title")), - description=str(entry.get("description", "")), - url=str(entry.get("webpage_url") or entry.get("url")), - preset=item.preset, - folder=item.folder, - download_dir=download_dir, - temp_dir=self.config.temp_path, - cookies=item.cookies, - template=item.template if item.template else self.config.output_template, - template_chapter=self.config.output_template_chapter, - datetime=formatdate(time.time()), - error=error, - is_live=is_live, - live_in=live_in if live_in else item.extras.get("live_in", None), - options=options, - cli=item.cli, - auto_start=item.auto_start, - extras=merge_dict(item.extras, get_extras(entry)), - ) - - try: - dlInfo: Download = Download(info=dl, info_dict=entry if item.auto_start else None, logs=logs) - nEvent: str | None = None - nTitle: str | None = None - nMessage: str | None = None - nStore: str = "queue" - hasFormats: bool = len(entry.get("formats", [])) > 0 or entry.get("url") - - text_logs: str = "" - if filtered_logs := extract_ytdlp_logs(logs): - text_logs = " " + ", ".join(filtered_logs) - - if "is_upcoming" == entry.get("live_status"): - nEvent = Events.ITEM_MOVED - nStore = "history" - nTitle = "Upcoming Premiere" if is_premiere else "Upcoming Live Stream" - nMessage = f"{'Premiere video' if is_premiere else 'Stream'} '{dlInfo.info.title}' is not available yet. {text_logs}" - - dlInfo.info.status = "not_live" - dlInfo.info.msg = nMessage.replace(f" '{dlInfo.info.title}'", "") - self._notify.emit( - Events.LOG_INFO, - data={"preset": dlInfo.info.preset, "lowPriority": True}, - title=nTitle, - message=nMessage, - ) - - itemDownload: Download = await self.done.put(dlInfo) - elif not hasFormats: - ava: str = entry.get("availability", "public") - nTitle = "Download Error" - nMessage: str = f"No formats for '{dl.title}'." - nEvent = Events.ITEM_MOVED - nStore = "history" - - if ava and ava not in ("public",): - nMessage += f" Availability is set for '{ava}'." - - dlInfo.info.error = nMessage.replace(f" for '{dl.title}'.", ".") + text_logs - dlInfo.info.status = "error" - itemDownload = await self.done.put(dlInfo) - - self._notify.emit( - Events.LOG_WARNING, - data={"preset": dlInfo.info.preset, "logs": text_logs}, - title=nTitle, - message=nMessage, - ) - elif is_premiere and self.config.prevent_live_premiere: - nStore = "history" - nTitle = "Premiere Video" - dlInfo.info.error = "Premiering right now." - - _requeue = True - if release_in: - try: - starts_in: datetime = str_to_dt(release_in) - starts_in: datetime = ( - starts_in.replace(tzinfo=UTC) if starts_in.tzinfo is None else starts_in.astimezone(UTC) - ) - starts_in = starts_in + timedelta(minutes=5, seconds=dl.extras.get("duration", 0)) - dlInfo.info.error += f" Download will start at {starts_in.astimezone().isoformat()}." - _requeue = False - except Exception as e: - LOG.error(f"Failed to parse live_in date '{release_in}'. {e!s}") - dlInfo.info.error += f" Failed to parse live_in date '{release_in}'." - else: - dlInfo.info.error += f" Delaying download by '{300 + dl.extras.get('duration', 0)}' seconds." - - nMessage = f"'{dlInfo.info.title}': '{dlInfo.info.error.strip()}'." - - if _requeue: - nEvent = Events.ITEM_ADDED - itemDownload = await self.queue.put(dlInfo) - if item.auto_start: - self.event.set() - else: - dlInfo.info.status = "not_live" - itemDownload = await self.done.put(dlInfo) - nStore = "history" - nEvent = Events.ITEM_MOVED - nTitle = "Premiering right now" - self._notify.emit( - Events.LOG_INFO, data={"preset": dlInfo.info.preset}, title=nTitle, message=nMessage - ) - else: - nEvent = Events.ITEM_ADDED - nTitle = "Item Added" - nMessage = f"Item '{dlInfo.info.title}' has been added to the download queue." - itemDownload = await self.queue.put(dlInfo) - if item.auto_start: - self.event.set() - else: - LOG.debug(f"Item {itemDownload.info.name()} is not set to auto-start.") - - self._notify.emit( - nEvent, - data={"to": nStore, "preset": itemDownload.info.preset, "item": itemDownload.info} - if Events.ITEM_MOVED == nEvent - else itemDownload.info, - title=nTitle, - message=nMessage, - ) - - return {"status": "ok"} - except Exception as e: - LOG.exception(e) - LOG.error(f"Failed to download item. '{e!s}'") - return {"status": "error", "msg": str(e)} - - async def _add_item( - self, entry: dict, item: Item, already=None, logs: list | None = None, yt_params: dict | None = None - ): - """ - Add an entry to the download queue. - - Args: - entry (dict): The entry to add to the download queue. - item (Item): The item to add to the download queue. - already (set): The set of already downloaded items. - logs (list): The list of logs generated during information extraction. - yt_params (dict): The parameters for yt-dlp. - - Returns: - dict: The status of the operation. - - """ - if not entry: - return {"status": "error", "msg": "Invalid/empty data was given."} - - event_type = entry.get("_type", "video") - - if event_type.startswith("playlist"): - return await self._process_playlist(entry=entry, item=item, already=already, yt_params=yt_params) - - if event_type.startswith("url"): - return await self.add(item=item.new_with(url=entry.get("url")), already=already) - - if event_type.startswith("video"): - return await self._add_video(entry=entry, item=item, logs=logs) - - return {"status": "error", "msg": f'Unsupported event type "{event_type}".'} - - async def add(self, item: Item, already: set | None = None, entry: dict | None = None) -> dict[str, str]: - """ - Add an item to the download queue. - - Args: - item (Item): The item to be added to the queue. - already (set): Set of already downloaded items. - entry (dict): The entry associated with the item. - - Returns: - dict[str, str]: The status of the download. - { "status": "text" } - - """ - _preset: Preset | None = Presets.get_instance().get(item.preset) - - if item.has_cli(): - try: - arg_converter(args=item.cli, level=True) - except Exception as e: - LOG.error(f"Invalid command options for yt-dlp '{item.cli}'. {e!s}") - return {"status": "error", "msg": f"Invalid command options for yt-dlp '{item.cli}'. {e!s}"} - - if _preset: - if _preset.folder and not item.folder: - item.folder = _preset.folder - - if _preset.template and not item.template: - item.template = _preset.template - - yt_conf = {} - cookie_file: Path = Path(self.config.temp_path) / f"c_{uuid.uuid4().hex}.txt" - - LOG.info(f"Adding '{item!r}'.") - - already = set() if already is None else already - - if item.url in already: - LOG.warning(f"Recursion detected with url '{item.url}' skipping.") - return {"status": "ok"} - - already.add(item.url) - - try: - logs: list = [] - - yt_conf: dict = { - "callback": { - "func": lambda _, msg: logs.append(msg), - "level": logging.WARNING, - "name": "callback-logger", - }, - **item.get_ytdlp_opts().get_all(), - } - - if yt_conf.get("external_downloader"): - LOG.warning(f"Using external downloader '{yt_conf.get('external_downloader')}' for '{item.url}'.") - item.extras.update({"external_downloader": True}) - - archive_id: str | None = item.get_archive_id() - - # Early archive check to avoid unnecessary extraction calls - # This sometimes can be different from the final extracted ID, so we need to verify again after extraction. - if archive_id and item.is_archived(): - store_type, dlInfo = await self.get_item(archive_id=archive_id) - if not store_type and not self.config.ignore_archived_items: - dlInfo = Download( - info=ItemDTO( - id=archive_id.split()[1], - title=archive_id, - url=item.url, - preset=item.preset, - folder=item.folder, - status="skip", - cookies=item.cookies, - template=item.template, - msg="URL is already downloaded.", - extras=item.extras, - ) - ) - - if archive_file := dlInfo.info.get_ytdlp_opts().get_all().get("download_archive"): - dlInfo.info.msg += f" Found in archive '{archive_file}'." - - await self.done.put(dlInfo) - - self._notify.emit( - Events.ITEM_MOVED, - data={"to": "history", "preset": dlInfo.info.preset, "item": dlInfo.info}, - title="Download History Update", - message=f"Download history updated with '{item.url}'.", - ) - return {"status": "ok"} - - message: str = f"The URL '{item.url}':'{archive_id}' is already downloaded and recorded in archive." - LOG.warning(message) - self._notify.emit( - Events.LOG_INFO, data={"preset": item.preset}, title="Already Downloaded", message=message - ) - return {"status": "error", "msg": message, "hidden": True} - - started: float = time.perf_counter() - - if item.cookies: - try: - yt_conf["cookiefile"] = str(create_cookies_file(item.cookies, cookie_file)) - except Exception as e: - msg = f"Failed to create cookie file for '{item.url}'. '{e!s}'." - LOG.error(msg) - return {"status": "error", "msg": msg} - - LOG.info(f"Extracting '{item.url}'{' with cookies' if yt_conf.get('cookiefile') else ''}.") - - if not entry: - entry: dict | None = await asyncio.wait_for( - fut=asyncio.get_running_loop().run_in_executor( - None, - functools.partial( - extract_info, - config=yt_conf, - url=item.url, - debug=bool(self.config.ytdlp_debug), - no_archive=False, - follow_redirect=True, - ), - ), - timeout=self.config.extract_info_timeout, - ) - - if not entry: - LOG.error(f"Unable to extract info for '{item.url}'. Logs: {logs}") - return {"status": "error", "msg": "Unable to extract info." + "\n".join(logs)} - - # Sometimes playlists or extractor returns different ID than what we get from the make_archive_id() - # So, we need to re-check the archive after extraction to be sure the item was not downloaded. - # This also apply to old archive IDs that might have been used before. - if _archive_file := item.get_archive_file(): - extra_ids: list[str] = [] - - if entry.get("_old_archive_ids") and isinstance(entry.get("_old_archive_ids"), list): - extra_ids.extend(entry.get("_old_archive_ids", [])) - - new_archive_id: str | None = None - - if entry.get("extractor_key") and entry.get("id"): - new_archive_id: str = f"{entry.get('extractor_key').lower()} {entry.get('id')}" - if new_archive_id != archive_id: - extra_ids.append(new_archive_id) - - if len(extra_ids) > 0: - archive_ids: list[str] = archive_read(_archive_file, extra_ids) - if len(archive_ids) > 0: - store_type = None - for n in archive_ids: - store_type, dlInfo = await self.get_item(archive_id=n) - if store_type: - break - - if not store_type and not self.config.ignore_archived_items: - new_archive_id = new_archive_id or extra_ids.pop(0) - dlInfo = Download( - info=ItemDTO( - id=new_archive_id.split()[1], - title=new_archive_id, - url=entry.get("url") or entry.get("webpage_url") or item.url, - preset=item.preset, - folder=item.folder, - status="skip", - cookies=item.cookies, - template=item.template, - msg="URL is already downloaded.", - extras=merge_dict(item.extras, get_extras(entry)), - ) - ) - - dlInfo.info.msg += f" Found in archive '{_archive_file}'." - await self.done.put(dlInfo) - - self._notify.emit( - Events.ITEM_MOVED, - data={"to": "history", "preset": dlInfo.info.preset, "item": dlInfo.info}, - title="Download History Update", - message=f"Download history updated with '{item.url}'.", - ) - return {"status": "ok"} - - message: str = f"The URL '{item.url}':'{archive_ids.pop(0)}' is already downloaded and recorded in archive." - LOG.warning(message) - self._notify.emit( - Events.LOG_INFO, data={"preset": item.preset}, title="Already Downloaded", message=message - ) - return {"status": "error", "msg": message, "hidden": True} - - if not item.requeued and (condition := Conditions.get_instance().match(info=entry)): - already.pop() - - message = f"Condition '{condition.name}' matched for '{item!r}'." - - if condition.cli: - message += f" Re-queuing with '{condition.cli}'." - - LOG.info(message) - - if condition.extras.get("ignore_download", False): - extra_msg: str = "" - if _archive_file and not condition.extras.get("no_archive", False): - archive_add(_archive_file, [archive_id]) - extra_msg = f" and added to archive file '{_archive_file}'" - - _name = entry.get("title", entry.get("id")) - log_message = f"Ignoring download of '{_name!r}' as per condition '{condition.name}'{extra_msg}." - - store_type, dlInfo = await self.get_item(archive_id=archive_id) - if not store_type: - dlInfo = Download( - info=ItemDTO( - id=entry.get("id"), - title=_name, - url=item.url, - preset=item.preset, - folder=item.folder, - status="skip", - cookies=item.cookies, - template=item.template, - msg=log_message, - extras=merge_dict(item.extras, get_extras(entry)), - ) - ) - await self.done.put(dlInfo) - - LOG.info(log_message) - self._notify.emit(Events.LOG_INFO, data={}, title="Ignored Download", message=log_message) - self._notify.emit( - Events.ITEM_MOVED, - data={"to": "history", "preset": dlInfo.info.preset, "item": dlInfo.info}, - title="Download History Update", - message=f"Download history updated with '{item.url}'.", - ) - return {"status": "ok"} - - if condition.extras.get("set_preset") and (target_preset := condition.extras.get("set_preset")): - if Presets.get_instance().has(target_preset): - log_message: str = f"Switching preset from '{item.preset}' to '{target_preset}' for '{item!r}' as per condition '{condition.name}'." - LOG.info(log_message) - self._notify.emit(Events.LOG_INFO, data={}, title="Preset Switched", message=log_message) - item = item.new_with(preset=target_preset) - else: - LOG.warning( - f"Preset '{target_preset}' specified in condition '{condition.name}' does not exist. Ignoring set_preset." - ) - - return await self.add(item=item.new_with(requeued=True, cli=condition.cli), already=already) - - _status, _msg = ytdlp_reject(entry=entry, yt_params=yt_conf) - if not _status: - LOG.debug(_msg) - return {"status": "error", "msg": _msg} - - end_time = time.perf_counter() - started - LOG.debug(f"extract_info: for 'URL: {item.url}' is done in '{end_time:.3f}'. Length: '{len(entry)}/keys'.") - except yt_dlp.utils.ExistingVideoReached as exc: - LOG.error(f"Video has been downloaded already and recorded in archive.log file. '{exc!s}'.") - return {"status": "error", "msg": "Video has been downloaded already and recorded in archive.log file."} - except yt_dlp.utils.YoutubeDLError as exc: - LOG.error(f"YoutubeDLError: Unable to extract info. '{exc!s}'.") - return {"status": "error", "msg": str(exc)} - except asyncio.exceptions.TimeoutError as exc: - LOG.error(f"TimeoutError: Unable to extract info. '{exc!s}'.") - return { - "status": "error", - "msg": f"TimeoutError: {self.config.extract_info_timeout}s reached Unable to extract info.", - } - finally: - if cookie_file and cookie_file.exists(): - try: - cookie_file.unlink(missing_ok=True) - yt_conf.pop("cookiefile", None) - except Exception as e: - LOG.error(f"Failed to remove cookie file '{yt_conf['cookiefile']}'. {e!s}") - - return await self._add_item(entry=entry, item=item, already=already, logs=logs, yt_params=yt_conf) - - async def cancel(self, ids: list[str]) -> dict[str, str]: - """ - Cancel the download. - - Args: - ids (list): The list of ids to cancel. - - Returns: - dict: The status of the operation. - - """ - status: dict[str, str] = {} - - for id in ids: - try: - item = await self.queue.get(key=id) - except KeyError as e: - status[id] = str(e) - status["status"] = "error" - LOG.warning(f"Requested cancel for non-existent download {id=}. {e!s}") - continue - - item_ref = f"{id=} {item.info.id=} {item.info.title=}" - - if item.running(): - LOG.debug(f"Canceling {item_ref}") - item.cancel() - LOG.info(f"Cancelled {item_ref}") - await item.close() - else: - await item.close() - LOG.debug(f"Deleting from queue {item_ref}") - await self.queue.delete(id) - self._notify.emit( - Events.ITEM_CANCELLED, - data=item.info, - title="Download Cancelled", - message=f"Download '{item.info.title}' has been cancelled.", - ) - item.info.status = "cancelled" - await self.done.put(item) - self._notify.emit( - Events.ITEM_MOVED, - data={"to": "history", "preset": item.info.preset, "item": item.info}, - title="Download Cancelled", - message=f"Download '{item.info.title}' has been cancelled.", - ) - LOG.info(f"Deleted from queue {item_ref}") - - status[id] = "ok" - - return status - - async def clear(self, ids: list[str], remove_file: bool = False) -> dict[str, str]: - """ - Clear the download history. - - Args: - ids (list): The list of ids to clear. - remove_file (bool): Only considered if config.remove_files is True. - - Returns: - dict: The status of the operation. - - """ - status: dict[str, str] = {} - - for id in ids: - try: - item: Download = await self.done.get(key=id) - except KeyError as e: - status[id] = str(e) - status["status"] = "error" - LOG.warning(f"Requested delete for non-existent download {id=}. {e!s}") - continue - - itemRef: str = f"{id=} {item.info.id=} {item.info.title=}" - removed_files = 0 - filename: str = "" - - if self.config.remove_files is not True: - remove_file = False - - LOG.debug(f"{remove_file=} {itemRef} - Removing local files: {item.info.status=}") - - if remove_file and "finished" == item.info.status and item.info.filename: - filename = str(item.info.filename) - if item.info.folder: - filename = f"{item.info.folder}/{item.info.filename}" - - try: - rf = Path( - calc_download_path( - base_path=Path(self.config.download_path), - folder=filename, - create_path=False, - ) - ) - if rf.is_file() and rf.exists(): - if rf.stem and rf.suffix: - for f in rf.parent.glob(f"{glob.escape(rf.stem)}.*"): - if f.is_file() and f.exists() and not f.name.startswith("."): - removed_files += 1 - LOG.debug(f"Removing '{itemRef}' local file '{f.name}'.") - f.unlink(missing_ok=True) - else: - LOG.debug(f"Removing '{itemRef}' local file '{rf.name}'.") - rf.unlink(missing_ok=True) - removed_files += 1 - else: - LOG.warning(f"Failed to remove '{itemRef}' local file '{filename}'. File not found.") - except Exception as e: - LOG.error(f"Unable to remove '{itemRef}' local file '{filename}'. {e!s}") - - await self.done.delete(id) - - _status: str = "Removed" if removed_files > 0 else "Cleared" - self._notify.emit( - Events.ITEM_DELETED, - data=item.info, - title=f"Download {_status}", - message=f"{_status} '{item.info.title}' from history.", - ) - - msg = f"Deleted completed download '{itemRef}'." - if removed_files > 0: - msg += f" and removed '{removed_files}' local files." - - LOG.info(msg=msg) - status[id] = "ok" - - return status - - async def get(self, mode: str = "all") -> dict[str, list[dict[str, ItemDTO]]]: - """ - Get the download queue and the download history. - - Args: - mode (str): The mode to get the items. Supported modes are 'all', 'queue', 'done'. - - Returns: - dict: The download queue and the download history. - - """ - items = {"queue": {}, "done": {}} - - if mode in ("all", "queue"): - for k, v in await self.queue.saved_items(): - items["queue"][k] = self._active[k].info if k in self._active else v - - if mode in ("all", "done"): - for k, v in await self.done.saved_items(): - items["done"][k] = v - - if mode in ("all", "queue"): - for k, v in self.queue.items(): - if k not in items["queue"]: - items["queue"][k] = self._active[k].info if k in self._active else v - - if mode in ("all", "done"): - for k, v in self.done.items(): - if k in items["done"]: - continue - - items["done"][k] = v.info - - return items - - async def get_item(self, **kwargs) -> tuple[StoreType, Download] | tuple[None, None]: - """ - Get a specific item from the download queue or history. - - Args: - **kwargs: The key-value pair to search for. Supported keys are 'id', 'url'. - - Returns: - (StoreType, Download) | None: The requested item if found, otherwise None. - - """ - if item := await self.queue.get_item(**kwargs): - return (StoreType.QUEUE, item) - - if item := await self.done.get_item(**kwargs): - return (StoreType.HISTORY, item) - - return (None, None) - - async def _download_pool(self) -> None: - """ - Create a pool of workers to download the files. - """ - adaptive_sleep = 0.2 # Start with base sleep - max_sleep = 5.0 # Maximum sleep to avoid excessive delays - - while True: - while not self.queue.has_downloads(): - LOG.info("Waiting for item to download.") - await self.event.wait() - self.event.clear() - adaptive_sleep = 0.2 - - if self.is_paused(): - LOG.warning("Download pool is paused.") - await self.paused.wait() - LOG.info("Download pool resumed downloading.") - adaptive_sleep = 0.2 - - items_processed = 0 - - for _id, entry in list(self.queue.items()): - if entry.started() or entry.is_cancelled() or entry.info.auto_start is False: - continue - - extractor: str = entry.info.get_extractor() or "unknown" - - # Live downloads bypass all limits. - if entry.is_live: - task: asyncio.Task[None] = asyncio.create_task( - self._download_live(_id, entry), name=f"download_live_{extractor}_{_id}" - ) - task.add_done_callback(self._handle_task_exception) - items_processed += 1 - else: - _limit: asyncio.Semaphore = self._get_limit(extractor) - - # Skip this item in this iteration if no slots are available. - if self.workers.locked() or _limit.locked(): - continue - - await self.workers.acquire() - await _limit.acquire() - - task: asyncio.Task[None] = asyncio.create_task( - self._download_file(_id, entry), name=f"download_file_{extractor}_{_id}" - ) - - def _release(t: asyncio.Task, sem=_limit) -> None: - sem.release() - self.workers.release() - self._handle_task_exception(t) - - task.add_done_callback(_release) - items_processed += 1 - - # No items could be processed, back off a bit to avoid busy-waiting. - if 0 == items_processed: - adaptive_sleep: float = min(adaptive_sleep * 1.5, max_sleep) - LOG.debug(f"No download slots available. Backing off for {adaptive_sleep:.2f}s before next attempt.") - else: - adaptive_sleep = 0.2 - - await asyncio.sleep(adaptive_sleep) - - async def _download_live(self, _id: str, entry: Download) -> None: - LOG.debug(f"Creating temporary worker for entry '{entry.info.name()}'.") - - try: - await self._download_file(_id, entry) - finally: - LOG.debug(f"Temporary worker for '{entry.info.name()}' completed.") - - async def _download_file(self, id: str, entry: Download) -> None: - """ - Download the file. - - Args: - id (str): The id of the download. - entry (Download): The download entry. - - Returns: - None - - """ - filePath: str = calc_download_path(base_path=self.config.download_path, folder=entry.info.folder) - LOG.info(f"Downloading 'id: {id}', 'Title: {entry.info.title}', 'URL: {entry.info.url}' To '{filePath}'.") - - try: - self._active[entry.info._id] = entry - await entry.start() - - if entry.info.status not in ("finished", "skip", "cancelled"): - if not entry.info.error: - entry.info.error = f"Download ended with unexpected status '{entry.info.status}'." - entry.info.status = "error" - except Exception as e: - entry.info.status = "error" - entry.info.error = str(e) - finally: - if entry.info._id in self._active: - self._active.pop(entry.info._id, None) - - await entry.close() - - if await self.queue.exists(key=id): - LOG.debug(f"Download Task '{id}' is completed. Removing from queue.") - await self.queue.delete(key=id) - - nTitle: str | None = None - nMessage: str | None = None - - if entry.is_cancelled() is True: - nTitle = "Download Cancelled" - nMessage = f"Cancelled '{entry.info.title}' download." - self._notify.emit(Events.ITEM_CANCELLED, data=entry.info, title=nTitle, message=nMessage) - entry.info.status = "cancelled" - - if entry.info.status == "finished" and entry.info.filename: - nTitle = "Download Completed" - nMessage = f"Completed '{entry.info.title}' download." - if entry.info.is_archivable and not entry.info.is_archived: - entry.info.is_archived = True - - self._notify.emit(Events.ITEM_COMPLETED, data=entry.info, title=nTitle, message=nMessage) - - await self.done.put(entry) - self._notify.emit( - Events.ITEM_MOVED, - data={"to": "history", "preset": entry.info.preset, "item": entry.info}, - title=nTitle, - message=nMessage, - ) - else: - LOG.warning(f"Download '{id}' not found in queue.") - - if self.event: - self.event.set() - - async def _check_for_stale(self): - """ - Monitor pool for stale downloads and cancel them if needed. - """ - if self.is_paused() or self.queue.empty(): - return - - for _id, item in list(self.queue.items()): - item_ref = f"{_id=} {item.info.id=} {item.info.title=}" - if not item.is_stale(): - continue - - try: - LOG.warning(f"Cancelling staled item '{item_ref}' from download queue.") - await self.cancel([_id]) - except Exception as e: - LOG.error(f"Failed to cancel staled item '{item_ref}'. {e!s}") - LOG.exception(e) - - async def _check_live(self): - """ - Monitor the queue for items marked as live events and queue them when time is reached. - """ - if self.is_paused() or self.done.empty(): - return - - if self.config.debug: - LOG.debug("Checking history queue for queued live stream links.") - - time_now = datetime.now(tz=UTC) - - status: list[str] = ["not_live", "is_upcoming", "is_live"] - - for id, item in list(self.done.items()): - if item.info.status not in status: - continue - - item_ref: str = f"{id=} {item.info.id=} {item.info.title=}" - if not item.is_live: - LOG.debug(f"Item '{item_ref}' is not a live stream.") - continue - - duration: int | None = item.info.extras.get("duration", None) - is_premiere: bool = item.info.extras.get("is_premiere", False) - - live_in: str | None = item.info.live_in or ag(item.info.extras, ["live_in", "release_in"], None) - if not live_in: - LOG.debug( - f"Item '{item_ref}' marked as {'premiere video' if is_premiere else 'live stream'}, but no date is set." - ) - continue - - starts_in = str_to_dt(live_in) - starts_in = starts_in.replace(tzinfo=UTC) if starts_in.tzinfo is None else starts_in.astimezone(UTC) - - if time_now < (starts_in + timedelta(minutes=1)): - LOG.debug(f"Item '{item_ref}' is not yet live. will start at '{dt_delta(starts_in - time_now)}'.") - continue - - if self.config.prevent_live_premiere and is_premiere and duration: - buffer_time = self.config.live_premiere_buffer if self.config.live_premiere_buffer >= 0 else 5 - premiere_ends: datetime = starts_in + timedelta(minutes=buffer_time, seconds=duration) - if time_now < premiere_ends: - LOG.debug( - f"Item '{item_ref}' is premiering, download will start in '{(starts_in + timedelta(minutes=buffer_time, seconds=duration)).astimezone().isoformat()}'" - ) - continue - - LOG.info(f"Retrying item '{item_ref} {item.info.extras=}' for download.") - - try: - await self.clear([item.info._id], remove_file=False) - except Exception as e: - LOG.error(f"Failed to clear item '{item_ref}'. {e!s}") - continue - - try: - await self.add( - item=Item( - url=item.info.url, - preset=item.info.preset, - folder=item.info.folder, - cookies=item.info.cookies, - template=item.info.template, - cli=item.info.cli, - extras=item.info.extras, - ) - ) - except Exception as e: - await self.done.put(item) - LOG.exception(e) - LOG.error(f"Failed to retry item '{item_ref}'. {e!s}") - - async def _delete_old_history(self) -> None: - """ - Automatically delete old download history based on user specified days. - 0 or negative value disables this feature. - """ - if self.config.auto_clear_history_days < 0 or self.is_paused() or self.done.empty(): - return - - cutoff_date: datetime = datetime.now(UTC) - timedelta(days=self.config.auto_clear_history_days) - items_to_delete: list[tuple[str, ItemDTO]] = [] - - for key, download in self.done.items(): - info: ItemDTO = download.info - if not info or not info.timestamp: - continue - - if "finished" != info.status or not info.filename: - continue - - try: - timestamp_seconds: float = info.timestamp / 1e9 - item_datetime: datetime = datetime.fromtimestamp(timestamp_seconds, tz=UTC) - if item_datetime < cutoff_date: - items_to_delete.append((key, info)) - except (OSError, ValueError, OverflowError) as e: - LOG.error(f"Failed to parse timestamp '{info.timestamp}' for item '{info.title}': {e}") - - titles: list[str] = [] - for key, info in items_to_delete: - item_name: str = info.title or info.id or info._id - self._notify.emit( - Events.ITEM_DELETED, - data=info, - title="Download Cleared", - message=f"'{item_name}' record removed from history.", - ) - titles.append(item_name) - await self.done.delete(key) - - if titles: - LOG.info(f"Automatically cleared '{', '.join(titles)}' from download history due to age.") - - def _handle_task_exception(self, task: asyncio.Task) -> None: - if task.cancelled(): - return - - if not (exc := task.exception()): - return - - task_name: str = task.get_name() if task.get_name() else "unknown_task" - tb = "".join(traceback.format_exception(type(exc), exc, exc.__traceback__)) - LOG.error(f"Unhandled exception in background task '{task_name}': {exc!s}. {tb}") diff --git a/app/library/Events.py b/app/library/Events.py index 39a0232a..05afb8cd 100644 --- a/app/library/Events.py +++ b/app/library/Events.py @@ -9,7 +9,7 @@ from typing import Any from .BackgroundWorker import BackgroundWorker from .Singleton import Singleton -LOG: logging.Logger = logging.getLogger("Events") +LOG: logging.Logger = logging.getLogger("library.events") class Events: diff --git a/app/library/ItemDTO.py b/app/library/ItemDTO.py index 05b5000c..d3faaf2f 100644 --- a/app/library/ItemDTO.py +++ b/app/library/ItemDTO.py @@ -399,6 +399,7 @@ class ItemDTO: percent: int | None = None speed: str | None = None eta: str | None = None + postprocessor: str | None = None _recomputed: bool = False _archive_file: str | None = None diff --git a/app/library/Tasks.py b/app/library/Tasks.py index 92a22c44..f1a78dcc 100644 --- a/app/library/Tasks.py +++ b/app/library/Tasks.py @@ -14,7 +14,7 @@ from typing import Any from aiohttp import web from .config import Config -from .DownloadQueue import DownloadQueue +from .downloads import DownloadQueue from .encoder import Encoder from .Events import EventBus, Events from .ItemDTO import Item, ItemDTO diff --git a/app/library/config.py b/app/library/config.py index 13b44051..18832e53 100644 --- a/app/library/config.py +++ b/app/library/config.py @@ -117,6 +117,9 @@ class Config(metaclass=Singleton): extract_info_timeout: int = 70 """The timeout to use for extracting video information.""" + extract_info_concurrency: int = 4 + """The number of concurrent extract_info calls allowed.""" + db_file: str = "{config_path}{os_sep}ytptube.db" """The path to the database file.""" diff --git a/app/library/downloads/__init__.py b/app/library/downloads/__init__.py new file mode 100644 index 00000000..2216f43b --- /dev/null +++ b/app/library/downloads/__init__.py @@ -0,0 +1,8 @@ +"""Downloads module - refactored download management system.""" + +from .core import Download +from .hooks import NestedLogger +from .queue_manager import DownloadQueue +from .types import Terminator + +__all__ = ["Download", "DownloadQueue", "NestedLogger", "Terminator"] diff --git a/app/library/downloads/core.py b/app/library/downloads/core.py new file mode 100644 index 00000000..416e80fd --- /dev/null +++ b/app/library/downloads/core.py @@ -0,0 +1,406 @@ +"""Core Download class - orchestrates the download process.""" + +from __future__ import annotations + +import asyncio +import logging +import os +import signal +import time +from datetime import UTC, datetime +from pathlib import Path +from typing import TYPE_CHECKING, Any + +import yt_dlp.utils + +from app.library.config import Config +from app.library.Events import EventBus, Events +from app.library.Utils import create_cookies_file, extract_info, extract_ytdlp_logs +from app.library.ytdlp import YTDLP + +from .hooks import HookHandlers, NestedLogger +from .process_manager import ProcessManager +from .status_tracker import StatusTracker +from .temp_manager import TempManager +from .types import Terminator +from .utils import BAD_LIVE_STREAM_OPTIONS, GENERIC_EXTRACTORS, is_download_stale + +if TYPE_CHECKING: + import multiprocessing + + from app.library.ItemDTO import ItemDTO + + +class Download: + update_task: asyncio.Task | None = None + + def __init__(self, info: ItemDTO, info_dict: dict | None = None, logs: list[str] | None = None): + """ + Initialize download task. + + Args: + info: ItemDTO object containing download parameters + info_dict: Pre-extracted yt-dlp metadata dict (optional) + logs: Pre-existing logs from yt-dlp (optional) + + """ + config: Config = Config.get_instance() + + self.download_dir: str = info.download_dir + self.temp_dir: str | None = info.temp_dir + self.template: str | None = info.template + self.template_chapter: str | None = info.template_chapter + self.download_info_expires = int(config.download_info_expires) + self.info: ItemDTO = info + self.id: str = info._id + self.debug = bool(config.debug) + self.debug_ytdl = bool(config.ytdlp_debug) + self.tmpfilename: str | None = None + self.status_queue: multiprocessing.Queue[Any] | None = None + self.max_workers = int(config.max_workers) + self.is_live: bool = bool(info.is_live) or info.live_in is not None + self.info_dict: dict | None = info_dict + self.logger: logging.Logger = logging.getLogger(f"Download.{info.id if info.id else info._id}") + self.started_time = 0 + self.queue_time: datetime = datetime.now(tz=UTC) + self.logs: list[str] = logs if logs else [] + + self._process_manager = ProcessManager( + download_id=self.id, + is_live=self.is_live, + logger=self.logger, + ) + + self._temp_manager = TempManager( + info=self.info, + temp_dir=self.temp_dir, + temp_disabled=bool(config.temp_disabled), + temp_keep=bool(config.temp_keep), + logger=self.logger, + ) + + self._status_tracker: StatusTracker | None = None + self._hook_handlers: HookHandlers | None = None + + def _download(self) -> None: + """ + Execute the download in a subprocess. + + This method runs in a separate process and performs the actual + download using yt-dlp. + """ + cookie_file: Path | None = None + + try: + params: dict = ( + self.info.get_ytdlp_opts() + .add( + config={ + "color": "no_color", + "paths": { + "home": str(self.download_dir), + "temp": str(self._temp_manager.temp_path) + if self._temp_manager.temp_path + else self.download_dir, + }, + "outtmpl": { + "default": self.template, + "chapter": self.template_chapter, + }, + "noprogress": True, + "ignoreerrors": False, + }, + from_user=False, + ) + .get_all() + ) + + params.update( + { + "progress_hooks": [self._hook_handlers.progress_hook], + "postprocessor_hooks": [self._hook_handlers.postprocessor_hook], + "post_hooks": [self._hook_handlers.post_hook], + } + ) + + if self.debug_ytdl: + params["verbose"] = True + params["noprogress"] = False + + if self.info.cookies: + try: + cookie_file = ( + Path(self._temp_manager.temp_path if self._temp_manager.temp_path else self.temp_dir) + / f"cookie_{self.info._id}.txt" + ) + self.logger.debug( + f"Creating cookie file for '{self.info.id}: {self.info.title}' - '{cookie_file}'." + ) + params["cookiefile"] = str(create_cookies_file(self.info.cookies, cookie_file)) + except Exception as e: + err_msg: str = f"Failed to create cookie file for '{self.info.id}: {self.info.title}'. '{e!s}'." + self.logger.error(err_msg) + raise ValueError(err_msg) from e + + if self.info_dict and isinstance(self.info_dict, dict): + if self.info_dict.get("extractor_key") in GENERIC_EXTRACTORS and self.info.get_preset().default: + self.logger.debug(f"Removing 'download_archive' for generic extractor. url={self.info.url}") + params.pop("download_archive", None) + + if self.download_info_expires > 0: + _ts: int | None = self.info_dict.get("epoch", self.info_dict.get("timestamp", None)) + _ts_dt = datetime.fromtimestamp(_ts, tz=UTC) if _ts else None + if not _ts_dt or (datetime.now(tz=UTC) - _ts_dt).total_seconds() > self.download_info_expires: + self.info_dict = None + self.logger.warning(f"Info for '{self.info.url}' has expired, re-extracting info.") + + if not self.info_dict or not isinstance(self.info_dict, dict): + self.logger.info(f"Extracting info for '{self.info.url}'.") + self.logs = [] + ie_params: dict = params.copy() + ie_params["callback"] = { + "func": lambda _, msg: self.logs.append(msg), + "level": logging.WARNING, + "name": "callback-logger", + } + + info: dict = extract_info( + config=ie_params, + url=self.info.url, + debug=self.debug, + no_archive=not params.get("download_archive", False), + follow_redirect=True, + ) + + if info: + self.info_dict = info + + if self.is_live: + deletedOpts: list[str] = [] + for opt in BAD_LIVE_STREAM_OPTIONS: + if opt in params: + params.pop(opt, None) + deletedOpts.append(opt) + + if len(deletedOpts) > 0: + self.logger.warning(f"Live stream detected for '{self.info.title}', deleted opts: {deletedOpts}") + + if isinstance(self.info_dict, dict) and not ( + len(self.info_dict.get("formats", [])) > 0 or self.info_dict.get("url") + ): + msg: str = f"Failed to extract any formats for '{self.info.url}'." + if filtered_logs := extract_ytdlp_logs(self.logs): + msg += " " + ", ".join(filtered_logs) + + raise ValueError(msg) # noqa: TRY301 + + self.logger.info( + f'Download {self.info.name()}, preset="{self.info.preset}", cookies="{bool(params.get("cookiefile"))}" started.' + ) + + if self.debug: + self.logger.debug(f"Params before passing to yt-dlp. {params}") + + params["logger"] = NestedLogger(self.logger) + + if "continuedl" in params and params["continuedl"] is False: + self._temp_manager.delete_temp(by_pass=True) + + cls = YTDLP(params=params) + + if "posix" == os.name: + + def mark_cancelled(*_) -> None: + cls._interrupted = True + cls.to_screen("[info] Interrupt received, exiting cleanly...") + raise SystemExit(130) # noqa: TRY301 + + signal.signal(signal.SIGUSR1, mark_cancelled) + + self.status_queue.put({"id": self.id, "status": "downloading"}) + + if isinstance(self.info_dict, dict) and len(self.info_dict) > 1: + self.logger.debug(f"Downloading '{self.info.url}' using pre-info.") + _dct: dict = self.info_dict.copy() + if isinstance(self.info.extras, dict) and len(self.info.extras) > 0: + _dct.update( + { + k: v + for k, v in self.info.extras.items() + if k not in _dct and v is not None and k not in ("is_live",) + } + ) + + cls.process_ie_result(ie_result=_dct, download=True) + ret: int = cls._download_retcode + else: + self.logger.debug(f"Downloading using url: {self.info.url}") + ret = cls.download(url_list=[self.info.url]) + + self.status_queue.put({"id": self.id, "status": "finished" if 0 == ret else "error"}) + except yt_dlp.utils.ExistingVideoReached as exc: + self.logger.error(exc) + self.status_queue.put({"id": self.id, "status": "skip", "msg": "Item has already been downloaded."}) + except Exception as exc: + self.logger.exception(exc) + self.logger.error(exc) + self.status_queue.put({"id": self.id, "status": "error", "msg": str(exc), "error": str(exc)}) + finally: + self.status_queue.put(Terminator()) + if cookie_file and cookie_file.exists(): + try: + cookie_file.unlink() + self.logger.debug(f"Deleted cookie file: {cookie_file}") + except Exception as e: + self.logger.error(f"Failed to delete cookie file: {cookie_file}. {e}") + + self.logger.info( + f'Task {self.info.name()} preset="{self.info.preset}" cookies="{bool(params.get("cookiefile"))}" completed.' + ) + + async def start(self) -> int | None: + """ + Start the download process. + + Creates the subprocess, initializes status tracking, and waits + for completion. + + Returns: + Process exit code or None + + """ + self.status_queue = Config.get_manager().Queue() + + if temp_path := self._temp_manager.create_temp_path(): + self.info.temp_path = str(temp_path) + + self._status_tracker = StatusTracker( + info=self.info, + download_id=self.id, + download_dir=self.download_dir, + temp_path=temp_path, + status_queue=self.status_queue, + logger=self.logger, + debug=self.debug, + ) + + self._hook_handlers = HookHandlers( + download_id=self.id, + status_queue=self.status_queue, + logger=self.logger, + debug=self.debug, + ) + + self._process_manager.create_process(target=self._download) + self._process_manager.start() + self.started_time = int(time.time()) + self.info.status = "preparing" + + EventBus.get_instance().emit(Events.ITEM_UPDATED, data=self.info) + asyncio.create_task(self._status_tracker.progress_update(), name=f"update-{self.id}") + + ret = await asyncio.get_running_loop().run_in_executor(None, self._process_manager.proc.join) + + if self._status_tracker.final_update or self._process_manager.is_cancelled(): + if self._process_manager.is_cancelled(): + self.info.status = "cancelled" + return ret + + self._status_tracker.put_terminator() + await self._status_tracker.drain_queue() + + return ret + + def started(self) -> bool: + """Check if download process has been started.""" + return self._process_manager.started() + + def cancel(self) -> bool: + """Cancel the download task.""" + return self._process_manager.cancel() + + async def close(self) -> bool: + """Close download process and clean up resources.""" + if not self.started() or self._process_manager.cancel_in_progress: + return False + + try: + if self._status_tracker: + self._status_tracker.cancel_update_task() + + await self._process_manager.close() + + if self._status_tracker: + self._status_tracker.put_terminator() + + self._temp_manager.delete_temp() + + return True + except Exception as e: + self.logger.error(f"Failed to close process. {e}") + self.logger.exception(e) + + return False + + def running(self) -> bool: + return self._process_manager.running() + + def is_cancelled(self) -> bool: + return self._process_manager.is_cancelled() + + def kill(self) -> bool: + return self._process_manager.kill() + + def delete_temp(self, by_pass: bool = False) -> None: + self._temp_manager.delete_temp(by_pass=by_pass) + + def is_stale(self) -> bool: + """ + Check if the download task is stale. + + A download task is considered stale if it hasn't been updated + for a certain period or is stuck in an unexpected state. + + Returns: + True if the download task is stale, False otherwise + + """ + if self.started_time < 1: + self.logger.debug(f"Download task '{self.info.name()}' not started yet.") + return False + + elapsed = int(time.time()) - self.started_time + if elapsed < 300: + self.logger.debug(f"Download task '{self.info.title}: {self.info.id}' started for '{elapsed}' seconds.") + return False + + status: str = self.info.status if self.info.status else "unknown" + is_stale = is_download_stale(self.started_time, status, self.running(), self.info.auto_start, min_elapsed=300) + + if is_stale: + if not self.running(): + self.logger.warning( + f"Download task '{self.info.title}: {self.info.id}' not started running for '{elapsed}' seconds." + ) + else: + self.logger.warning( + f"Download task '{self.info.title}: {self.info.id}' has been stuck in '{status}' state for '{elapsed}' seconds." + ) + + return is_stale + + def __getstate__(self) -> dict[str, Any]: + """ + Prepare state for pickling. + + Excludes unpickleable objects like EventBus instances, + primarily for Windows compatibility. + """ + state: dict[str, Any] = self.__dict__.copy() + + excluded_keys: tuple[str, ...] = ("_notify",) + for key in excluded_keys: + if key in state: + state[key] = None + + return state diff --git a/app/library/downloads/hooks.py b/app/library/downloads/hooks.py new file mode 100644 index 00000000..830056ec --- /dev/null +++ b/app/library/downloads/hooks.py @@ -0,0 +1,91 @@ +"""Hook handlers for download progress and postprocessing events.""" + +import logging +import re +from typing import TYPE_CHECKING, Any + +from .utils import DEBUG_MESSAGE_PREFIXES, YTDLP_PROGRESS_FIELDS, create_debug_safe_dict + +if TYPE_CHECKING: + from multiprocessing import Queue + + +class HookHandlers: + """Manages yt-dlp hook callbacks for progress tracking and postprocessing.""" + + def __init__(self, download_id: str, status_queue: "Queue[Any]", logger: logging.Logger, debug: bool = False): + """ + Initialize hook handlers. + + Args: + download_id: Unique identifier for the download + status_queue: Multiprocessing queue for status updates + logger: Logger instance for this download + debug: Whether to enable debug logging + + """ + self.id = download_id + self.status_queue = status_queue + self.logger = logger + self.debug = debug + + def progress_hook(self, data: dict[str, Any]) -> None: + if self.debug: + try: + d_safe = create_debug_safe_dict(data) + self.logger.debug(f"PG Hook: {d_safe}") + except Exception as e: + self.logger.debug(f"PG Hook: Error creating debug info: {e}") + + self.status_queue.put( + { + "id": self.id, + "action": "progress", + **{k: v for k, v in data.items() if k in YTDLP_PROGRESS_FIELDS}, + } + ) + + def postprocessor_hook(self, data: dict[str, Any]) -> None: + if self.debug: + try: + d_safe = create_debug_safe_dict(data) + d_safe["postprocessor"] = data.get("postprocessor") + self.logger.debug(f"PP Hook: {d_safe}") + except Exception as e: + self.logger.debug(f"PP Hook: Error creating debug info: {e}") + + self.status_queue.put( + { + "id": self.id, + "action": "postprocessing", + **{k: v for k, v in data.items() if k in YTDLP_PROGRESS_FIELDS}, + "status": "postprocessing", + } + ) + + def post_hook(self, filename: str) -> None: + self.status_queue.put({"id": self.id, "status": "finished", "final_name": filename}) + + +class NestedLogger: + """ + Logger adapter for yt-dlp that adjusts log levels based on message prefixes. + + yt-dlp logs everything through a custom logger. This adapter maps certain + message types to appropriate log levels and strips redundant prefixes. + """ + + def __init__(self, logger: logging.Logger) -> None: + self.logger: logging.Logger = logger + + def debug(self, msg: str) -> None: + levelno: int = logging.DEBUG if any(msg.startswith(x) for x in DEBUG_MESSAGE_PREFIXES) else logging.INFO + self.logger.log(level=levelno, msg=re.sub(r"^\[(debug|info)\] ", "", msg, flags=re.IGNORECASE)) + + def error(self, msg: str) -> None: + """Log an error message.""" + self.logger.error(msg) + + def warning(self, msg: str) -> None: + """Log a warning message.""" + self.logger.warning(msg) diff --git a/app/library/downloads/item_adder.py b/app/library/downloads/item_adder.py new file mode 100644 index 00000000..d0a23473 --- /dev/null +++ b/app/library/downloads/item_adder.py @@ -0,0 +1,378 @@ +""" +Item addition and entry routing. + +This module handles the complete flow of adding items to the download queue: +- Preset and CLI validation +- Cookie file management +- Archive checking (early and post-extraction) +- Info extraction with yt-dlp +- Conditions matching and re-queuing +- Entry type routing (playlist, video, URL) +""" + +import asyncio +import functools +import logging +import time +import uuid +from pathlib import Path +from typing import TYPE_CHECKING + +import yt_dlp.utils + +from app.library.conditions import Conditions +from app.library.Events import Events +from app.library.ItemDTO import ItemDTO +from app.library.Presets import Presets +from app.library.Utils import ( + archive_add, + archive_read, + arg_converter, + create_cookies_file, + extract_info, + get_extras, + merge_dict, + ytdlp_reject, +) + +from .core import Download +from .playlist_processor import process_playlist +from .video_processor import add_video + +if TYPE_CHECKING: + from app.library.ItemDTO import Item + from app.library.Presets import Preset + + from .queue_manager import DownloadQueue + +LOG: logging.Logger = logging.getLogger("downloads.add") + + +async def add_item( + queue: "DownloadQueue", + entry: dict, + item: "Item", + already=None, + logs: list | None = None, + yt_params: dict | None = None, +) -> dict[str, str]: + """ + Route an entry to the appropriate processor based on type. + + Args: + queue: DownloadQueue instance + entry: Entry dict from yt-dlp + item: Item configuration + already: Set of already processed URLs + logs: yt-dlp logs + yt_params: yt-dlp parameters + + Returns: + dict: Status dict with "status" and optional "msg" keys + + """ + if not entry: + return {"status": "error", "msg": "Invalid/empty data was given."} + + event_type = entry.get("_type", "video") + + if event_type.startswith("playlist"): + return await process_playlist(queue=queue, entry=entry, item=item, already=already, yt_params=yt_params) + + if event_type.startswith("url"): + return await add(queue=queue, item=item.new_with(url=entry.get("url")), already=already) + + if event_type.startswith("video"): + return await add_video(queue=queue, entry=entry, item=item, logs=logs) + + return {"status": "error", "msg": f'Unsupported event type "{event_type}".'} + + +async def add( + queue: "DownloadQueue", item: "Item", already: set | None = None, entry: dict | None = None +) -> dict[str, str]: + """ + Add an item to the download queue. + + Args: + queue: DownloadQueue instance + item: Item to be added to the queue + already: Set of already downloaded items + entry: Entry associated with the item (if already extracted) + + Returns: + dict[str, str]: Status dict with "status" and optional "msg" keys + + """ + _preset: Preset | None = Presets.get_instance().get(item.preset) + + if item.has_cli(): + try: + arg_converter(args=item.cli, level=True) + except Exception as e: + LOG.error(f"Invalid command options for yt-dlp '{item.cli}'. {e!s}") + return {"status": "error", "msg": f"Invalid command options for yt-dlp '{item.cli}'. {e!s}"} + + if _preset: + if _preset.folder and not item.folder: + item.folder = _preset.folder + + if _preset.template and not item.template: + item.template = _preset.template + + yt_conf = {} + cookie_file: Path = Path(queue.config.temp_path) / f"c_{uuid.uuid4().hex}.txt" + + LOG.info(f"Adding '{item!r}'.") + + already = set() if already is None else already + + if item.url in already: + LOG.warning(f"Recursion detected with url '{item.url}' skipping.") + return {"status": "ok"} + + already.add(item.url) + + try: + logs: list = [] + + yt_conf: dict = { + "callback": { + "func": lambda _, msg: logs.append(msg), + "level": logging.WARNING, + "name": "callback-logger", + }, + **item.get_ytdlp_opts().get_all(), + } + + if yt_conf.get("external_downloader"): + LOG.warning(f"Using external downloader '{yt_conf.get('external_downloader')}' for '{item.url}'.") + item.extras.update({"external_downloader": True}) + + archive_id: str | None = item.get_archive_id() + + # Early archive check to avoid unnecessary extraction calls + # This sometimes can be different from the final extracted ID, so we need to verify again after extraction. + if archive_id and item.is_archived(): + store_type, dlInfo = await queue.get_item(archive_id=archive_id) + if not store_type and not queue.config.ignore_archived_items: + dlInfo = Download( + info=ItemDTO( + id=archive_id.split()[1], + title=archive_id, + url=item.url, + preset=item.preset, + folder=item.folder, + status="skip", + cookies=item.cookies, + template=item.template, + msg="URL is already downloaded.", + extras=item.extras, + ) + ) + + if archive_file := dlInfo.info.get_ytdlp_opts().get_all().get("download_archive"): + dlInfo.info.msg += f" Found in archive '{archive_file}'." + + await queue.done.put(dlInfo) + + queue._notify.emit( + Events.ITEM_MOVED, + data={"to": "history", "preset": dlInfo.info.preset, "item": dlInfo.info}, + title="Download History Update", + message=f"Download history updated with '{item.url}'.", + ) + return {"status": "ok"} + + message: str = f"The URL '{item.url}':'{archive_id}' is already downloaded and recorded in archive." + LOG.warning(message) + queue._notify.emit( + Events.LOG_INFO, data={"preset": item.preset}, title="Already Downloaded", message=message + ) + return {"status": "error", "msg": message, "hidden": True} + + started: float = time.perf_counter() + + if item.cookies: + try: + yt_conf["cookiefile"] = str(create_cookies_file(item.cookies, cookie_file)) + except Exception as e: + msg = f"Failed to create cookie file for '{item.url}'. '{e!s}'." + LOG.error(msg) + return {"status": "error", "msg": msg} + + if entry: + LOG.info(f"[P] Extracting '{item.url}'{' with cookies' if yt_conf.get('cookiefile') else ''}.") + + if not entry: + async with queue.extractors: + LOG.info(f"Extracting '{item.url}'{' with cookies' if yt_conf.get('cookiefile') else ''}.") + entry: dict | None = await asyncio.wait_for( + fut=asyncio.get_running_loop().run_in_executor( + None, + functools.partial( + extract_info, + config=yt_conf, + url=item.url, + debug=bool(queue.config.ytdlp_debug), + no_archive=False, + follow_redirect=True, + ), + ), + timeout=queue.config.extract_info_timeout, + ) + + if not entry: + LOG.error(f"Unable to extract info for '{item.url}'. Logs: {logs}") + return {"status": "error", "msg": "Unable to extract info." + "\n".join(logs)} + + # Sometimes playlists or extractor returns different ID than what we get from the make_archive_id() + # So, we need to re-check the archive after extraction to be sure the item was not downloaded. + # This also apply to old archive IDs that might have been used before. + if _archive_file := item.get_archive_file(): + extra_ids: list[str] = [] + + if entry.get("_old_archive_ids") and isinstance(entry.get("_old_archive_ids"), list): + extra_ids.extend(entry.get("_old_archive_ids", [])) + + new_archive_id: str | None = None + + if entry.get("extractor_key") and entry.get("id"): + new_archive_id: str = f"{entry.get('extractor_key').lower()} {entry.get('id')}" + if new_archive_id != archive_id: + extra_ids.append(new_archive_id) + + if len(extra_ids) > 0: + archive_ids: list[str] = archive_read(_archive_file, extra_ids) + if len(archive_ids) > 0: + store_type = None + for n in archive_ids: + store_type, dlInfo = await queue.get_item(archive_id=n) + if store_type: + break + + if not store_type and not queue.config.ignore_archived_items: + new_archive_id = new_archive_id or extra_ids.pop(0) + dlInfo = Download( + info=ItemDTO( + id=new_archive_id.split()[1], + title=new_archive_id, + url=entry.get("url") or entry.get("webpage_url") or item.url, + preset=item.preset, + folder=item.folder, + status="skip", + cookies=item.cookies, + template=item.template, + msg="URL is already downloaded.", + extras=merge_dict(item.extras, get_extras(entry)), + ) + ) + + dlInfo.info.msg += f" Found in archive '{_archive_file}'." + await queue.done.put(dlInfo) + + queue._notify.emit( + Events.ITEM_MOVED, + data={"to": "history", "preset": dlInfo.info.preset, "item": dlInfo.info}, + title="Download History Update", + message=f"Download history updated with '{item.url}'.", + ) + return {"status": "ok"} + + message: str = ( + f"The URL '{item.url}':'{archive_ids.pop(0)}' is already downloaded and recorded in archive." + ) + LOG.warning(message) + queue._notify.emit( + Events.LOG_INFO, data={"preset": item.preset}, title="Already Downloaded", message=message + ) + return {"status": "error", "msg": message, "hidden": True} + + if not item.requeued and (condition := Conditions.get_instance().match(info=entry)): + already.pop() + + message = f"Condition '{condition.name}' matched for '{item!r}'." + + if condition.cli: + message += f" Re-queuing with '{condition.cli}'." + + LOG.info(message) + + if condition.extras.get("ignore_download", False): + extra_msg: str = "" + if _archive_file and not condition.extras.get("no_archive", False): + archive_add(_archive_file, [archive_id]) + extra_msg = f" and added to archive file '{_archive_file}'" + + _name = entry.get("title", entry.get("id")) + log_message = f"Ignoring download of '{_name!r}' as per condition '{condition.name}'{extra_msg}." + + store_type, dlInfo = await queue.get_item(archive_id=archive_id) + if not store_type: + dlInfo = Download( + info=ItemDTO( + id=entry.get("id"), + title=_name, + url=item.url, + preset=item.preset, + folder=item.folder, + status="skip", + cookies=item.cookies, + template=item.template, + msg=log_message, + extras=merge_dict(item.extras, get_extras(entry)), + ) + ) + await queue.done.put(dlInfo) + + LOG.info(log_message) + queue._notify.emit(Events.LOG_INFO, data={}, title="Ignored Download", message=log_message) + queue._notify.emit( + Events.ITEM_MOVED, + data={"to": "history", "preset": dlInfo.info.preset, "item": dlInfo.info}, + title="Download History Update", + message=f"Download history updated with '{item.url}'.", + ) + return {"status": "ok"} + + if condition.extras.get("set_preset") and (target_preset := condition.extras.get("set_preset")): + if Presets.get_instance().has(target_preset): + log_message: str = f"Switching preset from '{item.preset}' to '{target_preset}' for '{item!r}' as per condition '{condition.name}'." + LOG.info(log_message) + queue._notify.emit(Events.LOG_INFO, data={}, title="Preset Switched", message=log_message) + item = item.new_with(preset=target_preset) + else: + LOG.warning( + f"Preset '{target_preset}' specified in condition '{condition.name}' does not exist. Ignoring set_preset." + ) + + return await add(queue=queue, item=item.new_with(requeued=True, cli=condition.cli), already=already) + + _status, _msg = ytdlp_reject(entry=entry, yt_params=yt_conf) + if not _status: + LOG.debug(_msg) + return {"status": "error", "msg": _msg} + + end_time = time.perf_counter() - started + LOG.debug(f"extract_info: for 'URL: {item.url}' is done in '{end_time:.3f}'. Length: '{len(entry)}/keys'.") + except yt_dlp.utils.ExistingVideoReached as exc: + LOG.error(f"Video has been downloaded already and recorded in archive.log file. '{exc!s}'.") + return {"status": "error", "msg": "Video has been downloaded already and recorded in archive.log file."} + except yt_dlp.utils.YoutubeDLError as exc: + LOG.error(f"YoutubeDLError: Unable to extract info. '{exc!s}'.") + return {"status": "error", "msg": str(exc)} + except asyncio.exceptions.TimeoutError as exc: + LOG.error(f"TimeoutError: Unable to extract info. '{exc!s}'.") + return { + "status": "error", + "msg": f"TimeoutError: {queue.config.extract_info_timeout}s reached Unable to extract info.", + } + finally: + if cookie_file and cookie_file.exists(): + try: + cookie_file.unlink(missing_ok=True) + yt_conf.pop("cookiefile", None) + except Exception as e: + LOG.error(f"Failed to remove cookie file '{yt_conf['cookiefile']}'. {e!s}") + + return await add_item(queue=queue, entry=entry, item=item, already=already, logs=logs, yt_params=yt_conf) diff --git a/app/library/downloads/monitors.py b/app/library/downloads/monitors.py new file mode 100644 index 00000000..3938e580 --- /dev/null +++ b/app/library/downloads/monitors.py @@ -0,0 +1,173 @@ +"""Queue monitoring functions.""" + +import logging +from datetime import UTC, datetime, timedelta +from typing import TYPE_CHECKING + +from app.library.ag_utils import ag +from app.library.Events import Events +from app.library.ItemDTO import Item, ItemDTO +from app.library.Utils import dt_delta, str_to_dt + +if TYPE_CHECKING: + from .queue_manager import DownloadQueue + +LOG: logging.Logger = logging.getLogger("downloads.monitors") + + +async def check_for_stale(queue: "DownloadQueue") -> None: + """ + Monitor pool for stale downloads and cancel them if needed. + + Iterates through active queue items and cancels any that have become stale + (not making progress for an extended period). + + Args: + queue: DownloadQueue instance + + """ + if queue.is_paused() or queue.queue.empty(): + return + + for _id, item in list(queue.queue.items()): + item_ref = f"{_id=} {item.info.id=} {item.info.title=}" + if not item.is_stale(): + continue + + try: + LOG.warning(f"Cancelling staled item '{item_ref}' from download queue.") + await queue.cancel([_id]) + except Exception as e: + LOG.error(f"Failed to cancel staled item '{item_ref}'. {e!s}") + LOG.exception(e) + + +async def check_live(queue: "DownloadQueue") -> None: + """ + Monitor the queue for items marked as live events and queue them when time is reached. + + Checks history for live streams/premieres that are scheduled to start and + re-adds them to the download queue when their start time arrives. + + Args: + queue: DownloadQueue instance + + """ + if queue.is_paused() or queue.done.empty(): + return + + if queue.config.debug: + LOG.debug("Checking history queue for queued live stream links.") + + time_now = datetime.now(tz=UTC) + + status: list[str] = ["not_live", "is_upcoming", "is_live"] + + for id, item in list(queue.done.items()): + if item.info.status not in status: + continue + + item_ref: str = f"{id=} {item.info.id=} {item.info.title=}" + if not item.is_live: + LOG.debug(f"Item '{item_ref}' is not a live stream.") + continue + + duration: int | None = item.info.extras.get("duration", None) + is_premiere: bool = item.info.extras.get("is_premiere", False) + + live_in: str | None = item.info.live_in or ag(item.info.extras, ["live_in", "release_in"], None) + if not live_in: + LOG.debug( + f"Item '{item_ref}' marked as {'premiere video' if is_premiere else 'live stream'}, but no date is set." + ) + continue + + starts_in = str_to_dt(live_in) + starts_in = starts_in.replace(tzinfo=UTC) if starts_in.tzinfo is None else starts_in.astimezone(UTC) + + if time_now < (starts_in + timedelta(minutes=1)): + LOG.debug(f"Item '{item_ref}' is not yet live. will start at '{dt_delta(starts_in - time_now)}'.") + continue + + if queue.config.prevent_live_premiere and is_premiere and duration: + buffer_time = queue.config.live_premiere_buffer if queue.config.live_premiere_buffer >= 0 else 5 + premiere_ends: datetime = starts_in + timedelta(minutes=buffer_time, seconds=duration) + if time_now < premiere_ends: + LOG.debug( + f"Item '{item_ref}' is premiering, download will start in '{(starts_in + timedelta(minutes=buffer_time, seconds=duration)).astimezone().isoformat()}'" + ) + continue + + LOG.info(f"Retrying item '{item_ref} {item.info.extras=}' for download.") + + try: + await queue.clear([item.info._id], remove_file=False) + except Exception as e: + LOG.error(f"Failed to clear item '{item_ref}'. {e!s}") + continue + + try: + await queue.add( + item=Item( + url=item.info.url, + preset=item.info.preset, + folder=item.info.folder, + cookies=item.info.cookies, + template=item.info.template, + cli=item.info.cli, + extras=item.info.extras, + ) + ) + except Exception as e: + await queue.done.put(item) + LOG.exception(e) + LOG.error(f"Failed to retry item '{item_ref}'. {e!s}") + + +async def delete_old_history(queue: "DownloadQueue") -> None: + """ + Automatically delete old download history based on user specified days. + + Removes finished downloads from history that are older than the configured + auto_clear_history_days setting. 0 or negative value disables this feature. + + Args: + queue: DownloadQueue instance + + """ + if queue.config.auto_clear_history_days < 0 or queue.is_paused() or queue.done.empty(): + return + + cutoff_date: datetime = datetime.now(UTC) - timedelta(days=queue.config.auto_clear_history_days) + items_to_delete: list[tuple[str, ItemDTO]] = [] + + for key, download in queue.done.items(): + info: ItemDTO = download.info + if not info or not info.timestamp: + continue + + if "finished" != info.status or not info.filename: + continue + + try: + timestamp_seconds: float = info.timestamp / 1e9 + item_datetime: datetime = datetime.fromtimestamp(timestamp_seconds, tz=UTC) + if item_datetime < cutoff_date: + items_to_delete.append((key, info)) + except (OSError, ValueError, OverflowError) as e: + LOG.error(f"Failed to parse timestamp '{info.timestamp}' for item '{info.title}': {e}") + + titles: list[str] = [] + for key, info in items_to_delete: + item_name: str = info.title or info.id or info._id + queue._notify.emit( + Events.ITEM_DELETED, + data=info, + title="Download Cleared", + message=f"'{item_name}' record removed from history.", + ) + titles.append(item_name) + await queue.done.delete(key) + + if titles: + LOG.info(f"Automatically cleared '{', '.join(titles)}' from download history due to age.") diff --git a/app/library/downloads/playlist_processor.py b/app/library/downloads/playlist_processor.py new file mode 100644 index 00000000..2a7006d3 --- /dev/null +++ b/app/library/downloads/playlist_processor.py @@ -0,0 +1,157 @@ +"""Playlist processing.""" + +import asyncio +import logging +from typing import TYPE_CHECKING, Any + +from app.library.Utils import merge_dict, ytdlp_reject + +from .utils import handle_task_exception + +if TYPE_CHECKING: + from app.library.ItemDTO import Item + + from .queue_manager import DownloadQueue + +LOG: logging.Logger = logging.getLogger("downloads.playlist") + + +async def process_playlist( + queue: "DownloadQueue", entry: dict, item: "Item", already=None, yt_params: dict | None = None +) -> dict[str, str]: + """ + Process a playlist entry. + + Args: + queue: DownloadQueue instance for accessing queue/config + entry: Playlist entry dict from yt-dlp + item: Item configuration + already: Set of already processed IDs + yt_params: yt-dlp parameters + + Returns: + dict: Status dict with "status" and optional "msg" keys + + """ + if not yt_params: + yt_params = {} + + entries = entry.get("entries", []) + + playlist_name: str = f"{entry.get('id')}: {entry.get('title')}" + + LOG.info(f"Processing '{playlist_name} ({len(entries)})' Playlist.") + + playlistCount = entry.get("playlist_count") + playlistCount: int = int(playlistCount) if playlistCount else len(entries) + + playlist_keys: dict[str, Any] = { + "playlist_count": playlistCount, + "playlist": entry.get("title") or entry.get("id"), + "playlist_id": entry.get("id"), + "playlist_title": entry.get("title"), + "playlist_uploader": entry.get("uploader"), + "playlist_uploader_id": entry.get("uploader_id"), + "playlist_channel": entry.get("channel"), + "playlist_channel_id": entry.get("channel_id"), + "playlist_webpage_url": entry.get("webpage_url"), + "__last_playlist_index": playlistCount - 1, + "n_entries": len(entries), + } + + async def playlist_processor(i: int, etr: dict): + acquired = False + try: + item_name: str = ( + f"'{entry.get('title')}: {i}/{playlist_keys['n_entries']}' - '{etr.get('id')}: {etr.get('title')}'" + ) + LOG.debug(f"Waiting to acquire lock for {item_name}") + await queue.processors.acquire() + acquired = True + LOG.debug(f"Acquired lock for {item_name}") + + LOG.info(f"Processing '{item_name}'.") + + _status, _msg = ytdlp_reject(entry=etr, yt_params=yt_params) + if not _status: + return {"status": "error", "msg": _msg} + + extras: dict[str, Any] = { + **playlist_keys, + "playlist_index": i, + "playlist_index_number": i, + "playlist_autonumber": i, + } + + for property in ("id", "title", "uploader", "uploader_id"): + if property in entry: + extras[f"playlist_{property}"] = entry.get(property) + + extractor_key = entry.get("ie_key") or entry.get("extractor_key") or entry.get("extractor") or "" + if "thumbnail" not in etr and "youtube" in str(extractor_key).lower(): + extras["thumbnail"] = "https://img.youtube.com/vi/{id}/maxresdefault.jpg".format(**etr) + + newItem: Item = item.new_with(url=etr.get("url") or etr.get("webpage_url"), extras=extras) + + if ("video" == etr.get("_type") and etr.get("url")) or ( + "formats" in etr and isinstance(etr["formats"], list) and len(etr["formats"]) > 0 + ): + dct = merge_dict(merge_dict({"_type": "video"}, etr), entry) + dct.pop("entries", None) + return await queue.add(item=newItem, entry=dct, already=already) + + return await queue.add(item=newItem, already=already) + finally: + if acquired: + queue.processors.release() + + max_downloads: int = -1 + ytdlp_opts: dict[str, Any] = item.get_ytdlp_opts().get_all() + if ytdlp_opts.get("max_downloads") and isinstance(ytdlp_opts.get("max_downloads"), int): + max_downloads: int = ytdlp_opts.get("max_downloads") + + batch_size: int = max(50, int(queue.config.playlist_items_concurrency) * 10) + + async def run_batch(batch: list[tuple[int, dict]]) -> list[dict]: + tasks: list[asyncio.Task] = [] + for i, etr in batch: + task = asyncio.create_task( + playlist_processor(i, etr), + name=f"playlist_processor_{etr.get('id')}_{i}", + ) + task.add_done_callback(lambda t: handle_task_exception(t, LOG)) + tasks.append(task) + + batch_results: list[dict] = await asyncio.gather(*tasks) + await asyncio.sleep(0) + return batch_results + + results: list[dict] = [] + batch: list[tuple[int, dict]] = [] + + for i, etr in enumerate(entries, start=1): + if max_downloads > 0 and i > max_downloads: + break + + batch.append((i, etr)) + if len(batch) >= batch_size: + results.extend(await run_batch(batch)) + batch.clear() + + if batch: + results.extend(await run_batch(batch)) + + log_msg: str = f"Playlist '{playlist_name}' processing completed with '{len(results)}' entries." + if max_downloads > 0 and len(entries) > max_downloads: + skipped: int = len(entries) - max_downloads + log_msg += f" Limited to '{max_downloads}' items, skipped '{skipped}' remaining items." + + LOG.info(log_msg) + + if any("error" == res["status"] for res in results): + return { + "status": "error", + "msg": ", ".join(res["msg"] for res in results if "error" == res["status"] and "msg" in res), + } + + return {"status": "ok"} diff --git a/app/library/downloads/pool_manager.py b/app/library/downloads/pool_manager.py new file mode 100644 index 00000000..944cfe5e --- /dev/null +++ b/app/library/downloads/pool_manager.py @@ -0,0 +1,210 @@ +"""Download pool management - worker coordination and execution.""" + +import asyncio +import logging +from typing import TYPE_CHECKING + +from app.library.Events import EventBus, Events +from app.library.Utils import calc_download_path + +from .core import Download +from .utils import get_extractor_limit, handle_task_exception + +if TYPE_CHECKING: + from app.library.config import Config + + from .queue_manager import DownloadQueue + +LOG: logging.Logger = logging.getLogger("downloads.pool") + + +class PoolManager: + """ + Manages download worker pool and execution. + + Coordinates concurrent downloads with semaphore-based limits: + - Global worker limit (max_workers) + - Per-extractor limits (max_workers_per_extractor) + - Live streams bypass all limits + + Integrates with DownloadQueue for queue access and configuration. + """ + + def __init__(self, queue: "DownloadQueue", config: "Config"): + self.queue = queue + self.config = config + self._notify: EventBus = EventBus.get_instance() + + self.workers = asyncio.Semaphore(config.max_workers) + "Semaphore to limit the number of concurrent downloads." + + self.paused = asyncio.Event() + "Event to pause the download queue." + + self.event = asyncio.Event() + "Event to signal the download queue to start downloading." + + self._active: dict[str, Download] = {} + "Dictionary of active downloads." + + self.paused.set() + + def is_paused(self) -> bool: + return not self.paused.is_set() + + def pause(self) -> None: + if self.paused.is_set(): + self.paused.clear() + LOG.info("Download pool paused.") + + def resume(self) -> None: + if not self.paused.is_set(): + self.paused.set() + LOG.info("Download pool resumed.") + + def trigger_download(self) -> None: + """Trigger the download pool to check for new items.""" + if self.event: + self.event.set() + + def get_active_downloads(self) -> dict[str, Download]: + return self._active.copy() + + def get_active_download(self, download_id: str) -> Download | None: + return self._active.get(download_id) + + async def start_pool(self) -> None: + asyncio.create_task(self._download_pool(), name="download_pool") + + async def shutdown(self) -> None: + if self._active: + LOG.info(f"Cancelling '{len(self._active)}' active downloads.") + await self.queue.cancel(list(self._active.keys())) + + async def _download_pool(self) -> None: + adaptive_sleep = 0.2 + max_sleep = 5.0 + + while True: + while not self.queue.queue.has_downloads(): + LOG.info("Waiting for item to download.") + await self.event.wait() + self.event.clear() + adaptive_sleep = 0.2 + + if self.is_paused(): + LOG.warning("Download pool is paused.") + await self.paused.wait() + LOG.info("Download pool resumed downloading.") + adaptive_sleep = 0.2 + + items_processed = 0 + + for _id, entry in list(self.queue.queue.items()): + if entry.started() or entry.is_cancelled() or entry.info.auto_start is False: + continue + + extractor: str = entry.info.get_extractor() or "unknown" + + # Live downloads bypass all limits. + if entry.is_live: + task: asyncio.Task[None] = asyncio.create_task( + self._download_file(_id, entry), name=f"download_live_{extractor}_{_id}" + ) + task.add_done_callback(lambda t: handle_task_exception(t, LOG)) + items_processed += 1 + else: + _limit: asyncio.Semaphore = get_extractor_limit( + extractor, self.config.max_workers, self.config.max_workers_per_extractor, LOG + ) + + # Skip this item in this iteration if no slots are available. + if self.workers.locked() or _limit.locked(): + continue + + await self.workers.acquire() + await _limit.acquire() + + task: asyncio.Task[None] = asyncio.create_task( + self._download_file(_id, entry), name=f"download_file_{extractor}_{_id}" + ) + + def _release(t: asyncio.Task, sem=_limit) -> None: + sem.release() + self.workers.release() + handle_task_exception(t, LOG) + + task.add_done_callback(_release) + items_processed += 1 + + # No items could be processed, back off a bit to avoid busy-waiting. + if 0 == items_processed: + adaptive_sleep: float = min(adaptive_sleep * 1.5, max_sleep) + LOG.debug(f"No download slots available. Backing off for {adaptive_sleep:.2f}s before next attempt.") + else: + adaptive_sleep = 0.2 + + await asyncio.sleep(adaptive_sleep) + + async def _download_file(self, id: str, entry: Download) -> None: + """ + Download the file. + + Args: + id: The id of the download + entry: The download entry + + """ + filePath: str = calc_download_path(base_path=self.config.download_path, folder=entry.info.folder) + LOG.info(f"Downloading 'id: {entry.id}', 'Title: {entry.info.title}', 'URL: {entry.info.url}' To '{filePath}'.") + + try: + self._active[entry.info._id] = entry + await entry.start() + + if entry.info.status not in ("finished", "skip", "cancelled"): + if not entry.info.error: + entry.info.error = f"Download ended with unexpected status '{entry.info.status}'." + entry.info.status = "error" + except Exception as e: + entry.info.status = "error" + entry.info.error = str(e) + finally: + if entry.info._id in self._active: + self._active.pop(entry.info._id, None) + + await entry.close() + + if await self.queue.queue.exists(key=id): + LOG.debug(f"Download Task '{id}' is completed. Removing from queue.") + await self.queue.queue.delete(key=id) + + nTitle: str | None = None + nMessage: str | None = None + + if entry.is_cancelled() is True: + nTitle = "Download Cancelled" + nMessage = f"Cancelled '{entry.info.title}' download." + self._notify.emit(Events.ITEM_CANCELLED, data=entry.info, title=nTitle, message=nMessage) + entry.info.status = "cancelled" + + if entry.info.status == "finished" and entry.info.filename: + nTitle = "Download Completed" + nMessage = f"Completed '{entry.info.title}' download." + if entry.info.is_archivable and not entry.info.is_archived: + entry.info.is_archived = True + + self._notify.emit(Events.ITEM_COMPLETED, data=entry.info, title=nTitle, message=nMessage) + + await self.queue.done.put(entry) + self._notify.emit( + Events.ITEM_MOVED, + data={"to": "history", "preset": entry.info.preset, "item": entry.info}, + title=nTitle, + message=nMessage, + ) + else: + LOG.warning(f"Download '{id}' not found in queue.") + + if self.event: + self.event.set() diff --git a/app/library/downloads/process_manager.py b/app/library/downloads/process_manager.py new file mode 100644 index 00000000..330ecbd0 --- /dev/null +++ b/app/library/downloads/process_manager.py @@ -0,0 +1,184 @@ +"""Process lifecycle management for downloads.""" + +import logging +import multiprocessing +import os +import signal +from collections.abc import Callable + +from .utils import wait_for_process_with_timeout + + +class ProcessManager: + """ + Manages the lifecycle of a download subprocess. + + Handles process creation, monitoring, graceful termination, and + force-kill operations. Includes special handling for live streams + which may not respond to standard termination signals. + """ + + def __init__(self, download_id: str, is_live: bool, logger: logging.Logger): + """ + Initialize process manager. + + Args: + download_id: Unique identifier for this download + is_live: Whether this is a live stream download + logger: Logger instance + + """ + self.download_id = download_id + self.is_live = is_live + self.logger = logger + self.proc: multiprocessing.Process | None = None + self.cancelled = False + self.cancel_in_progress = False + + def create_process(self, target: Callable[[], None]) -> multiprocessing.Process: + """ + Create a new download process. + + Args: + target: Function to execute in the subprocess + + Returns: + Created but not started process + + """ + self.proc = multiprocessing.Process(name=f"download-{self.download_id}", target=target) + return self.proc + + def start(self) -> None: + if self.proc: + self.proc.start() + + def started(self) -> bool: + return self.proc is not None + + def running(self) -> bool: + try: + return self.proc is not None and self.proc.is_alive() + except ValueError: + return False + + def is_cancelled(self) -> bool: + return self.cancelled + + def cancel(self) -> bool: + """ + Mark download as cancelled and kill the process. + + Returns: + True if cancellation was initiated, False otherwise + + """ + if not self.started(): + return False + + self.cancelled = True + return self.kill() + + def kill(self) -> bool: + """ + Kill the download process. + + Attempts graceful termination via SIGUSR1 signal first (POSIX only), + then escalates to terminate() and finally kill() if needed. + Live streams get shorter timeouts as they're more likely to be unresponsive. + + Returns: + True if process was killed successfully, False otherwise + + """ + if not self.running(): + return False + + procId: int | None = self.proc.ident + try: + self.logger.info(f"Killing download process: PID={self.proc.pid}, ident={procId}.") + + if self.proc.pid and "posix" == os.name: + try: + self.logger.debug(f"Sending SIGUSR1 signal to PID={self.proc.pid}.") + os.kill(self.proc.pid, signal.SIGUSR1) + + if wait_for_process_with_timeout(self.proc, 2 if self.is_live else 5): + self.logger.debug(f"Process PID={self.proc.pid} terminated gracefully.") + return True + self.logger.warning( + f"Process PID={self.proc.pid} did not respond to SIGUSR1 " + f"({'live stream' if self.is_live else 'regular download'}), " + f"forcing termination." + ) + + except (OSError, AttributeError) as e: + self.logger.debug(f"Failed to send SIGUSR1 signal: {e}") + + if self.proc.is_alive(): + self.logger.info(f"Force-terminating process PID={self.proc.pid}.") + self.proc.terminate() + + if not wait_for_process_with_timeout(self.proc, 1 if self.is_live else 2): + self.logger.warning(f"Process PID={self.proc.pid} did not respond, killing forcefully.") + self.proc.kill() + wait_for_process_with_timeout(self.proc, 1) + + self.logger.info(f"Process PID={self.proc.pid} killed.") + return True + + except Exception as e: + self.logger.error(f"Failed to kill process PID={self.proc.pid}, ident={procId}. {e}") + self.logger.exception(e) + + return False + + async def close(self) -> bool: + """ + Close the download process and clean up resources. + + This method must be called to properly release resources. + + Returns: + True if close was successful, False otherwise + + """ + if not self.started() or self.cancel_in_progress: + return False + + self.cancel_in_progress = True + procId: int | None = self.proc.ident + + if not procId: + if self.proc: + self.proc.close() + self.proc = None + self.logger.warning("Attempted to close download process, but it is not running.") + return False + + self.logger.info(f"Closing PID='{procId}' download process.") + + try: + self.kill() + + import asyncio + + loop = asyncio.get_running_loop() + + if self.proc.is_alive(): + self.logger.debug(f"Waiting for PID='{procId}' to close.") + await loop.run_in_executor(None, self.proc.join) + self.logger.debug(f"PID='{procId}' closed.") + + if self.proc: + self.proc.close() + self.proc = None + + self.logger.debug(f"Closed PID='{procId}' download process.") + return True + + except Exception as e: + self.logger.error(f"Failed to close process: '{procId}'. {e}") + self.logger.exception(e) + + return False diff --git a/app/library/downloads/queue_manager.py b/app/library/downloads/queue_manager.py new file mode 100644 index 00000000..b9d66ad3 --- /dev/null +++ b/app/library/downloads/queue_manager.py @@ -0,0 +1,429 @@ +import asyncio +import functools +import glob +import logging +from datetime import UTC, datetime +from pathlib import Path +from typing import TYPE_CHECKING + +from aiohttp import web + +from app.library.config import Config +from app.library.Events import EventBus, Events +from app.library.ItemDTO import Item, ItemDTO +from app.library.Scheduler import Scheduler +from app.library.Services import Services +from app.library.Singleton import Singleton +from app.library.sqlite_store import SqliteStore +from app.library.Utils import calc_download_path + +from .core import Download +from .item_adder import add as add_impl +from .monitors import check_for_stale, check_live, delete_old_history +from .pool_manager import PoolManager + +if TYPE_CHECKING: + from app.library.DataStore import StoreType + +LOG: logging.Logger = logging.getLogger("downloads.queue") + + +class DownloadQueue(metaclass=Singleton): + def __init__(self, config: Config | None = None): + # Import here to avoid circular import with DataStore + from app.library.DataStore import DataStore, StoreType + + self.config: Config = config or Config.get_instance() + "Configuration instance." + self._notify: EventBus = EventBus.get_instance() + "Event bus instance." + self.done = DataStore(type=StoreType.HISTORY, connection=SqliteStore.get_instance()) + "DataStore for the completed downloads." + self.queue = DataStore(type=StoreType.QUEUE, connection=SqliteStore.get_instance()) + "DataStore for the download queue." + self.processors = asyncio.Semaphore(self.config.playlist_items_concurrency) + "Semaphore to limit the number of concurrent processors." + self.extractors = asyncio.Semaphore(self.config.extract_info_concurrency) + "Semaphore to limit the number of concurrent extract_info calls." + self.pool = PoolManager(queue=self, config=self.config) + "Pool manager for coordinating download execution." + + @staticmethod + def get_instance(config: Config | None = None) -> "DownloadQueue": + return DownloadQueue(config=config) + + def attach(self, _: web.Application) -> None: + Services.get_instance().add("queue", self) + + async def event_handler(_, __): + await self.initialize() + + self._notify.subscribe(Events.STARTED, event_handler, f"{__class__.__name__}.{__class__.initialize.__name__}") + + Scheduler.get_instance().add( + timer="* * * * *", + func=functools.partial(check_for_stale, self), + id=check_for_stale.__name__, + ) + + Scheduler.get_instance().add( + timer="* * * * *", + func=functools.partial(check_live, self), + id=check_live.__name__, + ) + + if self.config.auto_clear_history_days > 0: + Scheduler.get_instance().add( + timer="8 */1 * * *", + func=functools.partial(delete_old_history, self), + id=delete_old_history.__name__, + ) + + # app.on_shutdown.append(self.on_shutdown) + + async def test(self) -> bool: + await self.done.test() + return True + + async def initialize(self) -> None: + await self.queue.load() + LOG.info( + f"Using '{self.config.max_workers}' workers for downloading and '{self.config.max_workers_per_extractor}' per extractor." + ) + await self.pool.start_pool() + + async def start_items(self, ids: list[str]) -> dict[str, str]: + """ + Start one or more queued downloads that were added with auto_started=False. + + Args: + ids (list[str]): List of item IDs to start. + + Returns: + dict[str, str]: Dictionary of per-ID results and overall status. + + """ + status: dict[str, str] = {"status": "ok"} + started = False + + for item_id in ids: + try: + item: Download = await self.queue.get(key=item_id) + except KeyError as e: + status[item_id] = f"not found: {e!s}" + status["status"] = "error" + LOG.warning(f"Start requested for non-existent item {item_id=}.") + continue + + if item.info.auto_start: + status[item_id] = "already started" + continue + + item.info.auto_start = True + updated: Download = await self.queue.put(item) + self._notify.emit(Events.ITEM_UPDATED, data=updated.info) + self._notify.emit( + Events.ITEM_RESUMED, + data=item.info, + title="Download Resumed", + message=f"Download '{item.info.title}' has been resumed.", + ) + status[item_id] = "started" + started = True + + if started: + self.pool.trigger_download() + + return status + + async def pause_items(self, ids: list[str]) -> dict[str, str]: + """ + Pause one or more queued downloads that were added with auto_started=True. + + Args: + ids (list[str]): List of item IDs to pause. + + Returns: + dict[str, str]: Dictionary of per-ID results and overall status. + + """ + status: dict[str, str] = {"status": "ok"} + + for item_id in ids: + try: + item: Download = await self.queue.get(key=item_id) + except KeyError as e: + status[item_id] = f"not found: {e!s}" + status["status"] = "error" + LOG.warning(f"Start requested for non-existent item {item_id=}.") + continue + + if item.started() or item.is_cancelled(): + status[item_id] = "already started" + continue + + if item.info.auto_start is False: + status[item_id] = "not started" + continue + + item.info.auto_start = False + updated: Download = await self.queue.put(item) + self._notify.emit(Events.ITEM_UPDATED, data=updated.info) + self._notify.emit( + Events.ITEM_PAUSED, + data=item.info, + title="Download Paused", + message=f"Download '{item.info.title}' has been paused.", + ) + status[item_id] = "paused" + + return status + + def pause(self, shutdown: bool = False) -> bool: + """ + Pause the download queue. + + Returns: + bool: True if the download is paused, False otherwise + + """ + if not self.pool.is_paused(): + self.pool.pause() + if not shutdown: + LOG.warning(f"Download paused at. {datetime.now(tz=UTC).isoformat()}") + return True + + return False + + def resume(self) -> bool: + """ + Resume the download queue. + + Returns: + bool: True if the download is resumed, False otherwise + + """ + if self.pool.is_paused(): + self.pool.resume() + LOG.warning(f"Downloading resumed at. {datetime.now(tz=UTC).isoformat()}") + return True + + return False + + def is_paused(self) -> bool: + """ + Check if the download queue is paused. + + Returns: + bool: True if the download queue is paused, False otherwise + + """ + return self.pool.is_paused() + + async def on_shutdown(self, _: web.Application): + await self.pool.shutdown() + + async def add(self, item: Item, already: set | None = None, entry: dict | None = None) -> dict[str, str]: + """ + Add an item to the download queue. + + Args: + item: Item to be added to the queue + already: Set of already downloaded items + entry: Entry associated with the item (if already extracted) + + Returns: + dict[str, str]: Status dict with "status" and optional "msg" keys + + """ + return await add_impl(queue=self, item=item, already=already, entry=entry) + + async def cancel(self, ids: list[str]) -> dict[str, str]: + """ + Cancel the download. + + Args: + ids (list): The list of ids to cancel. + + Returns: + dict: The status of the operation. + + """ + status: dict[str, str] = {} + + for id in ids: + try: + item = await self.queue.get(key=id) + except KeyError as e: + status[id] = str(e) + status["status"] = "error" + LOG.warning(f"Requested cancel for non-existent download {id=}. {e!s}") + continue + + item_ref = f"{id=} {item.info.id=} {item.info.title=}" + + if item.running(): + LOG.debug(f"Canceling {item_ref}") + item.cancel() + LOG.info(f"Cancelled {item_ref}") + await item.close() + else: + await item.close() + LOG.debug(f"Deleting from queue {item_ref}") + await self.queue.delete(id) + self._notify.emit( + Events.ITEM_CANCELLED, + data=item.info, + title="Download Cancelled", + message=f"Download '{item.info.title}' has been cancelled.", + ) + item.info.status = "cancelled" + await self.done.put(item) + self._notify.emit( + Events.ITEM_MOVED, + data={"to": "history", "preset": item.info.preset, "item": item.info}, + title="Download Cancelled", + message=f"Download '{item.info.title}' has been cancelled.", + ) + LOG.info(f"Deleted from queue {item_ref}") + + status[id] = "ok" + + return status + + async def clear(self, ids: list[str], remove_file: bool = False) -> dict[str, str]: + """ + Clear the download history. + + Args: + ids (list): The list of ids to clear. + remove_file (bool): Only considered if config.remove_files is True. + + Returns: + dict: The status of the operation. + + """ + status: dict[str, str] = {} + + for id in ids: + try: + item: Download = await self.done.get(key=id) + except KeyError as e: + status[id] = str(e) + status["status"] = "error" + LOG.warning(f"Requested delete for non-existent download {id=}. {e!s}") + continue + + itemRef: str = f"{id=} {item.info.id=} {item.info.title=}" + removed_files = 0 + filename: str = "" + + if self.config.remove_files is not True: + remove_file = False + + LOG.debug(f"{remove_file=} {itemRef} - Removing local files: {item.info.status=}") + + if remove_file and "finished" == item.info.status and item.info.filename: + filename = str(item.info.filename) + if item.info.folder: + filename = f"{item.info.folder}/{item.info.filename}" + + try: + rf = Path( + calc_download_path( + base_path=Path(self.config.download_path), + folder=filename, + create_path=False, + ) + ) + if rf.is_file() and rf.exists(): + if rf.stem and rf.suffix: + for f in rf.parent.glob(f"{glob.escape(rf.stem)}.*"): + if f.is_file() and f.exists() and not f.name.startswith("."): + removed_files += 1 + LOG.debug(f"Removing '{itemRef}' local file '{f.name}'.") + f.unlink(missing_ok=True) + else: + LOG.debug(f"Removing '{itemRef}' local file '{rf.name}'.") + rf.unlink(missing_ok=True) + removed_files += 1 + else: + LOG.warning(f"Failed to remove '{itemRef}' local file '{filename}'. File not found.") + except Exception as e: + LOG.error(f"Unable to remove '{itemRef}' local file '{filename}'. {e!s}") + + await self.done.delete(id) + + _status: str = "Removed" if removed_files > 0 else "Cleared" + self._notify.emit( + Events.ITEM_DELETED, + data=item.info, + title=f"Download {_status}", + message=f"{_status} '{item.info.title}' from history.", + ) + + msg = f"Deleted completed download '{itemRef}'." + if removed_files > 0: + msg += f" and removed '{removed_files}' local files." + + LOG.info(msg=msg) + status[id] = "ok" + + return status + + async def get(self, mode: str = "all") -> dict[str, list[dict[str, ItemDTO]]]: + """ + Get the download queue and the download history. + + Args: + mode (str): The mode to get the items. Supported modes are 'all', 'queue', 'done'. + + Returns: + dict: The download queue and the download history. + + """ + items = {"queue": {}, "done": {}} + active = self.pool.get_active_downloads() + + if mode in ("all", "queue"): + for k, v in await self.queue.saved_items(): + items["queue"][k] = active[k].info if k in active else v + + if mode in ("all", "done"): + for k, v in await self.done.saved_items(): + items["done"][k] = v + + if mode in ("all", "queue"): + for k, v in self.queue.items(): + if k not in items["queue"]: + items["queue"][k] = active[k].info if k in active else v + + if mode in ("all", "done"): + for k, v in self.done.items(): + if k in items["done"]: + continue + + items["done"][k] = v.info + + return items + + async def get_item(self, **kwargs) -> tuple["StoreType", Download] | tuple[None, None]: + """ + Get a specific item from the download queue or history. + + Args: + **kwargs: The key-value pair to search for. Supported keys are 'id', 'url'. + + Returns: + (StoreType, Download) | None: The requested item if found, otherwise None. + + """ + from app.library.DataStore import StoreType + + if item := await self.queue.get_item(**kwargs): + return (StoreType.QUEUE, item) + + if item := await self.done.get_item(**kwargs): + return (StoreType.HISTORY, item) + + return (None, None) diff --git a/app/library/downloads/status_tracker.py b/app/library/downloads/status_tracker.py new file mode 100644 index 00000000..02671da3 --- /dev/null +++ b/app/library/downloads/status_tracker.py @@ -0,0 +1,202 @@ +"""Status tracking and update processing for downloads.""" + +import asyncio +import logging +import time +from email.utils import formatdate +from pathlib import Path +from typing import TYPE_CHECKING, Any + +from app.library.Events import EventBus, Events +from app.library.ffprobe import ffprobe + +from .types import StatusDict, Terminator +from .utils import safe_relative_path + +if TYPE_CHECKING: + from multiprocessing import Queue + + from app.library.ItemDTO import ItemDTO + + +class StatusTracker: + def __init__( + self, + info: "ItemDTO", + download_id: str, + download_dir: str, + temp_path: Path | None, + status_queue: "Queue[Any]", + logger: logging.Logger, + debug: bool = False, + ): + """ + Initialize status tracker. + + Args: + info: Download information object + download_id: Unique download identifier + download_dir: Base download directory + temp_path: Temporary directory for this download + status_queue: Multiprocessing queue for status updates + logger: Logger instance + debug: Enable debug logging + + """ + self.info = info + self.id = download_id + self.download_dir = download_dir + self.temp_path = temp_path + self.status_queue = status_queue + self.logger = logger + self.debug = debug + self._notify: EventBus = EventBus.get_instance() + self.tmpfilename: str | None = None + self.final_update = False + self.update_task: asyncio.Task | None = None + + async def process_status_update(self, status: StatusDict) -> None: + """ + Process a single status update from the download subprocess. + + Updates the ItemDTO with progress information, file paths, errors, + and completion status. + + Args: + status: Status dictionary from yt-dlp hooks + + """ + if self.final_update: + return + + if status.get("id") != self.id or len(status) < 2: + self.logger.warning(f"Received invalid status update. {status}") + return + + if self.debug: + self.logger.debug(f"Status Update: _id={self.info._id} status={status}") + + if isinstance(status, str): + self._notify.emit(Events.ITEM_UPDATED, data=self.info) + return + + self.tmpfilename = status.get("tmpfilename") + + self.info.status = status.get("status", self.info.status) + self.info.msg = status.get("msg") + self.info.postprocessor = status.get("postprocessor", None) + + if "error" == self.info.status and "error" in status: + self.info.error = status.get("error") + self._notify.emit( + Events.LOG_ERROR, + data=self.info, + title="Download Error", + message=f"'{self.info.title}' failed to download. {self.info.error}", + ) + + if "downloaded_bytes" in status and status.get("downloaded_bytes", 0) > 0: + self.info.downloaded_bytes = status.get("downloaded_bytes") + total: float | None = status.get("total_bytes") or status.get("total_bytes_estimate") + if total: + try: + self.info.percent = status["downloaded_bytes"] / total * 100 + except ZeroDivisionError: + self.info.percent = 0 + self.info.total_bytes = total + + self.info.speed = status.get("speed") + self.info.eta = status.get("eta") + + if final_name := status.get("final_name", None): + final_name = Path(final_name) + self.info.datetime = str(formatdate(time.time())) + self.info.status = "finished" + self.final_update = True + self.info.filename = safe_relative_path(final_name, Path(self.download_dir), self.temp_path) + self.logger.debug(f"Final file name: '{final_name}'.") + + if final_name.is_file() and final_name.exists(): + try: + self.info.file_size = final_name.stat().st_size + except FileNotFoundError: + self.info.file_size = 0 + + try: + ff = await ffprobe(final_name) + self.info.extras["is_video"] = ff.has_video() + self.info.extras["is_audio"] = ff.has_audio() + if ff.has_video() or ff.has_audio(): + self.info.extras["duration"] = int( + float(ff.metadata.get("duration", self.info.extras.get("duration", 0.0))) + ) + except Exception as e: + self.info.extras["is_video"] = True + self.info.extras["is_audio"] = True + self.logger.exception(e) + self.logger.error(f"Failed to run ffprobe. {e}") + + self._notify.emit(Events.ITEM_UPDATED, data=self.info) + + async def progress_update(self) -> None: + """ + Continuous loop that processes status updates from the queue. + + Runs until a Terminator sentinel is received or the task is cancelled. + """ + while True: + try: + self.update_task = asyncio.get_running_loop().run_in_executor(None, self.status_queue.get) + status = await self.update_task + if status is None or isinstance(status, Terminator): + return + await self.process_status_update(status) + except (asyncio.CancelledError, OSError, FileNotFoundError): + return + + async def drain_queue(self, max_iterations: int = 50) -> None: + """ + Drain remaining status updates from the queue. + + Called after download completion to ensure all status updates are processed. + + Args: + max_iterations: Maximum number of items to drain + + """ + self.logger.debug("Draining status queue.") + try: + drain_count: int = max_iterations + ( + self.status_queue.qsize() if hasattr(self.status_queue, "qsize") else 5 + ) + except Exception: + drain_count = max_iterations + 5 + + for i in range(drain_count): + try: + if self.debug: + self.logger.debug(f"({max_iterations}/{i}) Draining the status queue...") + if self.final_update: + if self.debug: + self.logger.debug(f"({max_iterations}/{i}) Draining stopped. Final update received.") + break + next_status = self.status_queue.get(timeout=0.1) + if next_status is None or isinstance(next_status, Terminator): + continue + await self.process_status_update(next_status) + except Exception as e: + self.logger.warning(f"Error processing status update during drain: {e}") + continue + + def cancel_update_task(self) -> None: + """Cancel the progress update task if it's running.""" + try: + if self.update_task and not self.update_task.done(): + self.update_task.cancel() + except Exception as e: + self.logger.error(f"Failed to cancel update task. {e}") + + def put_terminator(self) -> None: + """Put a terminator sentinel in the status queue.""" + if self.status_queue: + self.status_queue.put(Terminator()) diff --git a/app/library/downloads/temp_manager.py b/app/library/downloads/temp_manager.py new file mode 100644 index 00000000..52335137 --- /dev/null +++ b/app/library/downloads/temp_manager.py @@ -0,0 +1,100 @@ +"""Temporary file and directory management for downloads.""" + +import hashlib +import logging +from pathlib import Path +from typing import TYPE_CHECKING + +from app.library.Utils import delete_dir + +from .utils import is_safe_to_delete_dir + +if TYPE_CHECKING: + from app.library.ItemDTO import ItemDTO + + +class TempManager: + def __init__( + self, + info: "ItemDTO", + temp_dir: str | None, + temp_disabled: bool, + temp_keep: bool, + logger: logging.Logger, + ): + """ + Initialize temp manager. + + Args: + info: Download information object + temp_dir: Base temporary directory path + temp_disabled: Whether temporary directory feature is disabled + temp_keep: Whether to keep temp files after download + logger: Logger instance + + """ + self.info = info + self.temp_dir = temp_dir + self.temp_disabled = temp_disabled + self.temp_keep = temp_keep + self.logger = logger + self.temp_path: Path | None = None + + def create_temp_path(self) -> Path | None: + """ + Create unique temporary directory for this download. + + Returns: + Path to created temporary directory, or None if disabled + + """ + if self.temp_disabled or not self.temp_dir: + return None + + self.temp_path = Path(self.temp_dir) / hashlib.shake_256(f"D-{self.info.id}".encode()).hexdigest(5) + + if not self.temp_path.exists(): + self.temp_path.mkdir(parents=True, exist_ok=True) + + return self.temp_path + + def delete_temp(self, by_pass: bool = False) -> None: + """ + Delete temporary directory if conditions are met. + + Temp directory is kept if: + - temp_keep is enabled + - temp_disabled is enabled + - Download hasn't finished but has partial data (unless by_pass=True) + + Args: + by_pass: Force deletion regardless of download state + + """ + if self.temp_disabled or self.temp_keep is True or not self.temp_path: + return + + if ( + not by_pass + and "finished" != self.info.status + and self.info.downloaded_bytes + and self.info.downloaded_bytes > 0 + ): + self.logger.warning(f"Keeping temp folder '{self.temp_path}'. status={self.info.status}") + return + + tmp_dir = Path(self.temp_path) + + if not tmp_dir.exists(): + return + + if not self.temp_dir or not is_safe_to_delete_dir(tmp_dir, self.temp_dir): + self.logger.warning(f"Refusing to delete video temp folder '{self.temp_path}' as it's temp root.") + return + + status: bool = delete_dir(tmp_dir) + if by_pass: + tmp_dir.mkdir(parents=True, exist_ok=True) + self.logger.info(f"Temp folder '{self.temp_path}' emptied.") + else: + self.logger.info(f"Temp folder '{self.temp_path}' deletion is {'success' if status else 'failed'}.") diff --git a/app/library/downloads/types.py b/app/library/downloads/types.py new file mode 100644 index 00000000..dd5281d3 --- /dev/null +++ b/app/library/downloads/types.py @@ -0,0 +1,15 @@ +"""Shared types and protocols for the downloads module.""" + +from typing import Any + + +class Terminator: + """ + Sentinel class to signal termination of a status queue. + + Used to indicate that no more status updates will be sent through. + """ + + +StatusDict = dict[str, Any] +"""Type alias for status update dictionaries.""" diff --git a/app/library/downloads/utils.py b/app/library/downloads/utils.py new file mode 100644 index 00000000..898c0946 --- /dev/null +++ b/app/library/downloads/utils.py @@ -0,0 +1,253 @@ +"""Utility functions and constants for the downloads module.""" + +import asyncio +import logging +import os +import time +from pathlib import Path + +# Global limits container for per-extractor semaphores +LIMITS: dict[str, asyncio.Semaphore] = {} + +# Extractor Constants +GENERIC_EXTRACTORS = ("HTML5MediaEmbed", "generic") + +# yt-dlp Field Filtering +YTDLP_PROGRESS_FIELDS = ( + "tmpfilename", + "filename", + "status", + "msg", + "total_bytes", + "total_bytes_estimate", + "downloaded_bytes", + "speed", + "eta", + "postprocessor", +) + +# Live Stream Options (known to cause issues) +BAD_LIVE_STREAM_OPTIONS = [ + "concurrent_fragment_downloads", + "fragment_retries", + "skip_unavailable_fragments", +] + +# Debug Logging +DEBUG_MESSAGE_PREFIXES = ["[debug] ", "[download] "] + + +def safe_relative_path(file_path: Path, base_path: Path, fallback_path: Path | None = None) -> str: + """ + Get relative path with fallback handling. + + Attempts to compute the relative path from base_path. If that fails, + tries fallback_path. If both fail, returns absolute path as string. + + Args: + file_path: The file path to make relative + base_path: The base path to compute relative to + fallback_path: Optional fallback base path + + Returns: + str: Relative path string, or absolute path if relative computation fails + + """ + try: + return str(file_path.relative_to(base_path)) + except ValueError: + if fallback_path: + try: + return str(file_path.relative_to(fallback_path)) + except ValueError: + pass + return str(file_path) + + +def is_safe_to_delete_dir(path: Path, root_path: str | Path) -> bool: + """ + Safety check before deleting directories. + + Prevents accidental deletion of the root temporary directory. + + Args: + path: The path to check + root_path: The root path that should not be deleted + + Returns: + bool: True if safe to delete, False if it's the root + + """ + return str(path) != str(root_path) + + +def wait_for_process_with_timeout(proc, timeout: float, check_interval: float = 0.1) -> bool: + """ + Wait for process to terminate with timeout. + + Args: + proc: The multiprocessing.Process to wait for + timeout: Maximum time to wait in seconds + check_interval: How often to check process state + + Returns: + bool: True if process terminated, False if timeout reached + + """ + start_time = time.time() + while proc.is_alive() and (time.time() - start_time) < timeout: + time.sleep(check_interval) + return not proc.is_alive() + + +def parse_extractor_limit( + extractor: str, + default_limit: int, + max_workers: int, + logger: logging.Logger | None = None, +) -> int: + """ + Parse and validate environment variable limits for extractors. + + Checks for YTP_MAX_WORKERS_FOR_{EXTRACTOR} environment variable, + validates the value, and ensures it doesn't exceed max_workers. + + Args: + extractor: The extractor name + default_limit: Default limit if no valid env var found + max_workers: Maximum workers to never exceed + logger: Optional logger for warnings + + Returns: + int: The validated limit for this extractor + + """ + env_limit: str | None = os.environ.get(f"YTP_MAX_WORKERS_FOR_{extractor.upper()}") + + if env_limit and env_limit.isdigit() and 1 <= int(env_limit): + limit: int = min(int(env_limit), max_workers) + else: + if env_limit and logger: + logger.warning(f"Invalid extractor limit '{env_limit}' for '{extractor}', using default limit.") + limit = default_limit + + return min(limit, max_workers) + + +def get_extractor_limit( + extractor: str, + max_workers: int, + max_workers_per_extractor: int, + logger: logging.Logger, +) -> asyncio.Semaphore: + """ + Get or create a semaphore for the given extractor. + + Args: + extractor: The extractor name + max_workers: Maximum workers allowed + max_workers_per_extractor: Default per-extractor limit + logger: Logger for info messages + + Returns: + asyncio.Semaphore: The semaphore for this extractor + + """ + if extractor not in LIMITS: + limit = parse_extractor_limit(extractor, max_workers_per_extractor, max_workers, logger) + LIMITS[extractor] = asyncio.Semaphore(limit) + logger.info(f"Created limits container for extractor '{extractor}': {limit}") + + return LIMITS[extractor] + + +def create_debug_safe_dict(data: dict, exclude_keys: list[str] | None = None) -> dict: + """ + Create sanitized dict for debug logging. + + Args: + data: The data dict to sanitize + exclude_keys: Additional keys to exclude from info_dict + + Returns: + dict: Sanitized dict safe for logging + + """ + if exclude_keys is None: + exclude_keys = ["formats", "thumbnails", "description", "tags", "_format_sort_fields"] + + import types + + d_safe: dict = { + "status": data.get("status"), + "filename": data.get("filename"), + "info_dict": { + k: v + for k, v in data.get("info_dict", {}).items() + if k not in exclude_keys and v is not None and not isinstance(v, (types.FunctionType, types.LambdaType)) + }, + } + + return d_safe + + +def is_download_stale( + started_time: int, current_status: str, is_running: bool, auto_start: bool, min_elapsed: int = 300 +) -> bool: + """ + Determine if download task has become stale. + + A download is stale if it's been running too long without progress, + or if the process died unexpectedly. + + Terminal statuses (finished, error, cancelled, downloading, postprocessing) + are never stale - status is source of truth. + + Args: + started_time: Unix timestamp when download started + current_status: Current download status + is_running: Whether the process is currently running + auto_start: Whether download was set to auto-start + min_elapsed: Minimum seconds before considering stale + + Returns: + bool: True if download is stale and should be cancelled + + """ + terminal_statuses = ["finished", "error", "cancelled", "downloading", "postprocessing"] + + if current_status in terminal_statuses: + return False + + if not auto_start: + return False + + if is_running: + return False + + elapsed = int(time.time()) - started_time + return elapsed >= min_elapsed + + +def handle_task_exception(task: asyncio.Task, logger: logging.Logger) -> None: + """ + Handle exceptions from background tasks. + + Logs unhandled exceptions from asyncio tasks. Ignores cancelled tasks. + + Args: + task: The completed task to check + logger: Logger to use for error output + + """ + if task.cancelled(): + return + + if not (exc := task.exception()): + return + + task_name: str = task.get_name() if task.get_name() else "unknown_task" + import traceback + + tb = "".join(traceback.format_exception(type(exc), exc, exc.__traceback__)) + logger.error(f"Unhandled exception in background task '{task_name}': {exc!s}. {tb}") diff --git a/app/library/downloads/video_processor.py b/app/library/downloads/video_processor.py new file mode 100644 index 00000000..93c16844 --- /dev/null +++ b/app/library/downloads/video_processor.py @@ -0,0 +1,233 @@ +"""Video entry processing.""" + +import logging +import time +from datetime import UTC, datetime, timedelta +from email.utils import formatdate +from typing import TYPE_CHECKING + +from app.library.Events import Events +from app.library.ItemDTO import ItemDTO +from app.library.Utils import calc_download_path, extract_ytdlp_logs, get_extras, merge_dict, str_to_dt + +from .core import Download + +if TYPE_CHECKING: + from app.library.ItemDTO import Item + + from .queue_manager import DownloadQueue + +LOG: logging.Logger = logging.getLogger("downloads.video") + + +async def add_video(queue: "DownloadQueue", entry: dict, item: "Item", logs: list[str] | None = None) -> dict[str, str]: + """ + Process and add a video entry to the queue. + + Args: + queue: DownloadQueue instance + entry: Video entry dict from yt-dlp + item: Item configuration + logs: Optional yt-dlp logs + + Returns: + dict: Status dict with "status" and optional "msg" keys + + """ + if not logs: + logs: list[str] = [] + + options: dict = {} + error: str | None = None + live_in: str | None = None + is_premiere: bool = bool(entry.get("is_premiere", False)) + + release_in: str | None = None + if entry.get("release_timestamp"): + release_in = formatdate(entry.get("release_timestamp"), usegmt=True) + item.extras["release_in"] = release_in + + # check if the video is live stream. + if "is_upcoming" == entry.get("live_status"): + if release_in: + live_in = release_in + item.extras["live_in"] = live_in + else: + error = f"No start time is set for {'premiere' if is_premiere else 'live stream'}." + else: + error = entry.get("msg") + + LOG.debug(f"Entry id '{entry.get('id')}' url '{entry.get('webpage_url')} - {entry.get('url')}'.") + + try: + _item: Download = await queue.done.get(key=entry.get("id"), url=entry.get("webpage_url") or entry.get("url")) + err_msg: str = f"Removing {_item.info.name()} from history list." + LOG.warning(err_msg) + await queue.clear([_item.info._id], remove_file=False) + except KeyError: + pass + + try: + _item: Download = await queue.queue.get( + key=str(entry.get("id")), url=str(entry.get("webpage_url") or entry.get("url")) + ) + err_msg: str = f"Item {_item.info.name()} is already in download queue." + LOG.info(err_msg) + return {"status": "error", "msg": err_msg} + except KeyError: + pass + + live_status: list = ["is_live", "is_upcoming"] + is_live = bool(entry.get("is_live") or live_in or entry.get("live_status") in live_status) + + try: + download_dir: str = calc_download_path(base_path=queue.config.download_path, folder=item.folder) + except Exception as e: + LOG.exception(e) + return {"status": "error", "msg": str(e)} + + for field in ("uploader", "channel", "thumbnail"): + if entry.get(field): + item.extras[field] = entry.get(field) + + for key in entry: + if isinstance(key, str) and key.startswith("playlist") and entry.get(key): + item.extras[key] = entry.get(key) + + item.extras["duration"] = entry.get("duration", item.extras.get("duration")) + + if not item.extras.get("live_in") and live_in: + item.extras["live_in"] = live_in + + if not item.extras.get("is_premiere") and is_premiere: + item.extras["is_premiere"] = is_premiere + + dl = ItemDTO( + id=str(entry.get("id")), + title=str(entry.get("title")), + description=str(entry.get("description", "")), + url=str(entry.get("webpage_url") or entry.get("url")), + preset=item.preset, + folder=item.folder, + download_dir=download_dir, + temp_dir=queue.config.temp_path, + cookies=item.cookies, + template=item.template if item.template else queue.config.output_template, + template_chapter=queue.config.output_template_chapter, + datetime=formatdate(time.time()), + error=error, + is_live=is_live, + live_in=live_in if live_in else item.extras.get("live_in", None), + options=options, + cli=item.cli, + auto_start=item.auto_start, + extras=merge_dict(item.extras, get_extras(entry)), + ) + + try: + dlInfo: Download = Download(info=dl, info_dict=entry if item.auto_start else None, logs=logs) + nEvent: str | None = None + nTitle: str | None = None + nMessage: str | None = None + nStore: str = "queue" + hasFormats: bool = len(entry.get("formats", [])) > 0 or entry.get("url") + + text_logs: str = "" + if filtered_logs := extract_ytdlp_logs(logs): + text_logs = " " + ", ".join(filtered_logs) + + if "is_upcoming" == entry.get("live_status"): + nEvent = Events.ITEM_MOVED + nStore = "history" + nTitle = "Upcoming Premiere" if is_premiere else "Upcoming Live Stream" + nMessage = f"{'Premiere video' if is_premiere else 'Stream'} '{dlInfo.info.title}' is not available yet. {text_logs}" + + dlInfo.info.status = "not_live" + dlInfo.info.msg = nMessage.replace(f" '{dlInfo.info.title}'", "") + queue._notify.emit( + Events.LOG_INFO, + data={"preset": dlInfo.info.preset, "lowPriority": True}, + title=nTitle, + message=nMessage, + ) + + itemDownload: Download = await queue.done.put(dlInfo) + elif not hasFormats: + ava: str = entry.get("availability", "public") + nTitle = "Download Error" + nMessage: str = f"No formats for '{dl.title}'." + nEvent = Events.ITEM_MOVED + nStore = "history" + + if ava and ava not in ("public",): + nMessage += f" Availability is set for '{ava}'." + + dlInfo.info.error = nMessage.replace(f" for '{dl.title}'.", ".") + text_logs + dlInfo.info.status = "error" + itemDownload = await queue.done.put(dlInfo) + + queue._notify.emit( + Events.LOG_WARNING, + data={"preset": dlInfo.info.preset, "logs": text_logs}, + title=nTitle, + message=nMessage, + ) + elif is_premiere and queue.config.prevent_live_premiere: + nStore = "history" + nTitle = "Premiere Video" + dlInfo.info.error = "Premiering right now." + + _requeue = True + if release_in: + try: + starts_in: datetime = str_to_dt(release_in) + starts_in: datetime = ( + starts_in.replace(tzinfo=UTC) if starts_in.tzinfo is None else starts_in.astimezone(UTC) + ) + starts_in = starts_in + timedelta(minutes=5, seconds=dl.extras.get("duration", 0)) + dlInfo.info.error += f" Download will start at {starts_in.astimezone().isoformat()}." + _requeue = False + except Exception as e: + LOG.error(f"Failed to parse live_in date '{release_in}'. {e!s}") + dlInfo.info.error += f" Failed to parse live_in date '{release_in}'." + else: + dlInfo.info.error += f" Delaying download by '{300 + dl.extras.get('duration', 0)}' seconds." + + nMessage = f"'{dlInfo.info.title}': '{dlInfo.info.error.strip()}'." + + if _requeue: + nEvent = Events.ITEM_ADDED + itemDownload = await queue.queue.put(dlInfo) + if item.auto_start: + queue.pool.trigger_download() + else: + dlInfo.info.status = "not_live" + itemDownload = await queue.done.put(dlInfo) + nStore = "history" + nEvent = Events.ITEM_MOVED + nTitle = "Premiering right now" + queue._notify.emit(Events.LOG_INFO, data={"preset": dlInfo.info.preset}, title=nTitle, message=nMessage) + else: + nEvent = Events.ITEM_ADDED + nTitle = "Item Added" + nMessage = f"Item '{dlInfo.info.title}' has been added to the download queue." + itemDownload = await queue.queue.put(dlInfo) + if item.auto_start: + queue.pool.trigger_download() + else: + LOG.debug(f"Item {itemDownload.info.name()} is not set to auto-start.") + + queue._notify.emit( + nEvent, + data={"to": nStore, "preset": itemDownload.info.preset, "item": itemDownload.info} + if Events.ITEM_MOVED == nEvent + else itemDownload.info, + title=nTitle, + message=nMessage, + ) + + return {"status": "ok"} + except Exception as e: + LOG.exception(e) + LOG.error(f"Failed to download item. '{e!s}'") + return {"status": "error", "msg": str(e)} diff --git a/app/main.py b/app/main.py index 6fb6130e..c9860b63 100644 --- a/app/main.py +++ b/app/main.py @@ -18,7 +18,7 @@ from app.library.BackgroundWorker import BackgroundWorker from app.library.conditions import Conditions from app.library.config import Config from app.library.dl_fields import DLFields -from app.library.DownloadQueue import DownloadQueue +from app.library.downloads import DownloadQueue from app.library.Events import EventBus, Events from app.library.HttpAPI import HttpAPI from app.library.HttpSocket import HttpSocket diff --git a/app/routes/api/browser.py b/app/routes/api/browser.py index cafaa9b9..128c0e47 100644 --- a/app/routes/api/browser.py +++ b/app/routes/api/browser.py @@ -9,7 +9,7 @@ from aiohttp.web import Request, Response from app.library.cache import Cache from app.library.config import Config -from app.library.DownloadQueue import DownloadQueue +from app.library.downloads import DownloadQueue from app.library.encoder import Encoder from app.library.Events import EventBus, Events from app.library.ffprobe import ffprobe diff --git a/app/routes/api/history.py b/app/routes/api/history.py index 31d57015..f0e6e1be 100644 --- a/app/routes/api/history.py +++ b/app/routes/api/history.py @@ -7,8 +7,7 @@ from aiohttp.web import Request, Response from app.library.config import Config from app.library.DataStore import StoreType -from app.library.Download import Download -from app.library.DownloadQueue import DownloadQueue +from app.library.downloads import Download, DownloadQueue from app.library.encoder import Encoder from app.library.Events import EventBus, Events from app.library.ItemDTO import Item @@ -16,7 +15,7 @@ from app.library.Presets import Preset, Presets from app.library.router import route if TYPE_CHECKING: - from library.Download import Download + from library.downloads import Download LOG: logging.Logger = logging.getLogger(__name__) diff --git a/app/routes/api/ping.py b/app/routes/api/ping.py index a0da11bc..68cbf0f3 100644 --- a/app/routes/api/ping.py +++ b/app/routes/api/ping.py @@ -1,7 +1,7 @@ from aiohttp import web from aiohttp.web import Response -from app.library.DownloadQueue import DownloadQueue +from app.library.downloads import DownloadQueue from app.library.router import route diff --git a/app/routes/api/system.py b/app/routes/api/system.py index c0a5b3a6..490b98b8 100644 --- a/app/routes/api/system.py +++ b/app/routes/api/system.py @@ -7,7 +7,7 @@ from aiohttp.web import Request, Response from aiohttp.web_runner import GracefulExit from app.library.config import Config -from app.library.DownloadQueue import DownloadQueue +from app.library.downloads import DownloadQueue from app.library.encoder import Encoder from app.library.Events import EventBus, Events from app.library.router import route diff --git a/app/routes/socket/connection.py b/app/routes/socket/connection.py index 38e058ae..dd761fb9 100644 --- a/app/routes/socket/connection.py +++ b/app/routes/socket/connection.py @@ -7,7 +7,7 @@ import socketio from app.library.config import Config from app.library.dl_fields import DLFields -from app.library.DownloadQueue import DownloadQueue +from app.library.downloads import DownloadQueue from app.library.Events import EventBus, Events from app.library.Presets import Presets from app.library.router import RouteType, route diff --git a/app/routes/socket/history.py b/app/routes/socket/history.py index 6229235d..0508de8d 100644 --- a/app/routes/socket/history.py +++ b/app/routes/socket/history.py @@ -1,6 +1,6 @@ import logging -from app.library.DownloadQueue import DownloadQueue +from app.library.downloads import DownloadQueue from app.library.Events import EventBus, Events from app.library.ItemDTO import Item from app.library.router import RouteType, route diff --git a/app/tests/test_datastore_pagination.py b/app/tests/test_datastore_pagination.py index 0c8c280b..7c54b286 100644 --- a/app/tests/test_datastore_pagination.py +++ b/app/tests/test_datastore_pagination.py @@ -5,7 +5,7 @@ import pytest import pytest_asyncio from app.library.DataStore import DataStore, StoreType -from app.library.Download import Download +from app.library.downloads import Download from app.library.ItemDTO import ItemDTO from app.library.sqlite_store import SqliteStore diff --git a/app/tests/test_download.py b/app/tests/test_download.py index 6c2f24bb..e6bd8d21 100644 --- a/app/tests/test_download.py +++ b/app/tests/test_download.py @@ -1,10 +1,17 @@ import logging +import os +import signal from pathlib import Path from typing import Any +from unittest.mock import MagicMock, Mock, patch import pytest -from app.library.Download import Download, NestedLogger, Terminator +from app.library.downloads import Download, NestedLogger, Terminator +from app.library.downloads.hooks import HookHandlers +from app.library.downloads.process_manager import ProcessManager +from app.library.downloads.status_tracker import StatusTracker +from app.library.downloads.temp_manager import TempManager from app.library.ItemDTO import ItemDTO @@ -28,7 +35,7 @@ class DummyQueue: def put(self, obj: Any) -> None: self.items.append(obj) - def get(self) -> Any: + def get(self, timeout: float | None = None) -> Any: if not self.items: return None return self.items.pop(0) @@ -39,7 +46,6 @@ class TestNestedLogger: logger = logging.getLogger("nl_test") logger.setLevel(logging.DEBUG) cap = CaptureHandler() - # Remove existing handlers to avoid duplicates for h in list(logger.handlers): logger.removeHandler(h) logger.addHandler(cap) @@ -49,20 +55,18 @@ class TestNestedLogger: nl.debug("[download] progress") nl.debug("[info] info message") - # Two DEBUG, one INFO levels = [r.levelno for r in cap.records] - assert levels.count(logging.DEBUG) == 2 - assert levels.count(logging.INFO) == 1 + assert 2 == levels.count(logging.DEBUG), "Should have 2 debug messages" + assert 1 == levels.count(logging.INFO), "Should have 1 info message" msgs = [r.getMessage() for r in cap.records] - assert "[debug]" not in msgs[0] + assert "[debug]" not in msgs[0], "[debug] prefix should be stripped" assert msgs[1] == "[download] progress", "[download] prefix is not stripped by NestedLogger" - assert msgs[2] == "info message" + assert msgs[2] == "info message", "info message should have [info] prefix stripped" class TestDownloadHooks: @pytest.fixture(autouse=True) def cfg_and_bus(self, monkeypatch: pytest.MonkeyPatch): - # Minimal Config stub class Cfg: debug = False ytdlp_debug = False @@ -75,9 +79,8 @@ class TestDownloadHooks: def get_instance(): return Cfg - monkeypatch.setattr("app.library.Download.Config", Cfg) + monkeypatch.setattr("app.library.downloads.core.Config", Cfg) - # EventBus.get_instance is used during __init__ and start, we don't hit start here class EB: @staticmethod def get_instance(): @@ -87,12 +90,12 @@ class TestDownloadHooks: def emit(*_args, **_kwargs): return None - monkeypatch.setattr("app.library.Download.EventBus", EB) + monkeypatch.setattr("app.library.downloads.core.EventBus", EB) def test_progress_hook_filters_fields(self) -> None: d = Download(make_item()) q = DummyQueue() - d.status_queue = q + hooks = HookHandlers(d.id, q, d.logger, d.debug) payload = { "tmpfilename": "t", @@ -106,12 +109,12 @@ class TestDownloadHooks: "eta": 2, "other": "x", } - d._progress_hook(payload) - assert len(q.items) == 1 + hooks.progress_hook(payload) + assert 1 == len(q.items), "Should have 1 item in queue" ev = q.items[0] - assert ev["id"] == d.id - assert ev["action"] == "progress" - assert "other" not in ev, "ensure only whitelisted keys included" + assert ev["id"] == d.id, "Event should have correct download ID" + assert ev["action"] == "progress", "Action should be 'progress'" + assert "other" not in ev, "Non-whitelisted keys should not be included in event" for k in ( "tmpfilename", "filename", @@ -123,38 +126,15 @@ class TestDownloadHooks: "speed", "eta", ): - assert k in ev - - def test_postprocessor_hook_movefiles_sets_final_name(self, tmp_path: Path) -> None: - d = Download(make_item()) - q = DummyQueue() - d.status_queue = q - - finaldir = tmp_path / "out" - finaldir.mkdir() - path = finaldir / "file.mp4" - # we don't need it to exist for this branch; hook doesn't check file existence - payload = { - "postprocessor": "MoveFiles", - "status": "finished", - "info_dict": {"__finaldir": str(finaldir), "filepath": str(path)}, - } - d._postprocessor_hook(payload) - assert len(q.items) == 1 - ev = q.items[0] - assert ev["action"] == "moved" - assert ev["status"] == "finished" - assert ev["final_name"].endswith("file.mp4") + assert k in ev, f"Key '{k}' should be in event" def test_post_hooks_pushes_filename(self) -> None: d = Download(make_item()) q = DummyQueue() - d.status_queue = q - d._post_hooks(None) - assert len(q.items) == 0 - d._post_hooks("name.ext") - assert len(q.items) == 1 - assert q.items[0]["filename"] == "name.ext" + hooks = HookHandlers(d.id, q, d.logger, d.debug) + hooks.post_hook("name.ext") + assert 1 == len(q.items), "Should have 1 item when filename is provided" + assert q.items[0]["final_name"] == "name.ext", "Filename should match" class TestDownloadStale: @@ -172,7 +152,14 @@ class TestDownloadStale: def get_instance(): return Cfg - monkeypatch.setattr("app.library.Download.Config", Cfg) + @staticmethod + def get_manager(): + # Return a mock manager with Queue method + mock_manager = MagicMock() + mock_manager.Queue = MagicMock(return_value=DummyQueue()) + return mock_manager + + monkeypatch.setattr("app.library.downloads.core.Config", Cfg) class EB: @staticmethod @@ -183,203 +170,524 @@ class TestDownloadStale: def emit(*_args, **_kwargs): return None - monkeypatch.setattr("app.library.Download.EventBus", EB) + monkeypatch.setattr("app.library.downloads.core.EventBus", EB) def test_is_stale_conditions(self, monkeypatch: pytest.MonkeyPatch) -> None: d = Download(make_item()) - # Auto-start disabled is never stale + d.info.auto_start = False - assert d.is_stale() is False + assert d.is_stale() is False, "Download with auto_start disabled should not be stale" d.info.auto_start = True - # Not started yet -> not stale d.started_time = 0 - assert d.is_stale() is False + assert d.is_stale() is False, "Download that has not been started should not be stale" - # Less than 300 seconds elapsed -> not stale d.started_time = 1000 monkeypatch.setattr("time.time", lambda: 1200) - assert d.is_stale() is False + assert d.is_stale() is False, "Download running for less than 300 seconds should not be stale" - # Not running after 300s -> stale - d.started_time = 0 - d.started_time = 1000 monkeypatch.setattr("time.time", lambda: 1401) - monkeypatch.setattr("app.library.Download.Download.running", lambda _self: False) + + d.info.status = "finished" + assert d.is_stale() is False, "Download with status 'finished' should not be stale regardless of process state" + + d.info.status = "error" + assert d.is_stale() is False, "Download with status 'error' should not be stale regardless of process state" + + d.info.status = "cancelled" + assert d.is_stale() is False, "Download with status 'cancelled' should not be stale regardless of process state" + + d.info.status = "downloading" + assert d.is_stale() is False, ( + "Download with status 'downloading' should not be stale regardless of process state" + ) + + d.info.status = "postprocessing" + assert d.is_stale() is False, ( + "Download with status 'postprocessing' should not be stale regardless of process state" + ) + d.info.status = "preparing" - assert d.is_stale() is True + assert d.is_stale() is True, ( + "Download with status 'preparing' and no running process after 300s should be stale" + ) - # Running but status stuck -> stale - monkeypatch.setattr("app.library.Download.Download.running", lambda _self: True) d.info.status = "queued" - assert d.is_stale() is True + assert d.is_stale() is True, "Download with status 'queued' and no running process after 300s should be stale" - # Running and in allowed statuses -> not stale - for s in ("finished", "error", "cancelled", "downloading", "postprocessing"): - d.info.status = s - assert d.is_stale() is False + d.info.status = None + assert d.is_stale() is True, "Download with no status and no running process after 300s should be stale" + + @pytest.mark.asyncio + async def test_started_time_is_set_in_main_process(self, monkeypatch: pytest.MonkeyPatch) -> None: + d = Download(make_item()) + + # Create a mock process + mock_proc = MagicMock() + mock_proc.join = MagicMock(return_value=0) + + # Create a proper mock for create_task that consumes the coroutine + created_tasks = [] + + def mock_create_task(coro, **kwargs): + # Close the coroutine to avoid warning + coro.close() + task = MagicMock() + created_tasks.append(task) + return task + + # Mock process manager to avoid actually starting a subprocess + with ( + patch.object(d._process_manager, "create_process", return_value=mock_proc) as mock_create, + patch.object(d._process_manager, "start") as mock_start, + patch("asyncio.create_task", side_effect=mock_create_task) as mock_create_task_fn, + patch("asyncio.get_running_loop") as mock_loop, + ): + # Set the mock proc on the process manager + d._process_manager.proc = mock_proc + + # Mock the join to return immediately + async def mock_executor(*args): + return 0 + + mock_loop.return_value.run_in_executor = mock_executor + + # Mock status tracker to prevent actual status updates + d._status_tracker = MagicMock() + d._status_tracker.final_update = True + + assert d.started_time == 0, "started_time should be 0 before start() is called" + + # Call start() - this should set started_time in the main process + await d.start() + + # Verify started_time was set to a non-zero value + assert d.started_time > 0, "started_time should be set in main process after start() is called" + + # Verify process was actually started + mock_create.assert_called_once() + mock_start.assert_called_once() -@pytest.mark.asyncio -async def test_start_initializes_and_emits(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: - class Cfg: - debug = False - ytdlp_debug = False - max_workers = 1 - temp_keep = False - temp_disabled = False - download_info_expires = 3600 +class TestTempManager: + def test_create_temp_path_when_disabled(self) -> None: + info = make_item() + logger = logging.getLogger("test") + tm = TempManager(info, "/tmp", temp_disabled=True, temp_keep=False, logger=logger) - @staticmethod - def get_instance(): - return Cfg + result = tm.create_temp_path() + assert result is None, "Should return None when temp_disabled is True" + assert tm.temp_path is None, "temp_path should remain None when disabled" - class _Mgr: - @staticmethod - def Queue(): # noqa: N802 - return DummyQueue() + def test_create_temp_path_when_no_temp_dir(self) -> None: + info = make_item() + logger = logging.getLogger("test") + tm = TempManager(info, None, temp_disabled=False, temp_keep=False, logger=logger) - @staticmethod - def get_manager(): - return Cfg._Mgr + result = tm.create_temp_path() + assert result is None, "Should return None when temp_dir is None" + assert tm.temp_path is None, "temp_path should remain None when no temp_dir" - monkeypatch.setattr("app.library.Download.Config", Cfg) + def test_create_temp_path_creates_directory(self, tmp_path: Path) -> None: + info = make_item(id="test123") + logger = logging.getLogger("test") + tm = TempManager(info, str(tmp_path), temp_disabled=False, temp_keep=False, logger=logger) - events = [] + result = tm.create_temp_path() + assert result is not None, "Should return Path when enabled" + assert result.exists(), "Temporary directory should be created" + assert result.parent == tmp_path, "Temp directory should be created in temp_dir" + assert tm.temp_path == result, "temp_path should be set to created path" - class EB: - @staticmethod - def get_instance(): - return EB + def test_create_temp_path_uses_consistent_hash(self, tmp_path: Path) -> None: + info = make_item(id="test123") + logger = logging.getLogger("test") + tm1 = TempManager(info, str(tmp_path), temp_disabled=False, temp_keep=False, logger=logger) + tm2 = TempManager(info, str(tmp_path), temp_disabled=False, temp_keep=False, logger=logger) - @staticmethod - def emit(event, data=None, **_kwargs): - events.append((event, data)) + path1 = tm1.create_temp_path() + path2 = tm2.create_temp_path() + assert path1 == path2, "Same download ID should produce same temp path" - monkeypatch.setattr("app.library.Download.EventBus", EB) + def test_delete_temp_when_disabled(self, tmp_path: Path) -> None: + info = make_item() + logger = logging.getLogger("test") + tm = TempManager(info, str(tmp_path), temp_disabled=True, temp_keep=False, logger=logger) + tm.temp_path = tmp_path / "test" + tm.temp_path.mkdir() - class FakeProc: - def __init__(self, name: str, target, *_args, **_kwargs): - self.name = name - self.target = target - self.started = False + tm.delete_temp() + assert tm.temp_path.exists(), "Should not delete when temp_disabled is True" - def start(self) -> None: - self.started = True + def test_delete_temp_when_temp_keep_enabled(self, tmp_path: Path) -> None: + info = make_item() + logger = logging.getLogger("test") + tm = TempManager(info, str(tmp_path), temp_disabled=False, temp_keep=True, logger=logger) + tm.temp_path = tmp_path / "test" + tm.temp_path.mkdir() - def join(self) -> int: - return 0 + tm.delete_temp() + assert tm.temp_path.exists(), "Should not delete when temp_keep is True" - monkeypatch.setattr("app.library.Download.multiprocessing.Process", FakeProc) + def test_delete_temp_when_no_temp_path(self) -> None: + info = make_item() + logger = logging.getLogger("test") + tm = TempManager(info, "/tmp", temp_disabled=False, temp_keep=False, logger=logger) - item = make_item(id="start1") - d = Download(item) - d.temp_dir = str(tmp_path) - d.download_dir = str(tmp_path) + tm.delete_temp() - await d.start() + def test_delete_temp_keeps_partial_download(self, tmp_path: Path) -> None: + info = make_item() + info.status = "downloading" + info.downloaded_bytes = 1000 + logger = logging.getLogger("test") + tm = TempManager(info, str(tmp_path), temp_disabled=False, temp_keep=False, logger=logger) + tm.temp_path = tmp_path / "test" + tm.temp_path.mkdir() - assert d.info.status == "preparing" - assert isinstance(d.status_queue, DummyQueue) - assert d.temp_path is not None - assert Path(d.temp_path).exists() - assert len(events) >= 1 + tm.delete_temp() + assert tm.temp_path.exists(), "Should keep temp dir for partial download" + + def test_delete_temp_with_bypass(self, tmp_path: Path) -> None: + info = make_item() + info.status = "downloading" + info.downloaded_bytes = 1000 + logger = logging.getLogger("test") + tm = TempManager(info, str(tmp_path), temp_disabled=False, temp_keep=False, logger=logger) + tm.temp_path = tmp_path / "test" + tm.temp_path.mkdir() + (tm.temp_path / "file.txt").write_text("test") + + tm.delete_temp(by_pass=True) + assert tm.temp_path.exists(), "Directory should still exist with bypass" + assert not (tm.temp_path / "file.txt").exists(), "Contents should be deleted with bypass" + + def test_delete_temp_finished_download(self, tmp_path: Path) -> None: + info = make_item() + info.status = "finished" + logger = logging.getLogger("test") + tm = TempManager(info, str(tmp_path), temp_disabled=False, temp_keep=False, logger=logger) + tm.temp_path = tmp_path / "test" + tm.temp_path.mkdir() + + tm.delete_temp() + assert not tm.temp_path.exists(), "Should delete temp dir for finished download" + + def test_delete_temp_refuses_to_delete_temp_root(self, tmp_path: Path) -> None: + info = make_item() + info.status = "finished" + logger = logging.getLogger("test") + tm = TempManager(info, str(tmp_path), temp_disabled=False, temp_keep=False, logger=logger) + tm.temp_path = tmp_path + + tm.delete_temp() + assert tm.temp_path.exists(), "Should refuse to delete temp root directory" -@pytest.mark.asyncio -async def test_progress_update_processes_statuses(monkeypatch: pytest.MonkeyPatch) -> None: - class Cfg: - debug = False - ytdlp_debug = False - max_workers = 1 - temp_keep = False - temp_disabled = True - download_info_expires = 3600 +class TestProcessManager: + def test_create_process(self) -> None: + logger = logging.getLogger("test") + pm = ProcessManager("test-id", is_live=False, logger=logger) - @staticmethod - def get_instance(): - return Cfg + def dummy_target(): + pass - monkeypatch.setattr("app.library.Download.Config", Cfg) + proc = pm.create_process(dummy_target) + assert proc is not None, "Should create a process" + assert pm.proc is proc, "Should store process reference" + assert "download-test-id" == proc.name, "Process name should include download ID" - class EB: - @staticmethod - def get_instance(): - return EB + def test_started_returns_true_when_process_created(self) -> None: + logger = logging.getLogger("test") + pm = ProcessManager("test-id", is_live=False, logger=logger) - @staticmethod - def emit(*_a, **_kw): - return None + assert pm.started() is False, "Should return False when no process created" - monkeypatch.setattr("app.library.Download.EventBus", EB) + pm.create_process(lambda: None) + assert pm.started() is True, "Should return True after process created" - seen = [] + def test_running_returns_false_when_no_process(self) -> None: + logger = logging.getLogger("test") + pm = ProcessManager("test-id", is_live=False, logger=logger) - async def spy(_self, status): - seen.append(status) + assert pm.running() is False, "Should return False when no process" - d = Download(make_item(id="p1")) - d.status_queue = DummyQueue() - monkeypatch.setattr(Download, "_process_status_update", spy, raising=False) - d.status_queue.put({"id": d.id, "status": "downloading"}) - d.status_queue.put({"id": d.id, "status": "postprocessing"}) - d.status_queue.put(Terminator()) + def test_is_cancelled_returns_false_by_default(self) -> None: + logger = logging.getLogger("test") + pm = ProcessManager("test-id", is_live=False, logger=logger) - await d.progress_update() + assert pm.is_cancelled() is False, "Should return False by default" - assert [s["status"] for s in seen] == ["downloading", "postprocessing"] + def test_cancel_marks_as_cancelled(self) -> None: + logger = logging.getLogger("test") + pm = ProcessManager("test-id", is_live=False, logger=logger) + pm.proc = Mock() + pm.proc.is_alive = Mock(return_value=False) + + result = pm.cancel() + assert pm.is_cancelled() is True, "Should mark as cancelled" + + def test_cancel_returns_false_when_not_started(self) -> None: + logger = logging.getLogger("test") + pm = ProcessManager("test-id", is_live=False, logger=logger) + + result = pm.cancel() + assert result is False, "Should return False when process not started" + assert pm.is_cancelled() is False, "Should not mark as cancelled when not started" + + def test_kill_returns_false_when_not_running(self) -> None: + logger = logging.getLogger("test") + pm = ProcessManager("test-id", is_live=False, logger=logger) + + result = pm.kill() + assert result is False, "Should return False when process not running" + + def test_kill_sends_sigusr1_on_posix(self) -> None: + if "posix" != os.name: + pytest.skip("Test only runs on POSIX systems") + + logger = logging.getLogger("test") + pm = ProcessManager("test-id", is_live=False, logger=logger) + pm.proc = Mock() + pm.proc.pid = 12345 + pm.proc.ident = 67890 + pm.proc.is_alive = Mock(side_effect=[True, False, False]) + + with patch("app.library.downloads.process_manager.os.kill") as mock_kill: + result = pm.kill() + mock_kill.assert_called_once_with(12345, signal.SIGUSR1) + assert result is True, "Should return True when process killed successfully" + + def test_kill_uses_shorter_timeout_for_live_streams(self) -> None: + logger = logging.getLogger("test") + pm_live = ProcessManager("test-id", is_live=True, logger=logger) + pm_regular = ProcessManager("test-id", is_live=False, logger=logger) + + pm_live.proc = Mock() + pm_live.proc.pid = 12345 + pm_live.proc.ident = 67890 + pm_live.proc.is_alive = Mock(return_value=False) + + pm_regular.proc = Mock() + pm_regular.proc.pid = 12346 + pm_regular.proc.ident = 67891 + pm_regular.proc.is_alive = Mock(return_value=False) + + @pytest.mark.asyncio + async def test_close_returns_false_when_not_started(self) -> None: + logger = logging.getLogger("test") + pm = ProcessManager("test-id", is_live=False, logger=logger) + + result = await pm.close() + assert result is False, "Should return False when process not started" + + @pytest.mark.asyncio + async def test_close_returns_false_when_cancel_in_progress(self) -> None: + logger = logging.getLogger("test") + pm = ProcessManager("test-id", is_live=False, logger=logger) + pm.proc = Mock() + pm.cancel_in_progress = True + + result = await pm.close() + assert result is False, "Should return False when cancellation already in progress" + + @pytest.mark.asyncio + async def test_close_kills_and_joins_process(self) -> None: + logger = logging.getLogger("test") + pm = ProcessManager("test-id", is_live=False, logger=logger) + pm.proc = Mock() + pm.proc.ident = 12345 + pm.proc.is_alive = Mock(side_effect=[False, False]) + pm.proc.join = Mock() + pm.proc.close = Mock() + + result = await pm.close() + assert result is True, "Should return True on successful close" + assert pm.proc is None, "Process reference should be cleared" -@pytest.mark.asyncio -async def test__process_status_update_finish_sets_fields(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: - class Cfg: - debug = False - ytdlp_debug = False - max_workers = 1 - temp_keep = False - temp_disabled = True - download_info_expires = 3600 +class TestStatusTracker: + @pytest.fixture + def mock_config(self): + return { + "info": make_item(id="test-id"), + "download_id": "test-id", + "download_dir": "/downloads", + "temp_path": None, + "status_queue": DummyQueue(), + "logger": logging.getLogger("test"), + "debug": False, + } - @staticmethod - def get_instance(): - return Cfg + def test_init_sets_attributes(self, mock_config: dict) -> None: + st = StatusTracker(**mock_config) + assert st.id == "test-id", "Should set download ID" + assert st.info == mock_config["info"], "Should set info reference" + assert st.tmpfilename is None, "Should initialize tmpfilename as None" + assert st.final_update is False, "Should initialize final_update as False" - monkeypatch.setattr("app.library.Download.Config", Cfg) + @pytest.mark.asyncio + async def test_process_status_update_ignores_invalid_id(self, mock_config: dict) -> None: + st = StatusTracker(**mock_config) + status = {"id": "wrong-id", "status": "downloading"} - class EB: - @staticmethod - def get_instance(): - return EB + await st.process_status_update(status) + assert st.info.status != "downloading", "Should not update status for wrong ID" - @staticmethod - def emit(*_a, **_kw): - return None + @pytest.mark.asyncio + async def test_process_status_update_ignores_short_status(self, mock_config: dict) -> None: + st = StatusTracker(**mock_config) + status = {"id": "test-id"} - monkeypatch.setattr("app.library.Download.EventBus", EB) + await st.process_status_update(status) - class FF: - def has_video(self): - return True + @pytest.mark.asyncio + async def test_process_status_update_sets_status(self, mock_config: dict) -> None: + st = StatusTracker(**mock_config) + status = {"id": "test-id", "status": "downloading", "downloaded_bytes": 1000} - def has_audio(self): - return True + await st.process_status_update(status) + assert st.info.status == "downloading", "Should update info status" - metadata = {"duration": "7.9"} + @pytest.mark.asyncio + async def test_process_status_update_sets_tmpfilename(self, mock_config: dict) -> None: + st = StatusTracker(**mock_config) + status = {"id": "test-id", "status": "downloading", "tmpfilename": "/tmp/file.part"} - async def fake_ffprobe(_p): - return FF() + await st.process_status_update(status) + assert st.tmpfilename == "/tmp/file.part", "Should update tmpfilename" - monkeypatch.setattr("app.library.Download.ffprobe", fake_ffprobe) + @pytest.mark.asyncio + async def test_process_status_update_calculates_percent(self, mock_config: dict) -> None: + st = StatusTracker(**mock_config) + status = { + "id": "test-id", + "status": "downloading", + "downloaded_bytes": 50, + "total_bytes": 100, + } - d = Download(make_item(id="f1")) - d.download_dir = str(tmp_path) - path = Path(tmp_path) / "file.bin" - path.write_bytes(b"x" * 10) + await st.process_status_update(status) + assert st.info.downloaded_bytes == 50, "Should set downloaded_bytes" + assert st.info.total_bytes == 100, "Should set total_bytes" + assert st.info.percent == 50.0, "Should calculate percent correctly" - await d._process_status_update({"id": d.id, "status": "finished", "final_name": str(path)}) + @pytest.mark.asyncio + async def test_process_status_update_uses_estimated_total(self, mock_config: dict) -> None: + st = StatusTracker(**mock_config) + status = { + "id": "test-id", + "status": "downloading", + "downloaded_bytes": 30, + "total_bytes_estimate": 100, + } - assert d.info.file_size == 10 - assert d.info.extras["is_video"] is True - assert d.info.extras["is_audio"] is True - assert d.info.extras["duration"] == int(7.9) - assert isinstance(d.info.datetime, str) + await st.process_status_update(status) + assert st.info.total_bytes == 100, "Should use total_bytes_estimate when total_bytes not available" + assert st.info.percent == 30.0, "Should calculate percent from estimate" + + @pytest.mark.asyncio + async def test_process_status_update_handles_zero_division(self, mock_config: dict) -> None: + st = StatusTracker(**mock_config) + status = { + "id": "test-id", + "status": "downloading", + "downloaded_bytes": 50, + "total_bytes": 100, + } + + await st.process_status_update(status) + assert st.info.percent == 50.0, "Should calculate percent correctly with valid total" + + @pytest.mark.asyncio + async def test_process_status_update_sets_speed_and_eta(self, mock_config: dict) -> None: + st = StatusTracker(**mock_config) + status = {"id": "test-id", "status": "downloading", "speed": 1024000, "eta": 60} + + await st.process_status_update(status) + assert st.info.speed == 1024000, "Should set speed" + assert st.info.eta == 60, "Should set eta" + + @pytest.mark.asyncio + async def test_process_status_update_sets_error(self, mock_config: dict) -> None: + st = StatusTracker(**mock_config) + status = {"id": "test-id", "status": "error", "error": "Download failed"} + + await st.process_status_update(status) + assert st.info.status == "error", "Should set status to error" + assert st.info.error == "Download failed", "Should set error message" + + @pytest.mark.asyncio + async def test_process_status_update_sets_final_update(self, tmp_path: Path, mock_config: dict) -> None: + test_file = tmp_path / "test.mp4" + test_file.write_text("test content") + + st = StatusTracker(**mock_config) + st.download_dir = str(tmp_path) + status = {"id": "test-id", "status": "finished", "final_name": str(test_file)} + + await st.process_status_update(status) + assert st.final_update is True, "Should set final_update when final file exists" + assert st.info.filename == "test.mp4", "Should set relative filename" + + @pytest.mark.asyncio + async def test_drain_queue_processes_remaining_updates(self, mock_config: dict) -> None: + queue = DummyQueue() + queue.put({"id": "test-id", "status": "downloading", "downloaded_bytes": 100}) + queue.put({"id": "test-id", "status": "downloading", "downloaded_bytes": 200}) + queue.put(Terminator()) + + config = {**mock_config, "status_queue": queue} + st = StatusTracker(**config) + + await st.drain_queue(max_iterations=10) + assert st.info.downloaded_bytes == 200, "Should process all queued updates" + + @pytest.mark.asyncio + async def test_drain_queue_stops_on_final_update(self, tmp_path: Path, mock_config: dict) -> None: + test_file = tmp_path / "test.mp4" + test_file.write_text("test content") + + queue = DummyQueue() + queue.put({"id": "test-id", "status": "finished", "final_name": str(test_file)}) + queue.put({"id": "test-id", "status": "downloading", "downloaded_bytes": 999}) + + config = {**mock_config, "status_queue": queue, "download_dir": str(tmp_path)} + st = StatusTracker(**config) + + await st.drain_queue(max_iterations=10) + assert st.final_update is True, "Should stop draining after final update" + + @pytest.mark.asyncio + async def test_drain_queue_handles_errors_gracefully(self, mock_config: dict) -> None: + queue = DummyQueue() + queue.put({"id": "test-id", "status": "downloading"}) + queue.put(None) + + config = {**mock_config, "status_queue": queue} + st = StatusTracker(**config) + + await st.drain_queue(max_iterations=5) + + def test_cancel_update_task_cancels_running_task(self, mock_config: dict) -> None: + st = StatusTracker(**mock_config) + st.update_task = Mock() + st.update_task.done = Mock(return_value=False) + st.update_task.cancel = Mock() + + st.cancel_update_task() + st.update_task.cancel.assert_called_once() + + def test_cancel_update_task_handles_no_task(self, mock_config: dict) -> None: + st = StatusTracker(**mock_config) + + st.cancel_update_task() + + def test_put_terminator_adds_to_queue(self, mock_config: dict) -> None: + queue = DummyQueue() + config = {**mock_config, "status_queue": queue} + st = StatusTracker(**config) + + st.put_terminator() + assert 1 == len(queue.items), "Should add terminator to queue" + assert isinstance(queue.items[0], Terminator), "Should add Terminator instance" diff --git a/app/tests/test_download_utils.py b/app/tests/test_download_utils.py new file mode 100644 index 00000000..1bf8c460 --- /dev/null +++ b/app/tests/test_download_utils.py @@ -0,0 +1,377 @@ +import asyncio +import logging +import os +import time +from pathlib import Path +from unittest.mock import Mock, patch + +import pytest + +from app.library.downloads.utils import ( + BAD_LIVE_STREAM_OPTIONS, + DEBUG_MESSAGE_PREFIXES, + GENERIC_EXTRACTORS, + LIMITS, + YTDLP_PROGRESS_FIELDS, + create_debug_safe_dict, + get_extractor_limit, + handle_task_exception, + is_download_stale, + is_safe_to_delete_dir, + parse_extractor_limit, + safe_relative_path, + wait_for_process_with_timeout, +) + + +class TestConstants: + def test_generic_extractors_tuple(self) -> None: + assert isinstance(GENERIC_EXTRACTORS, tuple), "Should be a tuple" + assert "HTML5MediaEmbed" in GENERIC_EXTRACTORS, "Should contain HTML5MediaEmbed" + assert "generic" in GENERIC_EXTRACTORS, "Should contain generic" + + def test_ytdlp_progress_fields_tuple(self) -> None: + assert isinstance(YTDLP_PROGRESS_FIELDS, tuple), "Should be a tuple" + assert "status" in YTDLP_PROGRESS_FIELDS, "Should contain status field" + assert "downloaded_bytes" in YTDLP_PROGRESS_FIELDS, "Should contain downloaded_bytes field" + + def test_bad_live_stream_options_list(self) -> None: + assert isinstance(BAD_LIVE_STREAM_OPTIONS, list), "Should be a list" + assert "concurrent_fragment_downloads" in BAD_LIVE_STREAM_OPTIONS, "Should contain concurrent option" + + def test_debug_message_prefixes_list(self) -> None: + assert isinstance(DEBUG_MESSAGE_PREFIXES, list), "Should be a list" + assert "[debug] " in DEBUG_MESSAGE_PREFIXES, "Should contain debug prefix" + + +class TestPathUtilities: + def test_safe_relative_path_success(self) -> None: + base = Path("/downloads") + file = Path("/downloads/video.mp4") + result = safe_relative_path(file, base) + assert "video.mp4" == result, "Should return relative path" + + def test_safe_relative_path_nested(self) -> None: + base = Path("/downloads") + file = Path("/downloads/folder/subfolder/video.mp4") + result = safe_relative_path(file, base) + assert "folder/subfolder/video.mp4" == result, "Should return nested relative path" + + def test_safe_relative_path_with_fallback(self) -> None: + base = Path("/wrong/path") + fallback = Path("/temp") + file = Path("/temp/video.mp4") + result = safe_relative_path(file, base, fallback) + assert "video.mp4" == result, "Should use fallback path" + + def test_safe_relative_path_no_fallback_returns_absolute(self) -> None: + base = Path("/wrong/path") + file = Path("/downloads/video.mp4") + result = safe_relative_path(file, base) + assert "/downloads/video.mp4" == result, "Should return absolute path when both fail" + + def test_safe_relative_path_fallback_also_fails(self) -> None: + base = Path("/wrong/path") + fallback = Path("/also/wrong") + file = Path("/downloads/video.mp4") + result = safe_relative_path(file, base, fallback) + assert "/downloads/video.mp4" == result, "Should return absolute when both fail" + + def test_is_safe_to_delete_dir_root_protection(self) -> None: + path = Path("/tmp/downloads") + root = Path("/tmp/downloads") + result = is_safe_to_delete_dir(path, root) + assert result is False, "Should refuse to delete root directory" + + def test_is_safe_to_delete_dir_safe_path(self) -> None: + path = Path("/tmp/downloads/subfolder") + root = Path("/tmp/downloads") + result = is_safe_to_delete_dir(path, root) + assert result is True, "Should allow deleting subdirectory" + + def test_is_safe_to_delete_dir_string_comparison(self) -> None: + path = Path("/tmp/downloads") + root = "/tmp/downloads" + result = is_safe_to_delete_dir(path, root) + assert result is False, "Should handle string root path" + + +class TestProcessUtilities: + def test_wait_for_process_with_timeout_completes_immediately(self) -> None: + proc = Mock() + proc.is_alive = Mock(return_value=False) + result = wait_for_process_with_timeout(proc, timeout=1.0) + assert result is True, "Should return True when process terminates immediately" + + def test_wait_for_process_with_timeout_completes_after_delay(self) -> None: + proc = Mock() + call_count = [0] + + def is_alive_side_effect(): + call_count[0] += 1 + return call_count[0] < 3 + + proc.is_alive = Mock(side_effect=is_alive_side_effect) + + with patch("time.sleep"): + result = wait_for_process_with_timeout(proc, timeout=1.0, check_interval=0.1) + + assert result is True, "Should return True when process terminates during wait" + assert proc.is_alive.call_count >= 3, "Should check process multiple times" + + def test_wait_for_process_with_timeout_expires(self) -> None: + proc = Mock() + proc.is_alive = Mock(return_value=True) + + with patch("time.time") as mock_time, patch("time.sleep"): + mock_time.side_effect = [0, 0.5, 1.0, 1.5] + result = wait_for_process_with_timeout(proc, timeout=1.0, check_interval=0.1) + + assert result is False, "Should return False when timeout reached" + + def test_wait_for_process_with_custom_interval(self) -> None: + proc = Mock() + proc.is_alive = Mock(return_value=False) + result = wait_for_process_with_timeout(proc, timeout=5.0, check_interval=0.5) + assert result is True, "Should respect custom check interval" + + +class TestConfigUtilities: + def test_parse_extractor_limit_valid_env(self) -> None: + with patch.dict(os.environ, {"YTP_MAX_WORKERS_FOR_YOUTUBE": "3"}): + result = parse_extractor_limit("youtube", default_limit=5, max_workers=10) + assert 3 == result, "Should use environment variable value" + + def test_parse_extractor_limit_env_exceeds_max(self) -> None: + with patch.dict(os.environ, {"YTP_MAX_WORKERS_FOR_YOUTUBE": "15"}): + result = parse_extractor_limit("youtube", default_limit=5, max_workers=10) + assert 10 == result, "Should cap at max_workers" + + def test_parse_extractor_limit_invalid_env_not_digit(self) -> None: + logger = logging.getLogger("test") + with patch.dict(os.environ, {"YTP_MAX_WORKERS_FOR_YOUTUBE": "abc"}): + result = parse_extractor_limit("youtube", default_limit=5, max_workers=10, logger=logger) + assert 5 == result, "Should use default when env var is invalid" + + def test_parse_extractor_limit_invalid_env_zero(self) -> None: + with patch.dict(os.environ, {"YTP_MAX_WORKERS_FOR_YOUTUBE": "0"}): + result = parse_extractor_limit("youtube", default_limit=5, max_workers=10) + assert 5 == result, "Should use default when env var is zero" + + def test_parse_extractor_limit_no_env(self) -> None: + result = parse_extractor_limit("youtube", default_limit=5, max_workers=10) + assert 5 == result, "Should use default when no env var set" + + def test_parse_extractor_limit_respects_max_on_default(self) -> None: + result = parse_extractor_limit("youtube", default_limit=15, max_workers=10) + assert 10 == result, "Should cap default at max_workers" + + def test_parse_extractor_limit_logs_warning_on_invalid(self) -> None: + logger = Mock() + with patch.dict(os.environ, {"YTP_MAX_WORKERS_FOR_YOUTUBE": "invalid"}): + parse_extractor_limit("youtube", default_limit=5, max_workers=10, logger=logger) + logger.warning.assert_called_once() + assert "Invalid extractor limit" in logger.warning.call_args[0][0], "Should log warning" + + def test_get_extractor_limit_creates_new(self) -> None: + LIMITS.clear() + logger = logging.getLogger("test") + semaphore = get_extractor_limit("youtube", max_workers=10, max_workers_per_extractor=5, logger=logger) + assert isinstance(semaphore, asyncio.Semaphore), "Should return semaphore" + assert "youtube" in LIMITS, "Should store in LIMITS dict" + + def test_get_extractor_limit_reuses_existing(self) -> None: + LIMITS.clear() + logger = logging.getLogger("test") + sem1 = get_extractor_limit("youtube", max_workers=10, max_workers_per_extractor=5, logger=logger) + sem2 = get_extractor_limit("youtube", max_workers=10, max_workers_per_extractor=5, logger=logger) + assert sem1 is sem2, "Should reuse existing semaphore" + + def test_get_extractor_limit_respects_env_var(self) -> None: + LIMITS.clear() + logger = logging.getLogger("test") + with patch.dict(os.environ, {"YTP_MAX_WORKERS_FOR_TWITCH": "2"}): + semaphore = get_extractor_limit("twitch", max_workers=10, max_workers_per_extractor=5, logger=logger) + assert semaphore._value == 2, "Should respect environment variable" + + +class TestDataUtilities: + def test_create_debug_safe_dict_filters_formats(self) -> None: + data = { + "status": "downloading", + "filename": "video.mp4", + "info_dict": {"id": "123", "title": "Video", "formats": [{"format_id": "1"}], "description": "Long text"}, + } + result = create_debug_safe_dict(data) + assert "downloading" == result["status"], "Should include status" + assert "video.mp4" == result["filename"], "Should include filename" + assert "formats" not in result["info_dict"], "Should exclude formats" + assert "description" not in result["info_dict"], "Should exclude description" + assert "123" == result["info_dict"]["id"], "Should include id" + + def test_create_debug_safe_dict_custom_exclude(self) -> None: + data = { + "status": "downloading", + "filename": "video.mp4", + "info_dict": {"id": "123", "title": "Video", "custom_field": "value"}, + } + result = create_debug_safe_dict(data, exclude_keys=["custom_field"]) + assert "custom_field" not in result["info_dict"], "Should exclude custom field" + assert "123" == result["info_dict"]["id"], "Should include id" + + def test_create_debug_safe_dict_filters_none_and_functions(self) -> None: + data = { + "status": "downloading", + "info_dict": {"id": "123", "none_value": None, "lambda_value": lambda: None, "title": "Video"}, + } + result = create_debug_safe_dict(data) + assert "none_value" not in result["info_dict"], "Should filter None values" + assert "lambda_value" not in result["info_dict"], "Should filter lambda functions" + assert "Video" == result["info_dict"]["title"], "Should include regular values" + + def test_create_debug_safe_dict_empty_info_dict(self) -> None: + data = {"status": "downloading", "filename": "video.mp4"} + result = create_debug_safe_dict(data) + assert {} == result["info_dict"], "Should handle missing info_dict" + + +class TestStateUtilities: + def test_is_download_stale_terminal_status_finished(self) -> None: + result = is_download_stale( + started_time=int(time.time()) - 500, current_status="finished", is_running=False, auto_start=True + ) + assert result is False, "Finished downloads are never stale" + + def test_is_download_stale_terminal_status_error(self) -> None: + result = is_download_stale( + started_time=int(time.time()) - 500, current_status="error", is_running=False, auto_start=True + ) + assert result is False, "Error downloads are never stale" + + def test_is_download_stale_terminal_status_cancelled(self) -> None: + result = is_download_stale( + started_time=int(time.time()) - 500, current_status="cancelled", is_running=False, auto_start=True + ) + assert result is False, "Cancelled downloads are never stale" + + def test_is_download_stale_terminal_status_downloading(self) -> None: + result = is_download_stale( + started_time=int(time.time()) - 500, current_status="downloading", is_running=False, auto_start=True + ) + assert result is False, "Downloading status is never stale" + + def test_is_download_stale_terminal_status_postprocessing(self) -> None: + result = is_download_stale( + started_time=int(time.time()) - 500, current_status="postprocessing", is_running=False, auto_start=True + ) + assert result is False, "Postprocessing status is never stale" + + def test_is_download_stale_not_auto_start(self) -> None: + result = is_download_stale( + started_time=int(time.time()) - 500, current_status="pending", is_running=False, auto_start=False + ) + assert result is False, "Non-auto-start downloads are never stale" + + def test_is_download_stale_still_running(self) -> None: + result = is_download_stale( + started_time=int(time.time()) - 500, current_status="pending", is_running=True, auto_start=True + ) + assert result is False, "Running downloads are never stale" + + def test_is_download_stale_not_enough_time(self) -> None: + result = is_download_stale( + started_time=int(time.time()) - 100, + current_status="pending", + is_running=False, + auto_start=True, + min_elapsed=300, + ) + assert result is False, "Should not be stale before timeout" + + def test_is_download_stale_timeout_reached(self) -> None: + result = is_download_stale( + started_time=int(time.time()) - 400, + current_status="pending", + is_running=False, + auto_start=True, + min_elapsed=300, + ) + assert result is True, "Should be stale after timeout" + + def test_is_download_stale_custom_timeout(self) -> None: + result = is_download_stale( + started_time=int(time.time()) - 150, + current_status="pending", + is_running=False, + auto_start=True, + min_elapsed=100, + ) + assert result is True, "Should respect custom timeout" + + +class TestTaskExceptionHandling: + @pytest.mark.asyncio + async def test_handle_task_exception_ignores_cancelled(self) -> None: + logger = Mock() + + async def cancelled_task(): + raise asyncio.CancelledError() + + task = asyncio.create_task(cancelled_task()) + try: + await task + except asyncio.CancelledError: + pass + + handle_task_exception(task, logger) + logger.error.assert_not_called() + + @pytest.mark.asyncio + async def test_handle_task_exception_ignores_successful(self) -> None: + logger = Mock() + + async def successful_task(): + return "success" + + task = asyncio.create_task(successful_task()) + await task + + handle_task_exception(task, logger) + logger.error.assert_not_called() + + @pytest.mark.asyncio + async def test_handle_task_exception_logs_exception(self) -> None: + logger = Mock() + + async def failing_task(): + raise ValueError("Test error") + + task = asyncio.create_task(failing_task(), name="test_task") + try: + await task + except ValueError: + pass + + handle_task_exception(task, logger) + logger.error.assert_called_once() + error_msg = logger.error.call_args[0][0] + assert "test_task" in error_msg, "Should include task name" + assert "Test error" in error_msg, "Should include exception message" + + @pytest.mark.asyncio + async def test_handle_task_exception_unknown_task_name(self) -> None: + logger = Mock() + + async def failing_task(): + raise RuntimeError("Unknown task error") + + task = asyncio.create_task(failing_task()) + try: + await task + except RuntimeError: + pass + + handle_task_exception(task, logger) + logger.error.assert_called_once() + error_msg = logger.error.call_args[0][0] + assert "unknown_task" in error_msg or "Task" in error_msg, "Should handle unknown task name" diff --git a/ui/app/components/DLFields.vue b/ui/app/components/DLFields.vue index f84f4ef2..5e3b72a7 100644 --- a/ui/app/components/DLFields.vue +++ b/ui/app/components/DLFields.vue @@ -132,13 +132,9 @@
- - - - - - No custom fields found, you can add new fields using the button above. - + + + No custom fields found, you can add new fields using the button above.
@@ -250,7 +246,6 @@ const addNewField = () => items.value.push({ const deleteField = (index: number) => items.value.splice(index, 1) const validateItem = (item: DLField, index: number): boolean => { - const requiredFields = ['name', 'field', 'kind', 'description'] for (const field of requiredFields) { @@ -282,9 +277,10 @@ const validateItem = (item: DLField, index: number): boolean => { const sortedDLFields = computed(() => items.value.sort((a, b) => (a.order || 0) - (b.order || 0))) watch(() => config.ytdlp_options, newOptions => ytDlpOptions.value = newOptions - .filter(opt => !opt.ignored).flatMap(opt => opt.flags - .filter(flag => flag.startsWith('--')) - .map(flag => ({ value: flag, description: opt.description || '' }))), + .filter(opt => !opt.ignored) + .flatMap( + opt => opt.flags.filter(flag => flag.startsWith('--') + ).map(flag => ({ value: flag, description: opt.description || '' }))), { immediate: true } ) onMounted(async () => { diff --git a/ui/app/components/DeprecatedNotice.vue b/ui/app/components/DeprecatedNotice.vue deleted file mode 100644 index 9cad0fe1..00000000 --- a/ui/app/components/DeprecatedNotice.vue +++ /dev/null @@ -1,67 +0,0 @@ - - - diff --git a/ui/app/components/History.vue b/ui/app/components/History.vue index 187ec73f..2f0c0ee6 100644 --- a/ui/app/components/History.vue +++ b/ui/app/components/History.vue @@ -98,32 +98,36 @@ -
+
{{ formatTime(item.extras.duration) }} - - - + + + + +

+ Path: {{ getPath(config.app.download_path, item) }} +

+
+ +

{{ item.description }}

+
- - - - -

{{ item.description }}

- + + {{ item.title }}