diff --git a/API.md b/API.md index 907dd2d3..9257956a 100644 --- a/API.md +++ b/API.md @@ -96,6 +96,7 @@ This document describes the available endpoints and their usage. All endpoints r - [POST /api/notifications/test](#post-apinotificationstest) - [GET /api/yt-dlp/options](#get-apiyt-dlpoptions) - [GET /api/system/configuration](#get-apisystemconfiguration) + - [GET /api/system/limits](#get-apisystemlimits) - [POST /api/system/terminal](#post-apisystemterminal) - [GET /api/system/terminal/active](#get-apisystemterminalactive) - [GET /api/system/terminal/{session\_id}](#get-apisystemterminalsession_id) @@ -2479,6 +2480,55 @@ or an error: --- +### GET /api/system/limits +**Purpose**: Get the system limits. + +**Response**: +```json +{ + "downloads": { + "paused": false, + "live_bypasses_limits": true, + "global": { + "limit": 20, + "active": 3, + "available": 17, + "live_active": 1, + "queued": 8 + }, + "per_extractor": { + "default_limit": 2, + "items": [ + { + "name": "youtube", + "limit": 3, + "source": "env_override", + "active": 2, + "queued": 4, + "available": 1 + } + ] + } + }, + "extraction": { + "concurrency": 4, + "timeout_seconds": 70, + "info_cache_ttl_seconds": 10800 + }, + "live": { + "prevent_premiere": true, + "premiere_buffer_minutes": 5 + } +} +``` + +**Notes**: +- `downloads.global` counts only non-live downloads against the worker limit. +- `downloads.global.live_active` is reported separately because live downloads bypass the global and per-extractor worker limits. +- `downloads.per_extractor.items[*].source` is `default` unless an override was provided through `YTP_MAX_WORKERS_FOR_`. + +--- + ### POST /api/system/terminal **Purpose**: Start a yt-dlp terminal session. Requires `YTP_CONSOLE_ENABLED=true`. diff --git a/app/library/config.py b/app/library/config.py index b2619d18..49ab48e3 100644 --- a/app/library/config.py +++ b/app/library/config.py @@ -286,7 +286,7 @@ class Config(metaclass=Singleton): "console_enabled", "browser_control_enabled", "ytdlp_auto_update", - "prevent_premiere_live", + "prevent_live_premiere", "temp_disabled", "allow_internal_urls", "simple_mode", diff --git a/app/library/downloads/video_processor.py b/app/library/downloads/video_processor.py index b01d538e..424961d2 100644 --- a/app/library/downloads/video_processor.py +++ b/app/library/downloads/video_processor.py @@ -184,7 +184,8 @@ async def add_video(queue: "DownloadQueue", entry: dict, item: "Item", logs: lis starts_in: datetime = ( starts_in.replace(tzinfo=UTC) if starts_in.tzinfo is None else starts_in.astimezone(UTC) ) - starts_in = starts_in + timedelta(minutes=5, seconds=dl.extras.get("duration", 0)) + buffer_time = queue.config.live_premiere_buffer if queue.config.live_premiere_buffer >= 0 else 5 + starts_in = starts_in + timedelta(minutes=buffer_time, seconds=dl.extras.get("duration", 0)) dlInfo.info.error += f" Download will start at {starts_in.astimezone().isoformat()}." _requeue = False except Exception as e: diff --git a/app/routes/api/system.py b/app/routes/api/system.py index 59fb89fb..5d157156 100644 --- a/app/routes/api/system.py +++ b/app/routes/api/system.py @@ -1,5 +1,6 @@ import asyncio import logging +import os import time from pathlib import Path @@ -11,6 +12,7 @@ from app.features.dl_fields.service import DLFields from app.features.presets.service import Presets from app.library.config import Config from app.library.downloads import DownloadQueue +from app.library.downloads.core import Download from app.library.encoder import Encoder from app.library.Events import EventBus, Events from app.library.router import route @@ -363,3 +365,114 @@ async def stream_terminal_session( {"error": str(exc)}, status=web.HTTPBadRequest.status_code, ) + + +@route("GET", "api/system/limits", "system.limits") +async def system_limits(queue: DownloadQueue, config: Config, encoder: Encoder) -> Response: + """Return user-facing operational limits for downloads and extraction.""" + active_downloads: dict[str, Download] = queue.pool.get_active_downloads() + queue_items: list[tuple[str, Download]] = list(queue.queue.items()) + + active_non_live: list[Download] = [download for download in active_downloads.values() if not download.is_live] + active_live: list[Download] = [download for download in active_downloads.values() if download.is_live] + queued_non_live: list[Download] = [ + download + for _, download in queue_items + if not download.started() and not download.is_live and getattr(download.info, "auto_start", True) is not False + ] + + def _get_extractor_override_limits(config: Config) -> dict[str, dict[str, int | str]]: + overrides: dict[str, dict[str, int | str]] = {} + prefix = "YTP_MAX_WORKERS_FOR_" + + for env_key, raw_value in os.environ.items(): + if not env_key.startswith(prefix): + continue + + if not (extractor := env_key.removeprefix(prefix).strip().lower()): + continue + + if not raw_value.isdigit() or int(raw_value) < 1: + continue + + configured = int(raw_value) + overrides[extractor] = { + "configured": raw_value, + "effective": min(configured, config.max_workers), + "source": "env_override", + } + + return overrides + + overrides: dict[str, dict[str, int | str]] = _get_extractor_override_limits(config) + per_extractor: dict[str, dict[str, int | str]] = {} + + for download in [*active_non_live, *queued_non_live]: + extractor = (download.info.get_extractor() or "unknown").lower() + entry = per_extractor.setdefault( + extractor, + { + "name": extractor, + "limit": config.max_workers_per_extractor, + "source": "default", + "active": 0, + "queued": 0, + "available": config.max_workers_per_extractor, + }, + ) + + if extractor in overrides: + entry["limit"] = overrides[extractor]["effective"] + entry["source"] = overrides[extractor]["source"] + + if download in active_non_live: + entry["active"] += 1 + else: + entry["queued"] += 1 + + for extractor, override in overrides.items(): + per_extractor.setdefault( + extractor, + { + "name": extractor, + "limit": override["effective"], + "source": override["source"], + "active": 0, + "queued": 0, + "available": override["effective"], + }, + ) + + for entry in per_extractor.values(): + entry["available"] = max(int(entry["limit"]) - int(entry["active"]), 0) + + return web.json_response( + data={ + "downloads": { + "paused": queue.is_paused(), + "live_bypasses_limits": True, + "global": { + "limit": config.max_workers, + "active": len(active_non_live), + "available": max(config.max_workers - len(active_non_live), 0), + "live_active": len(active_live), + "queued": len(queued_non_live), + }, + "per_extractor": { + "default_limit": config.max_workers_per_extractor, + "items": sorted(per_extractor.values(), key=lambda item: str(item["name"])), + }, + }, + "extraction": { + "concurrency": config.extract_info_concurrency, + "timeout_seconds": config.extract_info_timeout, + "info_cache_ttl_seconds": config.download_info_expires, + }, + "live": { + "prevent_premiere": config.prevent_live_premiere, + "premiere_buffer_minutes": config.live_premiere_buffer, + }, + }, + status=web.HTTPOk.status_code, + dumps=encoder.encode, + ) diff --git a/app/tests/test_system_routes.py b/app/tests/test_system_routes.py index 85e97f8c..f578e248 100644 --- a/app/tests/test_system_routes.py +++ b/app/tests/test_system_routes.py @@ -1,10 +1,60 @@ -import pytest +import json +from dataclasses import dataclass from unittest.mock import AsyncMock, patch +import pytest + from app.library.config import Config from app.library.encoder import Encoder from app.library.UpdateChecker import UpdateChecker -from app.routes.api.system import check_updates +from app.routes.api.system import check_updates, system_limits + + +@dataclass +class DummyInfo: + extractor: str | None + + def get_extractor(self) -> str | None: + return self.extractor + + +@dataclass +class DummyDownload: + info: DummyInfo + is_live: bool = False + _started: bool = False + + def started(self) -> bool: + return self._started + + +class DummyPool: + def __init__(self, active: dict[str, DummyDownload] | None = None, paused: bool = False): + self._active = active or {} + self._paused = paused + + def get_active_downloads(self) -> dict[str, DummyDownload]: + return self._active.copy() + + def is_paused(self) -> bool: + return self._paused + + +class DummyStore: + def __init__(self, items: dict[str, DummyDownload] | None = None): + self._items = items or {} + + def items(self): + return self._items.items() + + +class DummyQueue: + def __init__(self, active: dict[str, DummyDownload] | None = None, queued: dict[str, DummyDownload] | None = None): + self.pool = DummyPool(active=active) + self.queue = DummyStore(items=queued) + + def is_paused(self) -> bool: + return self.pool.is_paused() class TestCheckUpdatesEndpoint: @@ -64,3 +114,83 @@ class TestCheckUpdatesEndpoint: body = response.body.decode("utf-8") assert "v1.0.5" in body, "Response should include new version" assert "update_available" in body, "Response should include update_available status" + + +class TestSystemLimitsEndpoint: + def setup_method(self): + Config._reset_singleton() + + @pytest.mark.asyncio + async def test_system_limits_returns_user_facing_limits(self): + config = Config.get_instance() + config.max_workers = 10 + config.max_workers_per_extractor = 3 + config.extract_info_concurrency = 4 + config.extract_info_timeout = 70 + config.download_info_expires = 10800 + config.prevent_live_premiere = True + config.live_premiere_buffer = 7 + config.default_pagination = 50 + + active = { + "a": DummyDownload(info=DummyInfo("youtube"), is_live=False, _started=True), + "b": DummyDownload(info=DummyInfo("youtube"), is_live=False, _started=True), + "c": DummyDownload(info=DummyInfo("twitch"), is_live=True, _started=True), + } + queued = { + "q1": DummyDownload(info=DummyInfo("youtube"), is_live=False, _started=False), + "q2": DummyDownload(info=DummyInfo("vimeo"), is_live=False, _started=False), + } + queue = DummyQueue(active=active, queued=queued) + encoder = Encoder() + + with patch.dict("os.environ", {"YTP_MAX_WORKERS_FOR_YOUTUBE": "2"}, clear=False): + response = await system_limits(queue, config, encoder) + + assert 200 == response.status + + body = json.loads(response.body.decode("utf-8")) + assert body["downloads"]["paused"] is False + assert body["downloads"]["live_bypasses_limits"] is True + assert body["downloads"]["global"] == { + "limit": 10, + "active": 2, + "available": 8, + "live_active": 1, + "queued": 2, + } + assert body["extraction"] == { + "concurrency": 4, + "timeout_seconds": 70, + "info_cache_ttl_seconds": 10800, + } + assert body["live"] == { + "prevent_premiere": True, + "premiere_buffer_minutes": 7, + } + + per_extractor = {item["name"]: item for item in body["downloads"]["per_extractor"]["items"]} + assert body["downloads"]["per_extractor"]["default_limit"] == 3 + assert per_extractor["youtube"] == { + "name": "youtube", + "limit": 2, + "source": "env_override", + "active": 2, + "queued": 1, + "available": 0, + } + assert per_extractor["vimeo"] == { + "name": "vimeo", + "limit": 3, + "source": "default", + "active": 0, + "queued": 1, + "available": 3, + } + + def test_config_reads_prevent_live_premiere_boolean_env(self): + with patch.dict("os.environ", {"YTP_PREVENT_LIVE_PREMIERE": "false"}, clear=False): + Config._reset_singleton() + config = Config.get_instance() + + assert config.prevent_live_premiere is False diff --git a/ui/app/components/LimitsPage.vue b/ui/app/components/LimitsPage.vue new file mode 100644 index 00000000..a424342d --- /dev/null +++ b/ui/app/components/LimitsPage.vue @@ -0,0 +1,415 @@ + + + diff --git a/ui/app/layouts/default.vue b/ui/app/layouts/default.vue index dfe6c4a6..a65a89df 100644 --- a/ui/app/layouts/default.vue +++ b/ui/app/layouts/default.vue @@ -167,6 +167,16 @@