diff --git a/API.md b/API.md index fc09f356..2e191ee4 100644 --- a/API.md +++ b/API.md @@ -700,10 +700,16 @@ GET /api/history?type=queue&status=pending&order=ASC This endpoint returns the current state of active downloads from memory. +**Query Parameters**: +- `limit` (optional): Override the configured queue display limit for this request. `0` means unlimited. + **Response**: ```json { "history_count": 0, // total number of completed items in history + "queue_count": 250, // total number of queued items + "queue_loaded": 100, // number of queued items included in this response + "queue_limit": 100, // effective display limit, 0 means unlimited "queue":{ "id": "abc123", "url": "https://example.com/video", @@ -2629,52 +2635,19 @@ or an error: --- ### GET /api/system/configuration -**Purpose**: Retrieve system configuration including app settings, presets, download fields, and queue status. +**Purpose**: Retrieve system configuration. **Response**: ```json { - "app": { - "version": "...", - "download_path": "/path/to/downloads", - "base_path": "/", - ... - }, - "presets": [ - { - "id": 1, - "name": "default", - "description": "...", - ... - } - ], - "dl_fields": [ - { - "id": 1, - "name": "Title", - "field": "title", - "kind": "text", - ... - } - ], + "app": {...}, + "presets": [...], + "dl_fields": [...], "paused": false, - "history_count": 150, - "queue": [ - { - "id": "abc123", - "url": "https://example.com/video", - "status": "pending", - ... - } - ] + "history_count": 150 } ``` -**Notes**: -- This endpoint combines multiple data sources into a single response for efficient initialization -- The `queue` array contains active download items -- Folder listing is available via the separate `/api/system/folders` endpoint - --- ### GET /api/system/folders @@ -3495,7 +3468,7 @@ Emitted when a download item is moved between queue and history. "data": { "id": "abc123", "from": "queue", - "to": "done" + "to": "history" } } ``` diff --git a/FAQ.md b/FAQ.md index 339bd87d..31a68061 100644 --- a/FAQ.md +++ b/FAQ.md @@ -44,6 +44,7 @@ or the `environment:` section in `compose.yaml` file. | YTP_FLARESOLVERR_CACHE_TTL | The cache TTL (in seconds) for FlareSolverr solutions | `600` | | YTP_BASE_PATH | Set this if you are serving YTPTube from sub-folder | `/` | | YTP_PREVENT_LIVE_PREMIERE | Prevents the initial youtube premiere stream from being downloaded | `false` | +| YTP_QUEUE_DISPLAY_LIMIT | Max queued downloads returned to the UI. `0` = unlimited | `100` | | YTP_LIVE_PREMIERE_BUFFER | buffer time in minutes to add to video duration | `5` | | YTP_TASKS_HANDLER_TIMER | The cron expression for the tasks handler timer | `15 */1 * * *` | | YTP_TEMP_DISABLED | Disable temp files handling. | `false` | diff --git a/app/library/DataStore.py b/app/library/DataStore.py index 4d3460ea..0e82d8ff 100644 --- a/app/library/DataStore.py +++ b/app/library/DataStore.py @@ -166,6 +166,12 @@ class DataStore: def items(self): return self._dict.items() + def __contains__(self, key: str) -> bool: + return key in self._dict + + def __len__(self) -> int: + return len(self._dict) + async def put(self, value: Download, no_notify: bool = False) -> Download: _ = no_notify self._dict[value.info._id] = value diff --git a/app/library/config.py b/app/library/config.py index 7cf49008..5b995f2d 100644 --- a/app/library/config.py +++ b/app/library/config.py @@ -227,6 +227,9 @@ class Config(metaclass=Singleton): default_pagination: int = 50 """The default number of items per page for pagination.""" + queue_display_limit: int = 100 + """Maximum number of queued downloads returned to the UI. 0 means unlimited.""" + task_handler_random_delay: float = 60.0 """The maximum random delay in seconds before starting a task handler.""" @@ -282,6 +285,7 @@ class Config(metaclass=Singleton): "download_info_expires", "auto_clear_history_days", "default_pagination", + "queue_display_limit", "extract_info_concurrency", "thumb_concurrency", "flaresolverr_max_timeout", @@ -342,6 +346,7 @@ class Config(metaclass=Singleton): "app_build_date", "app_branch", "default_pagination", + "queue_display_limit", "check_for_updates", "new_version", "yt_new_version", diff --git a/app/library/downloads/pool_manager.py b/app/library/downloads/pool_manager.py index 09336931..fcfd3c31 100644 --- a/app/library/downloads/pool_manager.py +++ b/app/library/downloads/pool_manager.py @@ -240,7 +240,7 @@ class PoolManager: await self.queue.done.put(entry) self._notify.emit( Events.ITEM_MOVED, - data={"to": "history", "preset": entry.info.preset, "item": entry.info}, + data={"from": "queue", "to": "history", "preset": entry.info.preset, "item": entry.info}, title=nTitle, message=nMessage, ) diff --git a/app/library/downloads/queue_manager.py b/app/library/downloads/queue_manager.py index 95fca5dc..53fd3b22 100644 --- a/app/library/downloads/queue_manager.py +++ b/app/library/downloads/queue_manager.py @@ -320,7 +320,7 @@ class DownloadQueue(metaclass=Singleton): await self.done.put(item) self._notify.emit( Events.ITEM_MOVED, - data={"to": "history", "preset": item.info.preset, "item": item.info}, + data={"from": "queue", "to": "history", "preset": item.info.preset, "item": item.info}, title="Download Cancelled", message=f"Download '{item.info.title}' has been cancelled.", ) @@ -500,16 +500,15 @@ class DownloadQueue(metaclass=Singleton): if not ids: return {"deleted": 0} - items = await self.done.get_many_by_ids(ids) + items: list[tuple[str, Download]] = await self.done.get_many_by_ids(ids) if not items: return {"deleted": 0} - if self.config.remove_files is not True: - remove_file = False + remove_file = False if self.config.remove_files is not True else remove_file removed_files = 0 deleted_ids: list[str] = [] - deleted_titles: list[str] = [] + item_summaries: list[dict] = [] for item_id, item in items: filename: str = "" @@ -524,6 +523,8 @@ class DownloadQueue(metaclass=Singleton): "title": item.info.title, "status": item.info.status, "remove_file": remove_file, + "preset": item.info.preset, + "path": str(p) if (p := item.info.get_file()) else None, } }, ) @@ -555,7 +556,9 @@ class DownloadQueue(metaclass=Singleton): "download_id": item_id, "item_id": item.info.id, "title": item.info.title, + "preset": item.info.preset, "filename": file_ref.name, + "path": str(file_ref), } }, ) @@ -571,6 +574,8 @@ class DownloadQueue(metaclass=Singleton): "item_id": item.info.id, "title": item.info.title, "filename": rf.name, + "preset": item.info.preset, + "path": str(rf), } }, ) @@ -587,6 +592,8 @@ class DownloadQueue(metaclass=Singleton): "item_id": item.info.id, "title": item.info.title, "filename": filename, + "preset": item.info.preset, + "path": str(rf), } }, ) @@ -601,6 +608,7 @@ class DownloadQueue(metaclass=Singleton): "item_id": item.info.id, "title": item.info.title, "filename": filename, + "preset": item.info.preset, "remove_file": remove_file, "exception_type": type(e).__name__, } @@ -608,61 +616,78 @@ class DownloadQueue(metaclass=Singleton): ) deleted_ids.append(item_id) - deleted_titles.append(item.info.title or item.info.id or item_id) + item_summaries.append( + { + "id": item_id, + "title": item.info.title, + "url": item.info.url, + "preset": item.info.preset, + "path": str(p) if (p := item.info.get_file()) else None, + } + ) - deleted_count = await self.done.bulk_delete(deleted_ids) + deleted_count: int = await self.done.bulk_delete(deleted_ids) if deleted_count < 1: return {"deleted": 0} - title = "History Removed" if removed_files > 0 else "History Cleared" - message = f"Removed {deleted_count} item{'s' if deleted_count != 1 else ''} from history." + message: list[str] = [f"Cleared {deleted_count} item{'s' if deleted_count != 1 else ''} from history."] if removed_files > 0: - message += f" Also removed {removed_files} local file{'s' if removed_files != 1 else ''}." + message.append(f"Also removed {removed_files} local file{'s' if removed_files != 1 else ''}.") self._notify.emit( Events.ITEM_BULK_DELETED, - data={"ids": deleted_ids, "count": deleted_count}, - title=title, - message=message, + data={"count": deleted_count, "removed_files": removed_files, "items": item_summaries}, + title="History Cleared", + message=" ".join(message), ) - summary = ", ".join(deleted_titles[:5]) - if deleted_count > 5: - summary += ", ..." LOG.info( - "Cleared %s history item(s), including %s.", + "Cleared %d history item(s).", deleted_count, - summary, - extra={"deleted_count": deleted_count, "removed_files": removed_files, "titles": deleted_titles[:5]}, + extra={"deleted_count": deleted_count, "removed_files": removed_files, "items": item_summaries}, ) return {"deleted": deleted_count} async def clear_by_status(self, status_filter: str, remove_file: bool = False) -> dict[str, int | str]: - if self.config.remove_files is not True: - remove_file = False + if not (items := await self.done.get_many_by_status(status_filter)): + return {"deleted": 0} - if not remove_file: - deleted_count = await self.done.bulk_delete_by_status(status_filter) - if deleted_count < 1: - return {"deleted": 0} + remove_file = False if self.config.remove_files is not True else remove_file - self._notify.emit( - Events.ITEM_BULK_DELETED, - data={"count": deleted_count, "status": status_filter}, - title="History Cleared", - message=f"Cleared {deleted_count} item{'s' if deleted_count != 1 else ''} from history.", - ) - LOG.info( - "Cleared %s history item(s) with status '%s'.", - deleted_count, - status_filter, - extra={"deleted_count": deleted_count, "status_filter": status_filter}, - ) - return {"deleted": deleted_count} + if remove_file: + return await self.clear_bulk([item_id for item_id, _ in items], remove_file=remove_file) - items = await self.done.get_many_by_status(status_filter) - return await self.clear_bulk([item_id for item_id, _ in items], remove_file=remove_file) + deleted_count: int = await self.done.bulk_delete_by_status(status_filter) + item_summaries: list[dict[str, str | None]] = [ + { + "id": item_id, + "title": dto.info.title, + "url": dto.info.url, + "preset": dto.info.preset, + "path": str(p) if (p := dto.info.get_file()) else None, + } + for item_id, dto in items + ] + + self._notify.emit( + Events.ITEM_BULK_DELETED, + data={"count": deleted_count, "status": status_filter, "items": item_summaries}, + title="History Cleared", + message=f"Cleared {deleted_count} item(s) from history.", + ) + + LOG.info( + "Cleared %d history item(s) with status '%s'.", + deleted_count, + status_filter, + extra={ + "deleted_count": deleted_count, + "status_filter": status_filter, + "items": item_summaries, + }, + ) + return {"deleted": deleted_count} async def get(self, mode: str = "all") -> dict[str, list[dict[str, ItemDTO]]]: """ @@ -700,6 +725,32 @@ class DownloadQueue(metaclass=Singleton): return items + def live_queue(self, limit: int = 0) -> dict[str, int | dict[str, ItemDTO]]: + """Return a display-limited live queue snapshot with total metadata.""" + limit = max(0, int(limit or 0)) + items: dict[str, ItemDTO] = {} + active = self.pool.get_active_downloads() + + for key, download in active.items(): + if key in self.queue: + items[key] = download.info + + for key, download in self.queue.items(): + if limit > 0 and len(items) >= limit: + break + + if key in items: + continue + + items[key] = active[key].info if key in active else download.info + + return { + "queue": items, + "queue_count": len(self.queue), + "queue_loaded": len(items), + "queue_limit": limit, + } + async def get_item(self, **kwargs) -> tuple["StoreType", Download] | tuple[None, None]: """ Get a specific item from the download queue or history. @@ -708,7 +759,7 @@ class DownloadQueue(metaclass=Singleton): **kwargs: The key-value pair to search for. Supported keys are 'id', 'url'. Returns: - (StoreType, Download) | None: The requested item if found, otherwise None. + tuple[StoreType, Download] | tuple[None, None]: The requested item if found, otherwise None. """ from app.library.DataStore import StoreType diff --git a/app/routes/api/history.py b/app/routes/api/history.py index bb0bdd60..d2df5242 100644 --- a/app/routes/api/history.py +++ b/app/routes/api/history.py @@ -123,23 +123,29 @@ async def items_list(request: Request, queue: DownloadQueue, encoder: Encoder, c @route("GET", "api/history/live", "items_live") -async def items_live(queue: DownloadQueue, encoder: Encoder) -> Response: +async def items_live(request: Request, queue: DownloadQueue, encoder: Encoder, config: Config) -> Response: """ Get live queue data Args: + request (Request): The request object. queue (DownloadQueue): The download queue instance. encoder (Encoder): The encoder instance. + config (Config): The configuration instance. Returns: Response: The response object with live queue items. """ + try: + limit = int(request.query.get("limit", config.queue_display_limit)) + except ValueError: + return web.json_response( + data={"error": "limit must be a valid integer."}, status=web.HTTPBadRequest.status_code + ) + return web.json_response( - data={ - "queue": (await queue.get("queue"))["queue"], - "history_count": await queue.done.get_total_count(), - }, + data={**queue.live_queue(limit), "history_count": await queue.done.get_total_count()}, status=web.HTTPOk.status_code, dumps=encoder.encode, ) diff --git a/app/routes/api/system.py b/app/routes/api/system.py index 3580cad8..acd323f1 100644 --- a/app/routes/api/system.py +++ b/app/routes/api/system.py @@ -45,7 +45,6 @@ async def system_config(queue: DownloadQueue, config: Config, encoder: Encoder) "dl_fields": await DLFields.get_instance().get_all_serialized(), "paused": queue.is_paused(), "history_count": await queue.done.get_total_count(), - "queue": (await queue.get("queue"))["queue"], }, status=web.HTTPOk.status_code, dumps=encoder.encode, diff --git a/app/tests/test_download.py b/app/tests/test_download.py index f5440d0e..a0614e70 100644 --- a/app/tests/test_download.py +++ b/app/tests/test_download.py @@ -1299,6 +1299,19 @@ class TestStatusTracker: class TestQueueManager: + class LiveStore: + def __init__(self, items: dict[str, Download]) -> None: + self._items = items + + def items(self): + return self._items.items() + + def __contains__(self, key: str) -> bool: + return key in self._items + + def __len__(self) -> int: + return len(self._items) + @staticmethod def _video_queue() -> Mock: async def put(item): @@ -1332,6 +1345,37 @@ class TestQueueManager: auto_start=True, ) + def test_live_queue_caps_visible_items(self) -> None: + queue_manager = object.__new__(DownloadQueue) + items = {f"id{i}": Mock(info=make_item(id=f"id{i}", title=f"Video {i}")) for i in range(5)} + queue_manager.queue = self.LiveStore(items) + queue_manager.pool = Mock() + queue_manager.pool.get_active_downloads.return_value = {} + + snapshot = DownloadQueue.live_queue(queue_manager, limit=2) + + assert list(snapshot["queue"].keys()) == ["id0", "id1"] + assert snapshot["queue_count"] == 5 + assert snapshot["queue_loaded"] == 2 + assert snapshot["queue_limit"] == 2 + + def test_live_queue_keeps_active(self) -> None: + queue_manager = object.__new__(DownloadQueue) + items = {f"id{i}": Mock(info=make_item(id=f"id{i}", title=f"Video {i}")) for i in range(5)} + queue_manager.queue = self.LiveStore(items) + queue_manager.pool = Mock() + queue_manager.pool.get_active_downloads.return_value = { + "id3": Mock(info=make_item(id="id3", title="Active 3")), + "id4": Mock(info=make_item(id="id4", title="Active 4")), + } + + snapshot = DownloadQueue.live_queue(queue_manager, limit=1) + + assert list(snapshot["queue"].keys()) == ["id3", "id4"] + assert snapshot["queue_count"] == 5 + assert snapshot["queue_loaded"] == 2 + assert snapshot["queue_limit"] == 1 + @pytest.mark.asyncio async def test_live_reextracts(self, monkeypatch: pytest.MonkeyPatch) -> None: seen: list[dict | None] = [] @@ -1501,7 +1545,11 @@ class TestQueueManager: done_store.bulk_delete.assert_awaited_once_with(["done-id-1", "done-id-2"]) queue_manager._notify.emit.assert_called_once() assert queue_manager._notify.emit.call_args.args[0] == Events.ITEM_BULK_DELETED - assert queue_manager._notify.emit.call_args.kwargs["data"] == {"ids": ["done-id-1", "done-id-2"], "count": 2} + assert queue_manager._notify.emit.call_args.kwargs["data"]["count"] == 2 + assert queue_manager._notify.emit.call_args.kwargs["data"]["removed_files"] == 0 + assert len(queue_manager._notify.emit.call_args.kwargs["data"]["items"]) == 2 + assert queue_manager._notify.emit.call_args.kwargs["data"]["items"][0]["id"] == "done-id-1" + assert queue_manager._notify.emit.call_args.kwargs["data"]["items"][1]["id"] == "done-id-2" @pytest.mark.asyncio async def test_clear_status_fetches(self) -> None: @@ -1509,19 +1557,25 @@ class TestQueueManager: queue_manager.config = Mock(remove_files=False, download_path="/tmp") queue_manager._notify = Mock() + item = Mock() + item.info = make_item(id="done-id", title="Cleared clip") + done_store = Mock() done_store.bulk_delete_by_status = AsyncMock(return_value=1) - done_store.get_many_by_status = AsyncMock() + done_store.get_many_by_status = AsyncMock(return_value=[("done-id", item)]) queue_manager.done = done_store result = await DownloadQueue.clear_by_status(queue_manager, "finished", remove_file=False) assert result == {"deleted": 1} + done_store.get_many_by_status.assert_awaited_once_with("finished") done_store.bulk_delete_by_status.assert_awaited_once_with("finished") - done_store.get_many_by_status.assert_not_called() queue_manager._notify.emit.assert_called_once() assert queue_manager._notify.emit.call_args.args[0] == Events.ITEM_BULK_DELETED - assert queue_manager._notify.emit.call_args.kwargs["data"] == {"count": 1, "status": "finished"} + assert queue_manager._notify.emit.call_args.kwargs["data"]["count"] == 1 + assert queue_manager._notify.emit.call_args.kwargs["data"]["status"] == "finished" + assert len(queue_manager._notify.emit.call_args.kwargs["data"]["items"]) == 1 + assert queue_manager._notify.emit.call_args.kwargs["data"]["items"][0]["id"] == "done-id" @pytest.mark.asyncio async def test_clear_status_files_fetch(self) -> None: diff --git a/app/tests/test_history_routes.py b/app/tests/test_history_routes.py index 06dc00c8..31d2f543 100644 --- a/app/tests/test_history_routes.py +++ b/app/tests/test_history_routes.py @@ -12,7 +12,7 @@ from app.library.cache import Cache from app.library.ItemDTO import ItemDTO from app.library.encoder import Encoder from app.routes.api import history -from app.routes.api.history import item_rename, item_thumbnail, items_delete +from app.routes.api.history import item_rename, item_thumbnail, items_delete, items_live from app.tests.helpers import temporary_test_dir @@ -84,6 +84,60 @@ async def test_items_delete_ids() -> None: assert body == {"items": {}, "deleted": 2} +@pytest.mark.asyncio +async def test_items_live_metadata() -> None: + request = _FakeRequest() + request.query = {} + queue = Mock() + queue.live_queue = Mock( + return_value={"queue": {"a": {"title": "A"}}, "queue_count": 3, "queue_loaded": 1, "queue_limit": 1} + ) + queue.done.get_total_count = AsyncMock(return_value=7) + encoder = Encoder() + config = SimpleNamespace(queue_display_limit=1) + + response = await items_live(request, queue, encoder, config) + + assert response.status == 200 + queue.live_queue.assert_called_once_with(1) + body = json.loads(response.body.decode("utf-8")) + assert body["queue_count"] == 3 + assert body["queue_loaded"] == 1 + assert body["queue_limit"] == 1 + assert body["history_count"] == 7 + + +@pytest.mark.asyncio +async def test_items_live_limit_query() -> None: + request = _FakeRequest() + request.query = {"limit": "25"} + queue = Mock() + queue.live_queue = Mock(return_value={"queue": {}, "queue_count": 0, "queue_loaded": 0, "queue_limit": 25}) + queue.done.get_total_count = AsyncMock(return_value=0) + encoder = Encoder() + config = SimpleNamespace(queue_display_limit=1) + + response = await items_live(request, queue, encoder, config) + + assert response.status == 200 + queue.live_queue.assert_called_once_with(25) + + +@pytest.mark.asyncio +async def test_items_live_bad_limit() -> None: + request = _FakeRequest() + request.query = {"limit": "many"} + queue = Mock() + queue.live_queue = Mock() + encoder = Encoder() + config = SimpleNamespace(queue_display_limit=1) + + response = await items_live(request, queue, encoder, config) + + assert response.status == 400 + queue.live_queue.assert_not_called() + + @pytest.mark.asyncio async def test_item_rename_needs_name() -> None: request = _FakeRequest(payload={}) diff --git a/app/tests/test_system_routes.py b/app/tests/test_system_routes.py index 680d158b..e21b8cf4 100644 --- a/app/tests/test_system_routes.py +++ b/app/tests/test_system_routes.py @@ -9,7 +9,7 @@ from app.library.config import Config from app.library.cache import Cache from app.library.encoder import Encoder from app.library.UpdateChecker import UpdateChecker -from app.routes.api.system import check_updates, system_diagnostics, system_folders, system_limits +from app.routes.api.system import check_updates, system_config, system_diagnostics, system_folders, system_limits @dataclass @@ -118,6 +118,38 @@ class TestCheckUpdatesEndpoint: assert "update_available" in body, "Response should include update_available status" +class TestSystemConfigEndpoint: + def setup_method(self): + Config._reset_singleton() + + @pytest.mark.asyncio + async def test_excludes_queue(self) -> None: + config = Config.get_instance() + encoder = Encoder() + queue = MagicMock() + queue.is_paused.return_value = False + queue.done.get_total_count = AsyncMock(return_value=0) + + with ( + patch("app.features.dl_fields.service.DLFields.get_instance") as mock_dlfields, + patch("app.features.presets.service.Presets.get_instance") as mock_presets, + ): + dl_fields = MagicMock() + dl_fields.get_all_serialized = AsyncMock(return_value=[]) + mock_dlfields.return_value = dl_fields + + presets = MagicMock() + presets.get_all.return_value = [] + mock_presets.return_value = presets + + response = await system_config(queue, config, encoder) + + assert response.status == 200 + body = json.loads(response.body.decode("utf-8")) + assert "queue" not in body + queue.get.assert_not_called() + + class TestSystemLimitsEndpoint: def setup_method(self): Config._reset_singleton() diff --git a/ui/app/components/Simple.vue b/ui/app/components/Simple.vue index 446366d5..a4f5f88d 100644 --- a/ui/app/components/Simple.vue +++ b/ui/app/components/Simple.vue @@ -193,7 +193,20 @@ Queue - {{ queueItems.length }} items +
+ {{ queueCountLabel }} + + + Show more + +
@@ -629,7 +642,7 @@ :isControls="true" :item="videoItem" class="w-full" - @closeModel="() => void requestCloseVideo()" + @closeModel="() => requestCloseVideo()" @error="async (error: string) => await box.alert(error)" @playback-state-change="(playing: boolean) => (playingNow = playing)" /> @@ -728,6 +741,7 @@ const embedUrl = ref(''); const videoItem = ref(null); const playingNow = ref(false); const autoRefreshInterval = ref | null>(null); +const hadSocketDisconnect = ref(false); const videoOpen = computed({ get: () => Boolean(videoItem.value), set: (value: boolean) => { @@ -778,6 +792,13 @@ const urlInputUi = { const historyPagination = computed(() => pagination.value); const historyIsLoading = computed(() => isLoading.value); const queueItems = computed(() => Object.values(stateStore.queue)); +const queueCountLabel = computed(() => { + if (stateStore.hasMore()) { + return `${stateStore.shown()} displayed / ${stateStore.count()} queued`; + } + + return `${stateStore.count()} queued`; +}); const historyEntries = computed(() => historyItems.value); const hasAnyItems = computed(() => queueItems.value.length > 0 || historyEntries.value.length > 0); const showSections = computed(() => hasAnyItems.value || historyIsLoading.value); @@ -863,6 +884,23 @@ const refreshQueue = async (): Promise => { } }; +const loadMoreQueue = async (): Promise => { + if (isRefreshing.value) { + return; + } + + isRefreshing.value = true; + + try { + await stateStore.loadMore(); + } catch (error) { + console.error('Failed to load more queue items:', error); + toast.error('Failed to load more queue items'); + } finally { + isRefreshing.value = false; + } +}; + const startAutoRefresh = (): void => { if (autoRefreshInterval.value) { clearInterval(autoRefreshInterval.value); @@ -1343,7 +1381,10 @@ onMounted(async () => { await normalizeSimpleRoute(); historyInitialized.value = true; socketStore.on('item_moved', handleHistoryItemMoved); - await load(1, { order: 'DESC', perPage: configStore.app.default_pagination }); + await Promise.allSettled([ + refreshQueue(), + load(1, { order: 'DESC', perPage: configStore.app.default_pagination }), + ]); if (!socketStore.isConnected && autoRefreshEnabled.value) { startAutoRefresh(); @@ -1374,7 +1415,7 @@ watch( () => route.path, () => { if (simpleMode.value) { - void normalizeSimpleRoute(); + normalizeSimpleRoute(); } }, ); @@ -1384,9 +1425,17 @@ watch( (connected) => { if (connected) { stopAutoRefresh(); + + if (hadSocketDisconnect.value) { + hadSocketDisconnect.value = false; + refreshQueue(); + } + return; } + hadSocketDisconnect.value = true; + if (autoRefreshEnabled.value) { startAutoRefresh(); } @@ -1429,7 +1478,7 @@ watch( return; } - void load(1, { order: 'DESC', perPage: configStore.app.default_pagination }); + load(1, { order: 'DESC', perPage: configStore.app.default_pagination }); }, ); diff --git a/ui/app/composables/useAppSocket.ts b/ui/app/composables/useAppSocket.ts index 36bd860e..429e9408 100644 --- a/ui/app/composables/useAppSocket.ts +++ b/ui/app/composables/useAppSocket.ts @@ -19,6 +19,20 @@ const getRuntimeConfig = () => useRuntimeConfig(); const getConfig = () => useYtpConfig(); const getQueueState = () => useQueueState(); const getToast = () => useNotification(); +let queueReloadTimer: ReturnType | null = null; + +const scheduleQueueBackfill = (): void => { + const queueState = getQueueState(); + + if (!queueState.needsBackfill() || queueReloadTimer) { + return; + } + + queueReloadTimer = setTimeout(() => { + queueReloadTimer = null; + queueState.loadQueue(queueState.limit || undefined); + }, 500); +}; const socket = ref(null); const isConnected = ref(false); @@ -278,8 +292,11 @@ on('item_added', (data: WSEP['item_added']) => { const queueState = getQueueState(); const toast = getToast(); - queueState.add(data.data._id, data.data); - toast.success(`Item queued: ${ag(queueState.get(data.data._id, {} as StoreItem), 'title')}`); + if (queueState.isLoaded()) { + queueState.add(data.data._id, data.data); + } + + toast.success(`Item queued: ${ag(data.data, 'title')}`); }); on( @@ -314,33 +331,39 @@ on('item_cancelled', (data: WSEP['item_cancelled']) => { const toast = getToast(); const id = data.data._id; + if (!queueState.isLoaded()) { + return; + } + if (true !== queueState.has(id)) { return; } toast.warning(`Download cancelled: ${ag(queueState.get(id, {} as StoreItem), 'title')}`); - - if (true === queueState.has(id)) { - queueState.remove(id); - } }); on('item_deleted', (data: WSEP['item_deleted']) => { const queueState = getQueueState(); const id = data.data._id; + if (!queueState.isLoaded()) { + return; + } + if (true === queueState.has(id)) { queueState.remove(id); + scheduleQueueBackfill(); } }); on('item_updated', (data: WSEP['item_updated']) => { const queueState = getQueueState(); - const id = data.data._id; - if (true === queueState.has(id)) { - queueState.update(id, data.data); + if (!queueState.isLoaded()) { + return; } + + queueState.update(data.data._id, data.data); }); on('item_progress', (data: WSEP['item_progress']) => { @@ -357,13 +380,24 @@ on('item_moved', (data: WSEP['item_moved']) => { const to = data.data.to; const id = data.data.item._id; + if (!queueState.isLoaded()) { + return; + } + if ('queue' === to) { queueState.add(id, data.data.item); } if ('history' === to) { + if ('queue' === data.data.from) { + queueState.drop(id); + scheduleQueueBackfill(); + return; + } + if (true === queueState.has(id)) { queueState.remove(id); + scheduleQueueBackfill(); } } }); diff --git a/ui/app/composables/useQueueState.ts b/ui/app/composables/useQueueState.ts index 27f12736..e1b897bc 100644 --- a/ui/app/composables/useQueueState.ts +++ b/ui/app/composables/useQueueState.ts @@ -7,18 +7,88 @@ type KeyType = string; interface QueueState { queue: Record; + total: number; + loaded: number; + limit: number; + is_loaded: boolean; + is_loading: boolean; } +type QueueMeta = { + queue_count?: number; + queue_loaded?: number; + queue_limit?: number; +}; + const state = reactive({ queue: {}, + total: 0, + loaded: 0, + limit: 0, + is_loaded: false, + is_loading: false, }); +const visibleCount = (): number => Object.keys(state.queue).length; + +const syncLoaded = (): void => { + state.loaded = visibleCount(); + if (state.total < state.loaded) { + state.total = state.loaded; + } +}; + +const canAddVisible = (): boolean => state.limit < 1 || visibleCount() < state.limit; + +const isActive = (item: StoreItem): boolean => { + return ['started', 'preparing', 'downloading', 'postprocessing'].includes(item.status || ''); +}; + +const applyMeta = (meta: QueueMeta = {}): void => { + state.loaded = Number(meta.queue_loaded ?? visibleCount()); + state.total = Number(meta.queue_count ?? state.loaded); + state.limit = Number(meta.queue_limit ?? 0); + syncLoaded(); +}; + const add = (key: KeyType, value: StoreItem): void => { + if (state.queue[key]) { + state.queue[key] = value; + syncLoaded(); + return; + } + + state.total += 1; + + if (!canAddVisible()) { + return; + } + state.queue[key] = value; + syncLoaded(); +}; + +const reveal = (key: KeyType, value: StoreItem): void => { + if (state.queue[key]) { + state.queue[key] = value; + syncLoaded(); + return; + } + + state.queue[key] = value; + syncLoaded(); }; const update = (key: KeyType, value: StoreItem): void => { + if (!state.queue[key]) { + if (isActive(value)) { + reveal(key, value); + } + return; + } + state.queue[key] = value; + syncLoaded(); }; const patch = (key: KeyType, fields: Partial): void => { @@ -34,6 +104,18 @@ const remove = (key: KeyType): void => { const { [key]: _, ...rest } = state.queue; state.queue = rest; + state.total = Math.max(0, state.total - 1); + syncLoaded(); +}; + +const drop = (key: KeyType): void => { + if (state.queue[key]) { + const { [key]: _, ...rest } = state.queue; + state.queue = rest; + } + + state.total = Math.max(0, state.total - 1); + syncLoaded(); }; const get = (key: KeyType, defaultValue: StoreItem | null = null): StoreItem | null => { @@ -46,30 +128,92 @@ const has = (key: KeyType): boolean => { const clearAll = (): void => { state.queue = {}; + state.total = 0; + state.loaded = 0; + state.limit = 0; + state.is_loaded = false; + state.is_loading = false; }; -const addAll = (data: Record): void => { +const addAll = (data: Record, meta: QueueMeta = {}): void => { state.queue = data; + state.is_loaded = true; + applyMeta(meta); }; const count = (): number => { - return Object.keys(state.queue).length; + return state.total; }; -const loadQueue = async (): Promise => { +const isLoaded = (): boolean => { + return state.is_loaded; +}; + +const shown = (): number => { + return visibleCount(); +}; + +const hasMore = (): boolean => { + if (!state.is_loaded) { + return false; + } + + return state.total > visibleCount(); +}; + +const needsBackfill = (): boolean => { + if (!state.is_loaded) { + return false; + } + + if (state.limit < 1) { + return false; + } + + return visibleCount() < Math.min(state.limit, state.total); +}; + +const loadQueue = async (limit?: number): Promise => { + if (state.is_loading) { + return; + } + + state.is_loading = true; + try { - const response = await request('/api/history/live'); + const params = new URLSearchParams(); + if (typeof limit === 'number') { + params.set('limit', String(limit)); + } + + const query = params.toString(); + const response = await request(`/api/history/live${query ? `?${query}` : ''}`); + if (!response.ok) { + throw new Error('Failed to load queue'); + } + const data = (await response.json()) as { queue: Record; - }; + } & QueueMeta; - state.queue = data.queue || {}; + addAll(data.queue || {}, data); } catch (error) { console.error('Failed to load queue:', error); throw error; + } finally { + state.is_loading = false; } }; +const loadMore = async (): Promise => { + if (!hasMore()) { + return; + } + + const step = state.limit > 0 ? state.limit : Math.max(visibleCount(), 100); + await loadQueue(Math.max(visibleCount() + 1, state.limit + step)); +}; + const addDownload = async (data: item_request): Promise => { const socket = useAppSocket(); const toast = useNotification(); @@ -235,12 +379,18 @@ const queueStateApi = proxyRefs({ update, patch, remove, + drop, get, has, clearAll, addAll, count, + isLoaded, + shown, + hasMore, + needsBackfill, loadQueue, + loadMore, addDownload, startItems, pauseItems, diff --git a/ui/app/composables/useYtpConfig.ts b/ui/app/composables/useYtpConfig.ts index 8dc65b9c..803ca2a7 100644 --- a/ui/app/composables/useYtpConfig.ts +++ b/ui/app/composables/useYtpConfig.ts @@ -4,7 +4,6 @@ import type { ConfigState } from '~/types/config'; import type { DLField } from '~/types/dl_fields'; import type { Preset } from '~/types/presets'; import type { ConfigFeature, ConfigUpdateAction } from '~/types/sockets'; -import { useQueueState } from '~/composables/useQueueState'; import { request } from '~/utils'; let last_reload = 0; @@ -33,6 +32,7 @@ const state = reactive({ app_env: 'production', simple_mode: false, default_pagination: 50, + queue_display_limit: 100, log_level: '', runtime_log_level: '', check_for_updates: true, @@ -79,17 +79,11 @@ const loadConfig = async (force: boolean = false) => { return; } const data = await resp.json(); - const queueState = useQueueState(); if ('number' === typeof data.history_count) { delete data.history_count; } - if (data.queue) { - queueState.addAll(data.queue); - delete data.queue; - } - setAll(data); state.is_loaded = true; last_reload = now; diff --git a/ui/app/pages/index.vue b/ui/app/pages/index.vue index cbf541bc..799254ac 100644 --- a/ui/app/pages/index.vue +++ b/ui/app/pages/index.vue @@ -38,7 +38,7 @@ variant="outline" size="sm" icon="i-lucide-pause" - @click="() => void pauseDownload()" + @click="() => pauseDownload()" > Pause @@ -49,7 +49,7 @@ variant="outline" size="sm" icon="i-lucide-play" - @click="() => void resumeDownload()" + @click="() => resumeDownload()" > Resume @@ -114,7 +114,7 @@ size="sm" icon="i-lucide-refresh-cw" :loading="isRefreshing" - @click="() => void refreshQueue()" + @click="() => refreshQueue()" > Refresh @@ -156,9 +156,20 @@ - Total: {{ stateStore.count() }} + {{ queueCountLabel }} + + + Show more + @@ -355,7 +366,7 @@ variant="outline" size="xs" icon="i-lucide-circle-play" - @click="() => void startItem(item)" + @click="() => startItem(item)" > Start @@ -366,7 +377,7 @@ variant="outline" size="xs" icon="i-lucide-pause" - @click="() => void pauseItem(item)" + @click="() => pauseItem(item)" > Pause @@ -618,7 +629,7 @@ variant="outline" icon="i-lucide-circle-play" class="w-full justify-center" - @click="() => void startItem(item)" + @click="() => startItem(item)" > Start @@ -629,7 +640,7 @@ variant="outline" icon="i-lucide-pause" class="w-full justify-center" - @click="() => void pauseItem(item)" + @click="() => pauseItem(item)" > Pause @@ -780,6 +791,7 @@ const masterSelectAll = ref(false); const embed_url = ref(''); const isRefreshing = ref(false); const autoRefreshInterval = ref | null>(null); +const hadSocketDisconnect = ref(false); const hasQueueContent = computed(() => stateStore.count() > 0 || query.value.trim().length > 0); const contentStyle = computed<'grid' | 'list'>(() => @@ -798,6 +810,14 @@ const displayedItems = computed(() => { return items.filter((item) => deepIncludes(item, normalizedQuery, new WeakSet())); }); +const queueCountLabel = computed(() => { + if (stateStore.hasMore()) { + return `Queued: ${stateStore.shown()} displayed / ${stateStore.count()} total`; + } + + return `Queued: ${stateStore.count()}`; +}); + const hasItems = computed(() => displayedItems.value.length > 0); const hasSelected = computed(() => selectedElms.value.length > 0); const displayedItemIds = computed(() => displayedItems.value.map((item) => item._id)); @@ -827,6 +847,22 @@ const refreshQueue = async (): Promise => { } }; +const loadMoreQueue = async (): Promise => { + if (isRefreshing.value) { + return; + } + + isRefreshing.value = true; + + try { + await stateStore.loadMore(); + } catch { + toast.error('Failed to load more queue items'); + } finally { + isRefreshing.value = false; + } +}; + const startAutoRefresh = (): void => { if (autoRefreshInterval.value) { clearInterval(autoRefreshInterval.value); @@ -867,6 +903,8 @@ onMounted(async () => { pendingDownloadFormItem.value = {}; } + await refreshQueue(); + if (!socket.isConnected && autoRefreshEnabled.value) { startAutoRefresh(); } @@ -885,9 +923,17 @@ watch( (connected) => { if (connected) { stopAutoRefresh(); + + if (hadSocketDisconnect.value) { + hadSocketDisconnect.value = false; + void refreshQueue(); + } + return; } + hadSocketDisconnect.value = true; + if (autoRefreshEnabled.value) { startAutoRefresh(); } @@ -1041,7 +1087,7 @@ const bulkActionGroups = computed(() => { label: 'Start', icon: 'i-lucide-circle-play', disabled: !hasSelected.value, - onSelect: () => void startItems(), + onSelect: () => startItems(), }); } @@ -1050,7 +1096,7 @@ const bulkActionGroups = computed(() => { label: 'Pause', icon: 'i-lucide-pause', disabled: !hasSelected.value, - onSelect: () => void pauseSelected(), + onSelect: () => pauseSelected(), }); } @@ -1058,7 +1104,7 @@ const bulkActionGroups = computed(() => { label: selectedLiveOnly ? 'Stop' : 'Cancel', icon: 'i-lucide-circle-off', disabled: !hasSelected.value, - onSelect: () => void cancelSelected(), + onSelect: () => cancelSelected(), }); return groups; @@ -1081,14 +1127,14 @@ const itemActionGroups = (item: StoreItem): Array> primaryActions.push({ label: item.is_live ? 'Stop Stream' : 'Cancel Download', icon: 'i-lucide-circle-off', - onSelect: () => void confirmCancel(item), + onSelect: () => confirmCancel(item), }); if (canStartItem(item)) { primaryActions.push({ label: 'Start Download', icon: 'i-lucide-circle-play', - onSelect: () => void startItem(item), + onSelect: () => startItem(item), }); } @@ -1096,7 +1142,7 @@ const itemActionGroups = (item: StoreItem): Array> primaryActions.push({ label: 'Pause Download', icon: 'i-lucide-pause', - onSelect: () => void pauseItem(item), + onSelect: () => pauseItem(item), }); } @@ -1351,7 +1397,7 @@ const cancelItems = (item: string | string[]): void => { return; } - void stateStore.cancelItems(items); + stateStore.cancelItems(items); }; const startItem = async (item: StoreItem): Promise => await stateStore.startItems([item._id]); diff --git a/ui/app/types/config.d.ts b/ui/app/types/config.d.ts index 09fce8a8..fec38156 100644 --- a/ui/app/types/config.d.ts +++ b/ui/app/types/config.d.ts @@ -43,6 +43,8 @@ type AppConfig = { app_env: 'production' | 'development'; /** Default number of items per page for pagination */ default_pagination: number; + /** Maximum number of queued downloads loaded into the UI. 0 means unlimited. */ + queue_display_limit: number; /** Configured default log level */ log_level: 'debug' | 'info' | 'warning' | 'error' | ''; /** Active runtime log level */ diff --git a/ui/app/types/sockets.d.ts b/ui/app/types/sockets.d.ts index 00c6c715..14dc54e3 100644 --- a/ui/app/types/sockets.d.ts +++ b/ui/app/types/sockets.d.ts @@ -38,7 +38,11 @@ export type WSEP = { item_cancelled: EventPayload; item_deleted: EventPayload; item_bulk_deleted: EventPayload<{ count: number; status?: string; ids?: string[] }>; - item_moved: EventPayload<{ to: 'queue' | 'history'; item: StoreItem }>; + item_moved: EventPayload<{ + from?: 'queue' | 'history'; + to: 'queue' | 'history'; + item: StoreItem; + }>; item_status: EventPayload<{ status?: string; msg?: string; preset?: string }>; paused: EventPayload<{ paused?: boolean }>; resumed: EventPayload<{ paused?: boolean }>; diff --git a/ui/tests/composables/useQueueState.test.ts b/ui/tests/composables/useQueueState.test.ts new file mode 100644 index 00000000..ad10b4b7 --- /dev/null +++ b/ui/tests/composables/useQueueState.test.ts @@ -0,0 +1,119 @@ +import { beforeEach, describe, expect, it, spyOn } from 'bun:test'; + +import * as utils from '~/utils/index'; +import { useQueueState } from '~/composables/useQueueState'; +import type { StoreItem } from '~/types/store'; + +const createMockResponse = (jsonData: unknown): Response => { + return { + ok: true, + status: 200, + headers: new Headers({ 'Content-Type': 'application/json' }), + redirected: false, + statusText: 'OK', + type: 'basic', + url: '', + body: null, + bodyUsed: false, + clone() { + return this; + }, + async json() { + return jsonData; + }, + text: async () => JSON.stringify(jsonData), + arrayBuffer: async () => new ArrayBuffer(0), + blob: async () => new Blob(), + formData: async () => new FormData(), + } as Response; +}; + +const makeItem = (id: string, status: StoreItem['status'] = null): StoreItem => { + return { + _id: id, + id, + title: `Video ${id}`, + description: '', + url: `https://example.com/${id}`, + preset: 'default', + folder: '', + download_dir: '/downloads', + temp_dir: '/tmp', + status, + error: null, + cookies: '', + template: '', + template_chapter: '', + timestamp: 0, + is_live: false, + datetime: '', + live_in: null, + file_size: null, + cli: '', + auto_start: true, + options: {}, + sidecar: {}, + extras: {}, + }; +}; + +describe('useQueueState', () => { + beforeEach(() => { + useQueueState().clearAll(); + }); + + it('load_metadata', async () => { + const requestSpy = spyOn(utils, 'request'); + requestSpy.mockResolvedValueOnce( + createMockResponse({ + queue: { a: makeItem('a') }, + queue_count: 3, + queue_loaded: 1, + queue_limit: 1, + }), + ); + + const queue = useQueueState(); + await queue.loadQueue(); + + expect(queue.count()).toBe(3); + expect(queue.shown()).toBe(1); + expect(queue.hasMore()).toBe(true); + expect(queue.limit).toBe(1); + expect(queue.get('a')?.title).toBe('Video a'); + + requestSpy.mockRestore(); + }); + + it('add_respects_limit', () => { + const queue = useQueueState(); + queue.addAll({ a: makeItem('a') }, { queue_count: 1, queue_loaded: 1, queue_limit: 1 }); + + queue.add('b', makeItem('b')); + + expect(queue.count()).toBe(2); + expect(queue.shown()).toBe(1); + expect(queue.get('b')).toBeNull(); + }); + + it('update_reveals_active', () => { + const queue = useQueueState(); + queue.addAll({ a: makeItem('a') }, { queue_count: 2, queue_loaded: 1, queue_limit: 1 }); + + queue.update('b', makeItem('b', 'downloading')); + + expect(queue.count()).toBe(2); + expect(queue.shown()).toBe(2); + expect(queue.get('b')?.status).toBe('downloading'); + }); + + it('drop_hidden_decrements_total', () => { + const queue = useQueueState(); + queue.addAll({ a: makeItem('a') }, { queue_count: 2, queue_loaded: 1, queue_limit: 1 }); + + queue.drop('b'); + + expect(queue.count()).toBe(1); + expect(queue.shown()).toBe(1); + }); +});