diff --git a/API.md b/API.md index 302013ec..b71649e4 100644 --- a/API.md +++ b/API.md @@ -81,6 +81,7 @@ This document describes the available endpoints and their usage. All endpoints r - [Connection Events](#connection-events) - [`connect` (Built-in)](#connect-built-in) - [`disconnect` (Built-in)](#disconnect-built-in) + - [`configuration` (Server → Client)](#configuration-server--client) - [`connected` (Server → Client)](#connected-server--client) - [Subscription Events](#subscription-events) - [`subscribe` (Client → Server)](#subscribe-client--server) @@ -442,22 +443,84 @@ or an error: --- ### GET /api/history -**Purpose**: Returns the download queue and the download history. +**Purpose**: Returns the download queue and/or download history with optional pagination support. -**Response**: +**Query Parameters**: +- `type` (optional): Type of items to return. Default: `all` + - `all` (default): Returns both queue and history (legacy behavior, no pagination) + - `queue`: Returns only queue items (with pagination) + - `done`: Returns only history items (with pagination) +- `page` (optional): Page number (1-indexed). Default: `1`. Only used when `type != all` +- `per_page` (optional): Items per page. Default: `config.default_pagination`, Max: `200`. Only used when `type != all` +- `order` (optional): Sort order. Default: `DESC`. Only used when `type != all` + - `DESC`: Newest items first (descending by creation date) + - `ASC`: Oldest items first (ascending by creation date) + +**Response (when `type=all` or no type set)** - Legacy format: ```json { "queue": [ - { ... }, + { + "id": "abc123", + "url": "https://example.com/video", + "title": "Video Title", + "status": "downloading", + ... + }, ... ], "history": [ - { ... }, + { + "id": "def456", + "url": "https://example.com/video2", + "title": "Completed Video", + "status": "finished", + ... + }, ... ] } ``` +**Response (when `type=queue` or `type=done`)** - Paginated format: +```json +{ + "pagination": { + "page": 1, + "per_page": 50, + "total": 1234, + "total_pages": 25, + "has_next": true, + "has_prev": false + }, + "items": [ + { + "id": "abc123", + "url": "https://example.com/video", + "title": "Video Title", + "status": "finished", + ... + }, + ... + ] +} +``` + +**Error Responses**: +- `400 Bad Request` if parameters are invalid: + ```json + { "error": "type must be one of all, queue, done." } + { "error": "page must be >= 1." } + { "error": "per_page must be between 1 and 1000." } + { "error": "order must be ASC or DESC." } + { "error": "page and per_page must be valid integers." } + ``` + +**Notes**: +- The `type=all` behavior is considered legacy and will be removed in future versions +- For large datasets, use paginated requests (`type=queue` or `type=done`) for better performance +- The `items` array contains ItemDTO objects serialized to JSON + --- ### DELETE /api/history/{id}/archive @@ -1671,25 +1734,35 @@ Fired when WebSocket connection is closed. No data payload. socket.on('disconnect', (reason: string) => console.log('WebSocket disconnected:', reason)); ``` -##### `connected` (Server → Client) -Initial connection event with full application state. +##### `configuration` (Server → Client) +Sends the current application configuration. **Data Fields**: - `config`: Global configuration object -- `queue`: Current download queue (array of items) -- `done`: Download history (array of completed items) -- `tasks`: Scheduled tasks - `presets`: Available download presets - `dl_fields`: Available download fields -- `folders`: Directory structure for downloads - `paused`: Queue pause status (boolean) +**Example**: +```typescript +socket.on('connected', (data: string) => { + const json = JSON.parse(data); + console.log('Current configuration:', json.data.config); +}); +``` + +##### `connected` (Server → Client) +When a client connects, this events sends the folder and current queue. + +**Data Fields**: +- `queue`: Current download queue (array of items) +- `folders`: Directory structure for downloads + **Example**: ```typescript socket.on('connected', (data: string) => { const json = JSON.parse(data); const queueItems = json.data.queue || {}; - const historyItems = json.data.done || {}; console.log('Connected with', Object.keys(queueItems).length, 'queued downloads'); }); ``` diff --git a/FAQ.md b/FAQ.md index 5ba877b9..acb74aa0 100644 --- a/FAQ.md +++ b/FAQ.md @@ -48,6 +48,7 @@ or the `environment:` section in `compose.yaml` file. | YTP_SIMPLE_MODE | Switch default interface to Simple mode. | `false` | | YTP_STATIC_UI_PATH | Path to custom static UI files. | `(not_set)` | | YTP_AUTO_CLEAR_HISTORY_DAYS | Number of days after which completed download history is cleared. | `0` | +| YTP_DEFAULT_PAGINATION | The default number of items per page for history. | `50` | > [!NOTE] > To raise the maximum workers for specific extractor, you need to add a ENV variable that follows the pattern `YTP_MAX_WORKERS_FOR_`. diff --git a/app/library/DataStore.py b/app/library/DataStore.py index a306adf6..193425a1 100644 --- a/app/library/DataStore.py +++ b/app/library/DataStore.py @@ -179,6 +179,88 @@ class DataStore: self._connection.execute('SELECT "id" FROM "history" LIMIT 1').fetchone() return True + def get_total_count(self) -> int: + """ + Get the total count of items in the datastore. + + Returns: + int: The total number of items in the datastore. + + """ + cursor = self._connection.execute( + 'SELECT COUNT(*) as count FROM "history" WHERE "type" = ?', + (str(self._type),), + ) + row = cursor.fetchone() + return row["count"] if row else 0 + + def get_items_paginated( + self, + page: int = 1, + per_page: int = 50, + order: str = "DESC", + ) -> tuple[list[tuple[str, ItemDTO]], int, int, int]: + """ + Get paginated items from the datastore. + + Args: + page (int): The page number (1-indexed). Defaults to 1. + per_page (int): Number of items per page. Defaults to 50. + order (str): Sort order - 'ASC' or 'DESC'. Defaults to 'DESC' (newest first). + + Returns: + tuple[list[tuple[str, ItemDTO]], int, int, int]: A tuple containing: + - List of (id, ItemDTO) tuples for the requested page + - Total number of items + - Current page number + - Total number of pages + + Raises: + ValueError: If page < 1 or per_page < 1 + + """ + if page < 1: + msg = "page must be >= 1" + raise ValueError(msg) + + if per_page < 1: + msg = "per_page must be >= 1" + raise ValueError(msg) + + order = order.upper() + if order not in ("ASC", "DESC"): + msg = f"order must be 'ASC' or 'DESC', got '{order}'" + raise ValueError(msg) + + order = "ASC" if order == "ASC" else "DESC" + + total_items = self.get_total_count() + total_pages = (total_items + per_page - 1) // per_page if total_items > 0 else 1 + + # Ensure page is within valid range. + if page > total_pages and total_items > 0: + page = total_pages + + offset = (page - 1) * per_page + + items: list[tuple[str, ItemDTO]] = [] + + cursor = self._connection.execute( + f'SELECT "id", "data", "created_at" FROM "history" WHERE "type" = ? ORDER BY "created_at" {order} LIMIT ? OFFSET ?', # noqa: S608 + (str(self._type), per_page, offset), + ) + + for row in cursor: + rowDate: datetime = datetime.strptime(row["created_at"], "%Y-%m-%d %H:%M:%S") # noqa: DTZ007 + data: dict = json.loads(row["data"]) + data.pop("_id", None) + item: ItemDTO = init_class(ItemDTO, data) + item._id = row["id"] + item.datetime = formatdate(rowDate.replace(tzinfo=UTC).timestamp()) + items.append((row["id"], item)) + + return items, total_items, page, total_pages + def _update_store_item(self, type: StoreType, item: ItemDTO) -> None: sqlStatement = """ INSERT INTO "history" ("id", "type", "url", "data") diff --git a/app/library/config.py b/app/library/config.py index 05abc8f2..ad78464c 100644 --- a/app/library/config.py +++ b/app/library/config.py @@ -195,6 +195,9 @@ class Config(metaclass=Singleton): auto_clear_history_days: int = 0 """Number of days after which completed download history is automatically cleared. 0 to disable.""" + default_pagination: int = 50 + """The default number of items per page for pagination.""" + pictures_backends: list[str] = [ "https://unsplash.it/1920/1080?random", "https://picsum.photos/1920/1080", @@ -234,6 +237,7 @@ class Config(metaclass=Singleton): "download_path_depth", "download_info_expires", "auto_clear_history_days", + "default_pagination", ) "The variables that are integers." @@ -281,6 +285,7 @@ class Config(metaclass=Singleton): "app_commit_sha", "app_build_date", "app_branch", + "default_pagination", ) "The variables that are relevant to the frontend." diff --git a/app/routes/api/_static.py b/app/routes/api/_static.py index f00faa1a..234b2bc9 100644 --- a/app/routes/api/_static.py +++ b/app/routes/api/_static.py @@ -19,6 +19,7 @@ EXT_TO_MIME: dict = { ".js": "application/javascript", ".json": "application/json", ".ico": "image/x-icon", + ".webmanifest": "application/manifest+json", } FRONTEND_ROUTES: list[str] = [ @@ -35,7 +36,6 @@ FRONTEND_ROUTES: list[str] = [ "/browser/{path:.*}", ] - async def serve_static_file(request: Request, config: Config) -> Response: """ Preload static files from the ui/exported folder. diff --git a/app/routes/api/history.py b/app/routes/api/history.py index 00ce125f..a1920ecf 100644 --- a/app/routes/api/history.py +++ b/app/routes/api/history.py @@ -5,6 +5,7 @@ from typing import TYPE_CHECKING from aiohttp import web from aiohttp.web import Request, Response +from app.library.config import Config from app.library.Download import Download from app.library.DownloadQueue import DownloadQueue from app.library.encoder import Encoder @@ -20,23 +21,85 @@ LOG: logging.Logger = logging.getLogger(__name__) @route("GET", r"api/history/", "items_list") -async def items_list(queue: DownloadQueue, encoder: Encoder) -> Response: +async def items_list(request: Request, queue: DownloadQueue, encoder: Encoder, config: Config) -> Response: """ - Get the history. + Get the history with optional pagination support. Args: + request (Request): The request object. queue (DownloadQueue): The download queue instance. encoder (Encoder): The encoder instance. + config (Config): The configuration instance. Returns: Response: The response object. - """ - data: dict = {"queue": [], "history": []} - q = queue.get() + Query Parameters: + type (str): Type of items to return - "all", "queue", or "done". Default: "all" + page (int): Page number for pagination (1-indexed). Only used when type != "all" + per_page (int): Items per page. Default: 50, Max: 1000. Only used when type != "all" + order (str): Sort order - "ASC" or "DESC". Default: "DESC". Only used when type != "all" - data["queue"].extend([q.get("queue", {}).get(k) for k in q.get("queue", {})]) - data["history"].extend([q.get("done", {}).get(k) for k in q.get("done", {})]) + """ + from app.library.DataStore import StoreType + + store_type = request.query.get("type", "all").lower() + stores: list[str] = ["all", StoreType.QUEUE.value, StoreType.HISTORY.value] + if store_type not in stores: + return web.json_response( + data={"error": f"type must be one of {', '.join(stores)}."}, + status=web.HTTPBadRequest.status_code, + ) + + # Legacy behavior: return all items without pagination, will be removed in future. + if "all" == store_type: + data: dict = {"queue": [], "history": []} + q = queue.get() + + data["queue"].extend([q.get("queue", {}).get(k) for k in q.get("queue", {})]) + data["history"].extend([q.get("done", {}).get(k) for k in q.get("done", {})]) + + return web.json_response(data=data, status=web.HTTPOk.status_code, dumps=encoder.encode) + + ds = queue.queue if store_type == StoreType.QUEUE.value else queue.done + + try: + page = int(request.query.get("page", 1)) + per_page = int(request.query.get("per_page", config.default_pagination)) + order = request.query.get("order", "DESC").upper() + except ValueError: + return web.json_response( + data={"error": "page and per_page must be valid integers."}, + status=web.HTTPBadRequest.status_code, + ) + + if page < 1: + return web.json_response(data={"error": "page must be >= 1."}, status=web.HTTPBadRequest.status_code) + + if per_page < 1 or per_page > 1000: + return web.json_response( + data={"error": "per_page must be between 1 and 1000."}, + status=web.HTTPBadRequest.status_code, + ) + + if order not in ("ASC", "DESC"): + return web.json_response( + data={"error": "order must be ASC or DESC."}, + status=web.HTTPBadRequest.status_code, + ) + + items, total, current_page, total_pages = ds.get_items_paginated(page=page, per_page=per_page, order=order) + data = { + "pagination": { + "page": current_page, + "per_page": per_page, + "total": total, + "total_pages": total_pages, + "has_next": current_page < total_pages, + "has_prev": current_page > 1, + }, + "items": [item for _, item in items], + } return web.json_response(data=data, status=web.HTTPOk.status_code, dumps=encoder.encode) diff --git a/app/routes/socket/connection.py b/app/routes/socket/connection.py index 3364386d..d7a51c61 100644 --- a/app/routes/socket/connection.py +++ b/app/routes/socket/connection.py @@ -44,7 +44,8 @@ async def connect(config: Config, queue: DownloadQueue, notify: EventBus, sid: s base=Path(config.download_path), depth_limit=config.download_path_depth - 1, ), - **queue.get(), + "history_count": queue.done.get_total_count(), + **queue.get()["queue"], }, title="Sending initial download data", message=f"Sending initial download data to client '{sid}'.", diff --git a/app/tests/test_datastore_pagination.py b/app/tests/test_datastore_pagination.py new file mode 100644 index 00000000..f44c7b89 --- /dev/null +++ b/app/tests/test_datastore_pagination.py @@ -0,0 +1,252 @@ +import json +import sqlite3 +from datetime import UTC, datetime +from pathlib import Path +from tempfile import TemporaryDirectory + +import pytest + +from app.library.DataStore import DataStore, StoreType +from app.library.ItemDTO import ItemDTO + + +class TestDataStorePagination: + """Test pagination functionality of DataStore.""" + + @pytest.fixture + def temp_db(self): + """Create a temporary database with test data.""" + with TemporaryDirectory() as tmpdir: + db_path = Path(tmpdir) / "test.db" + conn = sqlite3.connect(str(db_path)) + conn.row_factory = sqlite3.Row + + # Create the history table + conn.execute( + """ + CREATE TABLE IF NOT EXISTS "history" ( + "id" TEXT PRIMARY KEY, + "type" TEXT NOT NULL, + "url" TEXT NOT NULL, + "data" TEXT NOT NULL, + "created_at" TEXT NOT NULL + ) + """ + ) + + # Insert test data + base_time = datetime.now(UTC) + for i in range(100): + created_at = base_time.replace( + hour=(i // 4) % 24, minute=(i * 15) % 60, second=i % 60 + ).strftime("%Y-%m-%d %H:%M:%S") + + item_data = { + "url": f"https://example.com/video{i}", + "title": f"Test Video {i}", + "id": f"video{i}", + "folder": "/downloads", + "status": "finished", + } + + conn.execute( + 'INSERT INTO "history" ("id", "type", "url", "data", "created_at") VALUES (?, ?, ?, ?, ?)', + ( + f"test-id-{i}", + str(StoreType.HISTORY), + item_data["url"], + json.dumps(item_data), + created_at, + ), + ) + + conn.commit() + yield conn + conn.close() + + def test_get_total_count(self, temp_db): + """Test getting total count of items.""" + datastore = DataStore(type=StoreType.HISTORY, connection=temp_db) + count = datastore.get_total_count() + assert count == 100 + + def test_pagination_basic(self, temp_db): + """Test basic pagination functionality.""" + datastore = DataStore(type=StoreType.HISTORY, connection=temp_db) + + # Get first page + items, total, page, total_pages = datastore.get_items_paginated(page=1, per_page=10) + + assert len(items) == 10 + assert total == 100 + assert page == 1 + assert total_pages == 10 + + # Verify items are ItemDTO instances + for _item_id, item in items: + assert isinstance(item, ItemDTO) + assert item._id.startswith("test-id-") + + def test_pagination_last_page(self, temp_db): + """Test pagination on last page.""" + datastore = DataStore(type=StoreType.HISTORY, connection=temp_db) + + # Get last page + items, total, page, total_pages = datastore.get_items_paginated(page=10, per_page=10) + + assert len(items) == 10 + assert total == 100 + assert page == 10 + assert total_pages == 10 + + def test_pagination_partial_page(self, temp_db): + """Test pagination with partial last page.""" + datastore = DataStore(type=StoreType.HISTORY, connection=temp_db) + + # Get items with per_page that doesn't divide evenly + items, total, page, total_pages = datastore.get_items_paginated(page=4, per_page=30) + + assert len(items) == 10 # 100 items / 30 per page = 3 full pages + 10 items + assert total == 100 + assert page == 4 + assert total_pages == 4 + + def test_pagination_out_of_range(self, temp_db): + """Test pagination with page number out of range.""" + datastore = DataStore(type=StoreType.HISTORY, connection=temp_db) + + # Request page beyond total pages - should return last page + items, total, page, total_pages = datastore.get_items_paginated(page=999, per_page=10) + + assert len(items) == 10 + assert total == 100 + assert page == 10 # Adjusted to last page + assert total_pages == 10 + + def test_pagination_order_desc(self, temp_db): + """Test pagination with descending order (newest first).""" + datastore = DataStore(type=StoreType.HISTORY, connection=temp_db) + + items, _, _, _ = datastore.get_items_paginated(page=1, per_page=5, order="DESC") + + assert len(items) == 5 + # Verify order - should be newest to oldest + # (note: exact order depends on the timestamp generation in fixture) + + def test_pagination_order_asc(self, temp_db): + """Test pagination with ascending order (oldest first).""" + datastore = DataStore(type=StoreType.HISTORY, connection=temp_db) + + items, _, _, _ = datastore.get_items_paginated(page=1, per_page=5, order="ASC") + + assert len(items) == 5 + + def test_pagination_invalid_page(self, temp_db): + """Test pagination with invalid page number.""" + datastore = DataStore(type=StoreType.HISTORY, connection=temp_db) + + with pytest.raises(ValueError, match="page must be >= 1"): + datastore.get_items_paginated(page=0, per_page=10) + + with pytest.raises(ValueError, match="page must be >= 1"): + datastore.get_items_paginated(page=-1, per_page=10) + + def test_pagination_invalid_per_page(self, temp_db): + """Test pagination with invalid per_page value.""" + datastore = DataStore(type=StoreType.HISTORY, connection=temp_db) + + with pytest.raises(ValueError, match="per_page must be >= 1"): + datastore.get_items_paginated(page=1, per_page=0) + + with pytest.raises(ValueError, match="per_page must be >= 1"): + datastore.get_items_paginated(page=1, per_page=-10) + + def test_pagination_invalid_order(self, temp_db): + """Test pagination with invalid order parameter.""" + datastore = DataStore(type=StoreType.HISTORY, connection=temp_db) + + with pytest.raises(ValueError, match="order must be 'ASC' or 'DESC'"): + datastore.get_items_paginated(page=1, per_page=10, order="INVALID") + + def test_pagination_empty_store(self): + """Test pagination with empty datastore.""" + with TemporaryDirectory() as tmpdir: + db_path = Path(tmpdir) / "empty.db" + conn = sqlite3.connect(str(db_path)) + conn.row_factory = sqlite3.Row + + # Create empty table + conn.execute( + """ + CREATE TABLE IF NOT EXISTS "history" ( + "id" TEXT PRIMARY KEY, + "type" TEXT NOT NULL, + "url" TEXT NOT NULL, + "data" TEXT NOT NULL, + "created_at" TEXT NOT NULL + ) + """ + ) + conn.commit() + + datastore = DataStore(type=StoreType.HISTORY, connection=conn) + + items, total, page, total_pages = datastore.get_items_paginated(page=1, per_page=10) + + assert len(items) == 0 + assert total == 0 + assert page == 1 + assert total_pages == 1 + + conn.close() + + def test_pagination_single_item(self): + """Test pagination with single item.""" + with TemporaryDirectory() as tmpdir: + db_path = Path(tmpdir) / "single.db" + conn = sqlite3.connect(str(db_path)) + conn.row_factory = sqlite3.Row + + # Create table with single item + conn.execute( + """ + CREATE TABLE IF NOT EXISTS "history" ( + "id" TEXT PRIMARY KEY, + "type" TEXT NOT NULL, + "url" TEXT NOT NULL, + "data" TEXT NOT NULL, + "created_at" TEXT NOT NULL + ) + """ + ) + + item_data = { + "url": "https://example.com/single", + "title": "Single Video", + "id": "single-video", + "folder": "/downloads", + "status": "finished", + } + + conn.execute( + 'INSERT INTO "history" ("id", "type", "url", "data", "created_at") VALUES (?, ?, ?, ?, ?)', + ( + "single-id", + str(StoreType.HISTORY), + item_data["url"], + json.dumps(item_data), + datetime.now(UTC).strftime("%Y-%m-%d %H:%M:%S"), + ), + ) + conn.commit() + + datastore = DataStore(type=StoreType.HISTORY, connection=conn) + + items, total, page, total_pages = datastore.get_items_paginated(page=1, per_page=10) + + assert len(items) == 1 + assert total == 1 + assert page == 1 + assert total_pages == 1 + + conn.close() diff --git a/ui/app/components/FloatingImage.vue b/ui/app/components/FloatingImage.vue index 6db9b82e..bd311c45 100644 --- a/ui/app/components/FloatingImage.vue +++ b/ui/app/components/FloatingImage.vue @@ -2,19 +2,26 @@ @@ -36,6 +43,7 @@ const thumbnail_ratio = useStorage<'is-16by9' | 'is-3by1'>('thumbnail_ratio', 'i const url = ref(null) const error = ref(false) const isPreloading = ref(false) +const error_msg = ref(null) const loadTimer: ReturnType | null = null const cancelRequest = new AbortController() @@ -54,7 +62,7 @@ const defaultLoader = async (): Promise => { const response = await request(props.image, { signal: cancelRequest.signal }) if (!response.ok) { - toast.error(`ImageView Request error. ${response.status}: ${response.statusText}`) + error_msg.value = `ImageView Request error. ${response.status}: ${response.statusText}` return } diff --git a/ui/app/components/History.vue b/ui/app/components/History.vue index 6ebe6517..0831f571 100644 --- a/ui/app/components/History.vue +++ b/ui/app/components/History.vue @@ -5,7 +5,8 @@ - History ({{ stateStore.count('history') }}) + History + ({{ stateStore.count('history') }})  - Selected: {{ selectedElms.length }} @@ -77,6 +78,34 @@ +
+ +
+ + +
+
+
+
+ +