From 1107567f8672681c9e3c9adaca7b7913a1bbbb76 Mon Sep 17 00:00:00 2001 From: arabcoders Date: Thu, 13 Nov 2025 17:15:05 +0300 Subject: [PATCH] Feat: add paginated history endpoint --- API.md | 70 ++++++- app/library/DataStore.py | 82 ++++++++ app/routes/api/history.py | 75 +++++++- app/tests/test_datastore_pagination.py | 252 +++++++++++++++++++++++++ 4 files changed, 468 insertions(+), 11 deletions(-) create mode 100644 app/tests/test_datastore_pagination.py diff --git a/API.md b/API.md index 302013ec..d9f8053a 100644 --- a/API.md +++ b/API.md @@ -442,22 +442,84 @@ or an error: --- ### GET /api/history -**Purpose**: Returns the download queue and the download history. +**Purpose**: Returns the download queue and/or download history with optional pagination support. -**Response**: +**Query Parameters**: +- `type` (optional): Type of items to return. Default: `all` + - `all` (default): Returns both queue and history (legacy behavior, no pagination) + - `queue`: Returns only queue items (with pagination) + - `done`: Returns only history items (with pagination) +- `page` (optional): Page number (1-indexed). Default: `1`. Only used when `type != all` +- `per_page` (optional): Items per page. Default: `50`, Max: `200`. Only used when `type != all` +- `order` (optional): Sort order. Default: `DESC`. Only used when `type != all` + - `DESC`: Newest items first (descending by creation date) + - `ASC`: Oldest items first (ascending by creation date) + +**Response (when `type=all` or no type set)** - Legacy format: ```json { "queue": [ - { ... }, + { + "id": "abc123", + "url": "https://example.com/video", + "title": "Video Title", + "status": "downloading", + ... + }, ... ], "history": [ - { ... }, + { + "id": "def456", + "url": "https://example.com/video2", + "title": "Completed Video", + "status": "finished", + ... + }, ... ] } ``` +**Response (when `type=queue` or `type=done`)** - Paginated format: +```json +{ + "pagination": { + "page": 1, + "per_page": 50, + "total": 1234, + "total_pages": 25, + "has_next": true, + "has_prev": false + }, + "items": [ + { + "id": "abc123", + "url": "https://example.com/video", + "title": "Video Title", + "status": "finished", + ... + }, + ... + ] +} +``` + +**Error Responses**: +- `400 Bad Request` if parameters are invalid: + ```json + { "error": "type must be one of all, queue, done." } + { "error": "page must be >= 1." } + { "error": "per_page must be between 1 and 1000." } + { "error": "order must be ASC or DESC." } + { "error": "page and per_page must be valid integers." } + ``` + +**Notes**: +- The `type=all` behavior is considered legacy and will be removed in future versions +- For large datasets, use paginated requests (`type=queue` or `type=done`) for better performance +- The `items` array contains ItemDTO objects serialized to JSON + --- ### DELETE /api/history/{id}/archive diff --git a/app/library/DataStore.py b/app/library/DataStore.py index a306adf6..193425a1 100644 --- a/app/library/DataStore.py +++ b/app/library/DataStore.py @@ -179,6 +179,88 @@ class DataStore: self._connection.execute('SELECT "id" FROM "history" LIMIT 1').fetchone() return True + def get_total_count(self) -> int: + """ + Get the total count of items in the datastore. + + Returns: + int: The total number of items in the datastore. + + """ + cursor = self._connection.execute( + 'SELECT COUNT(*) as count FROM "history" WHERE "type" = ?', + (str(self._type),), + ) + row = cursor.fetchone() + return row["count"] if row else 0 + + def get_items_paginated( + self, + page: int = 1, + per_page: int = 50, + order: str = "DESC", + ) -> tuple[list[tuple[str, ItemDTO]], int, int, int]: + """ + Get paginated items from the datastore. + + Args: + page (int): The page number (1-indexed). Defaults to 1. + per_page (int): Number of items per page. Defaults to 50. + order (str): Sort order - 'ASC' or 'DESC'. Defaults to 'DESC' (newest first). + + Returns: + tuple[list[tuple[str, ItemDTO]], int, int, int]: A tuple containing: + - List of (id, ItemDTO) tuples for the requested page + - Total number of items + - Current page number + - Total number of pages + + Raises: + ValueError: If page < 1 or per_page < 1 + + """ + if page < 1: + msg = "page must be >= 1" + raise ValueError(msg) + + if per_page < 1: + msg = "per_page must be >= 1" + raise ValueError(msg) + + order = order.upper() + if order not in ("ASC", "DESC"): + msg = f"order must be 'ASC' or 'DESC', got '{order}'" + raise ValueError(msg) + + order = "ASC" if order == "ASC" else "DESC" + + total_items = self.get_total_count() + total_pages = (total_items + per_page - 1) // per_page if total_items > 0 else 1 + + # Ensure page is within valid range. + if page > total_pages and total_items > 0: + page = total_pages + + offset = (page - 1) * per_page + + items: list[tuple[str, ItemDTO]] = [] + + cursor = self._connection.execute( + f'SELECT "id", "data", "created_at" FROM "history" WHERE "type" = ? ORDER BY "created_at" {order} LIMIT ? OFFSET ?', # noqa: S608 + (str(self._type), per_page, offset), + ) + + for row in cursor: + rowDate: datetime = datetime.strptime(row["created_at"], "%Y-%m-%d %H:%M:%S") # noqa: DTZ007 + data: dict = json.loads(row["data"]) + data.pop("_id", None) + item: ItemDTO = init_class(ItemDTO, data) + item._id = row["id"] + item.datetime = formatdate(rowDate.replace(tzinfo=UTC).timestamp()) + items.append((row["id"], item)) + + return items, total_items, page, total_pages + def _update_store_item(self, type: StoreType, item: ItemDTO) -> None: sqlStatement = """ INSERT INTO "history" ("id", "type", "url", "data") diff --git a/app/routes/api/history.py b/app/routes/api/history.py index 00ce125f..9e162823 100644 --- a/app/routes/api/history.py +++ b/app/routes/api/history.py @@ -20,23 +20,84 @@ LOG: logging.Logger = logging.getLogger(__name__) @route("GET", r"api/history/", "items_list") -async def items_list(queue: DownloadQueue, encoder: Encoder) -> Response: +async def items_list(request: Request, queue: DownloadQueue, encoder: Encoder) -> Response: """ - Get the history. + Get the history with optional pagination support. Args: + request (Request): The request object. queue (DownloadQueue): The download queue instance. encoder (Encoder): The encoder instance. Returns: Response: The response object. - """ - data: dict = {"queue": [], "history": []} - q = queue.get() + Query Parameters: + type (str): Type of items to return - "all", "queue", or "done". Default: "all" + page (int): Page number for pagination (1-indexed). Only used when type != "all" + per_page (int): Items per page. Default: 50, Max: 1000. Only used when type != "all" + order (str): Sort order - "ASC" or "DESC". Default: "DESC". Only used when type != "all" - data["queue"].extend([q.get("queue", {}).get(k) for k in q.get("queue", {})]) - data["history"].extend([q.get("done", {}).get(k) for k in q.get("done", {})]) + """ + from app.library.DataStore import StoreType + + store_type = request.query.get("type", "all").lower() + stores: list[str] = ["all", StoreType.QUEUE.value, StoreType.HISTORY.value] + if store_type not in stores: + return web.json_response( + data={"error": f"type must be one of {', '.join(stores)}."}, + status=web.HTTPBadRequest.status_code, + ) + + # Legacy behavior: return all items without pagination, will be removed in future. + if "all" == store_type: + data: dict = {"queue": [], "history": []} + q = queue.get() + + data["queue"].extend([q.get("queue", {}).get(k) for k in q.get("queue", {})]) + data["history"].extend([q.get("done", {}).get(k) for k in q.get("done", {})]) + + return web.json_response(data=data, status=web.HTTPOk.status_code, dumps=encoder.encode) + + ds = queue.queue if store_type == StoreType.QUEUE.value else queue.done + + try: + page = int(request.query.get("page", 1)) + per_page = int(request.query.get("per_page", 50)) + order = request.query.get("order", "DESC").upper() + except ValueError: + return web.json_response( + data={"error": "page and per_page must be valid integers."}, + status=web.HTTPBadRequest.status_code, + ) + + if page < 1: + return web.json_response(data={"error": "page must be >= 1."}, status=web.HTTPBadRequest.status_code) + + if per_page < 1 or per_page > 200: + return web.json_response( + data={"error": "per_page must be between 1 and 1000."}, + status=web.HTTPBadRequest.status_code, + ) + + if order not in ("ASC", "DESC"): + return web.json_response( + data={"error": "order must be ASC or DESC."}, + status=web.HTTPBadRequest.status_code, + ) + + items, total, current_page, total_pages = ds.get_items_paginated(page=page, per_page=per_page, order=order) + data = { + "pagination": { + "page": current_page, + "per_page": per_page, + "total": total, + "total_pages": total_pages, + "has_next": current_page < total_pages, + "has_prev": current_page > 1, + }, + "items": [item for _, item in items], + } return web.json_response(data=data, status=web.HTTPOk.status_code, dumps=encoder.encode) diff --git a/app/tests/test_datastore_pagination.py b/app/tests/test_datastore_pagination.py new file mode 100644 index 00000000..f44c7b89 --- /dev/null +++ b/app/tests/test_datastore_pagination.py @@ -0,0 +1,252 @@ +import json +import sqlite3 +from datetime import UTC, datetime +from pathlib import Path +from tempfile import TemporaryDirectory + +import pytest + +from app.library.DataStore import DataStore, StoreType +from app.library.ItemDTO import ItemDTO + + +class TestDataStorePagination: + """Test pagination functionality of DataStore.""" + + @pytest.fixture + def temp_db(self): + """Create a temporary database with test data.""" + with TemporaryDirectory() as tmpdir: + db_path = Path(tmpdir) / "test.db" + conn = sqlite3.connect(str(db_path)) + conn.row_factory = sqlite3.Row + + # Create the history table + conn.execute( + """ + CREATE TABLE IF NOT EXISTS "history" ( + "id" TEXT PRIMARY KEY, + "type" TEXT NOT NULL, + "url" TEXT NOT NULL, + "data" TEXT NOT NULL, + "created_at" TEXT NOT NULL + ) + """ + ) + + # Insert test data + base_time = datetime.now(UTC) + for i in range(100): + created_at = base_time.replace( + hour=(i // 4) % 24, minute=(i * 15) % 60, second=i % 60 + ).strftime("%Y-%m-%d %H:%M:%S") + + item_data = { + "url": f"https://example.com/video{i}", + "title": f"Test Video {i}", + "id": f"video{i}", + "folder": "/downloads", + "status": "finished", + } + + conn.execute( + 'INSERT INTO "history" ("id", "type", "url", "data", "created_at") VALUES (?, ?, ?, ?, ?)', + ( + f"test-id-{i}", + str(StoreType.HISTORY), + item_data["url"], + json.dumps(item_data), + created_at, + ), + ) + + conn.commit() + yield conn + conn.close() + + def test_get_total_count(self, temp_db): + """Test getting total count of items.""" + datastore = DataStore(type=StoreType.HISTORY, connection=temp_db) + count = datastore.get_total_count() + assert count == 100 + + def test_pagination_basic(self, temp_db): + """Test basic pagination functionality.""" + datastore = DataStore(type=StoreType.HISTORY, connection=temp_db) + + # Get first page + items, total, page, total_pages = datastore.get_items_paginated(page=1, per_page=10) + + assert len(items) == 10 + assert total == 100 + assert page == 1 + assert total_pages == 10 + + # Verify items are ItemDTO instances + for _item_id, item in items: + assert isinstance(item, ItemDTO) + assert item._id.startswith("test-id-") + + def test_pagination_last_page(self, temp_db): + """Test pagination on last page.""" + datastore = DataStore(type=StoreType.HISTORY, connection=temp_db) + + # Get last page + items, total, page, total_pages = datastore.get_items_paginated(page=10, per_page=10) + + assert len(items) == 10 + assert total == 100 + assert page == 10 + assert total_pages == 10 + + def test_pagination_partial_page(self, temp_db): + """Test pagination with partial last page.""" + datastore = DataStore(type=StoreType.HISTORY, connection=temp_db) + + # Get items with per_page that doesn't divide evenly + items, total, page, total_pages = datastore.get_items_paginated(page=4, per_page=30) + + assert len(items) == 10 # 100 items / 30 per page = 3 full pages + 10 items + assert total == 100 + assert page == 4 + assert total_pages == 4 + + def test_pagination_out_of_range(self, temp_db): + """Test pagination with page number out of range.""" + datastore = DataStore(type=StoreType.HISTORY, connection=temp_db) + + # Request page beyond total pages - should return last page + items, total, page, total_pages = datastore.get_items_paginated(page=999, per_page=10) + + assert len(items) == 10 + assert total == 100 + assert page == 10 # Adjusted to last page + assert total_pages == 10 + + def test_pagination_order_desc(self, temp_db): + """Test pagination with descending order (newest first).""" + datastore = DataStore(type=StoreType.HISTORY, connection=temp_db) + + items, _, _, _ = datastore.get_items_paginated(page=1, per_page=5, order="DESC") + + assert len(items) == 5 + # Verify order - should be newest to oldest + # (note: exact order depends on the timestamp generation in fixture) + + def test_pagination_order_asc(self, temp_db): + """Test pagination with ascending order (oldest first).""" + datastore = DataStore(type=StoreType.HISTORY, connection=temp_db) + + items, _, _, _ = datastore.get_items_paginated(page=1, per_page=5, order="ASC") + + assert len(items) == 5 + + def test_pagination_invalid_page(self, temp_db): + """Test pagination with invalid page number.""" + datastore = DataStore(type=StoreType.HISTORY, connection=temp_db) + + with pytest.raises(ValueError, match="page must be >= 1"): + datastore.get_items_paginated(page=0, per_page=10) + + with pytest.raises(ValueError, match="page must be >= 1"): + datastore.get_items_paginated(page=-1, per_page=10) + + def test_pagination_invalid_per_page(self, temp_db): + """Test pagination with invalid per_page value.""" + datastore = DataStore(type=StoreType.HISTORY, connection=temp_db) + + with pytest.raises(ValueError, match="per_page must be >= 1"): + datastore.get_items_paginated(page=1, per_page=0) + + with pytest.raises(ValueError, match="per_page must be >= 1"): + datastore.get_items_paginated(page=1, per_page=-10) + + def test_pagination_invalid_order(self, temp_db): + """Test pagination with invalid order parameter.""" + datastore = DataStore(type=StoreType.HISTORY, connection=temp_db) + + with pytest.raises(ValueError, match="order must be 'ASC' or 'DESC'"): + datastore.get_items_paginated(page=1, per_page=10, order="INVALID") + + def test_pagination_empty_store(self): + """Test pagination with empty datastore.""" + with TemporaryDirectory() as tmpdir: + db_path = Path(tmpdir) / "empty.db" + conn = sqlite3.connect(str(db_path)) + conn.row_factory = sqlite3.Row + + # Create empty table + conn.execute( + """ + CREATE TABLE IF NOT EXISTS "history" ( + "id" TEXT PRIMARY KEY, + "type" TEXT NOT NULL, + "url" TEXT NOT NULL, + "data" TEXT NOT NULL, + "created_at" TEXT NOT NULL + ) + """ + ) + conn.commit() + + datastore = DataStore(type=StoreType.HISTORY, connection=conn) + + items, total, page, total_pages = datastore.get_items_paginated(page=1, per_page=10) + + assert len(items) == 0 + assert total == 0 + assert page == 1 + assert total_pages == 1 + + conn.close() + + def test_pagination_single_item(self): + """Test pagination with single item.""" + with TemporaryDirectory() as tmpdir: + db_path = Path(tmpdir) / "single.db" + conn = sqlite3.connect(str(db_path)) + conn.row_factory = sqlite3.Row + + # Create table with single item + conn.execute( + """ + CREATE TABLE IF NOT EXISTS "history" ( + "id" TEXT PRIMARY KEY, + "type" TEXT NOT NULL, + "url" TEXT NOT NULL, + "data" TEXT NOT NULL, + "created_at" TEXT NOT NULL + ) + """ + ) + + item_data = { + "url": "https://example.com/single", + "title": "Single Video", + "id": "single-video", + "folder": "/downloads", + "status": "finished", + } + + conn.execute( + 'INSERT INTO "history" ("id", "type", "url", "data", "created_at") VALUES (?, ?, ?, ?, ?)', + ( + "single-id", + str(StoreType.HISTORY), + item_data["url"], + json.dumps(item_data), + datetime.now(UTC).strftime("%Y-%m-%d %H:%M:%S"), + ), + ) + conn.commit() + + datastore = DataStore(type=StoreType.HISTORY, connection=conn) + + items, total, page, total_pages = datastore.get_items_paginated(page=1, per_page=10) + + assert len(items) == 1 + assert total == 1 + assert page == 1 + assert total_pages == 1 + + conn.close()