diff --git a/FAQ.md b/FAQ.md index 72e31a93..912b2c6b 100644 --- a/FAQ.md +++ b/FAQ.md @@ -52,6 +52,7 @@ or the `environment:` section in `compose.yaml` file. | YTP_SIMPLE_MODE | Switch default interface to Simple mode. | `false` | | YTP_STATIC_UI_PATH | Path to custom static UI files. | `(not_set)` | | YTP_AUTO_CLEAR_HISTORY_DAYS | Number of days after which completed download history is cleared. | `0` | +| YTP_MAX_HISTORY | Maximum number of history items to keep. `0` means unlimited. | `0` | | YTP_DEFAULT_PAGINATION | The default number of items per page for history. | `50` | | YTP_TASK_HANDLER_RANDOM_DELAY | The maximum random delay in seconds before starting a task handler. | `60` | | YTP_IGNORE_ARCHIVED_ITEMS | Don't report archived items in the download history. | `false` | @@ -71,6 +72,13 @@ or the `environment:` section in `compose.yaml` file. - `0` days means no automatic clearing of the download history. lowest value that will trigger the clearing is `1` day. - This setting will **NOT** delete the downloaded files, it will only clear the history from the database. +## Notes about YTP_MAX_HISTORY + +- `0` means no limit on the number of history items kept. +- When set to a positive value (e.g. `YTP_MAX_HISTORY=100`), the oldest history items will be automatically deleted to keep at most that many items. +- The check runs every 5 minutes. +- This setting will **NOT** delete the downloaded files, it will only remove the records from the database. + # Browser extensions & bookmarklets ## Simple bookmarklet diff --git a/app/library/config.py b/app/library/config.py index b153bcf7..4048e8c3 100644 --- a/app/library/config.py +++ b/app/library/config.py @@ -210,6 +210,9 @@ class Config(metaclass=Singleton): auto_clear_history_days: int = 0 """Number of days after which completed download history is automatically cleared. 0 to disable.""" + max_history: int = 0 + """Maximum number of history items to keep. 0 means unlimited.""" + default_pagination: int = 50 """The default number of items per page for pagination.""" @@ -268,6 +271,7 @@ class Config(metaclass=Singleton): "download_path_depth", "download_info_expires", "auto_clear_history_days", + "max_history", "default_pagination", "extract_info_concurrency", "flaresolverr_max_timeout", diff --git a/app/library/downloads/monitors.py b/app/library/downloads/monitors.py index 3938e580..b437721c 100644 --- a/app/library/downloads/monitors.py +++ b/app/library/downloads/monitors.py @@ -171,3 +171,26 @@ async def delete_old_history(queue: "DownloadQueue") -> None: if titles: LOG.info(f"Automatically cleared '{', '.join(titles)}' from download history due to age.") + + +async def enforce_max_history(queue: "DownloadQueue") -> None: + """ + Enforce the maximum number of history items by deleting the oldest excess items. + + When config.max_history is set to a positive value, this function trims the + history store so that only the most recent max_history items are kept. + + Args: + queue: DownloadQueue instance + + """ + if queue.config.max_history < 1 or queue.is_paused(): + return + + deleted_ids: list[str] = await queue.done._connection.trim_history("done", queue.config.max_history) + + for _id in deleted_ids: + queue.done._dict.pop(_id, None) + + if deleted_ids: + LOG.info(f"Trimmed '{len(deleted_ids)}' oldest items from history to enforce max_history={queue.config.max_history}.") diff --git a/app/library/downloads/queue_manager.py b/app/library/downloads/queue_manager.py index d68cc458..e406b187 100644 --- a/app/library/downloads/queue_manager.py +++ b/app/library/downloads/queue_manager.py @@ -18,7 +18,7 @@ from app.library.Utils import calc_download_path from .core import Download from .item_adder import add as add_impl -from .monitors import check_for_stale, check_live, delete_old_history +from .monitors import check_for_stale, check_live, delete_old_history, enforce_max_history from .pool_manager import PoolManager if TYPE_CHECKING: @@ -74,6 +74,13 @@ class DownloadQueue(metaclass=Singleton): id=delete_old_history.__name__, ) + if self.config.max_history > 0: + Scheduler.get_instance().add( + timer="*/5 * * * *", + func=functools.partial(enforce_max_history, self), + id=enforce_max_history.__name__, + ) + # app.on_shutdown.append(self.on_shutdown) async def test(self) -> bool: diff --git a/app/library/sqlite_store.py b/app/library/sqlite_store.py index 47ad0d90..72f1538c 100644 --- a/app/library/sqlite_store.py +++ b/app/library/sqlite_store.py @@ -296,6 +296,43 @@ class SqliteStore(metaclass=ThreadSafe): return row["count"] if row else 0 + async def trim_history(self, type_value: str, max_items: int) -> list[str]: + """ + Delete oldest history items exceeding the max_items limit. + + Args: + type_value: The history type to trim (e.g., "done"). + max_items: Maximum number of items to keep. Must be > 0. + + Returns: + list[str]: List of deleted item IDs. + + """ + if max_items < 1: + return [] + + await self.get_connection() + + total = await self.count(type_value) + if total <= max_items: + return [] + + excess = total - max_items + # Select the oldest excess items + result = await self._conn.execute( + text( + 'SELECT "id" FROM "history" WHERE "type" = :type_value ORDER BY "created_at" ASC LIMIT :limit' + ), + {"type_value": type_value, "limit": excess}, + ) + rows = result.mappings().all() + ids_to_delete = [row["id"] for row in rows] + + if ids_to_delete: + await self.bulk_delete(type_value, ids_to_delete) + + return ids_to_delete + async def paginate( self, type_value: str, diff --git a/app/tests/test_sqlite_store.py b/app/tests/test_sqlite_store.py index 3a52c3d8..35bace44 100644 --- a/app/tests/test_sqlite_store.py +++ b/app/tests/test_sqlite_store.py @@ -308,3 +308,78 @@ async def test_exists_and_get_raise_without_key_or_url(): with pytest.raises(KeyError): await store.get("queue") await store.close() + + +@pytest.mark.asyncio +async def test_trim_history_removes_oldest_items(): + """Test that trim_history deletes the oldest items when count exceeds limit.""" + store = await make_store() + base = datetime(2024, 1, 1, tzinfo=UTC) + + # Insert 10 items with known timestamps, track their IDs + items = [] + for i in range(10): + itm = make_item(i) + items.append(itm) + encoded = itm.json() + created_at = (base + timedelta(minutes=i)).strftime("%Y-%m-%d %H:%M:%S") + await store._conn.execute( + text( + 'INSERT INTO "history" ("id", "type", "url", "data", "created_at") ' + "VALUES (:id, :type, :url, :data, :created_at)" + ), + {"id": itm._id, "type": "done", "url": itm.url, "data": encoded, "created_at": created_at}, + ) + await store._conn.commit() + + assert await store.count("done") == 10 + + # Trim to keep only 7 items + deleted = await store.trim_history("done", max_items=7) + assert len(deleted) == 3 + assert await store.count("done") == 7 + + # Verify the oldest 3 items were deleted (items[0], items[1], items[2]) + remaining = await store.fetch_saved("done") + remaining_ids = [r[0] for r in remaining] + for i in range(3): + assert items[i]._id not in remaining_ids + for i in range(3, 10): + assert items[i]._id in remaining_ids + + await store.close() + + +@pytest.mark.asyncio +async def test_trim_history_no_op_when_under_limit(): + """Test that trim_history does nothing when item count is within the limit.""" + store = await make_store() + + # Use unique index offset to avoid URL conflicts with shared DB + for i in range(100, 105): + await store.enqueue_upsert("done", make_item(i)) + await store.flush() + + count_before = await store.count("done") + deleted = await store.trim_history("done", max_items=count_before + 10) + assert deleted == [] + assert await store.count("done") == count_before + + await store.close() + + +@pytest.mark.asyncio +async def test_trim_history_no_op_for_zero_or_negative_max(): + """Test that trim_history returns empty list for invalid max_items values.""" + store = await make_store() + + for i in range(200, 203): + await store.enqueue_upsert("done", make_item(i)) + await store.flush() + + count_before = await store.count("done") + assert await store.trim_history("done", max_items=0) == [] + assert await store.trim_history("done", max_items=-1) == [] + assert await store.count("done") == count_before + + await store.close()