From 687c82cb32bbd2e7be080e550c7419faccfb7436 Mon Sep 17 00:00:00 2001 From: Jesse Bate Date: Wed, 10 Jun 2026 20:29:18 +0930 Subject: [PATCH] feat: add YTP_MAX_HISTORY to cap history and prevent unbounded DB growth Long-running instances accumulate history indefinitely. As the history table grows, SQLite COUNT(*), paginated SELECT, and index scans degrade. YTP_MAX_HISTORY (default: 0 = unlimited) caps the number of completed download records kept. Enforcement is a hook in DataStore.put(): on each insertion, the oldest excess entries are immediately evicted from the in-memory OrderedDict and their deletes are enqueued to SQLite. DataStore.load() also trims on startup in case the limit was lowered between restarts. No background tasks, no DB reads needed for the check. --- app/library/DataStore.py | 16 +++++- app/library/config.py | 4 ++ app/library/downloads/queue_manager.py | 2 +- app/tests/test_datastore.py | 71 ++++++++++++++++++++++++++ 4 files changed, 91 insertions(+), 2 deletions(-) diff --git a/app/library/DataStore.py b/app/library/DataStore.py index 4d3460ea..ead5ce69 100644 --- a/app/library/DataStore.py +++ b/app/library/DataStore.py @@ -49,16 +49,23 @@ def _strip_transient_fields(item: ItemDTO) -> ItemDTO: class DataStore: - def __init__(self, type: StoreType, connection: SqliteStore): + def __init__(self, type: StoreType, connection: SqliteStore, max_history: int = 0): self._type = type self._connection: SqliteStore = connection self._dict: OrderedDict[str, Download] = OrderedDict() + self._max_history: int = max_history async def load(self) -> None: saved = await self._connection.fetch_saved(str(self._type)) for key, item in saved: self._dict[key] = Download(info=item) + if self._type == StoreType.HISTORY and self._max_history > 0 and len(self._dict) > self._max_history: + excess_keys = list(self._dict.keys())[: len(self._dict) - self._max_history] + for key in excess_keys: + self._dict.pop(key) + await self._connection.enqueue_bulk_delete(str(self._type), excess_keys) + async def saved_items(self) -> list[tuple[str, ItemDTO]]: return await self._connection.fetch_saved(str(self._type)) @@ -170,6 +177,13 @@ class DataStore: _ = no_notify self._dict[value.info._id] = value await self._connection.enqueue_upsert(str(self._type), _strip_transient_fields(value.info)) + + if self._type == StoreType.HISTORY and self._max_history > 0: + while len(self._dict) > self._max_history: + oldest_key, _ = next(iter(self._dict.items())) + self._dict.pop(oldest_key) + await self._connection.enqueue_delete(str(self._type), oldest_key) + return self._dict[value.info._id] async def delete(self, key: str) -> None: diff --git a/app/library/config.py b/app/library/config.py index 3baff0b5..a05c49d9 100644 --- a/app/library/config.py +++ b/app/library/config.py @@ -227,6 +227,9 @@ class Config(metaclass=Singleton): auto_clear_history_days: int = 0 """Number of days after which completed download history is automatically cleared. 0 to disable.""" + max_history: int = 0 + """Maximum number of history items to keep. 0 means unlimited.""" + default_pagination: int = 50 """The default number of items per page for pagination.""" @@ -285,6 +288,7 @@ class Config(metaclass=Singleton): "download_path_depth", "download_info_expires", "auto_clear_history_days", + "max_history", "default_pagination", "extract_info_concurrency", "thumb_concurrency", diff --git a/app/library/downloads/queue_manager.py b/app/library/downloads/queue_manager.py index 95fca5dc..4167d42d 100644 --- a/app/library/downloads/queue_manager.py +++ b/app/library/downloads/queue_manager.py @@ -36,7 +36,7 @@ class DownloadQueue(metaclass=Singleton): "Configuration instance." self._notify: EventBus = EventBus.get_instance() "Event bus instance." - self.done = DataStore(type=StoreType.HISTORY, connection=SqliteStore.get_instance()) + self.done = DataStore(type=StoreType.HISTORY, connection=SqliteStore.get_instance(), max_history=self.config.max_history) "DataStore for the completed downloads." self.queue = DataStore(type=StoreType.QUEUE, connection=SqliteStore.get_instance()) "DataStore for the download queue." diff --git a/app/tests/test_datastore.py b/app/tests/test_datastore.py index d44120d0..e0de8083 100644 --- a/app/tests/test_datastore.py +++ b/app/tests/test_datastore.py @@ -1153,3 +1153,74 @@ class TestDataStoreOperations: result = await store.get_item(nonexistent_field=(Operation.CONTAIN, "value")) assert result is None await db.close() + + +class TestMaxHistory: + @pytest.mark.asyncio + async def test_put_evicts_oldest_when_over_limit(self) -> None: + db = await make_db() + store = DataStore(StoreType.HISTORY, db, max_history=3) + + items = [] + for i in range(4): + item = make_item(id=f"id{i}", url=f"http://u/{i}") + item._id = f"id{i}" + items.append(StubDownload(info=item)) + for item in items: + await store.put(item) + + assert len(store._dict) == 3, "Should keep only 3 items" + assert "id0" not in store._dict, "Oldest item should be evicted" + assert "id3" in store._dict, "Newest item should be kept" + await db.flush() + await db.close() + + @pytest.mark.asyncio + async def test_put_evicts_multiple_when_already_over_limit(self) -> None: + db = await make_db() + store = DataStore(StoreType.HISTORY, db, max_history=2) + + for i in range(5): + item = make_item(id=f"id{i}", url=f"http://u/{i}") + item._id = f"id{i}" + await store.put(StubDownload(info=item)) + + assert len(store._dict) == 2 + assert "id3" in store._dict + assert "id4" in store._dict + await db.flush() + await db.close() + + @pytest.mark.asyncio + async def test_put_no_eviction_when_max_history_zero(self) -> None: + db = await make_db() + store = DataStore(StoreType.HISTORY, db, max_history=0) + + for i in range(5): + await store.put(StubDownload(info=make_item(id=f"id{i}", url=f"http://u/{i}"))) + + assert len(store._dict) == 5, "max_history=0 means unlimited" + await db.flush() + await db.close() + + @pytest.mark.asyncio + async def test_put_queue_type_not_affected_by_max_history(self) -> None: + db = await make_db() + store = DataStore(StoreType.QUEUE, db, max_history=2) + + for i in range(5): + await store.put(StubDownload(info=make_item(id=f"id{i}", url=f"http://u/{i}"))) + + assert len(store._dict) == 5, "max_history should not apply to QUEUE type" + await db.flush() + await db.close() + + @pytest.mark.asyncio + async def test_load_trims_on_startup_when_over_limit(self) -> None: + db = await make_db(data=5) + store = DataStore(StoreType.HISTORY, db, max_history=3) + await store.load() + + assert len(store._dict) == 3, "load() should trim to max_history on startup" + await db.flush() + await db.close()