diff --git a/app/library/DataStore.py b/app/library/DataStore.py index 4d3460ea..ead5ce69 100644 --- a/app/library/DataStore.py +++ b/app/library/DataStore.py @@ -49,16 +49,23 @@ def _strip_transient_fields(item: ItemDTO) -> ItemDTO: class DataStore: - def __init__(self, type: StoreType, connection: SqliteStore): + def __init__(self, type: StoreType, connection: SqliteStore, max_history: int = 0): self._type = type self._connection: SqliteStore = connection self._dict: OrderedDict[str, Download] = OrderedDict() + self._max_history: int = max_history async def load(self) -> None: saved = await self._connection.fetch_saved(str(self._type)) for key, item in saved: self._dict[key] = Download(info=item) + if self._type == StoreType.HISTORY and self._max_history > 0 and len(self._dict) > self._max_history: + excess_keys = list(self._dict.keys())[: len(self._dict) - self._max_history] + for key in excess_keys: + self._dict.pop(key) + await self._connection.enqueue_bulk_delete(str(self._type), excess_keys) + async def saved_items(self) -> list[tuple[str, ItemDTO]]: return await self._connection.fetch_saved(str(self._type)) @@ -170,6 +177,13 @@ class DataStore: _ = no_notify self._dict[value.info._id] = value await self._connection.enqueue_upsert(str(self._type), _strip_transient_fields(value.info)) + + if self._type == StoreType.HISTORY and self._max_history > 0: + while len(self._dict) > self._max_history: + oldest_key, _ = next(iter(self._dict.items())) + self._dict.pop(oldest_key) + await self._connection.enqueue_delete(str(self._type), oldest_key) + return self._dict[value.info._id] async def delete(self, key: str) -> None: diff --git a/app/library/config.py b/app/library/config.py index 3baff0b5..a05c49d9 100644 --- a/app/library/config.py +++ b/app/library/config.py @@ -227,6 +227,9 @@ class Config(metaclass=Singleton): auto_clear_history_days: int = 0 """Number of days after which completed download history is automatically cleared. 0 to disable.""" + max_history: int = 0 + """Maximum number of history items to keep. 0 means unlimited.""" + default_pagination: int = 50 """The default number of items per page for pagination.""" @@ -285,6 +288,7 @@ class Config(metaclass=Singleton): "download_path_depth", "download_info_expires", "auto_clear_history_days", + "max_history", "default_pagination", "extract_info_concurrency", "thumb_concurrency", diff --git a/app/library/downloads/queue_manager.py b/app/library/downloads/queue_manager.py index 95fca5dc..4167d42d 100644 --- a/app/library/downloads/queue_manager.py +++ b/app/library/downloads/queue_manager.py @@ -36,7 +36,7 @@ class DownloadQueue(metaclass=Singleton): "Configuration instance." self._notify: EventBus = EventBus.get_instance() "Event bus instance." - self.done = DataStore(type=StoreType.HISTORY, connection=SqliteStore.get_instance()) + self.done = DataStore(type=StoreType.HISTORY, connection=SqliteStore.get_instance(), max_history=self.config.max_history) "DataStore for the completed downloads." self.queue = DataStore(type=StoreType.QUEUE, connection=SqliteStore.get_instance()) "DataStore for the download queue." diff --git a/app/tests/test_datastore.py b/app/tests/test_datastore.py index d44120d0..e0de8083 100644 --- a/app/tests/test_datastore.py +++ b/app/tests/test_datastore.py @@ -1153,3 +1153,74 @@ class TestDataStoreOperations: result = await store.get_item(nonexistent_field=(Operation.CONTAIN, "value")) assert result is None await db.close() + + +class TestMaxHistory: + @pytest.mark.asyncio + async def test_put_evicts_oldest_when_over_limit(self) -> None: + db = await make_db() + store = DataStore(StoreType.HISTORY, db, max_history=3) + + items = [] + for i in range(4): + item = make_item(id=f"id{i}", url=f"http://u/{i}") + item._id = f"id{i}" + items.append(StubDownload(info=item)) + for item in items: + await store.put(item) + + assert len(store._dict) == 3, "Should keep only 3 items" + assert "id0" not in store._dict, "Oldest item should be evicted" + assert "id3" in store._dict, "Newest item should be kept" + await db.flush() + await db.close() + + @pytest.mark.asyncio + async def test_put_evicts_multiple_when_already_over_limit(self) -> None: + db = await make_db() + store = DataStore(StoreType.HISTORY, db, max_history=2) + + for i in range(5): + item = make_item(id=f"id{i}", url=f"http://u/{i}") + item._id = f"id{i}" + await store.put(StubDownload(info=item)) + + assert len(store._dict) == 2 + assert "id3" in store._dict + assert "id4" in store._dict + await db.flush() + await db.close() + + @pytest.mark.asyncio + async def test_put_no_eviction_when_max_history_zero(self) -> None: + db = await make_db() + store = DataStore(StoreType.HISTORY, db, max_history=0) + + for i in range(5): + await store.put(StubDownload(info=make_item(id=f"id{i}", url=f"http://u/{i}"))) + + assert len(store._dict) == 5, "max_history=0 means unlimited" + await db.flush() + await db.close() + + @pytest.mark.asyncio + async def test_put_queue_type_not_affected_by_max_history(self) -> None: + db = await make_db() + store = DataStore(StoreType.QUEUE, db, max_history=2) + + for i in range(5): + await store.put(StubDownload(info=make_item(id=f"id{i}", url=f"http://u/{i}"))) + + assert len(store._dict) == 5, "max_history should not apply to QUEUE type" + await db.flush() + await db.close() + + @pytest.mark.asyncio + async def test_load_trims_on_startup_when_over_limit(self) -> None: + db = await make_db(data=5) + store = DataStore(StoreType.HISTORY, db, max_history=3) + await store.load() + + assert len(store._dict) == 3, "load() should trim to max_history on startup" + await db.flush() + await db.close()