Refactor: swap sqlite to aiosqlite and add queued db updates

.vscode/settings.json

@@ -16,6 +16,7 @@
     "ahas",
     "ahash",
     "aiocron",
+    "aiosqlite",
     "anyio",
     "Archiver",
     "arrowdown",
@@ -73,6 +74,8 @@
     "falsey",
     "faststart",
     "Fetc",
+    "fetchall",
+    "fetchone",
     "filterwarnings",
     "finaldir",
     "flac",
@@ -91,6 +94,7 @@
     "hookspath",
     "httpx",
     "imagetools",
+    "isinstance",
     "jmespath",
     "jsonschema",
     "kibibytes",
@@ -112,7 +116,10 @@
     "mebibytes",
     "MEIPASS",
     "merch",
+    "metaclass",
+    "Metaclasses",
     "metadataparser",
+    "mgmt",
     "Microformat",
     "microformats",
     "mkvtoolsnix",
@@ -128,6 +135,7 @@
     "noninteractive",
     "noprogress",
     "onefile",
+    "oneline",
     "parsel",
     "pathex",
     "pickleable",
@@ -178,6 +186,7 @@
     "Unraid",
     "upgrader",
     "urandom",
+    "urlparse",
     "urlsafe",
     "usegmt",
     "usermod",
@@ -190,6 +199,7 @@
     "xerror",
     "youtu",
     "youtubepot",
+    "zstd",
     "русский",
     "العربية"
   ],

app/library/DataStore.py

@@ -1,27 +1,21 @@
+import asyncio
 import copy
-import json
 import logging
 from collections import OrderedDict
-from dataclasses import fields
+from collections.abc import Iterable
-from datetime import UTC, datetime
-from email.utils import formatdate
 from enum import Enum
-from sqlite3 import Connection
 
 from .Download import Download
 from .ItemDTO import ItemDTO
 from .operations import matches_condition
-from .Utils import init_class
+from .sqlite_store import SqliteStore
 
-LOG: logging.Logger = logging.getLogger("datastore")
+LOG = logging.getLogger("datastore")
-
-ITEM_DTO_FIELDS = {f.name for f in fields(ItemDTO)}
-"""Caching ItemDTO fields for performance."""
 
 
 class StoreType(str, Enum):
-    HISTORY: str = "done"
+    HISTORY = "done"
-    QUEUE: str = "queue"
+    QUEUE = "queue"
 
     @classmethod
     def all(cls) -> list[str]:
@@ -29,55 +23,57 @@ class StoreType(str, Enum):
 
     @classmethod
     def from_value(cls, value: str) -> "StoreType":
-        """
-        Returns the StoreType enum member corresponding to the given value.
-
-        Args:
-            value (str): The value to match against the enum members.
-
-        Returns:
-            StoreType: The enum member that matches the value.
-
-        Raises:
-            ValueError: If the value does not match any member.
-
-        """
         for member in cls:
             if member.value == value:
                 return member
-
         msg = f"Invalid StoreType value: {value}"
         raise ValueError(msg)
 
-    def __str__(self) -> str:
+    def __str__(self) -> str:  # pragma: no cover
         return self.value
 
 
+def _strip_transient_fields(item: ItemDTO) -> ItemDTO:
+    stored = copy.deepcopy(item)
+    if hasattr(stored, "datetime"):
+        try:
+            delattr(stored, "datetime")
+        except AttributeError:
+            pass
+    if stored.status == "finished" and hasattr(stored, "live_in"):
+        try:
+            delattr(stored, "live_in")
+        except AttributeError:
+            pass
+    return stored
+
+
 class DataStore:
     """
-    Persistent queue.
+    In-memory queue cache + optional write-behind to persistence.
+    History reads go straight to persistence.
     """
 
-    def __init__(self, type: StoreType, connection: Connection):
+    def __init__(self, type: StoreType, connection: SqliteStore):
+        self._type = type
+        self._connection: SqliteStore = connection
         self._dict: OrderedDict[str, Download] = OrderedDict()
-        "The dictionary of items."
-        self._type: StoreType = type
-        "The type of the datastore."
-        self._connection: Connection = connection
-        "The database connection."
 
-    def load(self) -> None:
+    # cache helpers
-        for id, item in self.saved_items():
+    async def load(self) -> None:
-            self._dict.update({id: Download(info=item)})
+        saved = await self._connection.fetch_saved(str(self._type))
+        for key, item in saved:
+            self._dict[key] = Download(info=item)
+
+    async def saved_items(self) -> list[tuple[str, ItemDTO]]:
+        return await self._connection.fetch_saved(str(self._type))
 
     def exists(self, key: str | None = None, url: str | None = None) -> bool:
         if not key and not url:
             msg = "key or url must be provided."
             raise KeyError(msg)
-
         if key and key in self._dict:
             return True
-
         return any(
             (key and self._dict[i].info._id == key) or (url and self._dict[i].info.url == url) for i in self._dict
         )
@@ -86,261 +82,88 @@ class DataStore:
         if not key and not url:
             msg = "key or url must be provided."
             raise KeyError(msg)
-
         for i in self._dict:
             if (key and self._dict[i].info._id == key) or (url and self._dict[i].info.url == url):
                 return self._dict[i]
-
         msg: str = f"{key=} or {url=} not found."
         raise KeyError(msg)
 
     def get_item(self, **kwargs) -> Download | None:
-        """
-        Get a specific item from the datastore based on provided attributes with optional operations.
-
-        Args:
-            **kwargs: Arbitrary keyword arguments representing attributes of the ItemDTO.
-                Each value can be either:
-                - A direct value (defaults to EQUAL operation): {"title": "test"}
-                - A tuple of (Operation, value): {"title": (Operation.CONTAIN, "test")}
-
-                If no attributes are provided, the method returns None.
-                If any attribute matches, the corresponding Download object is returned.
-
-        Returns:
-            Download | None: The requested item if found, otherwise None.
-
-        Examples:
-            # Direct equality check (default)
-            store.get_item(title="Video 1")
-
-            # Using operations
-            store.get_item(title=(Operation.CONTAIN, "test"))
-            store.get_item(id=(Operation.EQUAL, "123"), status=(Operation.NOT_EQUAL, "error"))
-
-            # Mixed usage
-            store.get_item(title=(Operation.CONTAIN, "test"), folder="downloads")
-
-        """
         if not kwargs:
             return None
-
         for i in self._dict:
             if not self._dict[i].info:
                 continue
-
             info = self._dict[i].info.__dict__
             if any(matches_condition(key, value, info) for key, value in kwargs.items()):
                 return self._dict[i]
+        return None
+
+    async def get_by_id(self, id: str) -> Download | None:
+        val = self._dict.get(id)
+        if val or self._type == StoreType.QUEUE:
+            return val
+
+        if item := await self._connection.get_by_id(str(self._type), id):
+            download = Download(info=item)
+            self._dict[id] = download
+            return download
 
         return None
 
-    def get_by_id(self, id: str) -> Download | None:
+    def items(self):
-        return self._dict.get(id, None)
-
-    def items(self) -> OrderedDict[tuple[str, Download]]:
         return self._dict.items()
 
-    def saved_items(self) -> list[tuple[str, ItemDTO]]:
+    async def put(self, value: Download, no_notify: bool = False) -> Download:
-        items: list[tuple[str, ItemDTO]] = []
+        _ = no_notify
-
+        self._dict[value.info._id] = value
-        cursor = self._connection.execute(
+        asyncio.create_task(self._connection.enqueue_upsert(str(self._type), _strip_transient_fields(value.info)))
-            'SELECT "id", "data", "created_at" FROM "history" WHERE "type" = ? ORDER BY "created_at" ASC',
+        return value
-            (str(self._type),),
-        )
-
-        for row in cursor:
-            rowDate: datetime = datetime.strptime(row["created_at"], "%Y-%m-%d %H:%M:%S")  # noqa: DTZ007
-            data: dict = json.loads(row["data"])
-            data.pop("_id", None)
-            item: ItemDTO = init_class(ItemDTO, data, ITEM_DTO_FIELDS)
-            item._id = row["id"]
-            item.datetime = formatdate(rowDate.replace(tzinfo=UTC).timestamp())
-            items.append((row["id"], item))
-
-        return items
-
-    def put(self, value: Download, no_notify: bool = False) -> Download:
-        if "error" == value.info.status and not no_notify:
-            from app.library.Events import EventBus, Events
-
-            EventBus.get_instance().emit(Events.ITEM_ERROR, value.info)
-
-        self._dict.update({value.info._id: value})
-        self._update_store_item(self._type, value.info)
-
-        return self._dict[value.info._id]
 
     def delete(self, key: str) -> None:
         self._dict.pop(key, None)
-        self._delete_store_item(key)
+        asyncio.create_task(self._connection.enqueue_delete(str(self._type), key))
 
-    def next(self) -> tuple[str, Download]:
+    def next(self):
         return next(iter(self._dict.items()))
 
-    def empty(self):
+    def empty(self) -> bool:
         return not bool(self._dict)
 
-    def has_downloads(self):
+    def has_downloads(self) -> bool:
-        if 0 == len(self._dict):
-            return False
-
         return any(self._dict[key].info.auto_start and self._dict[key].started() is False for key in self._dict)
 
-    def get_next_download(self) -> Download:
+    def get_next_download(self) -> Download | None:
         for key in self._dict:
             ref = self._dict[key]
-            if ref.info.auto_start and ref.started() is False and ref.is_cancelled() is False:
+            if ref.info.auto_start and not ref.started() and not ref.is_cancelled():
                 return ref
-
         return None
 
-    async def test(self) -> bool:
+    async def get_total_count(self, status_filter: str | None = None) -> int:
-        self._connection.execute('SELECT "id" FROM "history" LIMIT 1').fetchone()
+        return await self._connection.count(str(self._type), status_filter=status_filter)
-        return True
 
-    def get_total_count(self) -> int:
+    async def get_items_paginated(
-        """
+        self, page: int = 1, per_page: int = 50, order: str = "DESC", status_filter: str | None = None
-        Get the total count of items in the datastore.
+    ):
-
-        Returns:
-            int: The total number of items in the datastore.
-
-        """
-        cursor = self._connection.execute(
-            'SELECT COUNT(*) as count FROM "history" WHERE "type" = ?',
-            (str(self._type),),
-        )
-        row = cursor.fetchone()
-        return row["count"] if row else 0
-
-    def get_items_paginated(
-        self,
-        page: int = 1,
-        per_page: int = 50,
-        order: str = "DESC",
-        status_filter: str | None = None,
-    ) -> tuple[list[tuple[str, ItemDTO]], int, int, int]:
-        """
-        Get paginated items from the datastore.
-
-        Args:
-            page (int): The page number (1-indexed). Defaults to 1.
-            per_page (int): Number of items per page. Defaults to 50.
-            order (str): Sort order - 'ASC' or 'DESC'. Defaults to 'DESC' (newest first).
-            status_filter (str | None): Optional status filter. Can be a status value (e.g., 'finished')
-                to include only that status, or prefixed with '!' (e.g., '!finished') to exclude that status.
-
-        Returns:
-            tuple[list[tuple[str, ItemDTO]], int, int, int]: A tuple containing:
-                - List of (id, ItemDTO) tuples for the requested page
-                - Total number of items
-                - Current page number
-                - Total number of pages
-
-        Raises:
-            ValueError: If page < 1 or per_page < 1
-
-        """
         if page < 1:
             msg = "page must be >= 1"
             raise ValueError(msg)
-
         if per_page < 1:
             msg = "per_page must be >= 1"
             raise ValueError(msg)
-
         order = order.upper()
         if order not in ("ASC", "DESC"):
             msg = f"order must be 'ASC' or 'DESC', got '{order}'"
             raise ValueError(msg)
+        return await self._connection.paginate(str(self._type), page, per_page, order, status_filter)
 
-        order = "ASC" if order == "ASC" else "DESC"
+    async def bulk_delete(self, ids: Iterable[str]) -> int:
+        deleted = await self._connection.bulk_delete(str(self._type), ids)
+        for _id in ids:
+            self._dict.pop(_id, None)
+        return deleted
 
-        # Build SQL query with status filter if provided
+    async def test(self) -> bool:
-        where_clauses = ['"type" = ?']
+        await self._connection.count(str(self._type))
-        query_params: list = [str(self._type)]
+        return True
-
-        if status_filter:
-            # Check if it's an exclusion filter (starts with !)
-            if status_filter.startswith("!"):
-                status_value = status_filter[1:]
-                where_clauses.append("json_extract(data, '$.status') != ?")
-                query_params.append(status_value)
-            else:
-                where_clauses.append("json_extract(data, '$.status') = ?")
-                query_params.append(status_filter)
-
-        where_clause = " AND ".join(where_clauses)
-
-        # Get total count with filter
-        count_query = f'SELECT COUNT(*) as count FROM "history" WHERE {where_clause}'  # noqa: S608
-        count_result = self._connection.execute(count_query, tuple(query_params)).fetchone()
-        total_items: int = count_result["count"] if count_result else 0
-        total_pages: int = (total_items + per_page - 1) // per_page if total_items > 0 else 1
-
-        # Ensure page is within valid range.
-        if page > total_pages and total_items > 0:
-            page = total_pages
-
-        offset = (page - 1) * per_page
-
-        items: list[tuple[str, ItemDTO]] = []
-
-        query_params.extend([per_page, offset])
-
-        cursor = self._connection.execute(
-            f'SELECT "id", "data", "created_at" FROM "history" WHERE {where_clause} ORDER BY "created_at" {order} LIMIT ? OFFSET ?',  # noqa: S608
-            tuple(query_params),
-        )
-
-        for row in cursor:
-            rowDate: datetime = datetime.strptime(row["created_at"], "%Y-%m-%d %H:%M:%S")  # noqa: DTZ007
-            data: dict = json.loads(row["data"])
-            data.pop("_id", None)
-            item: ItemDTO = init_class(ItemDTO, data, ITEM_DTO_FIELDS)
-            item._id = row["id"]
-            item.datetime = formatdate(rowDate.replace(tzinfo=UTC).timestamp())
-            items.append((row["id"], item))
-
-        return items, total_items, page, total_pages
-
-    def _update_store_item(self, type: StoreType, item: ItemDTO) -> None:
-        sqlStatement = """
-        INSERT INTO "history" ("id", "type", "url", "data")
-        VALUES (?, ?, ?, ?)
-        ON CONFLICT DO UPDATE SET "type" = ?, "url" = ?, "data" = ?, created_at = ?
-        """
-
-        stored: ItemDTO = copy.deepcopy(item)
-
-        if hasattr(stored, "datetime"):
-            try:
-                delattr(stored, "datetime")
-            except AttributeError:
-                pass
-
-        if hasattr(stored, "live_in") and stored.status == "finished":
-            try:
-                delattr(stored, "live_in")
-            except AttributeError:
-                pass
-
-        encoded: str = stored.json()
-
-        self._connection.execute(
-            sqlStatement.strip(),
-            (
-                stored._id,
-                str(type),
-                stored.url,
-                encoded,
-                str(type),
-                stored.url,
-                encoded,
-                datetime.now(UTC).strftime("%Y-%m-%d %H:%M:%S"),
-            ),
-        )
-
-    def _delete_store_item(self, key: str) -> None:
-        self._connection.execute('DELETE FROM "history" WHERE "type" = ? AND "id" = ?', (str(self._type), key))

app/library/DownloadQueue.py

@@ -9,7 +9,6 @@ import uuid
 from datetime import UTC, datetime, timedelta
 from email.utils import formatdate
 from pathlib import Path
-from sqlite3 import Connection
 from typing import TYPE_CHECKING, Any
 
 import yt_dlp.utils
@@ -25,6 +24,7 @@ from .ItemDTO import Item, ItemDTO
 from .Presets import Presets
 from .Scheduler import Scheduler
 from .Singleton import Singleton
+from .sqlite_store import SqliteStore
 from .Utils import (
     archive_add,
     arg_converter,
@@ -49,14 +49,14 @@ class DownloadQueue(metaclass=Singleton):
     DownloadQueue class is a singleton class that manages the download queue and the download history.
     """
 
-    def __init__(self, connection: Connection, config: Config | None = None):
+    def __init__(self, config: Config | None = None):
         self.config: Config = config or Config.get_instance()
         "Configuration instance."
         self._notify: EventBus = EventBus.get_instance()
         "Event bus instance."
-        self.done = DataStore(type=StoreType.HISTORY, connection=connection)
+        self.done = DataStore(type=StoreType.HISTORY, connection=SqliteStore.get_instance(config.db_file))
         "DataStore for the completed downloads."
-        self.queue = DataStore(type=StoreType.QUEUE, connection=connection)
+        self.queue = DataStore(type=StoreType.QUEUE, connection=SqliteStore.get_instance(config.db_file))
         "DataStore for the download queue."
         self.workers = asyncio.Semaphore(self.config.max_workers)
         "Semaphore to limit the number of concurrent downloads."
@@ -71,8 +71,7 @@ class DownloadQueue(metaclass=Singleton):
         self._active: dict[str, Download] = {}
         """Dictionary of active downloads."""
 
-        self.done.load()
+        # Loading happens in async initialize to avoid blocking loop creation.
-        self.queue.load()
         self.paused.set()
 
     @staticmethod
@@ -166,6 +165,9 @@ class DownloadQueue(metaclass=Singleton):
         """
         Initialize the download queue.
         """
+        await self.done.test()
+        await self.done.load()
+        await self.queue.load()
         LOG.info(
             f"Using '{self.config.max_workers}' workers for downloading and '{self.config.max_workers_per_extractor}' per extractor."
         )
@@ -199,7 +201,7 @@ class DownloadQueue(metaclass=Singleton):
                 continue
 
             item.info.auto_start = True
-            updated: Download = self.queue.put(item)
+            updated: Download = await self.queue.put(item)
             self._notify.emit(Events.ITEM_UPDATED, data=updated.info)
             self._notify.emit(
                 Events.ITEM_RESUMED,
@@ -246,7 +248,7 @@ class DownloadQueue(metaclass=Singleton):
                 continue
 
             item.info.auto_start = False
-            updated: Download = self.queue.put(item)
+            updated: Download = await self.queue.put(item)
             self._notify.emit(Events.ITEM_UPDATED, data=updated.info)
             self._notify.emit(
                 Events.ITEM_PAUSED,
@@ -534,7 +536,7 @@ class DownloadQueue(metaclass=Singleton):
                     message=nMessage,
                 )
 
-                itemDownload: Download = self.done.put(dlInfo)
+                itemDownload: Download = await self.done.put(dlInfo)
             elif not hasFormats:
                 ava: str = entry.get("availability", "public")
                 nTitle = "Download Error"
@@ -547,7 +549,7 @@ class DownloadQueue(metaclass=Singleton):
 
                 dlInfo.info.error = nMessage.replace(f" for '{dl.title}'.", ".") + text_logs
                 dlInfo.info.status = "error"
-                itemDownload = self.done.put(dlInfo)
+                itemDownload = await self.done.put(dlInfo)
 
                 self._notify.emit(
                     Events.LOG_WARNING,
@@ -580,12 +582,12 @@ class DownloadQueue(metaclass=Singleton):
 
                 if _requeue:
                     nEvent = Events.ITEM_ADDED
-                    itemDownload = self.queue.put(dlInfo)
+                    itemDownload = await self.queue.put(dlInfo)
                     if item.auto_start:
                         self.event.set()
                 else:
                     dlInfo.info.status = "not_live"
-                    itemDownload = self.done.put(dlInfo)
+                    itemDownload = await self.done.put(dlInfo)
                     nStore = "history"
                     nEvent = Events.ITEM_MOVED
                     nTitle = "Premiering right now"
@@ -596,7 +598,7 @@ class DownloadQueue(metaclass=Singleton):
                 nEvent = Events.ITEM_ADDED
                 nTitle = "Item Added"
                 nMessage = f"Item '{dlInfo.info.title}' has been added to the download queue."
-                itemDownload = self.queue.put(dlInfo)
+                itemDownload = await self.queue.put(dlInfo)
                 if item.auto_start:
                     self.event.set()
                 else:
@@ -732,7 +734,7 @@ class DownloadQueue(metaclass=Singleton):
                         if archive_file := dlInfo.info.get_ytdlp_opts().get_all().get("download_archive"):
                             dlInfo.info.msg += f" Found in archive '{archive_file}'."
 
-                        self.done.put(dlInfo)
+                        await self.done.put(dlInfo)
 
                         self._notify.emit(
                             Events.ITEM_MOVED,
@@ -816,7 +818,7 @@ class DownloadQueue(metaclass=Singleton):
                                 extras=item.extras,
                             )
                         )
-                        self.done.put(dlInfo)
+                        await self.done.put(dlInfo)
 
                     LOG.info(log_message)
                     self._notify.emit(Events.LOG_INFO, data={}, title="Ignored Download", message=log_message)
@@ -910,7 +912,7 @@ class DownloadQueue(metaclass=Singleton):
                     message=f"Download '{item.info.title}' has been cancelled.",
                 )
                 item.info.status = "cancelled"
-                self.done.put(item)
+                await self.done.put(item)
                 self._notify.emit(
                     Events.ITEM_MOVED,
                     data={"to": "history", "preset": item.info.preset, "item": item.info},
@@ -1003,7 +1005,7 @@ class DownloadQueue(metaclass=Singleton):
 
         return status
 
-    def get(self, mode: str = "all") -> dict[str, list[dict[str, ItemDTO]]]:
+    async def get(self, mode: str = "all") -> dict[str, list[dict[str, ItemDTO]]]:
         """
         Get the download queue and the download history.
 
@@ -1017,11 +1019,11 @@ class DownloadQueue(metaclass=Singleton):
         items = {"queue": {}, "done": {}}
 
         if mode in ("all", "queue"):
-            for k, v in self.queue.saved_items():
+            for k, v in await self.queue.saved_items():
                 items["queue"][k] = self._active[k].info if k in self._active else v
 
         if mode in ("all", "done"):
-            for k, v in self.done.saved_items():
+            for k, v in await self.done.saved_items():
                 v.get_file_sidecar()
                 items["done"][k] = v
 
@@ -1186,7 +1188,7 @@ class DownloadQueue(metaclass=Singleton):
 
                 self._notify.emit(Events.ITEM_COMPLETED, data=entry.info, title=nTitle, message=nMessage)
 
-            self.done.put(entry)
+            await self.done.put(entry)
             self._notify.emit(
                 Events.ITEM_MOVED,
                 data={"to": "history", "preset": entry.info.preset, "item": entry.info},
@@ -1288,7 +1290,7 @@ class DownloadQueue(metaclass=Singleton):
                     )
                 )
             except Exception as e:
-                self.done.put(item)
+                await self.done.put(item)
                 LOG.exception(e)
                 LOG.error(f"Failed to retry item '{item_ref}'. {e!s}")
 

app/library/config.py

@@ -441,6 +441,7 @@ class Config(metaclass=Singleton):
             ("urllib3.connectionpool", logging.WARNING),
             ("apprise", logging.WARNING),
             ("httpcore", logging.INFO),
+            ("aiosqlite", logging.INFO),
         )
         for _tool, _level in _log_levels:
             logging.getLogger(_tool).setLevel(_level)

app/library/migrate.py

@@ -0,0 +1,321 @@
+# flake8: noqa: S608
+import contextlib
+import datetime
+import glob
+import os.path
+import traceback
+from collections.abc import AsyncIterator, Iterable, Sequence
+from importlib.machinery import ModuleSpec, SourceFileLoader
+from importlib.util import module_from_spec, spec_from_loader
+from typing import TYPE_CHECKING
+
+import aiosqlite
+
+if TYPE_CHECKING:
+    from types import ModuleType
+
+UTC_LENGTH = 14
+
+__version__ = "1.0.0"
+
+class Error(Exception):
+    """Base class for all Caribou errors."""
+
+
+class InvalidMigrationError(Error):
+    """Thrown when a client migration contains an error."""
+
+
+class InvalidNameError(Error):
+    """Thrown when a client migration has an invalid filename."""
+
+    def __init__(self, filename: str):
+        msg: str = (
+            f"Migration filenames must start with a UTC timestamp. The following file has an invalid name: {filename}"
+        )
+        super().__init__(msg)
+
+
+@contextlib.asynccontextmanager
+async def execute(
+    conn: aiosqlite.Connection, sql: str, params: Sequence[object] | None = None
+) -> AsyncIterator[aiosqlite.Cursor]:
+    params = [] if params is None else params
+    cursor = await conn.execute(sql, params)
+    try:
+        yield cursor
+    finally:
+        await cursor.close()
+
+
+@contextlib.asynccontextmanager
+async def transaction(conn: aiosqlite.Connection) -> AsyncIterator[None]:
+    try:
+        yield
+        await conn.commit()
+    except Exception:
+        await conn.rollback()
+        raise
+
+
+class Migration:
+    """This class represents a migration version."""
+
+    def __init__(self, path: str):
+        self.path: str = path
+        self.filename: str = os.path.basename(path)
+        self.module_name, _ = os.path.splitext(self.filename)
+        self.get_version()  # will assert the filename is valid
+        self.name: str = self.module_name[UTC_LENGTH:]
+        while self.name.startswith("_"):
+            self.name: str = self.name[1:]
+
+        loader = SourceFileLoader(self.module_name, path)
+        spec: ModuleSpec | None = spec_from_loader(self.module_name, loader)
+        if spec is None:
+            msg = f"Cannot create spec for {path}"
+            raise ImportError(msg)
+
+        try:
+            module: ModuleType = module_from_spec(spec)
+            loader.exec_module(module)
+            self.module: ModuleType = module
+        except Exception as e:
+            msg: str = f"Invalid migration {path}: {traceback.format_exc()}"
+            raise InvalidMigrationError(msg) from e
+
+        targets: list[str] = ["upgrade", "downgrade"]
+        missing: list[str] = [m for m in targets if not self.has_method(m)]
+        if missing:
+            msg = f"Migration '{self.path}' is missing required methods: {', '.join(missing)}."
+            raise InvalidMigrationError(msg)
+
+    def get_version(self) -> str:
+        if len(self.filename) < UTC_LENGTH:
+            raise InvalidNameError(self.filename)
+        timestamp: str = self.filename[:UTC_LENGTH]
+        if not timestamp.isdigit():
+            raise InvalidNameError(self.filename)
+        return timestamp
+
+    async def upgrade(self, conn: aiosqlite.Connection) -> None:
+        await self.module.upgrade(conn)
+
+    async def downgrade(self, conn: aiosqlite.Connection) -> None:
+        await self.module.downgrade(conn)
+
+    def has_method(self, name: str) -> bool:
+        return callable(getattr(self.module, name, None))
+
+    def __repr__(self) -> str:
+        return f"Migration(path={self.filename})"
+
+
+class Database:
+    def __init__(self, db_url: aiosqlite.Connection | str, version_table: str = "migration_version"):
+        if not db_url:
+            msg = "Database requires db_url."
+            raise ValueError(msg)
+
+        self.version_table: str = version_table
+
+        self._owns_connection = bool(isinstance(db_url, str))
+        if self._owns_connection:
+            self.conn: aiosqlite.Connection | None = None
+            self.db_url: str = db_url
+            self._ensure_connection()
+        else:
+            self.conn: aiosqlite.Connection = db_url
+
+    async def __aenter__(self) -> "Database":
+        if self._owns_connection:
+            await self._ensure_connection()
+
+        return self
+
+    async def __aexit__(self, exc_type, exc, tb) -> None:
+        if not self._owns_connection:
+            return
+        await self.close()
+
+    async def close(self) -> None:
+        if self._owns_connection and self.conn:
+            await self.conn.close()
+            self.conn = None
+
+    async def _ensure_connection(self) -> None:
+        if self.conn:
+            return
+        self.conn = await aiosqlite.connect(self.db_url)
+        self.conn.row_factory = aiosqlite.Row
+
+    async def is_version_controlled(self) -> bool:
+        await self._ensure_connection()
+        assert self.conn
+        sql = """SELECT * FROM sqlite_master WHERE type = 'table' AND name = ?"""
+        async with execute(self.conn, sql, [self.version_table]) as cursor:
+            return bool(await cursor.fetchall())
+
+    async def upgrade(self, migrations: list[Migration], target_version: str | None = None) -> None:
+        await self._ensure_connection()
+        assert self.conn
+        if target_version:
+            _assert_migration_exists(migrations, target_version)
+
+        migrations.sort(key=lambda x: x.get_version())
+        database_version = await self.get_version()
+
+        for migration in migrations:
+            current_version: str = migration.get_version()
+            if database_version is not None and current_version <= database_version:
+                continue
+            if target_version and current_version > target_version:
+                break
+            await migration.upgrade(self.conn)
+            new_version: str = migration.get_version()
+            await self.update_version(new_version)
+            database_version: str = new_version
+
+    async def downgrade(self, migrations: list[Migration], target_version: str | int) -> None:
+        await self._ensure_connection()
+        assert self.conn
+        if target_version not in (0, "0"):
+            _assert_migration_exists(migrations, target_version)
+
+        migrations.sort(key=lambda x: x.get_version(), reverse=True)
+        database_version: str | None = await self.get_version()
+
+        for i, migration in enumerate(migrations):
+            current_version: str = migration.get_version()
+            if database_version is not None and current_version > database_version:
+                continue
+            if current_version <= target_version:
+                break
+            await migration.downgrade(self.conn)
+            next_version: str | int = 0
+            if i < len(migrations) - 1:
+                next_migration: Migration = migrations[i + 1]
+                next_version = next_migration.get_version()
+            await self.update_version(str(next_version))
+            database_version = str(next_version)
+
+    async def get_version(self) -> str | None:
+        """
+        Return the database's version, or None if it is not under version
+        control.
+        """
+        await self._ensure_connection()
+        assert self.conn
+        if not await self.is_version_controlled():
+            return None
+        sql: str = f"SELECT version FROM {self.version_table}"
+        async with execute(self.conn, sql) as cursor:
+            result = await cursor.fetchall()
+            return result[0][0] if result else "0"
+
+    async def update_version(self, version: str) -> None:
+        await self._ensure_connection()
+        assert self.conn
+        sql: str = f"UPDATE {self.version_table} SET version = ?"
+        async with transaction(self.conn):
+            await self.conn.execute(sql, [version])
+
+    async def initialize_version_control(self) -> None:
+        await self._ensure_connection()
+        assert self.conn
+        sql: str = f"""CREATE TABLE IF NOT EXISTS {self.version_table} ( version TEXT ) """
+        async with transaction(self.conn):
+            await self.conn.execute(sql)
+            await self.conn.execute(f"INSERT INTO {self.version_table} VALUES (0)")
+
+    def __repr__(self) -> str:
+        return f'Database("{self.db_url if self._owns_connection else "external_connection"}")'
+
+
+def _assert_migration_exists(migrations: Iterable[Migration], version: str | int) -> None:
+    if version not in (m.get_version() for m in migrations):
+        msg: str = f"No migration with version {version} exists."
+        raise Error(msg)
+
+
+def load_migrations(directory: str) -> list[Migration]:
+    """Return the migrations contained in the given directory."""
+    directory = str(directory)
+    if not os.path.exists(directory) or not os.path.isdir(directory):
+        msg: str = f"{directory} is not a directory."
+        raise Error(msg)
+    return [Migration(f) for f in glob.glob(os.path.join(directory, "*.py"))]
+
+
+async def upgrade(db_url: aiosqlite.Connection | str, migration_dir: str, version: str | None = None) -> None:
+    """
+    Upgrade the given database with the migrations contained in the
+    migrations directory. If a version is not specified, upgrade
+    to the most recent version.
+    """
+    async with Database(db_url) as db:
+        if not await db.is_version_controlled():
+            await db.initialize_version_control()
+
+        await db.upgrade(load_migrations(migration_dir), version)
+
+
+async def downgrade(db_url: str | aiosqlite.Connection, migration_dir: str, version: str) -> None:
+    """
+    Downgrade the database to the given version with the migrations
+    contained in the given migration directory.
+    """
+    async with Database(db_url) as db:
+        if not await db.is_version_controlled():
+            msg = f"The database {db_url} is not version controlled."
+            raise Error(msg)
+        migrations = load_migrations(migration_dir)
+        await db.downgrade(migrations, version)
+
+
+async def get_version(db_url: aiosqlite.Connection | str) -> str | None:
+    """Return the migration version of the given database."""
+    async with Database(db_url) as db:
+        return await db.get_version()
+
+
+def create_migration(name: str, directory: str | None = None) -> str:
+    """
+    Create a migration with the given name. If no directory is specified,
+    the current working directory will be used.
+    """
+    directory = directory if directory else "."
+    if not os.path.exists(directory) or not os.path.isdir(directory):
+        msg: str = f"{directory} is not a directory."
+        raise Error(msg)
+
+    now: datetime.datetime = datetime.datetime.now(tz=datetime.UTC)
+    version: str = now.strftime("%Y%m%d%H%M%S")
+
+    contents: str = MIGRATION_TEMPLATE % {"name": name, "version": version}
+
+    sanitized: str = name.replace(" ", "_")
+    filename: str = f"{version}_{sanitized}.py"
+    path: str = os.path.join(directory, filename)
+    with open(path, "w") as migration_file:
+        migration_file.write(contents)
+
+    return path
+
+
+MIGRATION_TEMPLATE = """\
+\"\"\"
+This module contains a db migration.
+
+Migration Name: %(name)s
+Migration Version: %(version)s
+\"\"\"
+
+async def upgrade(c):
+    # add your upgrade step here
+    await c.execute("SELECT 1")
+
+async def downgrade(c):
+    # add your downgrade step here
+    await c.execute("SELECT 1")
+"""

app/library/sqlite_store.py

@@ -0,0 +1,348 @@
+import asyncio
+import contextlib
+import json
+import logging
+import os
+from collections.abc import Iterable
+from dataclasses import fields
+from datetime import UTC, datetime
+from email.utils import formatdate
+
+import aiosqlite
+from aiohttp import web
+
+from .ItemDTO import ItemDTO
+from .Singleton import ThreadSafe
+from .Utils import init_class
+
+LOG: logging.Logger = logging.getLogger(__name__)
+
+ITEM_DTO_FIELDS: set[str] = {f.name for f in fields(ItemDTO)}
+
+
+class Terminator:
+    pass
+
+
+class _Op:
+    """Queued write operation."""
+
+    __slots__ = ("item", "key", "keys", "op", "type_value")
+
+    def __init__(
+        self, op: Terminator | str, type_value: str, item: ItemDTO | None, key: str | None, keys: list[str] | None
+    ):
+        self.op = op
+        self.type_value = type_value
+        self.item = item
+        self.key = key
+        self.keys = keys
+
+
+class SqliteStore(metaclass=ThreadSafe):
+    """
+    Async persistence layer with back-pressure and write-behind queue.
+    Singleton per process (ThreadSafe). Owns its aiosqlite connection.
+    """
+
+    @staticmethod
+    def get_instance(db_path: str | None = None) -> "SqliteStore":
+        return SqliteStore(db_path)
+
+    def attach(self, app: web.Application):
+        """Get/create singleton bound to db_path."""
+        app.on_shutdown.append(self.on_shutdown)
+
+    async def on_shutdown(self, _: web.Application):
+        """Close singleton on app shutdown."""
+        LOG.debug("Shutting down SqliteStore...")
+        await self.close()
+        LOG.debug("SqliteStore shut down complete.")
+
+    def __init__(self, db_path: str | None = None, *, max_pending: int = 200, flush_interval: float = 0.05):
+        self._db_path = db_path
+        self._conn: aiosqlite.Connection | None = None
+        self._queue: asyncio.Queue[_Op] | None = None
+        self._task: asyncio.Task | None = None
+        self._flush_interval = flush_interval
+        self._max_pending = max_pending
+        self._lock = asyncio.Lock()
+
+    async def __aenter__(self) -> "SqliteStore":
+        await self._ensure_conn()
+        return self
+
+    async def __aexit__(self, exc_type, exc, tb) -> None:
+        await self.close()
+
+    # ---------- public API ----------
+    async def fetch_saved(self, type_value: str) -> list[tuple[str, ItemDTO]]:
+        await self._ensure_conn()
+        cursor = await self._conn.execute(
+            'SELECT "id", "data", "created_at" FROM "history" WHERE "type" = ? ORDER BY "created_at" ASC',
+            (type_value,),
+        )
+        items: list[tuple[str, ItemDTO]] = []
+        async with cursor:
+            async for row in cursor:
+                row_date = datetime.strptime(row["created_at"], "%Y-%m-%d %H:%M:%S")  # noqa: DTZ007
+                data = json.loads(row["data"])
+                data.pop("_id", None)
+                item = init_class(ItemDTO, data, ITEM_DTO_FIELDS)
+                item._id = row["id"]
+                item.datetime = formatdate(row_date.replace(tzinfo=UTC).timestamp())
+                items.append((row["id"], item))
+        return items
+
+    async def get_by_id(self, type_value: str, id: str) -> ItemDTO | None:
+        await self._ensure_conn()
+        cursor = await self._conn.execute(
+            'SELECT "id", "data", "created_at" FROM "history" WHERE "type" = ? AND "id" = ?',
+            (type_value, id),
+        )
+        row = await cursor.fetchone()
+        if not row:
+            return None
+
+        row_date = datetime.strptime(row["created_at"], "%Y-%m-%d %H:%M:%S")  # noqa: DTZ007
+        data = json.loads(row["data"])
+        data.pop("_id", None)
+        item = init_class(ItemDTO, data, ITEM_DTO_FIELDS)
+        item._id = row["id"]
+        item.datetime = formatdate(row_date.replace(tzinfo=UTC).timestamp())
+        return item
+
+    async def count(self, type_value: str, status_filter: str | None = None) -> int:
+        await self._ensure_conn()
+        where_clauses = ['"type" = ?']
+        query_params: list[str] = [type_value]
+
+        if status_filter:
+            if status_filter.startswith("!"):
+                status_value = status_filter[1:]
+                where_clauses.append("json_extract(data, '$.status') != ?")
+                query_params.append(status_value)
+            else:
+                where_clauses.append("json_extract(data, '$.status') = ?")
+                query_params.append(status_filter)
+
+        where_clause = " AND ".join(where_clauses)
+        count_query = f'SELECT COUNT(*) as count FROM "history" WHERE {where_clause}'  # noqa: S608
+        cursor = await self._conn.execute(count_query, tuple(query_params))
+        row = await cursor.fetchone()
+        return row["count"] if row else 0
+
+    async def paginate(
+        self,
+        type_value: str,
+        page: int,
+        per_page: int,
+        order: str,
+        status_filter: str | None = None,
+    ) -> tuple[list[tuple[str, ItemDTO]], int, int, int]:
+        await self._ensure_conn()
+        where_clauses = ['"type" = ?']
+        query_params: list[str | int] = [type_value]
+
+        if status_filter:
+            if status_filter.startswith("!"):
+                status_value = status_filter[1:]
+                where_clauses.append("json_extract(data, '$.status') != ?")
+                query_params.append(status_value)
+            else:
+                where_clauses.append("json_extract(data, '$.status') = ?")
+                query_params.append(status_filter)
+
+        where_clause = " AND ".join(where_clauses)
+        count_query = f'SELECT COUNT(*) as count FROM "history" WHERE {where_clause}'  # noqa: S608
+        count_cursor = await self._conn.execute(count_query, tuple(query_params))
+        count_row = await count_cursor.fetchone()
+        total_items = count_row["count"] if count_row else 0
+        total_pages = (total_items + per_page - 1) // per_page if total_items > 0 else 1
+
+        if page > total_pages and total_items > 0:
+            page = total_pages
+
+        offset = (page - 1) * per_page
+        query_params.extend([per_page, offset])
+
+        cursor = await self._conn.execute(
+            f'SELECT "id", "data", "created_at" FROM "history" WHERE {where_clause} ORDER BY "created_at" {order} LIMIT ? OFFSET ?',  # noqa: S608
+            tuple(query_params),
+        )
+
+        items: list[tuple[str, ItemDTO]] = []
+        async with cursor:
+            async for row in cursor:
+                row_date = datetime.strptime(row["created_at"], "%Y-%m-%d %H:%M:%S")  # noqa: DTZ007
+                data = json.loads(row["data"])
+                data.pop("_id", None)
+                item = init_class(ItemDTO, data, ITEM_DTO_FIELDS)
+                item._id = row["id"]
+                item.datetime = formatdate(row_date.replace(tzinfo=UTC).timestamp())
+                items.append((row["id"], item))
+
+        return items, total_items, page, total_pages
+
+    # direct CRUD helpers (used by DataStore)
+    async def upsert(self, type_value: str, item: ItemDTO) -> None:
+        await self._ensure_conn()
+        await self._upsert_now(type_value, item)
+
+    async def delete(self, type_value: str, key: str) -> None:
+        await self._ensure_conn()
+        await self._conn.execute('DELETE FROM "history" WHERE "type" = ? AND "id" = ?', (type_value, key))
+
+    async def bulk_delete(self, type_value: str, keys: Iterable[str]) -> int:
+        await self._ensure_conn()
+        keys_list = list(keys)
+        if not keys_list:
+            return 0
+        placeholders = ",".join("?" for _ in keys_list)
+        params = [type_value, *keys_list]
+        cur = await self._conn.execute(
+            f'DELETE FROM "history" WHERE "type" = ? AND "id" IN ({placeholders})',  # noqa: S608
+            tuple(params),
+        )
+        return cur.rowcount if cur else 0
+
+    async def commit(self) -> None:
+        if self._conn:
+            await self._conn.commit()
+
+    async def enqueue_upsert(self, type_value: str, item: ItemDTO) -> None:
+        await self._enqueue(_Op("upsert", type_value, item, None, None))
+
+    async def enqueue_delete(self, type_value: str, key: str) -> None:
+        await self._enqueue(_Op("delete", type_value, None, key, None))
+
+    async def enqueue_bulk_delete(self, type_value: str, keys: Iterable[str]) -> None:
+        await self._enqueue(_Op("bulk_delete", type_value, None, None, list(keys)))
+
+    async def flush(self) -> None:
+        if self._queue:
+            await self._queue.join()
+        if self._conn:
+            await self._conn.commit()
+
+    async def shutdown(self) -> None:
+        """Flush pending writes, stop writer task, and commit."""
+        if self._queue:
+            try:
+                await self._queue.put(_Op(Terminator(), "", None, None, None))
+                LOG.debug("Waiting for SqliteStore queue to drain...")
+                await asyncio.wait_for(self._queue.join(), timeout=2)
+            except TimeoutError:
+                LOG.warning("SqliteStore queue did not drain within timeout; forcing writer shutdown.")
+
+        if self._task:
+            LOG.debug("Waiting for SqliteStore writer task to finish...")
+            if not self._task.done():
+                self._task.cancel()
+            with contextlib.suppress(asyncio.CancelledError):
+                await asyncio.wait_for(self._task, timeout=2)
+
+        if self._conn:
+            LOG.debug("Committing final changes to SqliteStore...")
+            await self._conn.commit()
+
+        self._task = None
+        self._queue = None
+
+    async def close(self) -> None:
+        await self.shutdown()
+        if self._conn:
+            LOG.debug("Closing SqliteStore connection...")
+            await self._conn.close()
+            self._conn = None
+
+    async def _enqueue(self, op: _Op) -> None:
+        self._ensure_worker()
+        await self._queue.put(op)
+
+    def _ensure_worker(self) -> None:
+        if self._queue is None:
+            self._queue = asyncio.Queue(maxsize=self._max_pending)
+            self._task = asyncio.create_task(self._writer(), name="sqlite-store-writer")
+
+    async def _writer(self) -> None:
+        while True:
+            op = await self._queue.get()
+            if isinstance(op.op, Terminator):
+                self._queue.task_done()
+                break
+            try:
+                async with self._lock:
+                    await self._apply(op)
+            except Exception as ex:
+                LOG.exception(ex)
+            finally:
+                self._queue.task_done()
+                await asyncio.sleep(self._flush_interval)
+
+    async def _apply(self, op: _Op) -> None:
+        await self._ensure_conn()
+        if op.op == "upsert" and op.item:
+            await self._upsert_now(op.type_value, op.item)
+        elif op.op == "delete" and op.key:
+            await self._conn.execute('DELETE FROM "history" WHERE "type" = ? AND "id" = ?', (op.type_value, op.key))
+        elif op.op == "bulk_delete" and op.keys:
+            placeholders = ",".join("?" for _ in op.keys)
+            params = [op.type_value, *op.keys]
+            await self._conn.execute(
+                f'DELETE FROM "history" WHERE "type" = ? AND "id" IN ({placeholders})',  # noqa: S608
+                tuple(params),
+            )
+        await self._conn.commit()
+
+    async def _upsert_now(self, type_value: str, item: ItemDTO) -> None:
+        await self._ensure_conn()
+        sql = """
+        INSERT INTO "history" ("id", "type", "url", "data")
+        VALUES (?, ?, ?, ?)
+        ON CONFLICT DO UPDATE SET "type" = ?, "url" = ?, "data" = ?, created_at = ?
+        """
+        encoded = item.json()
+        await self._conn.execute(
+            sql.strip(),
+            (
+                item._id,
+                type_value,
+                item.url,
+                encoded,
+                type_value,
+                item.url,
+                encoded,
+                datetime.now(UTC).strftime("%Y-%m-%d %H:%M:%S"),
+            ),
+        )
+
+    async def _ensure_conn(self) -> None:
+        if self._conn:
+            return
+
+        if not self._db_path:
+            msg = "SqliteStore requires db_path or injected connection."
+            raise RuntimeError(msg)
+
+        from app.library import migrate
+        from app.main import ROOT_PATH
+
+        if not self._db_path.startswith(":memory"):
+            os.makedirs(os.path.dirname(self._db_path) or ".", exist_ok=True)
+
+        self._conn = await aiosqlite.connect(database=self._db_path, isolation_level=None)
+        self._conn.row_factory = aiosqlite.Row
+        version = await migrate.get_version(self._conn)
+        if version:
+            LOG.debug(f"DB Version: '{version}'.")
+
+        await migrate.upgrade(self._conn, ROOT_PATH / "migrations")
+        if not version:
+            version = await migrate.get_version(self._conn)
+            LOG.debug(f"DB Version after initial migration: '{version}'.")
+
+        await self._conn.execute("PRAGMA journal_mode=wal")
+        await self._conn.execute("PRAGMA busy_timeout=5000")
+        await self._conn.execute("PRAGMA foreign_keys=ON")
+        await self._conn.commit()

app/main.py

@@ -9,16 +9,15 @@ if APP_ROOT not in sys.path:
 
 import asyncio
 import logging
-import sqlite3
 from pathlib import Path
 
-import caribou
 import magic
 from aiohttp import web
 
 from app.library.BackgroundWorker import BackgroundWorker
 from app.library.conditions import Conditions
 from app.library.config import Config
+from app.library.DataStore import SqliteStore
 from app.library.dl_fields import DLFields
 from app.library.DownloadQueue import DownloadQueue
 from app.library.Events import EventBus, Events
@@ -50,18 +49,6 @@ class Main:
 
         self._check_folders()
 
-        LOG.debug(f"DB Version: '{caribou.get_version(self._config.db_file)}'.")
-        caribou.upgrade(self._config.db_file, ROOT_PATH / "migrations")
-
-        connection = sqlite3.connect(database=self._config.db_file, isolation_level=None)
-        connection.row_factory = sqlite3.Row
-        connection.execute("PRAGMA journal_mode=wal")
-
-        async def _close_connection(_):
-            LOG.debug("Closing database connection.")
-            connection.close()
-            LOG.debug("Database connection closed.")
-
         try:
             db_file = Path(self._config.db_file)
             if "600" != oct(db_file.stat().st_mode)[-3:]:
@@ -69,12 +56,10 @@ class Main:
         except Exception:
             pass
 
-        self._queue = DownloadQueue(connection=connection)
+        self._queue = DownloadQueue(config=self._config)
         self._http = HttpAPI(root_path=ROOT_PATH, queue=self._queue)
         self._socket = HttpSocket(root_path=ROOT_PATH, queue=self._queue)
 
-        self._app.on_cleanup.append(_close_connection)
-
     def _check_folders(self):
         """Check if the required folders exist and create them if they do not."""
         folders = (self._config.download_path, self._config.temp_path, self._config.config_path)
@@ -136,6 +121,7 @@ class Main:
         Conditions.get_instance().attach(self._app)
         DLFields.get_instance().attach(self._app)
         TaskDefinitions.get_instance().attach(self._app)
+        SqliteStore.get_instance().attach(self._app)
         self._background_worker.attach(self._app)
 
         EventBus.get_instance().emit(

app/migrations/20231115152938_initial.py

@@ -1,12 +1,12 @@
 """
-This module contains a Caribou migration.
+This module contains a db migration.
 
 Migration Name: initial
 Migration Version: 20231115152938
 """
 
 
-def upgrade(connection):
+async def upgrade(c):
     sql = """
     CREATE TABLE "history" (
         "id" TEXT PRIMARY KEY UNIQUE NOT NULL,
@@ -16,24 +16,24 @@ def upgrade(connection):
         "created_at" TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
     );
     """
-    connection.execute(sql)
+    await c.execute(sql)
 
     sql = """
     CREATE INDEX "history_type" ON "history" ("type");
     """
-    connection.execute(sql)
+    await c.execute(sql)
 
     sql = """
     CREATE UNIQUE INDEX "history_url" ON "history" ("url");
     """
-    connection.execute(sql)
+    await c.execute(sql)
 
-    connection.commit()
+    await c.commit()
 
 
-def downgrade(connection):
+async def downgrade(c):
     sql = """
     DROP TABLE IF EXISTS "history";
     """
 
-    connection.execute(sql)
+    await c.execute(sql)

app/migrations/20251116173731_add_status_index.py

@@ -1,11 +1,11 @@
 """
-This module contains a Caribou migration.
+This module contains a db migration.
 
 Migration Name: add_status_index
 Migration Version: 20251116173731
 """
 
-def upgrade(connection):
+async def upgrade(connection):
     """
     Add index on json_extract(data, '$.status') for better query performance.
 
@@ -15,13 +15,13 @@ def upgrade(connection):
     sql = """
     CREATE INDEX IF NOT EXISTS "history_status" ON "history" (json_extract("data", '$.status'));
     """
-    connection.execute(sql)
+    await connection.execute(sql)
 
-def downgrade(connection):
+async def downgrade(connection):
     """
     Remove the status index.
     """
     sql = """
     DROP INDEX IF EXISTS "history_status";
     """
-    connection.execute(sql)
+    await connection.execute(sql)

app/routes/api/browser.py

@@ -402,7 +402,7 @@ async def path_actions(request: Request, config: Config, queue: DownloadQueue, n
                     if sidecar_renamed:
                         item.info.get_file_sidecar()
 
-                    queue.done.put(item)
+                    await queue.done.put(item)
                     notify.emit(Events.ITEM_UPDATED, data=item.info)
 
         if "delete" == action:
@@ -508,7 +508,7 @@ async def path_actions(request: Request, config: Config, queue: DownloadQueue, n
                     if sidecar_moved:
                         item.info.get_file_sidecar()
 
-                    queue.done.put(item)
+                    await queue.done.put(item)
                     notify.emit(Events.ITEM_UPDATED, data=item.info)
 
     return web.json_response(data=operations_status, status=web.HTTPOk.status_code)

app/routes/api/history.py

@@ -47,30 +47,21 @@ async def items_list(request: Request, queue: DownloadQueue, encoder: Encoder, c
     """
     from app.library.DataStore import StoreType
 
-    store_type = request.query.get("type", "all").lower()
+    store_type: str = request.query.get("type", "queue").lower()
-    stores: list[str] = ["all", StoreType.QUEUE.value, StoreType.HISTORY.value]
+    try:
-    if store_type not in stores:
+        store_type: StoreType = StoreType.from_value(store_type)
+    except ValueError:
         return web.json_response(
-            data={"error": f"type must be one of {', '.join(stores)}."},
+            data={"error": f"type must be one of {', '.join(StoreType.all())}."},
             status=web.HTTPBadRequest.status_code,
         )
 
-    # Legacy behavior: return all items without pagination, will be removed in future.
+    ds = queue.queue if store_type == StoreType.QUEUE else queue.done
-    if "all" == store_type:
-        data: dict = {"queue": [], "history": []}
-        q = queue.get()
-
-        data["queue"].extend([q.get("queue", {}).get(k) for k in q.get("queue", {})])
-        data["history"].extend([q.get("done", {}).get(k) for k in q.get("done", {})])
-
-        return web.json_response(data=data, status=web.HTTPOk.status_code, dumps=encoder.encode)
-
-    ds = queue.queue if store_type == StoreType.QUEUE.value else queue.done
 
     try:
         page = int(request.query.get("page", 1))
         per_page = int(request.query.get("per_page", config.default_pagination))
-        order = request.query.get("order", "DESC").upper()
+        order: str = request.query.get("order", "DESC").upper()
     except ValueError:
         return web.json_response(
             data={"error": "page and per_page must be valid integers."},
@@ -92,13 +83,15 @@ async def items_list(request: Request, queue: DownloadQueue, encoder: Encoder, c
             status=web.HTTPBadRequest.status_code,
         )
 
-    # Parse status filter
     status_filter = request.query.get("status", None)
 
-    items, total, current_page, total_pages = ds.get_items_paginated(
+    items, total, current_page, total_pages = await ds.get_items_paginated(
         page=page, per_page=per_page, order=order, status_filter=status_filter
     )
+
+    return web.json_response(
         data={
+            "type": store_type.value,
             "pagination": {
                 "page": current_page,
                 "per_page": per_page,
@@ -108,9 +101,10 @@ async def items_list(request: Request, queue: DownloadQueue, encoder: Encoder, c
                 "has_prev": current_page > 1,
             },
             "items": [item for _, item in items],
-    }
+        },
-
+        status=web.HTTPOk.status_code,
-    return web.json_response(data=data, status=web.HTTPOk.status_code, dumps=encoder.encode)
+        dumps=encoder.encode,
+    )
 
 
 @route("DELETE", "api/history/", "items_delete")
@@ -137,24 +131,26 @@ async def items_delete(request: Request, queue: DownloadQueue, encoder: Encoder)
 
     ids = data.get("ids")
     remove_file: bool = bool(data.get("remove_file", True))
-    _type = data.get("type", data.get("where"))
+    storeType = data.get("type", data.get("where"))
 
-    if not _type:
+    if not storeType:
         return web.json_response(data={"error": "Type is required."}, status=web.HTTPBadRequest.status_code)
 
-    if _type not in [StoreType.QUEUE.value, StoreType.HISTORY.value]:
+    try:
+        storeType: StoreType = StoreType.from_value(storeType)
+    except ValueError:
         return web.json_response(
-            data={"error": f"type must be '{StoreType.QUEUE.value}' or '{StoreType.HISTORY.value}'."},
+            data={"error": f"type must be one of '{StoreType.all()}'."},
             status=web.HTTPBadRequest.status_code,
         )
 
-    ds = queue.queue if _type == StoreType.QUEUE.value else queue.done
+    ds = queue.queue if storeType == StoreType.QUEUE else queue.done
 
     if ids:
         return web.json_response(
             data={
                 "items": await (
-                    queue.cancel(ids) if _type == StoreType.QUEUE.value else queue.clear(ids, remove_file=remove_file)
+                    queue.cancel(ids) if storeType == StoreType.QUEUE else queue.clear(ids, remove_file=remove_file)
                 ),
                 "deleted": len(ids),
             },
@@ -174,7 +170,7 @@ async def items_delete(request: Request, queue: DownloadQueue, encoder: Encoder)
     per_page = 1000
 
     while True:
-        items, _, current_page, total_pages = ds.get_items_paginated(
+        items, _, current_page, total_pages = await ds.get_items_paginated(
             page=page, per_page=per_page, order="DESC", status_filter=status_filter
         )
 
@@ -192,7 +188,7 @@ async def items_delete(request: Request, queue: DownloadQueue, encoder: Encoder)
         data={
             "items": await (
                 queue.cancel(items_to_delete)
-                if _type == StoreType.QUEUE.value
+                if storeType == StoreType.QUEUE
                 else queue.clear(items_to_delete, remove_file=remove_file)
             ),
             "deleted": len(items_to_delete),
@@ -221,7 +217,7 @@ async def item_view(request: Request, queue: DownloadQueue, encoder: Encoder) ->
     if not id:
         return web.json_response(data={"error": "id is required."}, status=web.HTTPBadRequest.status_code)
 
-    item: Download | None = queue.done.get_by_id(id) or queue.queue.get_by_id(id)
+    item: Download | None = await queue.done.get_by_id(id) or await queue.queue.get_by_id(id)
     if not item:
         return web.json_response(data={"error": "item not found."}, status=web.HTTPNotFound.status_code)
 
@@ -263,7 +259,7 @@ async def item_update(request: Request, queue: DownloadQueue, encoder: Encoder,
     if not id:
         return web.json_response(data={"error": "id is required."}, status=web.HTTPBadRequest.status_code)
 
-    item: Download | None = queue.done.get_by_id(id)
+    item: Download | None = await queue.done.get_by_id(id)
     if not item:
         return web.json_response(data={"error": "item not found."}, status=web.HTTPNotFound.status_code)
 
@@ -285,7 +281,7 @@ async def item_update(request: Request, queue: DownloadQueue, encoder: Encoder,
         LOG.debug(f"Updated '{k}' to '{v}' for '{item.info.id}'")
 
     if updated:
-        queue.done.put(item)
+        await queue.done.put(item)
         notify.emit(Events.ITEM_UPDATED, data=item.info)
 
     return web.json_response(
@@ -394,7 +390,7 @@ async def item_archive_add(request: Request, queue: DownloadQueue, notify: Event
         return web.json_response(data={"error": "id is required."}, status=web.HTTPBadRequest.status_code)
 
     try:
-        item: Download | None = queue.done.get_by_id(id)
+        item: Download | None = await queue.done.get_by_id(id)
         if not item:
             return web.json_response(data={"error": f"item '{id}' not found."}, status=web.HTTPNotFound.status_code)
     except KeyError:
@@ -425,7 +421,7 @@ async def item_archive_add(request: Request, queue: DownloadQueue, notify: Event
         )
 
     item.info.archive_status(force=True)
-    queue.done.put(item, no_notify=True)
+    await queue.done.put(item, no_notify=True)
     notify.emit(Events.ITEM_UPDATED, data=item.info)
 
     return web.json_response(
@@ -452,7 +448,7 @@ async def item_archive_delete(request: Request, queue: DownloadQueue, notify: Ev
         return web.json_response(data={"error": "id is required."}, status=web.HTTPBadRequest.status_code)
 
     try:
-        item: Download | None = queue.done.get_by_id(id)
+        item: Download | None = await queue.done.get_by_id(id)
         if not item:
             return web.json_response(data={"error": f"item '{id}' not found."}, status=web.HTTPNotFound.status_code)
     except KeyError:
@@ -483,7 +479,7 @@ async def item_archive_delete(request: Request, queue: DownloadQueue, notify: Ev
         )
 
     item.info.archive_status(force=True)
-    queue.done.put(item, no_notify=True)
+    await queue.done.put(item, no_notify=True)
     notify.emit(Events.ITEM_UPDATED, data=item.info)
 
     return web.json_response(

app/routes/socket/connection.py

@@ -44,8 +44,8 @@ async def connect(config: Config, queue: DownloadQueue, notify: EventBus, sid: s
                 base=Path(config.download_path),
                 depth_limit=config.download_path_depth - 1,
             ),
-            "history_count": queue.done.get_total_count(),
+            "history_count": await queue.done.get_total_count(),
-            "queue": queue.get("queue")["queue"],
+            "queue": (await queue.get("queue"))["queue"],
         },
         title="Sending initial download data",
         message=f"Sending initial download data to client '{sid}'.",

app/scripts/create_migration.py

@@ -0,0 +1,147 @@
+#!/usr/bin/env python3
+"""
+Caribou migrations CLI.
+
+Provides commands to create, list, and apply database migrations.
+"""
+
+from __future__ import annotations
+
+import argparse
+import sys
+import traceback
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+APP_ROOT: Path = Path(__file__).resolve().parents[2]
+if str(APP_ROOT) not in sys.path:
+    sys.path.insert(0, str(APP_ROOT))
+
+from app.library import migrate
+
+if TYPE_CHECKING:
+    from collections.abc import Callable
+
+
+def e_print(msg: str = "") -> None:
+    sys.stderr.write(f"{msg}\n")
+
+
+def o_print(msg: str = "") -> None:
+    sys.stdout.write(f"{msg}\n")
+
+
+def cmd_info(_: argparse.Namespace) -> None:
+    o_print(f"Version: {migrate.__version__}")
+
+
+def cmd_create(args: argparse.Namespace) -> None:
+    path = migrate.create_migration(args.name, args.migration_dir)
+    o_print(f"created migration {path}")
+
+
+def cmd_version(args: argparse.Namespace) -> None:
+    version = migrate.get_version(args.database_path)
+    if version:
+        o_print(f"the db [{args.database_path}] is at version {version}")
+    else:
+        o_print(f"the db [{args.database_path}] is not under version control")
+
+
+def cmd_upgrade(args: argparse.Namespace) -> None:
+    if args.version:
+        o_print(f"upgrading db [{args.database_path}] to version [{args.version}]")
+    else:
+        o_print(f"upgrading db [{args.database_path}] to most recent version")
+
+    migrate.upgrade(args.database_path, args.migration_dir, args.version)
+
+    new_version = migrate.get_version(args.database_path)
+    if args.version is not None:
+        assert new_version == args.version
+
+    o_print(f"upgraded [{args.database_path}] successfully to version [{new_version}]")
+
+
+def cmd_downgrade(args: argparse.Namespace) -> None:
+    o_print(f"downgrading db [{args.database_path}] to version [{args.version}]")
+    migrate.downgrade(args.database_path, args.migration_dir, args.version)
+    o_print(f"downgraded [{args.database_path}] successfully to version [{args.version}]")
+
+
+def cmd_list(args: argparse.Namespace) -> None:
+    o_print(f"Migrations in [{args.migration_dir}]:\n")
+    for m in migrate.load_migrations(args.migration_dir):
+        o_print(f"{m.get_version()}\t{m.name}\t{m.path}")
+
+
+def build_parser() -> argparse.ArgumentParser:
+    p = argparse.ArgumentParser(prog="caribou")
+    sub = p.add_subparsers(dest="command", required=True)
+
+    info = sub.add_parser("info", help="print library version")
+    info.set_defaults(func=cmd_info)
+
+    create = sub.add_parser("create", help="create a new migration file")
+    create.add_argument("name", help="the name of migration")
+    create.add_argument("-d", "--migration-dir", default=".", help="the migration directory")
+    create.set_defaults(func=cmd_create)
+
+    ver = sub.add_parser("version", help="return the migration version of the database")
+    ver.add_argument("database_path", help="path to the sqlite database")
+    ver.set_defaults(func=cmd_version)
+
+    up = sub.add_parser(
+        "upgrade",
+        help="upgrade the db (if version isn't specified, upgrade to the most recent)",
+    )
+    up.add_argument("database_path", help="path to the sqlite database")
+    up.add_argument("migration_dir", help="the migration directory")
+    up.add_argument("-v", "--version", type=int, default=None, help="the target migration version")
+    up.set_defaults(func=cmd_upgrade)
+
+    down = sub.add_parser(
+        "downgrade",
+        help="downgrade the db to a particular version (use 0 to rollback all changes)",
+    )
+    down.add_argument("database_path", help="path to the sqlite database")
+    down.add_argument("migration_dir", help="the migration directory")
+    down.add_argument("version", type=int, help="the target migration version")
+    down.set_defaults(func=cmd_downgrade)
+
+    lst = sub.add_parser("list", help="list the migration versions")
+    lst.add_argument("migration_dir", help="the migration directory")
+    lst.set_defaults(func=cmd_list)
+
+    return p
+
+
+def run(argv: list[str]) -> int:
+    parser = build_parser()
+    args = parser.parse_args(argv)
+
+    try:
+        func: Callable[[argparse.Namespace], None] = args.func
+        func(args)
+        return 0
+
+    except migrate.InvalidMigrationError:
+        e_print(traceback.format_exc())
+        return 1
+
+    except migrate.Error as err:
+        e_print(f"Error: {err}")
+        return 1
+
+    except Exception:
+        e_print("an unexpected error occurred:")
+        e_print(traceback.format_exc())
+        return 1
+
+
+def main() -> int:
+    return run(sys.argv[1:])
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

app/tests/test_datastore_pagination.py

@@ -1,45 +1,25 @@
 import json
-import sqlite3
 from datetime import UTC, datetime
-from pathlib import Path
-from tempfile import TemporaryDirectory
 
 import pytest
+import pytest_asyncio
 
 from app.library.DataStore import DataStore, StoreType
 from app.library.ItemDTO import ItemDTO
+from app.library.sqlite_store import SqliteStore
 
 
-class TestDataStorePagination:
+async def make_db(data: int = 100) -> SqliteStore:
-    """Test pagination functionality of DataStore."""
-
-    @pytest.fixture
-    def temp_db(self):
     """Create a temporary database with test data."""
-        with TemporaryDirectory() as tmpdir:
+    SqliteStore._reset_singleton()
-            db_path = Path(tmpdir) / "test.db"
+    ins = SqliteStore.get_instance(db_path=":memory:")
-            conn = sqlite3.connect(str(db_path))
+    await ins._ensure_conn()
-            conn.row_factory = sqlite3.Row
 
-            # Create the history table
-            conn.execute(
-                """
-                CREATE TABLE IF NOT EXISTS "history" (
-                    "id" TEXT PRIMARY KEY,
-                    "type" TEXT NOT NULL,
-                    "url" TEXT NOT NULL,
-                    "data" TEXT NOT NULL,
-                    "created_at" TEXT NOT NULL
-                )
-                """
-            )
-
-            # Insert test data
     base_time = datetime.now(UTC)
-            for i in range(100):
+    for i in range(data):
-                created_at = base_time.replace(
+        created_at = base_time.replace(hour=(i // 4) % 24, minute=(i * 15) % 60, second=i % 60).strftime(
-                    hour=(i // 4) % 24, minute=(i * 15) % 60, second=i % 60
+            "%Y-%m-%d %H:%M:%S"
-                ).strftime("%Y-%m-%d %H:%M:%S")
+        )
 
         item_data = {
             "url": f"https://example.com/video{i}",
@@ -49,7 +29,7 @@ class TestDataStorePagination:
             "status": "finished",
         }
 
-                conn.execute(
+        await ins._conn.execute(
             'INSERT INTO "history" ("id", "type", "url", "data", "created_at") VALUES (?, ?, ?, ?, ?)',
             (
                 f"test-id-{i}",
@@ -59,199 +39,172 @@ class TestDataStorePagination:
                 created_at,
             ),
         )
+    if data > 0:
+        await ins._conn.commit()
 
-            conn.commit()
+    return ins
-            yield conn
-            conn.close()
 
-    def test_get_total_count(self, temp_db):
+
+@pytest.mark.asyncio
+class TestDataStorePagination:
+    """Test pagination functionality of DataStore."""
+
+    @pytest_asyncio.fixture
+    async def make_db(self, data: int = 100):
+        """Fixture to provide a temporary database with test data."""
+        return await make_db(data=data)
+
+    async def test_get_total_count(self):
         """Test getting total count of items."""
-        datastore = DataStore(type=StoreType.HISTORY, connection=temp_db)
+        db = await make_db(data=100)
-        count = datastore.get_total_count()
+        try:
+            datastore = DataStore(type=StoreType.HISTORY, connection=db)
+            count = await datastore.get_total_count()
             assert count == 100
+        finally:
+            await db.close()
 
-    def test_pagination_basic(self, temp_db):
+    async def test_pagination_basic(self):
         """Test basic pagination functionality."""
-        datastore = DataStore(type=StoreType.HISTORY, connection=temp_db)
+        db = await make_db(data=100)
-
+        try:
-        # Get first page
+            datastore = DataStore(type=StoreType.HISTORY, connection=db)
-        items, total, page, total_pages = datastore.get_items_paginated(page=1, per_page=10)
+            items, total, page, total_pages = await datastore.get_items_paginated(page=1, per_page=10)
-
             assert len(items) == 10
             assert total == 100
             assert page == 1
             assert total_pages == 10
-
+            for _, item in items:
-        # Verify items are ItemDTO instances
-        for _item_id, item in items:
                 assert isinstance(item, ItemDTO)
                 assert item._id.startswith("test-id-")
+        finally:
+            await db.close()
 
-    def test_pagination_last_page(self, temp_db):
+    async def test_pagination_last_page(self):
         """Test pagination on last page."""
-        datastore = DataStore(type=StoreType.HISTORY, connection=temp_db)
+        db = await make_db(data=100)
-
+        try:
-        # Get last page
+            datastore = DataStore(type=StoreType.HISTORY, connection=db)
-        items, total, page, total_pages = datastore.get_items_paginated(page=10, per_page=10)
+            items, total, page, total_pages = await datastore.get_items_paginated(page=10, per_page=10)
-
             assert len(items) == 10
             assert total == 100
             assert page == 10
             assert total_pages == 10
+        finally:
+            await db.close()
 
-    def test_pagination_partial_page(self, temp_db):
+    async def test_pagination_partial_page(self):
         """Test pagination with partial last page."""
-        datastore = DataStore(type=StoreType.HISTORY, connection=temp_db)
+        db = await make_db(data=100)
-
+        try:
-        # Get items with per_page that doesn't divide evenly
+            datastore = DataStore(type=StoreType.HISTORY, connection=db)
-        items, total, page, total_pages = datastore.get_items_paginated(page=4, per_page=30)
+            items, total, page, total_pages = await datastore.get_items_paginated(page=4, per_page=30)
-
             assert len(items) == 10  # 100 items / 30 per page = 3 full pages + 10 items
             assert total == 100
             assert page == 4
             assert total_pages == 4
+        finally:
+            await db.close()
 
-    def test_pagination_out_of_range(self, temp_db):
+    async def test_pagination_out_of_range(self):
         """Test pagination with page number out of range."""
-        datastore = DataStore(type=StoreType.HISTORY, connection=temp_db)
+        db = await make_db(data=100)
-
+        try:
-        # Request page beyond total pages - should return last page
+            datastore = DataStore(type=StoreType.HISTORY, connection=db)
-        items, total, page, total_pages = datastore.get_items_paginated(page=999, per_page=10)
+            items, total, page, total_pages = await datastore.get_items_paginated(page=999, per_page=10)
-
             assert len(items) == 10
             assert total == 100
             assert page == 10  # Adjusted to last page
             assert total_pages == 10
+        finally:
+            await db.close()
 
-    def test_pagination_order_desc(self, temp_db):
+    async def test_pagination_order_desc(self):
         """Test pagination with descending order (newest first)."""
-        datastore = DataStore(type=StoreType.HISTORY, connection=temp_db)
+        db = await make_db(data=100)
-
+        try:
-        items, _, _, _ = datastore.get_items_paginated(page=1, per_page=5, order="DESC")
+            datastore = DataStore(type=StoreType.HISTORY, connection=db)
-
+            items, _, _, _ = await datastore.get_items_paginated(page=1, per_page=5, order="DESC")
             assert len(items) == 5
-        # Verify order - should be newest to oldest
+        finally:
-        # (note: exact order depends on the timestamp generation in fixture)
+            await db.close()
 
-    def test_pagination_order_asc(self, temp_db):
+    async def test_pagination_order_asc(self):
         """Test pagination with ascending order (oldest first)."""
-        datastore = DataStore(type=StoreType.HISTORY, connection=temp_db)
+        db = await make_db(data=100)
-
+        try:
-        items, _, _, _ = datastore.get_items_paginated(page=1, per_page=5, order="ASC")
+            datastore = DataStore(type=StoreType.HISTORY, connection=db)
-
+            items, _, _, _ = await datastore.get_items_paginated(page=1, per_page=5, order="ASC")
             assert len(items) == 5
+        finally:
+            await db.close()
 
-    def test_pagination_invalid_page(self, temp_db):
+    async def test_pagination_invalid_page(self):
         """Test pagination with invalid page number."""
-        datastore = DataStore(type=StoreType.HISTORY, connection=temp_db)
+        db = await make_db(data=100)
+        try:
+            datastore = DataStore(type=StoreType.HISTORY, connection=db)
+            with pytest.raises(ValueError, match="page must be >= 1"):
+                await datastore.get_items_paginated(page=0, per_page=10)
 
             with pytest.raises(ValueError, match="page must be >= 1"):
-            datastore.get_items_paginated(page=0, per_page=10)
+                await datastore.get_items_paginated(page=-1, per_page=10)
+        finally:
+            await db.close()
 
-        with pytest.raises(ValueError, match="page must be >= 1"):
+    async def test_pagination_invalid_per_page(self):
-            datastore.get_items_paginated(page=-1, per_page=10)
-
-    def test_pagination_invalid_per_page(self, temp_db):
         """Test pagination with invalid per_page value."""
-        datastore = DataStore(type=StoreType.HISTORY, connection=temp_db)
+        db = await make_db(data=100)
+        try:
+            datastore = DataStore(type=StoreType.HISTORY, connection=db)
 
             with pytest.raises(ValueError, match="per_page must be >= 1"):
-            datastore.get_items_paginated(page=1, per_page=0)
+                await datastore.get_items_paginated(page=1, per_page=0)
 
             with pytest.raises(ValueError, match="per_page must be >= 1"):
-            datastore.get_items_paginated(page=1, per_page=-10)
+                await datastore.get_items_paginated(page=1, per_page=-10)
+        finally:
+            await db.close()
 
-    def test_pagination_invalid_order(self, temp_db):
+    async def test_pagination_invalid_order(self):
         """Test pagination with invalid order parameter."""
-        datastore = DataStore(type=StoreType.HISTORY, connection=temp_db)
+        db = await make_db(data=100)
-
+        try:
+            datastore = DataStore(type=StoreType.HISTORY, connection=db)
             with pytest.raises(ValueError, match="order must be 'ASC' or 'DESC'"):
-            datastore.get_items_paginated(page=1, per_page=10, order="INVALID")
+                await datastore.get_items_paginated(page=1, per_page=10, order="INVALID")
+        finally:
+            await db.close()
 
-    def test_pagination_empty_store(self):
+    async def test_pagination_empty_store(self):
         """Test pagination with empty datastore."""
-        with TemporaryDirectory() as tmpdir:
+        db = await make_db(data=0)
-            db_path = Path(tmpdir) / "empty.db"
+        try:
-            conn = sqlite3.connect(str(db_path))
+            datastore = DataStore(type=StoreType.HISTORY, connection=db)
-            conn.row_factory = sqlite3.Row
+            items, total, page, total_pages = await datastore.get_items_paginated(page=1, per_page=10)
-
-            # Create empty table
-            conn.execute(
-                """
-                CREATE TABLE IF NOT EXISTS "history" (
-                    "id" TEXT PRIMARY KEY,
-                    "type" TEXT NOT NULL,
-                    "url" TEXT NOT NULL,
-                    "data" TEXT NOT NULL,
-                    "created_at" TEXT NOT NULL
-                )
-                """
-            )
-            conn.commit()
-
-            datastore = DataStore(type=StoreType.HISTORY, connection=conn)
-
-            items, total, page, total_pages = datastore.get_items_paginated(page=1, per_page=10)
 
             assert len(items) == 0
             assert total == 0
             assert page == 1
             assert total_pages == 1
+        finally:
+            await db.close()
 
-            conn.close()
+    async def test_pagination_single_item(self):
-
-    def test_pagination_single_item(self):
         """Test pagination with single item."""
-        with TemporaryDirectory() as tmpdir:
+        db = await make_db(data=1)
-            db_path = Path(tmpdir) / "single.db"
+        try:
-            conn = sqlite3.connect(str(db_path))
+            datastore = DataStore(type=StoreType.HISTORY, connection=db)
-            conn.row_factory = sqlite3.Row
+            items, total, page, total_pages = await datastore.get_items_paginated(page=1, per_page=10)
-
-            # Create table with single item
-            conn.execute(
-                """
-                CREATE TABLE IF NOT EXISTS "history" (
-                    "id" TEXT PRIMARY KEY,
-                    "type" TEXT NOT NULL,
-                    "url" TEXT NOT NULL,
-                    "data" TEXT NOT NULL,
-                    "created_at" TEXT NOT NULL
-                )
-                """
-            )
-
-            item_data = {
-                "url": "https://example.com/single",
-                "title": "Single Video",
-                "id": "single-video",
-                "folder": "/downloads",
-                "status": "finished",
-            }
-
-            conn.execute(
-                'INSERT INTO "history" ("id", "type", "url", "data", "created_at") VALUES (?, ?, ?, ?, ?)',
-                (
-                    "single-id",
-                    str(StoreType.HISTORY),
-                    item_data["url"],
-                    json.dumps(item_data),
-                    datetime.now(UTC).strftime("%Y-%m-%d %H:%M:%S"),
-                ),
-            )
-            conn.commit()
-
-            datastore = DataStore(type=StoreType.HISTORY, connection=conn)
-
-            items, total, page, total_pages = datastore.get_items_paginated(page=1, per_page=10)
 
             assert len(items) == 1
             assert total == 1
             assert page == 1
             assert total_pages == 1
+        finally:
+            await db.close()
 
-            conn.close()
+    async def test_pagination_status_filter_include(self):
-
-    def test_pagination_status_filter_include(self, temp_db):
         """Test pagination with status filter (inclusion)."""
         # Add some items with different status values
         item_data_pending = {
@@ -269,7 +222,9 @@ class TestDataStorePagination:
             "status": "downloading",
         }
 
-        temp_db.execute(
+        db = await make_db(data=100)
+        try:
+            await db._conn.execute(
                 'INSERT INTO "history" ("id", "type", "url", "data", "created_at") VALUES (?, ?, ?, ?, ?)',
                 (
                     "pending-id",
@@ -279,7 +234,7 @@ class TestDataStorePagination:
                     datetime.now(UTC).strftime("%Y-%m-%d %H:%M:%S"),
                 ),
             )
-        temp_db.execute(
+            await db._conn.execute(
                 'INSERT INTO "history" ("id", "type", "url", "data", "created_at") VALUES (?, ?, ?, ?, ?)',
                 (
                     "downloading-id",
@@ -289,14 +244,11 @@ class TestDataStorePagination:
                     datetime.now(UTC).strftime("%Y-%m-%d %H:%M:%S"),
                 ),
             )
-        temp_db.commit()
+            await db._conn.commit()
-
+            datastore = DataStore(type=StoreType.HISTORY, connection=db)
-        datastore = DataStore(type=StoreType.HISTORY, connection=temp_db)
 
             # Filter for finished items only
-        items, total, _page, _total_pages = datastore.get_items_paginated(
+            items, total, _, __ = await datastore.get_items_paginated(page=1, per_page=50, status_filter="finished")
-            page=1, per_page=50, status_filter="finished"
-        )
 
             assert len(items) == 50  # First page of finished items
             assert total == 100  # Only 100 finished items in fixture
@@ -304,15 +256,17 @@ class TestDataStorePagination:
                 assert item.status == "finished"
 
             # Filter for pending items only
-        items, total, _page, _total_pages = datastore.get_items_paginated(
+            items, total, _page, _total_pages = await datastore.get_items_paginated(
                 page=1, per_page=50, status_filter="pending"
             )
 
             assert len(items) == 1
             assert total == 1
             assert items[0][1].status == "pending"
+        finally:
+            await db.close()
 
-    def test_pagination_status_filter_exclude(self, temp_db):
+    async def test_pagination_status_filter_exclude(self):
         """Test pagination with status filter (exclusion)."""
         # Add some items with different status values
         item_data_pending = {
@@ -329,8 +283,9 @@ class TestDataStorePagination:
             "folder": "/downloads",
             "status": "error",
         }
-
+        db = await make_db(data=0)
-        temp_db.execute(
+        try:
+            await db._conn.execute(
                 'INSERT INTO "history" ("id", "type", "url", "data", "created_at") VALUES (?, ?, ?, ?, ?)',
                 (
                     "pending-id-2",
@@ -340,7 +295,7 @@ class TestDataStorePagination:
                     datetime.now(UTC).strftime("%Y-%m-%d %H:%M:%S"),
                 ),
             )
-        temp_db.execute(
+            await db._conn.execute(
                 'INSERT INTO "history" ("id", "type", "url", "data", "created_at") VALUES (?, ?, ?, ?, ?)',
                 (
                     "error-id",
@@ -350,12 +305,12 @@ class TestDataStorePagination:
                     datetime.now(UTC).strftime("%Y-%m-%d %H:%M:%S"),
                 ),
             )
-        temp_db.commit()
+            await db._conn.commit()
 
-        datastore = DataStore(type=StoreType.HISTORY, connection=temp_db)
+            datastore = DataStore(type=StoreType.HISTORY, connection=db)
 
             # Exclude finished items
-        items, total, _page, _total_pages = datastore.get_items_paginated(
+            items, total, _page, _total_pages = await datastore.get_items_paginated(
                 page=1, per_page=50, status_filter="!finished"
             )
 
@@ -367,13 +322,17 @@ class TestDataStorePagination:
             # Verify we have pending and error
             statuses = {item.status for _, item in items}
             assert statuses == {"pending", "error"}
+        finally:
+            await db.close()
 
-    def test_pagination_status_filter_none_matching(self, temp_db):
+    async def test_pagination_status_filter_none_matching(self):
         """Test pagination with status filter that matches no items."""
-        datastore = DataStore(type=StoreType.HISTORY, connection=temp_db)
+        db = await make_db(data=0)
+        try:
+            datastore = DataStore(type=StoreType.HISTORY, connection=db)
 
             # Filter for a status that doesn't exist
-        items, total, page, total_pages = datastore.get_items_paginated(
+            items, total, page, total_pages = await datastore.get_items_paginated(
                 page=1, per_page=50, status_filter="nonexistent"
             )
 
@@ -381,3 +340,5 @@ class TestDataStorePagination:
             assert total == 0
             assert page == 1
             assert total_pages == 1
+        finally:
+            await db.close()

app/tests/test_sqlite_store.py

@@ -0,0 +1,183 @@
+from datetime import UTC, datetime, timedelta
+
+import pytest
+
+from app.library.ItemDTO import ItemDTO
+from app.library.sqlite_store import SqliteStore
+
+
+async def make_store() -> SqliteStore:
+    """Create an isolated in-memory SqliteStore instance with schema."""
+    SqliteStore._reset_singleton()
+    store = SqliteStore.get_instance(db_path=":memory:")
+    await store._ensure_conn()
+    assert store._conn is not None
+    return store
+
+
+def make_item(idx: int, *, status: str = "finished") -> ItemDTO:
+    return ItemDTO(
+        id=f"vid{idx}",
+        title=f"Video {idx}",
+        url=f"https://example.com/{idx}",
+        folder="/downloads",
+        status=status,
+    )
+
+
+@pytest.mark.asyncio
+async def test_enqueue_upsert_and_fetch_saved():
+    store = await make_store()
+    item = make_item(1)
+
+    await store.enqueue_upsert("queue", item)
+    await store.flush()
+
+    saved = await store.fetch_saved("queue")
+    assert len(saved) == 1
+    key, loaded = saved[0]
+    assert key == item._id
+    assert loaded.title == item.title
+    await store.close()
+
+
+@pytest.mark.asyncio
+async def test_delete_and_bulk_delete():
+    store = await make_store()
+    items = [make_item(i) for i in range(3)]
+    for itm in items:
+        await store.enqueue_upsert("queue", itm)
+    await store.flush()
+
+    await store.enqueue_delete("queue", items[0]._id)
+    await store.enqueue_bulk_delete("queue", [items[1]._id])
+    await store.flush()
+
+    remaining = await store.fetch_saved("queue")
+    assert len(remaining) == 1
+    assert remaining[0][0] == items[2]._id
+    await store.close()
+
+
+@pytest.mark.asyncio
+async def test_count_with_status_filters():
+    store = await make_store()
+    finished_items = [make_item(i, status="finished") for i in range(2)]
+    pending_items = [make_item(i + 10, status="pending") for i in range(3)]
+
+    for itm in finished_items + pending_items:
+        await store.enqueue_upsert("history", itm)
+    await store.flush()
+
+    assert await store.count("history", status_filter="finished") == 2
+    assert await store.count("history", status_filter="pending") == 3
+    assert await store.count("history", status_filter="!finished") == 3
+    await store.close()
+
+
+@pytest.mark.asyncio
+async def test_paginate_order_and_bounds():
+    store = await make_store()
+    base = datetime(2024, 1, 1, tzinfo=UTC)
+    _list: list[ItemDTO] = []
+    for i in range(15):
+        itm = make_item(i)
+        encoded = itm.json()
+        created_at = (base + timedelta(minutes=i)).strftime("%Y-%m-%d %H:%M:%S")
+        _list.append(itm)
+        await store._conn.execute(
+            'INSERT INTO "history" ("id", "type", "url", "data", "created_at") VALUES (?, ?, ?, ?, ?)',
+            (itm._id, "history", itm.url, encoded, created_at),
+        )
+    await store._conn.commit()
+
+    items, total, page, total_pages = await store.paginate("history", page=1, per_page=5, order="ASC")
+    assert total == 15
+    assert page == 1
+    assert total_pages == 3
+    assert items[0][0] == _list[0]._id
+
+    items_desc, *_ = await store.paginate("history", page=1, per_page=3, order="DESC")
+    assert len(items_desc) == 3
+    assert items_desc[0][0] == _list[14]._id
+    await store.close()
+
+@pytest.mark.asyncio
+async def test_get_by_id():
+    store = await make_store()
+    item = make_item(99)
+    await store.enqueue_upsert("queue", item)
+    await store.flush()
+    assert item._id == (await store.get_by_id("queue", item._id))._id
+    await store.close()
+
+@pytest.mark.asyncio
+async def test_shutdown_closes_worker_queue():
+    store = await make_store()
+    await store.enqueue_upsert("queue", make_item(99))
+    await store.shutdown()
+    assert store._queue is None
+    await store.close()
+
+
+@pytest.mark.asyncio
+async def test_enqueue_delete_removes_row():
+    store = await make_store()
+    item = make_item(5)
+    await store.enqueue_upsert("queue", item)
+    await store.flush()
+
+    await store.enqueue_delete("queue", item._id)
+    await store.flush()
+
+    rows = await store.fetch_saved("queue")
+    assert rows == []
+    await store.close()
+
+
+@pytest.mark.asyncio
+async def test_enqueue_bulk_delete_returns_count_and_bulk_path():
+    store = await make_store()
+    items = [make_item(i) for i in range(3)]
+    for itm in items:
+        await store.enqueue_upsert("history", itm)
+    await store.flush()
+
+    await store.enqueue_bulk_delete("history", [items[0]._id, items[1]._id])
+    await store.flush()
+
+    remaining = await store.fetch_saved("history")
+    assert len(remaining) == 1
+    assert remaining[0][0] == items[2]._id
+
+    # direct bulk_delete should report affected rows
+    count = await store.bulk_delete("history", [items[2]._id])
+    assert count == 1
+    await store.close()
+
+
+@pytest.mark.asyncio
+async def test_paginate_out_of_range_returns_last_page():
+    store = await make_store()
+    for i in range(7):
+        await store.enqueue_upsert("history", make_item(i))
+    await store.flush()
+
+    items, total, page, total_pages = await store.paginate("history", page=10, per_page=3, order="DESC")
+    assert total == 7
+    assert total_pages == 3
+    assert page == 3
+    assert len(items) == 1
+    await store.close()
+
+
+@pytest.mark.asyncio
+async def test_on_shutdown_closes_connection():
+    store = await make_store()
+    await store.enqueue_upsert("queue", make_item(1))
+    await store.flush()
+
+    await store.on_shutdown(None)
+    assert store._conn is None
+
+

pyproject.toml

@@ -15,7 +15,6 @@ dependencies = [
   # Cross‐platform
   "python-socketio>=5.11.1",
   "aiohttp>=3.9.3",
-  "caribou>=0.3.0",
   "coloredlogs>=15.0.1",
   "aiocron>=1.8",
   "python-dotenv>=1.0.1",
@@ -44,6 +43,7 @@ dependencies = [
   "parsel>=1.10.0",
   "jmespath>=1.0.1",
   "jsonschema>=4.23.0",
+  "aiosqlite>=0.22.0",
 ]
 
 [tool.ruff]

uv.lock

@@ -105,6 +105,15 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/fb/76/641ae371508676492379f16e2fa48f4e2c11741bd63c48be4b12a6b09cba/aiosignal-1.4.0-py3-none-any.whl", hash = "sha256:053243f8b92b990551949e63930a839ff0cf0b0ebbe0597b0f3fb19e1a0fe82e", size = 7490 },
 ]
 
+[[package]]
+name = "aiosqlite"
+version = "0.22.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/3a/0d/449c024bdabd0678ae07d804e60ed3b9786facd3add66f51eee67a0fccea/aiosqlite-0.22.0.tar.gz", hash = "sha256:7e9e52d72b319fcdeac727668975056c49720c995176dc57370935e5ba162bb9", size = 14707 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/ef/39/b2181148075272edfbbd6d87e6cd78cc71dca243446fa3b381fd4116950b/aiosqlite-0.22.0-py3-none-any.whl", hash = "sha256:96007fac2ce70eda3ca1bba7a3008c435258a592b8fbf2ee3eeaa36d33971a09", size = 17263 },
+]
+
 [[package]]
 name = "altgraph"
 version = "0.17.5"
@@ -224,15 +233,6 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/07/6b/6e92009df3b8b7272f85a0992b306b61c34b7ea1c4776643746e61c380ac/brotlicffi-1.2.0.0-cp38-abi3-win_amd64.whl", hash = "sha256:f139a7cdfe4ae7859513067b736eb44d19fae1186f9e99370092f6915216451b", size = 378586 },
 ]
 
-[[package]]
-name = "caribou"
-version = "0.4.1"
-source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/6c/74/de2c20c5a24811d69a2a6150d5cd841f2af21ffc65ff964a73f46738bb68/caribou-0.4.1.tar.gz", hash = "sha256:13a66fc9ef9b1c9e9ef220876d59a44ee5eff9e579655252271b7514fc0ad787", size = 14019 }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/b5/91/795eccdad6abd41b3634502d0971cb696d2e5c9cede67f8b38ebe30d2b1e/caribou-0.4.1-py2.py3-none-any.whl", hash = "sha256:5c7a036584b34021011f1512620152272924e55ca87c7c805073aeef31c5baed", size = 7203 },
-]
-
 [[package]]
 name = "certifi"
 version = "2025.11.12"
@@ -1823,13 +1823,13 @@ source = { virtual = "." }
 dependencies = [
     { name = "aiocron" },
     { name = "aiohttp" },
+    { name = "aiosqlite" },
     { name = "anyio" },
     { name = "apprise" },
     { name = "async-timeout" },
     { name = "bgutil-ytdlp-pot-provider" },
     { name = "brotli" },
     { name = "brotlicffi" },
-    { name = "caribou" },
     { name = "coloredlogs" },
     { name = "curl-cffi" },
     { name = "dateparser" },
@@ -1874,13 +1874,13 @@ dev = [
 requires-dist = [
     { name = "aiocron", specifier = ">=1.8" },
     { name = "aiohttp", specifier = ">=3.9.3" },
+    { name = "aiosqlite", specifier = ">=0.22.0" },
     { name = "anyio" },
     { name = "apprise", specifier = ">=1.9.3" },
     { name = "async-timeout" },
     { name = "bgutil-ytdlp-pot-provider", specifier = ">=1.2.1" },
     { name = "brotli" },
     { name = "brotlicffi" },
-    { name = "caribou", specifier = ">=0.3.0" },
     { name = "coloredlogs", specifier = ">=15.0.1" },
     { name = "curl-cffi", specifier = ">=0.13" },
     { name = "dateparser", specifier = ">=1.2.1" },