diff --git a/.vscode/settings.json b/.vscode/settings.json index e2d04f93..1fb8c714 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -16,6 +16,7 @@ "ahas", "ahash", "aiocron", + "aiosqlite", "anyio", "Archiver", "arrowdown", @@ -73,6 +74,8 @@ "falsey", "faststart", "Fetc", + "fetchall", + "fetchone", "filterwarnings", "finaldir", "flac", @@ -91,6 +94,7 @@ "hookspath", "httpx", "imagetools", + "isinstance", "jmespath", "jsonschema", "kibibytes", @@ -112,7 +116,10 @@ "mebibytes", "MEIPASS", "merch", + "metaclass", + "Metaclasses", "metadataparser", + "mgmt", "Microformat", "microformats", "mkvtoolsnix", @@ -128,6 +135,7 @@ "noninteractive", "noprogress", "onefile", + "oneline", "parsel", "pathex", "pickleable", @@ -178,6 +186,7 @@ "Unraid", "upgrader", "urandom", + "urlparse", "urlsafe", "usegmt", "usermod", @@ -190,6 +199,7 @@ "xerror", "youtu", "youtubepot", + "zstd", "русский", "العربية" ], diff --git a/app/library/DataStore.py b/app/library/DataStore.py index 7d668189..00642512 100644 --- a/app/library/DataStore.py +++ b/app/library/DataStore.py @@ -1,27 +1,21 @@ +import asyncio import copy -import json import logging from collections import OrderedDict -from dataclasses import fields -from datetime import UTC, datetime -from email.utils import formatdate +from collections.abc import Iterable from enum import Enum -from sqlite3 import Connection from .Download import Download from .ItemDTO import ItemDTO from .operations import matches_condition -from .Utils import init_class +from .sqlite_store import SqliteStore -LOG: logging.Logger = logging.getLogger("datastore") - -ITEM_DTO_FIELDS = {f.name for f in fields(ItemDTO)} -"""Caching ItemDTO fields for performance.""" +LOG = logging.getLogger("datastore") class StoreType(str, Enum): - HISTORY: str = "done" - QUEUE: str = "queue" + HISTORY = "done" + QUEUE = "queue" @classmethod def all(cls) -> list[str]: @@ -29,55 +23,57 @@ class StoreType(str, Enum): @classmethod def from_value(cls, value: str) -> "StoreType": - """ - Returns the StoreType enum member corresponding to the given value. - - Args: - value (str): The value to match against the enum members. - - Returns: - StoreType: The enum member that matches the value. - - Raises: - ValueError: If the value does not match any member. - - """ for member in cls: if member.value == value: return member - msg = f"Invalid StoreType value: {value}" raise ValueError(msg) - def __str__(self) -> str: + def __str__(self) -> str: # pragma: no cover return self.value +def _strip_transient_fields(item: ItemDTO) -> ItemDTO: + stored = copy.deepcopy(item) + if hasattr(stored, "datetime"): + try: + delattr(stored, "datetime") + except AttributeError: + pass + if stored.status == "finished" and hasattr(stored, "live_in"): + try: + delattr(stored, "live_in") + except AttributeError: + pass + return stored + + class DataStore: """ - Persistent queue. + In-memory queue cache + optional write-behind to persistence. + History reads go straight to persistence. """ - def __init__(self, type: StoreType, connection: Connection): + def __init__(self, type: StoreType, connection: SqliteStore): + self._type = type + self._connection: SqliteStore = connection self._dict: OrderedDict[str, Download] = OrderedDict() - "The dictionary of items." - self._type: StoreType = type - "The type of the datastore." - self._connection: Connection = connection - "The database connection." - def load(self) -> None: - for id, item in self.saved_items(): - self._dict.update({id: Download(info=item)}) + # cache helpers + async def load(self) -> None: + saved = await self._connection.fetch_saved(str(self._type)) + for key, item in saved: + self._dict[key] = Download(info=item) + + async def saved_items(self) -> list[tuple[str, ItemDTO]]: + return await self._connection.fetch_saved(str(self._type)) def exists(self, key: str | None = None, url: str | None = None) -> bool: if not key and not url: msg = "key or url must be provided." raise KeyError(msg) - if key and key in self._dict: return True - return any( (key and self._dict[i].info._id == key) or (url and self._dict[i].info.url == url) for i in self._dict ) @@ -86,261 +82,88 @@ class DataStore: if not key and not url: msg = "key or url must be provided." raise KeyError(msg) - for i in self._dict: if (key and self._dict[i].info._id == key) or (url and self._dict[i].info.url == url): return self._dict[i] - msg: str = f"{key=} or {url=} not found." raise KeyError(msg) def get_item(self, **kwargs) -> Download | None: - """ - Get a specific item from the datastore based on provided attributes with optional operations. - - Args: - **kwargs: Arbitrary keyword arguments representing attributes of the ItemDTO. - Each value can be either: - - A direct value (defaults to EQUAL operation): {"title": "test"} - - A tuple of (Operation, value): {"title": (Operation.CONTAIN, "test")} - - If no attributes are provided, the method returns None. - If any attribute matches, the corresponding Download object is returned. - - Returns: - Download | None: The requested item if found, otherwise None. - - Examples: - # Direct equality check (default) - store.get_item(title="Video 1") - - # Using operations - store.get_item(title=(Operation.CONTAIN, "test")) - store.get_item(id=(Operation.EQUAL, "123"), status=(Operation.NOT_EQUAL, "error")) - - # Mixed usage - store.get_item(title=(Operation.CONTAIN, "test"), folder="downloads") - - """ if not kwargs: return None - for i in self._dict: if not self._dict[i].info: continue - info = self._dict[i].info.__dict__ if any(matches_condition(key, value, info) for key, value in kwargs.items()): return self._dict[i] + return None + + async def get_by_id(self, id: str) -> Download | None: + val = self._dict.get(id) + if val or self._type == StoreType.QUEUE: + return val + + if item := await self._connection.get_by_id(str(self._type), id): + download = Download(info=item) + self._dict[id] = download + return download return None - def get_by_id(self, id: str) -> Download | None: - return self._dict.get(id, None) - - def items(self) -> OrderedDict[tuple[str, Download]]: + def items(self): return self._dict.items() - def saved_items(self) -> list[tuple[str, ItemDTO]]: - items: list[tuple[str, ItemDTO]] = [] - - cursor = self._connection.execute( - 'SELECT "id", "data", "created_at" FROM "history" WHERE "type" = ? ORDER BY "created_at" ASC', - (str(self._type),), - ) - - for row in cursor: - rowDate: datetime = datetime.strptime(row["created_at"], "%Y-%m-%d %H:%M:%S") # noqa: DTZ007 - data: dict = json.loads(row["data"]) - data.pop("_id", None) - item: ItemDTO = init_class(ItemDTO, data, ITEM_DTO_FIELDS) - item._id = row["id"] - item.datetime = formatdate(rowDate.replace(tzinfo=UTC).timestamp()) - items.append((row["id"], item)) - - return items - - def put(self, value: Download, no_notify: bool = False) -> Download: - if "error" == value.info.status and not no_notify: - from app.library.Events import EventBus, Events - - EventBus.get_instance().emit(Events.ITEM_ERROR, value.info) - - self._dict.update({value.info._id: value}) - self._update_store_item(self._type, value.info) - - return self._dict[value.info._id] + async def put(self, value: Download, no_notify: bool = False) -> Download: + _ = no_notify + self._dict[value.info._id] = value + asyncio.create_task(self._connection.enqueue_upsert(str(self._type), _strip_transient_fields(value.info))) + return value def delete(self, key: str) -> None: self._dict.pop(key, None) - self._delete_store_item(key) + asyncio.create_task(self._connection.enqueue_delete(str(self._type), key)) - def next(self) -> tuple[str, Download]: + def next(self): return next(iter(self._dict.items())) - def empty(self): + def empty(self) -> bool: return not bool(self._dict) - def has_downloads(self): - if 0 == len(self._dict): - return False - + def has_downloads(self) -> bool: return any(self._dict[key].info.auto_start and self._dict[key].started() is False for key in self._dict) - def get_next_download(self) -> Download: + def get_next_download(self) -> Download | None: for key in self._dict: ref = self._dict[key] - if ref.info.auto_start and ref.started() is False and ref.is_cancelled() is False: + if ref.info.auto_start and not ref.started() and not ref.is_cancelled(): return ref - return None - async def test(self) -> bool: - self._connection.execute('SELECT "id" FROM "history" LIMIT 1').fetchone() - return True + async def get_total_count(self, status_filter: str | None = None) -> int: + return await self._connection.count(str(self._type), status_filter=status_filter) - def get_total_count(self) -> int: - """ - Get the total count of items in the datastore. - - Returns: - int: The total number of items in the datastore. - - """ - cursor = self._connection.execute( - 'SELECT COUNT(*) as count FROM "history" WHERE "type" = ?', - (str(self._type),), - ) - row = cursor.fetchone() - return row["count"] if row else 0 - - def get_items_paginated( - self, - page: int = 1, - per_page: int = 50, - order: str = "DESC", - status_filter: str | None = None, - ) -> tuple[list[tuple[str, ItemDTO]], int, int, int]: - """ - Get paginated items from the datastore. - - Args: - page (int): The page number (1-indexed). Defaults to 1. - per_page (int): Number of items per page. Defaults to 50. - order (str): Sort order - 'ASC' or 'DESC'. Defaults to 'DESC' (newest first). - status_filter (str | None): Optional status filter. Can be a status value (e.g., 'finished') - to include only that status, or prefixed with '!' (e.g., '!finished') to exclude that status. - - Returns: - tuple[list[tuple[str, ItemDTO]], int, int, int]: A tuple containing: - - List of (id, ItemDTO) tuples for the requested page - - Total number of items - - Current page number - - Total number of pages - - Raises: - ValueError: If page < 1 or per_page < 1 - - """ + async def get_items_paginated( + self, page: int = 1, per_page: int = 50, order: str = "DESC", status_filter: str | None = None + ): if page < 1: msg = "page must be >= 1" raise ValueError(msg) - if per_page < 1: msg = "per_page must be >= 1" raise ValueError(msg) - order = order.upper() if order not in ("ASC", "DESC"): msg = f"order must be 'ASC' or 'DESC', got '{order}'" raise ValueError(msg) + return await self._connection.paginate(str(self._type), page, per_page, order, status_filter) - order = "ASC" if order == "ASC" else "DESC" + async def bulk_delete(self, ids: Iterable[str]) -> int: + deleted = await self._connection.bulk_delete(str(self._type), ids) + for _id in ids: + self._dict.pop(_id, None) + return deleted - # Build SQL query with status filter if provided - where_clauses = ['"type" = ?'] - query_params: list = [str(self._type)] - - if status_filter: - # Check if it's an exclusion filter (starts with !) - if status_filter.startswith("!"): - status_value = status_filter[1:] - where_clauses.append("json_extract(data, '$.status') != ?") - query_params.append(status_value) - else: - where_clauses.append("json_extract(data, '$.status') = ?") - query_params.append(status_filter) - - where_clause = " AND ".join(where_clauses) - - # Get total count with filter - count_query = f'SELECT COUNT(*) as count FROM "history" WHERE {where_clause}' # noqa: S608 - count_result = self._connection.execute(count_query, tuple(query_params)).fetchone() - total_items: int = count_result["count"] if count_result else 0 - total_pages: int = (total_items + per_page - 1) // per_page if total_items > 0 else 1 - - # Ensure page is within valid range. - if page > total_pages and total_items > 0: - page = total_pages - - offset = (page - 1) * per_page - - items: list[tuple[str, ItemDTO]] = [] - - query_params.extend([per_page, offset]) - - cursor = self._connection.execute( - f'SELECT "id", "data", "created_at" FROM "history" WHERE {where_clause} ORDER BY "created_at" {order} LIMIT ? OFFSET ?', # noqa: S608 - tuple(query_params), - ) - - for row in cursor: - rowDate: datetime = datetime.strptime(row["created_at"], "%Y-%m-%d %H:%M:%S") # noqa: DTZ007 - data: dict = json.loads(row["data"]) - data.pop("_id", None) - item: ItemDTO = init_class(ItemDTO, data, ITEM_DTO_FIELDS) - item._id = row["id"] - item.datetime = formatdate(rowDate.replace(tzinfo=UTC).timestamp()) - items.append((row["id"], item)) - - return items, total_items, page, total_pages - - def _update_store_item(self, type: StoreType, item: ItemDTO) -> None: - sqlStatement = """ - INSERT INTO "history" ("id", "type", "url", "data") - VALUES (?, ?, ?, ?) - ON CONFLICT DO UPDATE SET "type" = ?, "url" = ?, "data" = ?, created_at = ? - """ - - stored: ItemDTO = copy.deepcopy(item) - - if hasattr(stored, "datetime"): - try: - delattr(stored, "datetime") - except AttributeError: - pass - - if hasattr(stored, "live_in") and stored.status == "finished": - try: - delattr(stored, "live_in") - except AttributeError: - pass - - encoded: str = stored.json() - - self._connection.execute( - sqlStatement.strip(), - ( - stored._id, - str(type), - stored.url, - encoded, - str(type), - stored.url, - encoded, - datetime.now(UTC).strftime("%Y-%m-%d %H:%M:%S"), - ), - ) - - def _delete_store_item(self, key: str) -> None: - self._connection.execute('DELETE FROM "history" WHERE "type" = ? AND "id" = ?', (str(self._type), key)) + async def test(self) -> bool: + await self._connection.count(str(self._type)) + return True diff --git a/app/library/DownloadQueue.py b/app/library/DownloadQueue.py index 544415f6..0e6f2716 100644 --- a/app/library/DownloadQueue.py +++ b/app/library/DownloadQueue.py @@ -9,7 +9,6 @@ import uuid from datetime import UTC, datetime, timedelta from email.utils import formatdate from pathlib import Path -from sqlite3 import Connection from typing import TYPE_CHECKING, Any import yt_dlp.utils @@ -25,6 +24,7 @@ from .ItemDTO import Item, ItemDTO from .Presets import Presets from .Scheduler import Scheduler from .Singleton import Singleton +from .sqlite_store import SqliteStore from .Utils import ( archive_add, arg_converter, @@ -49,14 +49,14 @@ class DownloadQueue(metaclass=Singleton): DownloadQueue class is a singleton class that manages the download queue and the download history. """ - def __init__(self, connection: Connection, config: Config | None = None): + def __init__(self, config: Config | None = None): self.config: Config = config or Config.get_instance() "Configuration instance." self._notify: EventBus = EventBus.get_instance() "Event bus instance." - self.done = DataStore(type=StoreType.HISTORY, connection=connection) + self.done = DataStore(type=StoreType.HISTORY, connection=SqliteStore.get_instance(config.db_file)) "DataStore for the completed downloads." - self.queue = DataStore(type=StoreType.QUEUE, connection=connection) + self.queue = DataStore(type=StoreType.QUEUE, connection=SqliteStore.get_instance(config.db_file)) "DataStore for the download queue." self.workers = asyncio.Semaphore(self.config.max_workers) "Semaphore to limit the number of concurrent downloads." @@ -71,8 +71,7 @@ class DownloadQueue(metaclass=Singleton): self._active: dict[str, Download] = {} """Dictionary of active downloads.""" - self.done.load() - self.queue.load() + # Loading happens in async initialize to avoid blocking loop creation. self.paused.set() @staticmethod @@ -166,6 +165,9 @@ class DownloadQueue(metaclass=Singleton): """ Initialize the download queue. """ + await self.done.test() + await self.done.load() + await self.queue.load() LOG.info( f"Using '{self.config.max_workers}' workers for downloading and '{self.config.max_workers_per_extractor}' per extractor." ) @@ -199,7 +201,7 @@ class DownloadQueue(metaclass=Singleton): continue item.info.auto_start = True - updated: Download = self.queue.put(item) + updated: Download = await self.queue.put(item) self._notify.emit(Events.ITEM_UPDATED, data=updated.info) self._notify.emit( Events.ITEM_RESUMED, @@ -246,7 +248,7 @@ class DownloadQueue(metaclass=Singleton): continue item.info.auto_start = False - updated: Download = self.queue.put(item) + updated: Download = await self.queue.put(item) self._notify.emit(Events.ITEM_UPDATED, data=updated.info) self._notify.emit( Events.ITEM_PAUSED, @@ -534,7 +536,7 @@ class DownloadQueue(metaclass=Singleton): message=nMessage, ) - itemDownload: Download = self.done.put(dlInfo) + itemDownload: Download = await self.done.put(dlInfo) elif not hasFormats: ava: str = entry.get("availability", "public") nTitle = "Download Error" @@ -547,7 +549,7 @@ class DownloadQueue(metaclass=Singleton): dlInfo.info.error = nMessage.replace(f" for '{dl.title}'.", ".") + text_logs dlInfo.info.status = "error" - itemDownload = self.done.put(dlInfo) + itemDownload = await self.done.put(dlInfo) self._notify.emit( Events.LOG_WARNING, @@ -580,12 +582,12 @@ class DownloadQueue(metaclass=Singleton): if _requeue: nEvent = Events.ITEM_ADDED - itemDownload = self.queue.put(dlInfo) + itemDownload = await self.queue.put(dlInfo) if item.auto_start: self.event.set() else: dlInfo.info.status = "not_live" - itemDownload = self.done.put(dlInfo) + itemDownload = await self.done.put(dlInfo) nStore = "history" nEvent = Events.ITEM_MOVED nTitle = "Premiering right now" @@ -596,7 +598,7 @@ class DownloadQueue(metaclass=Singleton): nEvent = Events.ITEM_ADDED nTitle = "Item Added" nMessage = f"Item '{dlInfo.info.title}' has been added to the download queue." - itemDownload = self.queue.put(dlInfo) + itemDownload = await self.queue.put(dlInfo) if item.auto_start: self.event.set() else: @@ -732,7 +734,7 @@ class DownloadQueue(metaclass=Singleton): if archive_file := dlInfo.info.get_ytdlp_opts().get_all().get("download_archive"): dlInfo.info.msg += f" Found in archive '{archive_file}'." - self.done.put(dlInfo) + await self.done.put(dlInfo) self._notify.emit( Events.ITEM_MOVED, @@ -816,7 +818,7 @@ class DownloadQueue(metaclass=Singleton): extras=item.extras, ) ) - self.done.put(dlInfo) + await self.done.put(dlInfo) LOG.info(log_message) self._notify.emit(Events.LOG_INFO, data={}, title="Ignored Download", message=log_message) @@ -910,7 +912,7 @@ class DownloadQueue(metaclass=Singleton): message=f"Download '{item.info.title}' has been cancelled.", ) item.info.status = "cancelled" - self.done.put(item) + await self.done.put(item) self._notify.emit( Events.ITEM_MOVED, data={"to": "history", "preset": item.info.preset, "item": item.info}, @@ -1003,7 +1005,7 @@ class DownloadQueue(metaclass=Singleton): return status - def get(self, mode: str = "all") -> dict[str, list[dict[str, ItemDTO]]]: + async def get(self, mode: str = "all") -> dict[str, list[dict[str, ItemDTO]]]: """ Get the download queue and the download history. @@ -1017,11 +1019,11 @@ class DownloadQueue(metaclass=Singleton): items = {"queue": {}, "done": {}} if mode in ("all", "queue"): - for k, v in self.queue.saved_items(): + for k, v in await self.queue.saved_items(): items["queue"][k] = self._active[k].info if k in self._active else v if mode in ("all", "done"): - for k, v in self.done.saved_items(): + for k, v in await self.done.saved_items(): v.get_file_sidecar() items["done"][k] = v @@ -1186,7 +1188,7 @@ class DownloadQueue(metaclass=Singleton): self._notify.emit(Events.ITEM_COMPLETED, data=entry.info, title=nTitle, message=nMessage) - self.done.put(entry) + await self.done.put(entry) self._notify.emit( Events.ITEM_MOVED, data={"to": "history", "preset": entry.info.preset, "item": entry.info}, @@ -1288,7 +1290,7 @@ class DownloadQueue(metaclass=Singleton): ) ) except Exception as e: - self.done.put(item) + await self.done.put(item) LOG.exception(e) LOG.error(f"Failed to retry item '{item_ref}'. {e!s}") diff --git a/app/library/config.py b/app/library/config.py index f7d3aa05..a4a98b2e 100644 --- a/app/library/config.py +++ b/app/library/config.py @@ -441,6 +441,7 @@ class Config(metaclass=Singleton): ("urllib3.connectionpool", logging.WARNING), ("apprise", logging.WARNING), ("httpcore", logging.INFO), + ("aiosqlite", logging.INFO), ) for _tool, _level in _log_levels: logging.getLogger(_tool).setLevel(_level) diff --git a/app/library/migrate.py b/app/library/migrate.py new file mode 100644 index 00000000..1eca8b8f --- /dev/null +++ b/app/library/migrate.py @@ -0,0 +1,321 @@ +# flake8: noqa: S608 +import contextlib +import datetime +import glob +import os.path +import traceback +from collections.abc import AsyncIterator, Iterable, Sequence +from importlib.machinery import ModuleSpec, SourceFileLoader +from importlib.util import module_from_spec, spec_from_loader +from typing import TYPE_CHECKING + +import aiosqlite + +if TYPE_CHECKING: + from types import ModuleType + +UTC_LENGTH = 14 + +__version__ = "1.0.0" + +class Error(Exception): + """Base class for all Caribou errors.""" + + +class InvalidMigrationError(Error): + """Thrown when a client migration contains an error.""" + + +class InvalidNameError(Error): + """Thrown when a client migration has an invalid filename.""" + + def __init__(self, filename: str): + msg: str = ( + f"Migration filenames must start with a UTC timestamp. The following file has an invalid name: {filename}" + ) + super().__init__(msg) + + +@contextlib.asynccontextmanager +async def execute( + conn: aiosqlite.Connection, sql: str, params: Sequence[object] | None = None +) -> AsyncIterator[aiosqlite.Cursor]: + params = [] if params is None else params + cursor = await conn.execute(sql, params) + try: + yield cursor + finally: + await cursor.close() + + +@contextlib.asynccontextmanager +async def transaction(conn: aiosqlite.Connection) -> AsyncIterator[None]: + try: + yield + await conn.commit() + except Exception: + await conn.rollback() + raise + + +class Migration: + """This class represents a migration version.""" + + def __init__(self, path: str): + self.path: str = path + self.filename: str = os.path.basename(path) + self.module_name, _ = os.path.splitext(self.filename) + self.get_version() # will assert the filename is valid + self.name: str = self.module_name[UTC_LENGTH:] + while self.name.startswith("_"): + self.name: str = self.name[1:] + + loader = SourceFileLoader(self.module_name, path) + spec: ModuleSpec | None = spec_from_loader(self.module_name, loader) + if spec is None: + msg = f"Cannot create spec for {path}" + raise ImportError(msg) + + try: + module: ModuleType = module_from_spec(spec) + loader.exec_module(module) + self.module: ModuleType = module + except Exception as e: + msg: str = f"Invalid migration {path}: {traceback.format_exc()}" + raise InvalidMigrationError(msg) from e + + targets: list[str] = ["upgrade", "downgrade"] + missing: list[str] = [m for m in targets if not self.has_method(m)] + if missing: + msg = f"Migration '{self.path}' is missing required methods: {', '.join(missing)}." + raise InvalidMigrationError(msg) + + def get_version(self) -> str: + if len(self.filename) < UTC_LENGTH: + raise InvalidNameError(self.filename) + timestamp: str = self.filename[:UTC_LENGTH] + if not timestamp.isdigit(): + raise InvalidNameError(self.filename) + return timestamp + + async def upgrade(self, conn: aiosqlite.Connection) -> None: + await self.module.upgrade(conn) + + async def downgrade(self, conn: aiosqlite.Connection) -> None: + await self.module.downgrade(conn) + + def has_method(self, name: str) -> bool: + return callable(getattr(self.module, name, None)) + + def __repr__(self) -> str: + return f"Migration(path={self.filename})" + + +class Database: + def __init__(self, db_url: aiosqlite.Connection | str, version_table: str = "migration_version"): + if not db_url: + msg = "Database requires db_url." + raise ValueError(msg) + + self.version_table: str = version_table + + self._owns_connection = bool(isinstance(db_url, str)) + if self._owns_connection: + self.conn: aiosqlite.Connection | None = None + self.db_url: str = db_url + self._ensure_connection() + else: + self.conn: aiosqlite.Connection = db_url + + async def __aenter__(self) -> "Database": + if self._owns_connection: + await self._ensure_connection() + + return self + + async def __aexit__(self, exc_type, exc, tb) -> None: + if not self._owns_connection: + return + await self.close() + + async def close(self) -> None: + if self._owns_connection and self.conn: + await self.conn.close() + self.conn = None + + async def _ensure_connection(self) -> None: + if self.conn: + return + self.conn = await aiosqlite.connect(self.db_url) + self.conn.row_factory = aiosqlite.Row + + async def is_version_controlled(self) -> bool: + await self._ensure_connection() + assert self.conn + sql = """SELECT * FROM sqlite_master WHERE type = 'table' AND name = ?""" + async with execute(self.conn, sql, [self.version_table]) as cursor: + return bool(await cursor.fetchall()) + + async def upgrade(self, migrations: list[Migration], target_version: str | None = None) -> None: + await self._ensure_connection() + assert self.conn + if target_version: + _assert_migration_exists(migrations, target_version) + + migrations.sort(key=lambda x: x.get_version()) + database_version = await self.get_version() + + for migration in migrations: + current_version: str = migration.get_version() + if database_version is not None and current_version <= database_version: + continue + if target_version and current_version > target_version: + break + await migration.upgrade(self.conn) + new_version: str = migration.get_version() + await self.update_version(new_version) + database_version: str = new_version + + async def downgrade(self, migrations: list[Migration], target_version: str | int) -> None: + await self._ensure_connection() + assert self.conn + if target_version not in (0, "0"): + _assert_migration_exists(migrations, target_version) + + migrations.sort(key=lambda x: x.get_version(), reverse=True) + database_version: str | None = await self.get_version() + + for i, migration in enumerate(migrations): + current_version: str = migration.get_version() + if database_version is not None and current_version > database_version: + continue + if current_version <= target_version: + break + await migration.downgrade(self.conn) + next_version: str | int = 0 + if i < len(migrations) - 1: + next_migration: Migration = migrations[i + 1] + next_version = next_migration.get_version() + await self.update_version(str(next_version)) + database_version = str(next_version) + + async def get_version(self) -> str | None: + """ + Return the database's version, or None if it is not under version + control. + """ + await self._ensure_connection() + assert self.conn + if not await self.is_version_controlled(): + return None + sql: str = f"SELECT version FROM {self.version_table}" + async with execute(self.conn, sql) as cursor: + result = await cursor.fetchall() + return result[0][0] if result else "0" + + async def update_version(self, version: str) -> None: + await self._ensure_connection() + assert self.conn + sql: str = f"UPDATE {self.version_table} SET version = ?" + async with transaction(self.conn): + await self.conn.execute(sql, [version]) + + async def initialize_version_control(self) -> None: + await self._ensure_connection() + assert self.conn + sql: str = f"""CREATE TABLE IF NOT EXISTS {self.version_table} ( version TEXT ) """ + async with transaction(self.conn): + await self.conn.execute(sql) + await self.conn.execute(f"INSERT INTO {self.version_table} VALUES (0)") + + def __repr__(self) -> str: + return f'Database("{self.db_url if self._owns_connection else "external_connection"}")' + + +def _assert_migration_exists(migrations: Iterable[Migration], version: str | int) -> None: + if version not in (m.get_version() for m in migrations): + msg: str = f"No migration with version {version} exists." + raise Error(msg) + + +def load_migrations(directory: str) -> list[Migration]: + """Return the migrations contained in the given directory.""" + directory = str(directory) + if not os.path.exists(directory) or not os.path.isdir(directory): + msg: str = f"{directory} is not a directory." + raise Error(msg) + return [Migration(f) for f in glob.glob(os.path.join(directory, "*.py"))] + + +async def upgrade(db_url: aiosqlite.Connection | str, migration_dir: str, version: str | None = None) -> None: + """ + Upgrade the given database with the migrations contained in the + migrations directory. If a version is not specified, upgrade + to the most recent version. + """ + async with Database(db_url) as db: + if not await db.is_version_controlled(): + await db.initialize_version_control() + + await db.upgrade(load_migrations(migration_dir), version) + + +async def downgrade(db_url: str | aiosqlite.Connection, migration_dir: str, version: str) -> None: + """ + Downgrade the database to the given version with the migrations + contained in the given migration directory. + """ + async with Database(db_url) as db: + if not await db.is_version_controlled(): + msg = f"The database {db_url} is not version controlled." + raise Error(msg) + migrations = load_migrations(migration_dir) + await db.downgrade(migrations, version) + + +async def get_version(db_url: aiosqlite.Connection | str) -> str | None: + """Return the migration version of the given database.""" + async with Database(db_url) as db: + return await db.get_version() + + +def create_migration(name: str, directory: str | None = None) -> str: + """ + Create a migration with the given name. If no directory is specified, + the current working directory will be used. + """ + directory = directory if directory else "." + if not os.path.exists(directory) or not os.path.isdir(directory): + msg: str = f"{directory} is not a directory." + raise Error(msg) + + now: datetime.datetime = datetime.datetime.now(tz=datetime.UTC) + version: str = now.strftime("%Y%m%d%H%M%S") + + contents: str = MIGRATION_TEMPLATE % {"name": name, "version": version} + + sanitized: str = name.replace(" ", "_") + filename: str = f"{version}_{sanitized}.py" + path: str = os.path.join(directory, filename) + with open(path, "w") as migration_file: + migration_file.write(contents) + + return path + + +MIGRATION_TEMPLATE = """\ +\"\"\" +This module contains a db migration. + +Migration Name: %(name)s +Migration Version: %(version)s +\"\"\" + +async def upgrade(c): + # add your upgrade step here + await c.execute("SELECT 1") + +async def downgrade(c): + # add your downgrade step here + await c.execute("SELECT 1") +""" diff --git a/app/library/sqlite_store.py b/app/library/sqlite_store.py new file mode 100644 index 00000000..36a9e63f --- /dev/null +++ b/app/library/sqlite_store.py @@ -0,0 +1,348 @@ +import asyncio +import contextlib +import json +import logging +import os +from collections.abc import Iterable +from dataclasses import fields +from datetime import UTC, datetime +from email.utils import formatdate + +import aiosqlite +from aiohttp import web + +from .ItemDTO import ItemDTO +from .Singleton import ThreadSafe +from .Utils import init_class + +LOG: logging.Logger = logging.getLogger(__name__) + +ITEM_DTO_FIELDS: set[str] = {f.name for f in fields(ItemDTO)} + + +class Terminator: + pass + + +class _Op: + """Queued write operation.""" + + __slots__ = ("item", "key", "keys", "op", "type_value") + + def __init__( + self, op: Terminator | str, type_value: str, item: ItemDTO | None, key: str | None, keys: list[str] | None + ): + self.op = op + self.type_value = type_value + self.item = item + self.key = key + self.keys = keys + + +class SqliteStore(metaclass=ThreadSafe): + """ + Async persistence layer with back-pressure and write-behind queue. + Singleton per process (ThreadSafe). Owns its aiosqlite connection. + """ + + @staticmethod + def get_instance(db_path: str | None = None) -> "SqliteStore": + return SqliteStore(db_path) + + def attach(self, app: web.Application): + """Get/create singleton bound to db_path.""" + app.on_shutdown.append(self.on_shutdown) + + async def on_shutdown(self, _: web.Application): + """Close singleton on app shutdown.""" + LOG.debug("Shutting down SqliteStore...") + await self.close() + LOG.debug("SqliteStore shut down complete.") + + def __init__(self, db_path: str | None = None, *, max_pending: int = 200, flush_interval: float = 0.05): + self._db_path = db_path + self._conn: aiosqlite.Connection | None = None + self._queue: asyncio.Queue[_Op] | None = None + self._task: asyncio.Task | None = None + self._flush_interval = flush_interval + self._max_pending = max_pending + self._lock = asyncio.Lock() + + async def __aenter__(self) -> "SqliteStore": + await self._ensure_conn() + return self + + async def __aexit__(self, exc_type, exc, tb) -> None: + await self.close() + + # ---------- public API ---------- + async def fetch_saved(self, type_value: str) -> list[tuple[str, ItemDTO]]: + await self._ensure_conn() + cursor = await self._conn.execute( + 'SELECT "id", "data", "created_at" FROM "history" WHERE "type" = ? ORDER BY "created_at" ASC', + (type_value,), + ) + items: list[tuple[str, ItemDTO]] = [] + async with cursor: + async for row in cursor: + row_date = datetime.strptime(row["created_at"], "%Y-%m-%d %H:%M:%S") # noqa: DTZ007 + data = json.loads(row["data"]) + data.pop("_id", None) + item = init_class(ItemDTO, data, ITEM_DTO_FIELDS) + item._id = row["id"] + item.datetime = formatdate(row_date.replace(tzinfo=UTC).timestamp()) + items.append((row["id"], item)) + return items + + async def get_by_id(self, type_value: str, id: str) -> ItemDTO | None: + await self._ensure_conn() + cursor = await self._conn.execute( + 'SELECT "id", "data", "created_at" FROM "history" WHERE "type" = ? AND "id" = ?', + (type_value, id), + ) + row = await cursor.fetchone() + if not row: + return None + + row_date = datetime.strptime(row["created_at"], "%Y-%m-%d %H:%M:%S") # noqa: DTZ007 + data = json.loads(row["data"]) + data.pop("_id", None) + item = init_class(ItemDTO, data, ITEM_DTO_FIELDS) + item._id = row["id"] + item.datetime = formatdate(row_date.replace(tzinfo=UTC).timestamp()) + return item + + async def count(self, type_value: str, status_filter: str | None = None) -> int: + await self._ensure_conn() + where_clauses = ['"type" = ?'] + query_params: list[str] = [type_value] + + if status_filter: + if status_filter.startswith("!"): + status_value = status_filter[1:] + where_clauses.append("json_extract(data, '$.status') != ?") + query_params.append(status_value) + else: + where_clauses.append("json_extract(data, '$.status') = ?") + query_params.append(status_filter) + + where_clause = " AND ".join(where_clauses) + count_query = f'SELECT COUNT(*) as count FROM "history" WHERE {where_clause}' # noqa: S608 + cursor = await self._conn.execute(count_query, tuple(query_params)) + row = await cursor.fetchone() + return row["count"] if row else 0 + + async def paginate( + self, + type_value: str, + page: int, + per_page: int, + order: str, + status_filter: str | None = None, + ) -> tuple[list[tuple[str, ItemDTO]], int, int, int]: + await self._ensure_conn() + where_clauses = ['"type" = ?'] + query_params: list[str | int] = [type_value] + + if status_filter: + if status_filter.startswith("!"): + status_value = status_filter[1:] + where_clauses.append("json_extract(data, '$.status') != ?") + query_params.append(status_value) + else: + where_clauses.append("json_extract(data, '$.status') = ?") + query_params.append(status_filter) + + where_clause = " AND ".join(where_clauses) + count_query = f'SELECT COUNT(*) as count FROM "history" WHERE {where_clause}' # noqa: S608 + count_cursor = await self._conn.execute(count_query, tuple(query_params)) + count_row = await count_cursor.fetchone() + total_items = count_row["count"] if count_row else 0 + total_pages = (total_items + per_page - 1) // per_page if total_items > 0 else 1 + + if page > total_pages and total_items > 0: + page = total_pages + + offset = (page - 1) * per_page + query_params.extend([per_page, offset]) + + cursor = await self._conn.execute( + f'SELECT "id", "data", "created_at" FROM "history" WHERE {where_clause} ORDER BY "created_at" {order} LIMIT ? OFFSET ?', # noqa: S608 + tuple(query_params), + ) + + items: list[tuple[str, ItemDTO]] = [] + async with cursor: + async for row in cursor: + row_date = datetime.strptime(row["created_at"], "%Y-%m-%d %H:%M:%S") # noqa: DTZ007 + data = json.loads(row["data"]) + data.pop("_id", None) + item = init_class(ItemDTO, data, ITEM_DTO_FIELDS) + item._id = row["id"] + item.datetime = formatdate(row_date.replace(tzinfo=UTC).timestamp()) + items.append((row["id"], item)) + + return items, total_items, page, total_pages + + # direct CRUD helpers (used by DataStore) + async def upsert(self, type_value: str, item: ItemDTO) -> None: + await self._ensure_conn() + await self._upsert_now(type_value, item) + + async def delete(self, type_value: str, key: str) -> None: + await self._ensure_conn() + await self._conn.execute('DELETE FROM "history" WHERE "type" = ? AND "id" = ?', (type_value, key)) + + async def bulk_delete(self, type_value: str, keys: Iterable[str]) -> int: + await self._ensure_conn() + keys_list = list(keys) + if not keys_list: + return 0 + placeholders = ",".join("?" for _ in keys_list) + params = [type_value, *keys_list] + cur = await self._conn.execute( + f'DELETE FROM "history" WHERE "type" = ? AND "id" IN ({placeholders})', # noqa: S608 + tuple(params), + ) + return cur.rowcount if cur else 0 + + async def commit(self) -> None: + if self._conn: + await self._conn.commit() + + async def enqueue_upsert(self, type_value: str, item: ItemDTO) -> None: + await self._enqueue(_Op("upsert", type_value, item, None, None)) + + async def enqueue_delete(self, type_value: str, key: str) -> None: + await self._enqueue(_Op("delete", type_value, None, key, None)) + + async def enqueue_bulk_delete(self, type_value: str, keys: Iterable[str]) -> None: + await self._enqueue(_Op("bulk_delete", type_value, None, None, list(keys))) + + async def flush(self) -> None: + if self._queue: + await self._queue.join() + if self._conn: + await self._conn.commit() + + async def shutdown(self) -> None: + """Flush pending writes, stop writer task, and commit.""" + if self._queue: + try: + await self._queue.put(_Op(Terminator(), "", None, None, None)) + LOG.debug("Waiting for SqliteStore queue to drain...") + await asyncio.wait_for(self._queue.join(), timeout=2) + except TimeoutError: + LOG.warning("SqliteStore queue did not drain within timeout; forcing writer shutdown.") + + if self._task: + LOG.debug("Waiting for SqliteStore writer task to finish...") + if not self._task.done(): + self._task.cancel() + with contextlib.suppress(asyncio.CancelledError): + await asyncio.wait_for(self._task, timeout=2) + + if self._conn: + LOG.debug("Committing final changes to SqliteStore...") + await self._conn.commit() + + self._task = None + self._queue = None + + async def close(self) -> None: + await self.shutdown() + if self._conn: + LOG.debug("Closing SqliteStore connection...") + await self._conn.close() + self._conn = None + + async def _enqueue(self, op: _Op) -> None: + self._ensure_worker() + await self._queue.put(op) + + def _ensure_worker(self) -> None: + if self._queue is None: + self._queue = asyncio.Queue(maxsize=self._max_pending) + self._task = asyncio.create_task(self._writer(), name="sqlite-store-writer") + + async def _writer(self) -> None: + while True: + op = await self._queue.get() + if isinstance(op.op, Terminator): + self._queue.task_done() + break + try: + async with self._lock: + await self._apply(op) + except Exception as ex: + LOG.exception(ex) + finally: + self._queue.task_done() + await asyncio.sleep(self._flush_interval) + + async def _apply(self, op: _Op) -> None: + await self._ensure_conn() + if op.op == "upsert" and op.item: + await self._upsert_now(op.type_value, op.item) + elif op.op == "delete" and op.key: + await self._conn.execute('DELETE FROM "history" WHERE "type" = ? AND "id" = ?', (op.type_value, op.key)) + elif op.op == "bulk_delete" and op.keys: + placeholders = ",".join("?" for _ in op.keys) + params = [op.type_value, *op.keys] + await self._conn.execute( + f'DELETE FROM "history" WHERE "type" = ? AND "id" IN ({placeholders})', # noqa: S608 + tuple(params), + ) + await self._conn.commit() + + async def _upsert_now(self, type_value: str, item: ItemDTO) -> None: + await self._ensure_conn() + sql = """ + INSERT INTO "history" ("id", "type", "url", "data") + VALUES (?, ?, ?, ?) + ON CONFLICT DO UPDATE SET "type" = ?, "url" = ?, "data" = ?, created_at = ? + """ + encoded = item.json() + await self._conn.execute( + sql.strip(), + ( + item._id, + type_value, + item.url, + encoded, + type_value, + item.url, + encoded, + datetime.now(UTC).strftime("%Y-%m-%d %H:%M:%S"), + ), + ) + + async def _ensure_conn(self) -> None: + if self._conn: + return + + if not self._db_path: + msg = "SqliteStore requires db_path or injected connection." + raise RuntimeError(msg) + + from app.library import migrate + from app.main import ROOT_PATH + + if not self._db_path.startswith(":memory"): + os.makedirs(os.path.dirname(self._db_path) or ".", exist_ok=True) + + self._conn = await aiosqlite.connect(database=self._db_path, isolation_level=None) + self._conn.row_factory = aiosqlite.Row + version = await migrate.get_version(self._conn) + if version: + LOG.debug(f"DB Version: '{version}'.") + + await migrate.upgrade(self._conn, ROOT_PATH / "migrations") + if not version: + version = await migrate.get_version(self._conn) + LOG.debug(f"DB Version after initial migration: '{version}'.") + + await self._conn.execute("PRAGMA journal_mode=wal") + await self._conn.execute("PRAGMA busy_timeout=5000") + await self._conn.execute("PRAGMA foreign_keys=ON") + await self._conn.commit() diff --git a/app/main.py b/app/main.py index 121e4484..9ec841dc 100644 --- a/app/main.py +++ b/app/main.py @@ -9,16 +9,15 @@ if APP_ROOT not in sys.path: import asyncio import logging -import sqlite3 from pathlib import Path -import caribou import magic from aiohttp import web from app.library.BackgroundWorker import BackgroundWorker from app.library.conditions import Conditions from app.library.config import Config +from app.library.DataStore import SqliteStore from app.library.dl_fields import DLFields from app.library.DownloadQueue import DownloadQueue from app.library.Events import EventBus, Events @@ -50,18 +49,6 @@ class Main: self._check_folders() - LOG.debug(f"DB Version: '{caribou.get_version(self._config.db_file)}'.") - caribou.upgrade(self._config.db_file, ROOT_PATH / "migrations") - - connection = sqlite3.connect(database=self._config.db_file, isolation_level=None) - connection.row_factory = sqlite3.Row - connection.execute("PRAGMA journal_mode=wal") - - async def _close_connection(_): - LOG.debug("Closing database connection.") - connection.close() - LOG.debug("Database connection closed.") - try: db_file = Path(self._config.db_file) if "600" != oct(db_file.stat().st_mode)[-3:]: @@ -69,12 +56,10 @@ class Main: except Exception: pass - self._queue = DownloadQueue(connection=connection) + self._queue = DownloadQueue(config=self._config) self._http = HttpAPI(root_path=ROOT_PATH, queue=self._queue) self._socket = HttpSocket(root_path=ROOT_PATH, queue=self._queue) - self._app.on_cleanup.append(_close_connection) - def _check_folders(self): """Check if the required folders exist and create them if they do not.""" folders = (self._config.download_path, self._config.temp_path, self._config.config_path) @@ -136,6 +121,7 @@ class Main: Conditions.get_instance().attach(self._app) DLFields.get_instance().attach(self._app) TaskDefinitions.get_instance().attach(self._app) + SqliteStore.get_instance().attach(self._app) self._background_worker.attach(self._app) EventBus.get_instance().emit( diff --git a/app/migrations/20231115152938_initial.py b/app/migrations/20231115152938_initial.py index 119de811..043327db 100644 --- a/app/migrations/20231115152938_initial.py +++ b/app/migrations/20231115152938_initial.py @@ -1,12 +1,12 @@ """ -This module contains a Caribou migration. +This module contains a db migration. Migration Name: initial Migration Version: 20231115152938 """ -def upgrade(connection): +async def upgrade(c): sql = """ CREATE TABLE "history" ( "id" TEXT PRIMARY KEY UNIQUE NOT NULL, @@ -16,24 +16,24 @@ def upgrade(connection): "created_at" TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ); """ - connection.execute(sql) + await c.execute(sql) sql = """ CREATE INDEX "history_type" ON "history" ("type"); """ - connection.execute(sql) + await c.execute(sql) sql = """ CREATE UNIQUE INDEX "history_url" ON "history" ("url"); """ - connection.execute(sql) + await c.execute(sql) - connection.commit() + await c.commit() -def downgrade(connection): +async def downgrade(c): sql = """ DROP TABLE IF EXISTS "history"; """ - connection.execute(sql) + await c.execute(sql) diff --git a/app/migrations/20251116173731_add_status_index.py b/app/migrations/20251116173731_add_status_index.py index 51b9d7f6..ff1e1f08 100644 --- a/app/migrations/20251116173731_add_status_index.py +++ b/app/migrations/20251116173731_add_status_index.py @@ -1,11 +1,11 @@ """ -This module contains a Caribou migration. +This module contains a db migration. Migration Name: add_status_index Migration Version: 20251116173731 """ -def upgrade(connection): +async def upgrade(connection): """ Add index on json_extract(data, '$.status') for better query performance. @@ -15,13 +15,13 @@ def upgrade(connection): sql = """ CREATE INDEX IF NOT EXISTS "history_status" ON "history" (json_extract("data", '$.status')); """ - connection.execute(sql) + await connection.execute(sql) -def downgrade(connection): +async def downgrade(connection): """ Remove the status index. """ sql = """ DROP INDEX IF EXISTS "history_status"; """ - connection.execute(sql) + await connection.execute(sql) diff --git a/app/routes/api/browser.py b/app/routes/api/browser.py index 4066f0dd..8266f583 100644 --- a/app/routes/api/browser.py +++ b/app/routes/api/browser.py @@ -402,7 +402,7 @@ async def path_actions(request: Request, config: Config, queue: DownloadQueue, n if sidecar_renamed: item.info.get_file_sidecar() - queue.done.put(item) + await queue.done.put(item) notify.emit(Events.ITEM_UPDATED, data=item.info) if "delete" == action: @@ -508,7 +508,7 @@ async def path_actions(request: Request, config: Config, queue: DownloadQueue, n if sidecar_moved: item.info.get_file_sidecar() - queue.done.put(item) + await queue.done.put(item) notify.emit(Events.ITEM_UPDATED, data=item.info) return web.json_response(data=operations_status, status=web.HTTPOk.status_code) diff --git a/app/routes/api/history.py b/app/routes/api/history.py index 4a4ef53a..7229f4fc 100644 --- a/app/routes/api/history.py +++ b/app/routes/api/history.py @@ -47,30 +47,21 @@ async def items_list(request: Request, queue: DownloadQueue, encoder: Encoder, c """ from app.library.DataStore import StoreType - store_type = request.query.get("type", "all").lower() - stores: list[str] = ["all", StoreType.QUEUE.value, StoreType.HISTORY.value] - if store_type not in stores: + store_type: str = request.query.get("type", "queue").lower() + try: + store_type: StoreType = StoreType.from_value(store_type) + except ValueError: return web.json_response( - data={"error": f"type must be one of {', '.join(stores)}."}, + data={"error": f"type must be one of {', '.join(StoreType.all())}."}, status=web.HTTPBadRequest.status_code, ) - # Legacy behavior: return all items without pagination, will be removed in future. - if "all" == store_type: - data: dict = {"queue": [], "history": []} - q = queue.get() - - data["queue"].extend([q.get("queue", {}).get(k) for k in q.get("queue", {})]) - data["history"].extend([q.get("done", {}).get(k) for k in q.get("done", {})]) - - return web.json_response(data=data, status=web.HTTPOk.status_code, dumps=encoder.encode) - - ds = queue.queue if store_type == StoreType.QUEUE.value else queue.done + ds = queue.queue if store_type == StoreType.QUEUE else queue.done try: page = int(request.query.get("page", 1)) per_page = int(request.query.get("per_page", config.default_pagination)) - order = request.query.get("order", "DESC").upper() + order: str = request.query.get("order", "DESC").upper() except ValueError: return web.json_response( data={"error": "page and per_page must be valid integers."}, @@ -92,25 +83,28 @@ async def items_list(request: Request, queue: DownloadQueue, encoder: Encoder, c status=web.HTTPBadRequest.status_code, ) - # Parse status filter status_filter = request.query.get("status", None) - items, total, current_page, total_pages = ds.get_items_paginated( + items, total, current_page, total_pages = await ds.get_items_paginated( page=page, per_page=per_page, order=order, status_filter=status_filter ) - data = { - "pagination": { - "page": current_page, - "per_page": per_page, - "total": total, - "total_pages": total_pages, - "has_next": current_page < total_pages, - "has_prev": current_page > 1, - }, - "items": [item for _, item in items], - } - return web.json_response(data=data, status=web.HTTPOk.status_code, dumps=encoder.encode) + return web.json_response( + data={ + "type": store_type.value, + "pagination": { + "page": current_page, + "per_page": per_page, + "total": total, + "total_pages": total_pages, + "has_next": current_page < total_pages, + "has_prev": current_page > 1, + }, + "items": [item for _, item in items], + }, + status=web.HTTPOk.status_code, + dumps=encoder.encode, + ) @route("DELETE", "api/history/", "items_delete") @@ -137,24 +131,26 @@ async def items_delete(request: Request, queue: DownloadQueue, encoder: Encoder) ids = data.get("ids") remove_file: bool = bool(data.get("remove_file", True)) - _type = data.get("type", data.get("where")) + storeType = data.get("type", data.get("where")) - if not _type: + if not storeType: return web.json_response(data={"error": "Type is required."}, status=web.HTTPBadRequest.status_code) - if _type not in [StoreType.QUEUE.value, StoreType.HISTORY.value]: + try: + storeType: StoreType = StoreType.from_value(storeType) + except ValueError: return web.json_response( - data={"error": f"type must be '{StoreType.QUEUE.value}' or '{StoreType.HISTORY.value}'."}, + data={"error": f"type must be one of '{StoreType.all()}'."}, status=web.HTTPBadRequest.status_code, ) - ds = queue.queue if _type == StoreType.QUEUE.value else queue.done + ds = queue.queue if storeType == StoreType.QUEUE else queue.done if ids: return web.json_response( data={ "items": await ( - queue.cancel(ids) if _type == StoreType.QUEUE.value else queue.clear(ids, remove_file=remove_file) + queue.cancel(ids) if storeType == StoreType.QUEUE else queue.clear(ids, remove_file=remove_file) ), "deleted": len(ids), }, @@ -174,7 +170,7 @@ async def items_delete(request: Request, queue: DownloadQueue, encoder: Encoder) per_page = 1000 while True: - items, _, current_page, total_pages = ds.get_items_paginated( + items, _, current_page, total_pages = await ds.get_items_paginated( page=page, per_page=per_page, order="DESC", status_filter=status_filter ) @@ -192,7 +188,7 @@ async def items_delete(request: Request, queue: DownloadQueue, encoder: Encoder) data={ "items": await ( queue.cancel(items_to_delete) - if _type == StoreType.QUEUE.value + if storeType == StoreType.QUEUE else queue.clear(items_to_delete, remove_file=remove_file) ), "deleted": len(items_to_delete), @@ -221,7 +217,7 @@ async def item_view(request: Request, queue: DownloadQueue, encoder: Encoder) -> if not id: return web.json_response(data={"error": "id is required."}, status=web.HTTPBadRequest.status_code) - item: Download | None = queue.done.get_by_id(id) or queue.queue.get_by_id(id) + item: Download | None = await queue.done.get_by_id(id) or await queue.queue.get_by_id(id) if not item: return web.json_response(data={"error": "item not found."}, status=web.HTTPNotFound.status_code) @@ -263,7 +259,7 @@ async def item_update(request: Request, queue: DownloadQueue, encoder: Encoder, if not id: return web.json_response(data={"error": "id is required."}, status=web.HTTPBadRequest.status_code) - item: Download | None = queue.done.get_by_id(id) + item: Download | None = await queue.done.get_by_id(id) if not item: return web.json_response(data={"error": "item not found."}, status=web.HTTPNotFound.status_code) @@ -285,7 +281,7 @@ async def item_update(request: Request, queue: DownloadQueue, encoder: Encoder, LOG.debug(f"Updated '{k}' to '{v}' for '{item.info.id}'") if updated: - queue.done.put(item) + await queue.done.put(item) notify.emit(Events.ITEM_UPDATED, data=item.info) return web.json_response( @@ -394,7 +390,7 @@ async def item_archive_add(request: Request, queue: DownloadQueue, notify: Event return web.json_response(data={"error": "id is required."}, status=web.HTTPBadRequest.status_code) try: - item: Download | None = queue.done.get_by_id(id) + item: Download | None = await queue.done.get_by_id(id) if not item: return web.json_response(data={"error": f"item '{id}' not found."}, status=web.HTTPNotFound.status_code) except KeyError: @@ -425,7 +421,7 @@ async def item_archive_add(request: Request, queue: DownloadQueue, notify: Event ) item.info.archive_status(force=True) - queue.done.put(item, no_notify=True) + await queue.done.put(item, no_notify=True) notify.emit(Events.ITEM_UPDATED, data=item.info) return web.json_response( @@ -452,7 +448,7 @@ async def item_archive_delete(request: Request, queue: DownloadQueue, notify: Ev return web.json_response(data={"error": "id is required."}, status=web.HTTPBadRequest.status_code) try: - item: Download | None = queue.done.get_by_id(id) + item: Download | None = await queue.done.get_by_id(id) if not item: return web.json_response(data={"error": f"item '{id}' not found."}, status=web.HTTPNotFound.status_code) except KeyError: @@ -483,7 +479,7 @@ async def item_archive_delete(request: Request, queue: DownloadQueue, notify: Ev ) item.info.archive_status(force=True) - queue.done.put(item, no_notify=True) + await queue.done.put(item, no_notify=True) notify.emit(Events.ITEM_UPDATED, data=item.info) return web.json_response( diff --git a/app/routes/socket/connection.py b/app/routes/socket/connection.py index 64586289..d4fc85ba 100644 --- a/app/routes/socket/connection.py +++ b/app/routes/socket/connection.py @@ -44,8 +44,8 @@ async def connect(config: Config, queue: DownloadQueue, notify: EventBus, sid: s base=Path(config.download_path), depth_limit=config.download_path_depth - 1, ), - "history_count": queue.done.get_total_count(), - "queue": queue.get("queue")["queue"], + "history_count": await queue.done.get_total_count(), + "queue": (await queue.get("queue"))["queue"], }, title="Sending initial download data", message=f"Sending initial download data to client '{sid}'.", diff --git a/app/scripts/create_migration.py b/app/scripts/create_migration.py new file mode 100644 index 00000000..c3ac4447 --- /dev/null +++ b/app/scripts/create_migration.py @@ -0,0 +1,147 @@ +#!/usr/bin/env python3 +""" +Caribou migrations CLI. + +Provides commands to create, list, and apply database migrations. +""" + +from __future__ import annotations + +import argparse +import sys +import traceback +from pathlib import Path +from typing import TYPE_CHECKING + +APP_ROOT: Path = Path(__file__).resolve().parents[2] +if str(APP_ROOT) not in sys.path: + sys.path.insert(0, str(APP_ROOT)) + +from app.library import migrate + +if TYPE_CHECKING: + from collections.abc import Callable + + +def e_print(msg: str = "") -> None: + sys.stderr.write(f"{msg}\n") + + +def o_print(msg: str = "") -> None: + sys.stdout.write(f"{msg}\n") + + +def cmd_info(_: argparse.Namespace) -> None: + o_print(f"Version: {migrate.__version__}") + + +def cmd_create(args: argparse.Namespace) -> None: + path = migrate.create_migration(args.name, args.migration_dir) + o_print(f"created migration {path}") + + +def cmd_version(args: argparse.Namespace) -> None: + version = migrate.get_version(args.database_path) + if version: + o_print(f"the db [{args.database_path}] is at version {version}") + else: + o_print(f"the db [{args.database_path}] is not under version control") + + +def cmd_upgrade(args: argparse.Namespace) -> None: + if args.version: + o_print(f"upgrading db [{args.database_path}] to version [{args.version}]") + else: + o_print(f"upgrading db [{args.database_path}] to most recent version") + + migrate.upgrade(args.database_path, args.migration_dir, args.version) + + new_version = migrate.get_version(args.database_path) + if args.version is not None: + assert new_version == args.version + + o_print(f"upgraded [{args.database_path}] successfully to version [{new_version}]") + + +def cmd_downgrade(args: argparse.Namespace) -> None: + o_print(f"downgrading db [{args.database_path}] to version [{args.version}]") + migrate.downgrade(args.database_path, args.migration_dir, args.version) + o_print(f"downgraded [{args.database_path}] successfully to version [{args.version}]") + + +def cmd_list(args: argparse.Namespace) -> None: + o_print(f"Migrations in [{args.migration_dir}]:\n") + for m in migrate.load_migrations(args.migration_dir): + o_print(f"{m.get_version()}\t{m.name}\t{m.path}") + + +def build_parser() -> argparse.ArgumentParser: + p = argparse.ArgumentParser(prog="caribou") + sub = p.add_subparsers(dest="command", required=True) + + info = sub.add_parser("info", help="print library version") + info.set_defaults(func=cmd_info) + + create = sub.add_parser("create", help="create a new migration file") + create.add_argument("name", help="the name of migration") + create.add_argument("-d", "--migration-dir", default=".", help="the migration directory") + create.set_defaults(func=cmd_create) + + ver = sub.add_parser("version", help="return the migration version of the database") + ver.add_argument("database_path", help="path to the sqlite database") + ver.set_defaults(func=cmd_version) + + up = sub.add_parser( + "upgrade", + help="upgrade the db (if version isn't specified, upgrade to the most recent)", + ) + up.add_argument("database_path", help="path to the sqlite database") + up.add_argument("migration_dir", help="the migration directory") + up.add_argument("-v", "--version", type=int, default=None, help="the target migration version") + up.set_defaults(func=cmd_upgrade) + + down = sub.add_parser( + "downgrade", + help="downgrade the db to a particular version (use 0 to rollback all changes)", + ) + down.add_argument("database_path", help="path to the sqlite database") + down.add_argument("migration_dir", help="the migration directory") + down.add_argument("version", type=int, help="the target migration version") + down.set_defaults(func=cmd_downgrade) + + lst = sub.add_parser("list", help="list the migration versions") + lst.add_argument("migration_dir", help="the migration directory") + lst.set_defaults(func=cmd_list) + + return p + + +def run(argv: list[str]) -> int: + parser = build_parser() + args = parser.parse_args(argv) + + try: + func: Callable[[argparse.Namespace], None] = args.func + func(args) + return 0 + + except migrate.InvalidMigrationError: + e_print(traceback.format_exc()) + return 1 + + except migrate.Error as err: + e_print(f"Error: {err}") + return 1 + + except Exception: + e_print("an unexpected error occurred:") + e_print(traceback.format_exc()) + return 1 + + +def main() -> int: + return run(sys.argv[1:]) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/app/tests/test_datastore.py b/app/tests/test_datastore.py index d7f8a717..b2e5be9c 100644 --- a/app/tests/test_datastore.py +++ b/app/tests/test_datastore.py @@ -1,5 +1,5 @@ +import asyncio import json -import sqlite3 from collections import OrderedDict from dataclasses import asdict from datetime import UTC, datetime @@ -10,13 +10,50 @@ import pytest from app.library.DataStore import DataStore, StoreType from app.library.ItemDTO import ItemDTO from app.library.operations import Operation +from app.library.sqlite_store import SqliteStore + + +async def make_db(data: int = 0) -> SqliteStore: + """Create a temporary database with test data.""" + SqliteStore._reset_singleton() + ins = SqliteStore.get_instance(db_path=":memory:") + await ins._ensure_conn() + + base_time = datetime.now(UTC) + for i in range(data): + created_at: str = base_time.replace(hour=(i // 4) % 24, minute=(i * 15) % 60, second=i % 60).strftime( + "%Y-%m-%d %H:%M:%S" + ) + + item_data = { + "url": f"https://example.com/video{i}", + "title": f"Test Video {i}", + "id": f"video{i}", + "folder": "/downloads", + "status": "finished", + } + + await ins._conn.execute( + 'INSERT INTO "history" ("id", "type", "url", "data", "created_at") VALUES (?, ?, ?, ?, ?)', + ( + f"test-id-{i}", + str(StoreType.HISTORY), + item_data["url"], + json.dumps(item_data), + created_at, + ), + ) + if data > 0: + await ins._conn.commit() + + return ins class StubDownload: def __init__(self, info: ItemDTO, started: bool = False, cancelled: bool = False): self.info = info - self._started = started - self._cancelled = cancelled + self._started: bool = started + self._cancelled: bool = cancelled def started(self) -> bool: return self._started @@ -25,11 +62,8 @@ class StubDownload: return self._cancelled -def make_conn() -> sqlite3.Connection: - conn = sqlite3.connect(":memory:") - conn.row_factory = sqlite3.Row - conn.execute("CREATE TABLE history (id TEXT PRIMARY KEY, type TEXT, url TEXT, data TEXT, created_at TEXT)") - return conn +async def make_store_async(store_type: StoreType) -> DataStore: + return DataStore(store_type, await make_db()) def make_item(id: str, url: str = "http://u", title: str = "t", folder: str = "f") -> ItemDTO: @@ -47,20 +81,23 @@ class TestStoreType: class TestDataStore: - def test_saved_items_parses_rows(self) -> None: - conn = make_conn() - # Prepare a stored item JSON (without _id, it will be set from the row) + @pytest.mark.asyncio + async def test_saved_items_parses_rows(self) -> None: + db = await make_db() + store = DataStore(StoreType.QUEUE, db) + db = store._connection + dto = make_item(id="ignore", url="http://x", title="Title", folder="F") data = asdict(dto) data.pop("_id", None) created = datetime(2024, 1, 2, 3, 4, 5, tzinfo=UTC) - conn.execute( + await db._conn.execute( "INSERT INTO history (id, type, url, data, created_at) VALUES (?, ?, ?, ?, ?)", ("abc", "queue", "http://x", json.dumps(data), created.strftime("%Y-%m-%d %H:%M:%S")), ) - store = DataStore(StoreType.QUEUE, conn) + await db._conn.commit() - items = store.saved_items() + items = await store.saved_items() assert len(items) == 1 key, item = items[0] assert key == "abc" @@ -69,35 +106,44 @@ class TestDataStore: assert isinstance(item.datetime, str) assert item.datetime == formatdate(created.timestamp()) - def test_put_and_delete_persist(self) -> None: - conn = make_conn() - store = DataStore(StoreType.QUEUE, conn) + @pytest.mark.asyncio + async def test_put_and_delete_persist(self) -> None: + store = await make_store_async(StoreType.QUEUE) + db = store._connection item = make_item(id="vid1") d = StubDownload(info=item) - ret = store.put(d) + ret = await store.put(d) + await db.flush() assert ret is store._dict[item._id] # Verify row written; JSON should not contain datetime field - row = conn.execute("SELECT * FROM history WHERE id=?", (item._id,)).fetchone() + cur = await db._conn.execute("SELECT * FROM history WHERE id=?", (item._id,)) + row = await cur.fetchone() assert row is not None assert row["type"] == "queue" assert row["url"] == item.url assert '"datetime"' not in row["data"] + assert row["id"] == item._id # Delete and ensure removal store.delete(item._id) - row2 = conn.execute("SELECT * FROM history WHERE id=?", (item._id,)).fetchone() + await asyncio.sleep(0) + await db.flush() + cur2 = await db._conn.execute("SELECT * FROM history WHERE id=?", (item._id,)) + row2 = await cur2.fetchone() assert row2 is None + await db.close() - def test_exists_and_get(self) -> None: - conn = make_conn() - store = DataStore(StoreType.QUEUE, conn) + @pytest.mark.asyncio + async def test_exists_and_get(self) -> None: + store = await make_store_async(StoreType.QUEUE) item = make_item(id="a1", url="http://u1") d = StubDownload(info=item) - store.put(d) + await store.put(d) + await store._connection.flush() assert store.exists(key=item._id) is True assert store.exists(url=item.url) is True @@ -112,24 +158,27 @@ class TestDataStore: store.get() with pytest.raises(KeyError): store.get(key="missing") + await store._connection.close() - def test_next_and_empty(self) -> None: - conn = make_conn() - store = DataStore(StoreType.QUEUE, conn) + @pytest.mark.asyncio + async def test_next_and_empty(self) -> None: + store = await make_store_async(StoreType.QUEUE) assert store.empty() is True d1 = StubDownload(info=make_item(id="x1")) d2 = StubDownload(info=make_item(id="x2")) - store.put(d1) - store.put(d2) + await store.put(d1) + await store.put(d2) + await store._connection.flush() assert store.empty() is False first_key, first_val = store.next() assert first_key == d1.info._id assert first_val.info._id == d1.info._id + await store._connection.close() - def test_has_downloads_and_get_next_download(self) -> None: - conn = make_conn() - store = DataStore(StoreType.QUEUE, conn) + @pytest.mark.asyncio + async def test_has_downloads_and_get_next_download(self) -> None: + store = await make_store_async(StoreType.QUEUE) # One non-auto-start, one started, one cancelled, and one eligible i1 = make_item(id="n1") @@ -138,36 +187,40 @@ class TestDataStore: i3 = make_item(id="c1") i4 = make_item(id="ok1") - store.put(StubDownload(info=i1, started=False)) - store.put(StubDownload(info=i2, started=True)) - store.put(StubDownload(info=i3, started=False, cancelled=True)) - store.put(StubDownload(info=i4, started=False)) + await store.put(StubDownload(info=i1, started=False)) + await store.put(StubDownload(info=i2, started=True)) + await store.put(StubDownload(info=i3, started=False, cancelled=True)) + await store.put(StubDownload(info=i4, started=False)) + await store._connection.flush() assert store.has_downloads() is True nxt = store.get_next_download() assert isinstance(nxt, StubDownload) assert nxt.info._id == i4._id + await store._connection.close() @pytest.mark.asyncio async def test_test_method_executes_query(self) -> None: - conn = make_conn() - store = DataStore(StoreType.QUEUE, conn) + store = await make_store_async(StoreType.QUEUE) # Should not raise ok = await store.test() assert ok is True - def test_get_item_returns_none_when_no_kwargs(self) -> None: + @pytest.mark.asyncio + async def test_get_item_returns_none_when_no_kwargs(self) -> None: """Test that get_item returns None when no kwargs provided.""" - conn = make_conn() - store = DataStore(StoreType.QUEUE, conn) + db = await make_db() + store = DataStore(StoreType.QUEUE, db) result = store.get_item() assert result is None + await db.close() - def test_get_item_finds_by_single_attribute(self) -> None: + @pytest.mark.asyncio + async def test_get_item_finds_by_single_attribute(self) -> None: """Test that get_item correctly finds item by a single attribute.""" - conn = make_conn() - store = DataStore(StoreType.QUEUE, conn) + db = await make_db() + store = DataStore(StoreType.QUEUE, db) # Create items with different attributes item1 = make_item(id="vid1", url="http://example.com/1", title="Video 1", folder="folder1") @@ -178,8 +231,9 @@ class TestDataStore: d1 = StubDownload(info=item1) d2 = StubDownload(info=item2) - store.put(d1) - store.put(d2) + await store.put(d1) + await store.put(d2) + await store._connection.flush() # Test finding by title result = store.get_item(title="Video 1") @@ -197,11 +251,13 @@ class TestDataStore: result = store.get_item(url="http://example.com/1") assert result is not None assert result.info._id == "id1" + await db.close() - def test_get_item_finds_by_multiple_attributes(self) -> None: + @pytest.mark.asyncio + async def test_get_item_finds_by_multiple_attributes(self) -> None: """Test that get_item finds item when ANY of the provided attributes match.""" - conn = make_conn() - store = DataStore(StoreType.QUEUE, conn) + db = await make_db() + store = DataStore(StoreType.QUEUE, db) item1 = make_item(id="vid1", url="http://example.com/1", title="Video 1", folder="folder1") item1._id = "id1" @@ -211,8 +267,9 @@ class TestDataStore: d1 = StubDownload(info=item1) d2 = StubDownload(info=item2) - store.put(d1) - store.put(d2) + await store.put(d1) + await store.put(d2) + await store._connection.flush() # Test finding by multiple attributes where one matches result = store.get_item(title="Video 1", folder="wrong_folder") @@ -223,16 +280,19 @@ class TestDataStore: result = store.get_item(title="Wrong Title", folder="folder2") assert result is not None assert result.info._id == "id2" + await db.close() - def test_get_item_returns_none_when_no_match(self) -> None: + @pytest.mark.asyncio + async def test_get_item_returns_none_when_no_match(self) -> None: """Test that get_item returns None when no attributes match.""" - conn = make_conn() - store = DataStore(StoreType.QUEUE, conn) + db = await make_db() + store = DataStore(StoreType.QUEUE, db) item = make_item(id="vid1", url="http://example.com/1", title="Video 1", folder="folder1") item._id = "id1" d = StubDownload(info=item) - store.put(d) + await store.put(d) + await store._connection.flush() # Test with non-matching attribute result = store.get_item(title="Nonexistent Video") @@ -241,17 +301,20 @@ class TestDataStore: # Test with non-existent attribute key result = store.get_item(nonexistent_field="value") assert result is None + await db.close() - def test_get_item_skips_items_with_no_info(self) -> None: + @pytest.mark.asyncio + async def test_get_item_skips_items_with_no_info(self) -> None: """Test that get_item skips items that have no info attribute.""" - conn = make_conn() - store = DataStore(StoreType.QUEUE, conn) + db = await make_db() + store = DataStore(StoreType.QUEUE, db) # Create a valid item item = make_item(id="vid1", url="http://example.com/1", title="Video 1") item._id = "id1" d = StubDownload(info=item) - store.put(d) + await store.put(d) + await store._connection.flush() # Manually add an item with None info class BrokenDownload: @@ -264,11 +327,13 @@ class TestDataStore: result = store.get_item(title="Video 1") assert result is not None assert result.info._id == "id1" + await db.close() - def test_get_item_returns_first_match(self) -> None: + @pytest.mark.asyncio + async def test_get_item_returns_first_match(self) -> None: """Test that get_item returns the first matching item when multiple match.""" - conn = make_conn() - store = DataStore(StoreType.QUEUE, conn) + db = await make_db() + store = DataStore(StoreType.QUEUE, db) # Create multiple items with same title item1 = make_item(id="vid1", url="http://example.com/1", title="Same Title", folder="folder1") @@ -279,27 +344,31 @@ class TestDataStore: d1 = StubDownload(info=item1) d2 = StubDownload(info=item2) - store.put(d1) - store.put(d2) + await store.put(d1) + await store.put(d2) + await store._connection.flush() # Should return first match (note: OrderedDict maintains insertion order) result = store.get_item(title="Same Title") assert result is not None assert result.info._id == "id1" + await db.close() - def test_init(self) -> None: + @pytest.mark.asyncio + async def test_init(self) -> None: """Test DataStore initialization.""" - conn = make_conn() - store = DataStore(StoreType.QUEUE, conn) + store = await make_store_async(StoreType.QUEUE) assert store._type == StoreType.QUEUE - assert store._connection is conn + assert store._connection is not None assert isinstance(store._dict, OrderedDict) assert len(store._dict) == 0 - def test_load(self) -> None: + @pytest.mark.asyncio + async def test_load(self) -> None: """Test loading items from database into memory.""" - conn = make_conn() + db = await make_db() + conn = db._conn # Insert items directly into database item1_data = asdict(make_item(id="vid1", url="http://example.com/1", title="Video 1")) @@ -308,7 +377,7 @@ class TestDataStore: item2_data.pop("_id", None) created = datetime(2024, 1, 1, 12, 0, 0, tzinfo=UTC) - conn.execute( + await conn.execute( "INSERT INTO history (id, type, url, data, created_at) VALUES (?, ?, ?, ?, ?)", ( "id1", @@ -318,7 +387,7 @@ class TestDataStore: created.strftime("%Y-%m-%d %H:%M:%S"), ), ) - conn.execute( + await conn.execute( "INSERT INTO history (id, type, url, data, created_at) VALUES (?, ?, ?, ?, ?)", ( "id2", @@ -328,21 +397,24 @@ class TestDataStore: created.strftime("%Y-%m-%d %H:%M:%S"), ), ) + await conn.commit() - # Create store and load - store = DataStore(StoreType.QUEUE, conn) + store = DataStore(StoreType.QUEUE, db) assert len(store._dict) == 0 - store.load() + await store.load() assert len(store._dict) == 2 assert "id1" in store._dict assert "id2" in store._dict assert store._dict["id1"].info.url == "http://example.com/1" assert store._dict["id2"].info.url == "http://example.com/2" + await db.close() - def test_load_with_different_store_types(self) -> None: + @pytest.mark.asyncio + async def test_load_with_different_store_types(self) -> None: """Test that load only loads items matching the store type.""" - conn = make_conn() + db = await make_db() + conn = db._conn # Insert items with different types item1_data = asdict(make_item(id="vid1", url="http://example.com/1")) @@ -351,7 +423,7 @@ class TestDataStore: item2_data.pop("_id", None) created = datetime(2024, 1, 1, 12, 0, 0, tzinfo=UTC) - conn.execute( + await conn.execute( "INSERT INTO history (id, type, url, data, created_at) VALUES (?, ?, ?, ?, ?)", ( "id1", @@ -361,7 +433,7 @@ class TestDataStore: created.strftime("%Y-%m-%d %H:%M:%S"), ), ) - conn.execute( + await conn.execute( "INSERT INTO history (id, type, url, data, created_at) VALUES (?, ?, ?, ?, ?)", ( "id2", @@ -371,23 +443,26 @@ class TestDataStore: created.strftime("%Y-%m-%d %H:%M:%S"), ), ) + await conn.commit() # Load QUEUE store - should only get queue items - queue_store = DataStore(StoreType.QUEUE, conn) - queue_store.load() + queue_store = DataStore(StoreType.QUEUE, db) + await queue_store.load() assert len(queue_store._dict) == 1 assert "id1" in queue_store._dict # Load HISTORY store - should only get history items - history_store = DataStore(StoreType.HISTORY, conn) - history_store.load() + history_store = DataStore(StoreType.HISTORY, db) + await history_store.load() assert len(history_store._dict) == 1 assert "id2" in history_store._dict + await db.close() - def test_get_by_id(self) -> None: + @pytest.mark.asyncio + async def test_get_by_id(self) -> None: """Test getting item by ID.""" - conn = make_conn() - store = DataStore(StoreType.QUEUE, conn) + db = await make_db() + store = DataStore(StoreType.QUEUE, db) item1 = make_item(id="vid1", url="http://example.com/1", title="Video 1") item1._id = "id1" @@ -397,27 +472,30 @@ class TestDataStore: d1 = StubDownload(info=item1) d2 = StubDownload(info=item2) - store.put(d1) - store.put(d2) + await store.put(d1) + await store.put(d2) + await store._connection.flush() # Test getting existing items - result = store.get_by_id("id1") + result = await store.get_by_id("id1") assert result is not None assert result.info._id == "id1" assert result.info.title == "Video 1" - result = store.get_by_id("id2") + result = await store.get_by_id("id2") assert result is not None assert result.info._id == "id2" # Test getting non-existent item - result = store.get_by_id("nonexistent") + result = await store.get_by_id("nonexistent") assert result is None + await db.close() - def test_items(self) -> None: + @pytest.mark.asyncio + async def test_items(self) -> None: """Test getting all items as list of tuples.""" - conn = make_conn() - store = DataStore(StoreType.QUEUE, conn) + db = await make_db() + store = DataStore(StoreType.QUEUE, db) # Empty store result = store.items() @@ -432,8 +510,9 @@ class TestDataStore: d1 = StubDownload(info=item1) d2 = StubDownload(info=item2) - store.put(d1) - store.put(d2) + await store.put(d1) + await store.put(d2) + await store._connection.flush() # Test getting all items result = list(store.items()) @@ -447,36 +526,44 @@ class TestDataStore: # Verify order is maintained (OrderedDict) assert result[0][0] == "id1" assert result[1][0] == "id2" + await db.close() - def test_get_total_count_with_empty_store(self) -> None: + @pytest.mark.asyncio + async def test_get_total_count_with_empty_store(self) -> None: """Test get_total_count with empty datastore.""" - conn = make_conn() - store = DataStore(StoreType.QUEUE, conn) + store = await make_store_async(StoreType.QUEUE) - count = store.get_total_count() + count = await store.get_total_count() assert count == 0 + await store._connection.close() - def test_get_total_count_with_items(self) -> None: + @pytest.mark.asyncio + async def test_get_total_count_with_items(self) -> None: """Test get_total_count with items in database.""" - conn = make_conn() - store = DataStore(StoreType.QUEUE, conn) + store = await make_store_async(StoreType.QUEUE) + db = store._connection + conn = db._conn # Add items directly to database created = datetime.now(UTC).strftime("%Y-%m-%d %H:%M:%S") for i in range(5): item_data = asdict(make_item(id=f"vid{i}")) item_data.pop("_id", None) - conn.execute( + await conn.execute( "INSERT INTO history (id, type, url, data, created_at) VALUES (?, ?, ?, ?, ?)", (f"id{i}", str(StoreType.QUEUE), f"http://example.com/{i}", json.dumps(item_data), created), ) + await conn.commit() - count = store.get_total_count() + count = await store.get_total_count() assert count == 5 + await store._connection.close() - def test_get_total_count_respects_store_type(self) -> None: + @pytest.mark.asyncio + async def test_get_total_count_respects_store_type(self) -> None: """Test that get_total_count only counts items of the correct type.""" - conn = make_conn() + db = await make_db() + conn = db._conn created = datetime.now(UTC).strftime("%Y-%m-%d %H:%M:%S") @@ -484,30 +571,33 @@ class TestDataStore: for i in range(3): item_data = asdict(make_item(id=f"vid{i}")) item_data.pop("_id", None) - conn.execute( + await conn.execute( "INSERT INTO history (id, type, url, data, created_at) VALUES (?, ?, ?, ?, ?)", - (f"q{i}", str(StoreType.QUEUE), f"http://example.com/{i}", json.dumps(item_data), created), + (f"q{i}", str(StoreType.QUEUE), f"http://example.com/queue/{i}", json.dumps(item_data), created), ) # Add 2 HISTORY items for i in range(2): item_data = asdict(make_item(id=f"vid{i}")) item_data.pop("_id", None) - conn.execute( + await conn.execute( "INSERT INTO history (id, type, url, data, created_at) VALUES (?, ?, ?, ?, ?)", - (f"h{i}", str(StoreType.HISTORY), f"http://example.com/{i}", json.dumps(item_data), created), + (f"h{i}", str(StoreType.HISTORY), f"http://example.com/history/{i}", json.dumps(item_data), created), ) + await conn.commit() - queue_store = DataStore(StoreType.QUEUE, conn) - assert queue_store.get_total_count() == 3 + queue_store = DataStore(StoreType.QUEUE, db) + assert await queue_store.get_total_count() == 3 - history_store = DataStore(StoreType.HISTORY, conn) - assert history_store.get_total_count() == 2 + history_store = DataStore(StoreType.HISTORY, db) + assert await history_store.get_total_count() == 2 + await db.close() - def test_put_with_error_status_emits_event(self) -> None: + @pytest.mark.asyncio + async def test_put_with_error_status_emits_event(self) -> None: """Test that put() emits an event when item has error status.""" - conn = make_conn() - store = DataStore(StoreType.QUEUE, conn) + db = await make_db() + store = DataStore(StoreType.QUEUE, db) item = make_item(id="vid1") item.status = "error" @@ -517,13 +607,16 @@ class TestDataStore: # We can't easily test event emission without mocking EventBus # Just verify it doesn't crash - result = store.put(d) + result = await store.put(d) + await store._connection.flush() assert result is not None + await db.close() - def test_put_with_no_notify_skips_event(self) -> None: + @pytest.mark.asyncio + async def test_put_with_no_notify_skips_event(self) -> None: """Test that put() with no_notify=True doesn't emit events.""" - conn = make_conn() - store = DataStore(StoreType.QUEUE, conn) + db = await make_db() + store = DataStore(StoreType.QUEUE, db) item = make_item(id="vid1") item.status = "error" @@ -532,138 +625,167 @@ class TestDataStore: d = StubDownload(info=item) # Should not emit event when no_notify=True - result = store.put(d, no_notify=True) + result = await store.put(d, no_notify=True) + await store._connection.flush() assert result is not None assert result.info._id == item._id + await db.close() - def test_delete_nonexistent_item(self) -> None: + @pytest.mark.asyncio + async def test_delete_nonexistent_item(self) -> None: """Test that deleting non-existent item doesn't raise error.""" - conn = make_conn() - store = DataStore(StoreType.QUEUE, conn) + db = await make_db() + store = DataStore(StoreType.QUEUE, db) # Should not raise error store.delete("nonexistent_id") # Verify nothing was deleted from database - row = conn.execute("SELECT * FROM history WHERE id=?", ("nonexistent_id",)).fetchone() + await store._connection.flush() + cursor = await db._conn.execute("SELECT * FROM history WHERE id=?", ("nonexistent_id",)) + row = await cursor.fetchone() assert row is None + await db.close() - def test_has_downloads_with_empty_dict(self) -> None: + @pytest.mark.asyncio + async def test_has_downloads_with_empty_dict(self) -> None: """Test has_downloads returns False when dict is empty.""" - conn = make_conn() - store = DataStore(StoreType.QUEUE, conn) + db = await make_db() + store = DataStore(StoreType.QUEUE, db) assert store.has_downloads() is False + await db.close() - def test_has_downloads_with_no_eligible_downloads(self) -> None: + @pytest.mark.asyncio + async def test_has_downloads_with_no_eligible_downloads(self) -> None: """Test has_downloads returns False when no downloads are eligible.""" - conn = make_conn() - store = DataStore(StoreType.QUEUE, conn) + db = await make_db() + store = DataStore(StoreType.QUEUE, db) # Add item that's already started item1 = make_item(id="vid1") - store.put(StubDownload(info=item1, started=True)) + await store.put(StubDownload(info=item1, started=True)) # Add item with auto_start=False item2 = make_item(id="vid2") item2.auto_start = False - store.put(StubDownload(info=item2, started=False)) + await store.put(StubDownload(info=item2, started=False)) + await store._connection.flush() assert store.has_downloads() is False + await db.close() - def test_get_next_download_returns_none_when_empty(self) -> None: + @pytest.mark.asyncio + async def test_get_next_download_returns_none_when_empty(self) -> None: """Test get_next_download returns None when no eligible downloads.""" - conn = make_conn() - store = DataStore(StoreType.QUEUE, conn) + db = await make_db() + store = DataStore(StoreType.QUEUE, db) result = store.get_next_download() assert result is None + await db.close() - def test_get_next_download_skips_cancelled(self) -> None: + @pytest.mark.asyncio + async def test_get_next_download_skips_cancelled(self) -> None: """Test get_next_download skips cancelled downloads.""" - conn = make_conn() - store = DataStore(StoreType.QUEUE, conn) + db = await make_db() + store = DataStore(StoreType.QUEUE, db) # Add cancelled download item1 = make_item(id="vid1") item1._id = "id1" - store.put(StubDownload(info=item1, started=False, cancelled=True)) + await store.put(StubDownload(info=item1, started=False, cancelled=True)) # Add eligible download item2 = make_item(id="vid2") item2._id = "id2" - store.put(StubDownload(info=item2, started=False, cancelled=False)) + await store.put(StubDownload(info=item2, started=False, cancelled=False)) + await store._connection.flush() result = store.get_next_download() assert result is not None assert result.info._id == "id2" + await db.close() - def test_update_store_item_removes_datetime_field(self) -> None: + @pytest.mark.asyncio + async def test_update_store_item_removes_datetime_field(self) -> None: """Test that _update_store_item removes datetime field before storage.""" - conn = make_conn() - store = DataStore(StoreType.QUEUE, conn) + db = await make_db() + store = DataStore(StoreType.QUEUE, db) item = make_item(id="vid1") item.datetime = "Thu, 01 Jan 2024 12:00:00 GMT" # Add datetime field d = StubDownload(info=item) - store.put(d) + await store.put(d) + await store._connection.flush() # Verify datetime field is not in stored JSON - row = conn.execute("SELECT data FROM history WHERE id=?", (item._id,)).fetchone() + cursor = await db._conn.execute("SELECT data FROM history WHERE id=?", (item._id,)) + row = await cursor.fetchone() assert row is not None data = json.loads(row["data"]) assert "datetime" not in data + await db.close() - def test_update_store_item_removes_live_in_when_finished(self) -> None: + @pytest.mark.asyncio + async def test_update_store_item_removes_live_in_when_finished(self) -> None: """Test that _update_store_item removes live_in field when status is finished.""" - conn = make_conn() - store = DataStore(StoreType.QUEUE, conn) + store = await make_store_async(StoreType.QUEUE) + conn = store._connection item = make_item(id="vid1") item.status = "finished" item.live_in = "PT5M" # Add live_in field d = StubDownload(info=item) - store.put(d) + await store.put(d) + await store._connection.flush() # Verify live_in field is not in stored JSON when status is finished - row = conn.execute("SELECT data FROM history WHERE id=?", (item._id,)).fetchone() + cursor = await conn._conn.execute("SELECT data FROM history WHERE id=?", (item._id,)) + row = await cursor.fetchone() assert row is not None data = json.loads(row["data"]) assert "live_in" not in data + await store._connection.close() - def test_update_store_item_keeps_live_in_when_not_finished(self) -> None: + @pytest.mark.asyncio + async def test_update_store_item_keeps_live_in_when_not_finished(self) -> None: """Test that _update_store_item keeps live_in field when status is not finished.""" - conn = make_conn() - store = DataStore(StoreType.QUEUE, conn) + store = await make_store_async(StoreType.QUEUE) + conn = store._connection item = make_item(id="vid1") item.status = "downloading" item.live_in = "PT5M" # Add live_in field d = StubDownload(info=item) - store.put(d) + await store.put(d) + await store._connection.flush() # Verify live_in field IS in stored JSON when status is not finished - row = conn.execute("SELECT data FROM history WHERE id=?", (item._id,)).fetchone() + cursor = await conn._conn.execute("SELECT data FROM history WHERE id=?", (item._id,)) + row = await cursor.fetchone() assert row is not None data = json.loads(row["data"]) assert "live_in" in data assert data["live_in"] == "PT5M" + await store._connection.close() +@pytest.mark.asyncio class TestDataStoreOperations: """Test get_item with different comparison operations.""" - def test_operation_equal(self) -> None: + async def test_operation_equal(self) -> None: """Test EQUAL operation (default behavior).""" - conn = make_conn() - store = DataStore(StoreType.QUEUE, conn) + db = await make_db() + store = DataStore(StoreType.QUEUE, db) item1 = make_item(id="vid1", title="Exact Match", folder="folder1") item1._id = "id1" - store.put(StubDownload(info=item1)) + await store.put(StubDownload(info=item1)) # Test with explicit EQUAL operation result = store.get_item(title=(Operation.EQUAL, "Exact Match")) @@ -678,37 +800,39 @@ class TestDataStoreOperations: # Test no match result = store.get_item(title=(Operation.EQUAL, "No Match")) assert result is None + await db.close() - def test_operation_not_equal(self) -> None: + async def test_operation_not_equal(self) -> None: """Test NOT_EQUAL operation.""" - conn = make_conn() - store = DataStore(StoreType.QUEUE, conn) + db = await make_db() + store = DataStore(StoreType.QUEUE, db) item1 = make_item(id="vid1", title="Video 1", folder="folder1") item1._id = "id1" item2 = make_item(id="vid2", title="Video 2", folder="folder2") item2._id = "id2" - store.put(StubDownload(info=item1)) - store.put(StubDownload(info=item2)) + await store.put(StubDownload(info=item1)) + await store.put(StubDownload(info=item2)) # Find item where title is not "Video 1" result = store.get_item(title=(Operation.NOT_EQUAL, "Video 1")) assert result is not None assert result.info._id == "id2" + await db.close() - def test_operation_contain(self) -> None: + async def test_operation_contain(self) -> None: """Test CONTAIN operation (substring match).""" - conn = make_conn() - store = DataStore(StoreType.QUEUE, conn) + db = await make_db() + store = DataStore(StoreType.QUEUE, db) item1 = make_item(id="vid1", title="Python Tutorial Video", folder="folder1") item1._id = "id1" item2 = make_item(id="vid2", title="JavaScript Course", folder="folder2") item2._id = "id2" - store.put(StubDownload(info=item1)) - store.put(StubDownload(info=item2)) + await store.put(StubDownload(info=item1)) + await store.put(StubDownload(info=item2)) # Find item with "Python" in title result = store.get_item(title=(Operation.CONTAIN, "Python")) @@ -723,37 +847,39 @@ class TestDataStoreOperations: # No match result = store.get_item(title=(Operation.CONTAIN, "Rust")) assert result is None + await db.close() - def test_operation_not_contain(self) -> None: + async def test_operation_not_contain(self) -> None: """Test NOT_CONTAIN operation.""" - conn = make_conn() - store = DataStore(StoreType.QUEUE, conn) + db = await make_db() + store = DataStore(StoreType.QUEUE, db) item1 = make_item(id="vid1", title="Python Tutorial", folder="folder1") item1._id = "id1" item2 = make_item(id="vid2", title="JavaScript Course", folder="folder2") item2._id = "id2" - store.put(StubDownload(info=item1)) - store.put(StubDownload(info=item2)) + await store.put(StubDownload(info=item1)) + await store.put(StubDownload(info=item2)) # Find item that doesn't contain "Python" result = store.get_item(title=(Operation.NOT_CONTAIN, "Python")) assert result is not None assert result.info._id == "id2" + await db.close() - def test_operation_starts_with(self) -> None: + async def test_operation_starts_with(self) -> None: """Test STARTS_WITH operation.""" - conn = make_conn() - store = DataStore(StoreType.QUEUE, conn) + db = await make_db() + store = DataStore(StoreType.QUEUE, db) item1 = make_item(id="vid1", title="Tutorial: Python Basics", folder="folder1") item1._id = "id1" item2 = make_item(id="vid2", title="Course: JavaScript", folder="folder2") item2._id = "id2" - store.put(StubDownload(info=item1)) - store.put(StubDownload(info=item2)) + await store.put(StubDownload(info=item1)) + await store.put(StubDownload(info=item2)) # Find item starting with "Tutorial" result = store.get_item(title=(Operation.STARTS_WITH, "Tutorial")) @@ -768,19 +894,20 @@ class TestDataStoreOperations: # No match result = store.get_item(title=(Operation.STARTS_WITH, "Video")) assert result is None + await db.close() - def test_operation_ends_with(self) -> None: + async def test_operation_ends_with(self) -> None: """Test ENDS_WITH operation.""" - conn = make_conn() - store = DataStore(StoreType.QUEUE, conn) + db = await make_db() + store = DataStore(StoreType.QUEUE, db) item1 = make_item(id="vid1", title="Learn Python", folder="folder1") item1._id = "id1" item2 = make_item(id="vid2", title="Learn JavaScript", folder="folder2") item2._id = "id2" - store.put(StubDownload(info=item1)) - store.put(StubDownload(info=item2)) + await store.put(StubDownload(info=item1)) + await store.put(StubDownload(info=item2)) # Find item ending with "Python" result = store.get_item(title=(Operation.ENDS_WITH, "Python")) @@ -795,11 +922,12 @@ class TestDataStoreOperations: # No match result = store.get_item(title=(Operation.ENDS_WITH, "Course")) assert result is None + await db.close() - def test_operation_greater_than(self) -> None: + async def test_operation_greater_than(self) -> None: """Test GREATER_THAN operation with numeric values.""" - conn = make_conn() - store = DataStore(StoreType.QUEUE, conn) + db = await make_db() + store = DataStore(StoreType.QUEUE, db) item1 = make_item(id="vid1", title="Video 1") item1._id = "id1" @@ -808,8 +936,8 @@ class TestDataStoreOperations: item2._id = "id2" item2.filesize = 2000 - store.put(StubDownload(info=item1)) - store.put(StubDownload(info=item2)) + await store.put(StubDownload(info=item1)) + await store.put(StubDownload(info=item2)) # Find item with filesize > 1500 result = store.get_item(filesize=(Operation.GREATER_THAN, 1500)) @@ -820,11 +948,12 @@ class TestDataStoreOperations: result = store.get_item(filesize=(Operation.GREATER_THAN, 500)) assert result is not None assert result.info._id == "id1" + await db.close() - def test_operation_less_than(self) -> None: + async def test_operation_less_than(self) -> None: """Test LESS_THAN operation.""" - conn = make_conn() - store = DataStore(StoreType.QUEUE, conn) + db = await make_db() + store = DataStore(StoreType.QUEUE, db) item1 = make_item(id="vid1", title="Video 1") item1._id = "id1" @@ -833,24 +962,25 @@ class TestDataStoreOperations: item2._id = "id2" item2.filesize = 2000 - store.put(StubDownload(info=item1)) - store.put(StubDownload(info=item2)) + await store.put(StubDownload(info=item1)) + await store.put(StubDownload(info=item2)) # Find item with filesize < 1500 result = store.get_item(filesize=(Operation.LESS_THAN, 1500)) assert result is not None assert result.info._id == "id1" + await db.close() - def test_operation_greater_equal(self) -> None: + async def test_operation_greater_equal(self) -> None: """Test GREATER_EQUAL operation.""" - conn = make_conn() - store = DataStore(StoreType.QUEUE, conn) + db = await make_db() + store = DataStore(StoreType.QUEUE, db) item1 = make_item(id="vid1", title="Video 1") item1._id = "id1" item1.filesize = 1000 - store.put(StubDownload(info=item1)) + await store.put(StubDownload(info=item1)) # Test >= with exact match result = store.get_item(filesize=(Operation.GREATER_EQUAL, 1000)) @@ -865,17 +995,18 @@ class TestDataStoreOperations: # Test >= with greater than result = store.get_item(filesize=(Operation.GREATER_EQUAL, 1500)) assert result is None + await db.close() - def test_operation_less_equal(self) -> None: + async def test_operation_less_equal(self) -> None: """Test LESS_EQUAL operation.""" - conn = make_conn() - store = DataStore(StoreType.QUEUE, conn) + db = await make_db() + store = DataStore(StoreType.QUEUE, db) item1 = make_item(id="vid1", title="Video 1") item1._id = "id1" item1.filesize = 1000 - store.put(StubDownload(info=item1)) + await store.put(StubDownload(info=item1)) # Test <= with exact match result = store.get_item(filesize=(Operation.LESS_EQUAL, 1000)) @@ -888,11 +1019,12 @@ class TestDataStoreOperations: # Test <= with less than result = store.get_item(filesize=(Operation.LESS_EQUAL, 500)) assert result is None + await db.close() - def test_mixed_operations(self) -> None: + async def test_mixed_operations(self) -> None: """Test using multiple operations in a single query.""" - conn = make_conn() - store = DataStore(StoreType.QUEUE, conn) + db = await make_db() + store = DataStore(StoreType.QUEUE, db) item1 = make_item(id="vid1", title="Python Tutorial", folder="tutorials") item1._id = "id1" @@ -901,9 +1033,9 @@ class TestDataStoreOperations: item3 = make_item(id="vid3", title="JavaScript Basics", folder="tutorials") item3._id = "id3" - store.put(StubDownload(info=item1)) - store.put(StubDownload(info=item2)) - store.put(StubDownload(info=item3)) + await store.put(StubDownload(info=item1)) + await store.put(StubDownload(info=item2)) + await store.put(StubDownload(info=item3)) # Mix of operation and default (any match returns true) result = store.get_item(title=(Operation.CONTAIN, "Python"), folder="tutorials") @@ -914,17 +1046,18 @@ class TestDataStoreOperations: result = store.get_item(title=(Operation.CONTAIN, "JavaScript"), folder="nonexistent") assert result is not None assert result.info._id == "id3" + await db.close() - def test_operation_with_none_values(self) -> None: + async def test_operation_with_none_values(self) -> None: """Test operations handle None values gracefully.""" - conn = make_conn() - store = DataStore(StoreType.QUEUE, conn) + db = await make_db() + store = DataStore(StoreType.QUEUE, db) item1 = make_item(id="vid1", title="Video 1") item1._id = "id1" item1.description = None - store.put(StubDownload(info=item1)) + await store.put(StubDownload(info=item1)) # CONTAIN with None field should return False result = store.get_item(description=(Operation.CONTAIN, "test")) @@ -937,30 +1070,32 @@ class TestDataStoreOperations: # GREATER_THAN with None should return False result = store.get_item(description=(Operation.GREATER_THAN, 100)) assert result is None + await db.close() - def test_operation_with_invalid_comparisons(self) -> None: + async def test_operation_with_invalid_comparisons(self) -> None: """Test that invalid comparisons are handled gracefully.""" - conn = make_conn() - store = DataStore(StoreType.QUEUE, conn) + db = await make_db() + store = DataStore(StoreType.QUEUE, db) item1 = make_item(id="vid1", title="Video 1") item1._id = "id1" - store.put(StubDownload(info=item1)) + await store.put(StubDownload(info=item1)) # Try to compare string with number using > (should return False/None) result = store.get_item(title=(Operation.GREATER_THAN, 100)) assert result is None + await db.close() - def test_operation_backward_compatibility(self) -> None: + async def test_operation_backward_compatibility(self) -> None: """Test that string operation names work for backward compatibility.""" - conn = make_conn() - store = DataStore(StoreType.QUEUE, conn) + db = await make_db() + store = DataStore(StoreType.QUEUE, db) item1 = make_item(id="vid1", title="Python Tutorial") item1._id = "id1" - store.put(StubDownload(info=item1)) + await store.put(StubDownload(info=item1)) # Using string operation value result = store.get_item(title=("in", "Python")) @@ -970,16 +1105,17 @@ class TestDataStoreOperations: # Using string for EQUAL result = store.get_item(title=("==", "Python Tutorial")) assert result is not None + await db.close() - def test_operation_with_nonexistent_field(self) -> None: + async def test_operation_with_nonexistent_field(self) -> None: """Test operations with fields that don't exist.""" - conn = make_conn() - store = DataStore(StoreType.QUEUE, conn) + db = await make_db() + store = DataStore(StoreType.QUEUE, db) item1 = make_item(id="vid1", title="Video 1") item1._id = "id1" - store.put(StubDownload(info=item1)) + await store.put(StubDownload(info=item1)) # Try to match on non-existent field result = store.get_item(nonexistent_field=(Operation.EQUAL, "value")) @@ -987,3 +1123,4 @@ class TestDataStoreOperations: result = store.get_item(nonexistent_field=(Operation.CONTAIN, "value")) assert result is None + await db.close() diff --git a/app/tests/test_datastore_pagination.py b/app/tests/test_datastore_pagination.py index da38f3f1..da827824 100644 --- a/app/tests/test_datastore_pagination.py +++ b/app/tests/test_datastore_pagination.py @@ -1,257 +1,210 @@ import json -import sqlite3 from datetime import UTC, datetime -from pathlib import Path -from tempfile import TemporaryDirectory import pytest +import pytest_asyncio from app.library.DataStore import DataStore, StoreType from app.library.ItemDTO import ItemDTO +from app.library.sqlite_store import SqliteStore +async def make_db(data: int = 100) -> SqliteStore: + """Create a temporary database with test data.""" + SqliteStore._reset_singleton() + ins = SqliteStore.get_instance(db_path=":memory:") + await ins._ensure_conn() + + base_time = datetime.now(UTC) + for i in range(data): + created_at = base_time.replace(hour=(i // 4) % 24, minute=(i * 15) % 60, second=i % 60).strftime( + "%Y-%m-%d %H:%M:%S" + ) + + item_data = { + "url": f"https://example.com/video{i}", + "title": f"Test Video {i}", + "id": f"video{i}", + "folder": "/downloads", + "status": "finished", + } + + await ins._conn.execute( + 'INSERT INTO "history" ("id", "type", "url", "data", "created_at") VALUES (?, ?, ?, ?, ?)', + ( + f"test-id-{i}", + str(StoreType.HISTORY), + item_data["url"], + json.dumps(item_data), + created_at, + ), + ) + if data > 0: + await ins._conn.commit() + + return ins + + +@pytest.mark.asyncio class TestDataStorePagination: """Test pagination functionality of DataStore.""" - @pytest.fixture - def temp_db(self): - """Create a temporary database with test data.""" - with TemporaryDirectory() as tmpdir: - db_path = Path(tmpdir) / "test.db" - conn = sqlite3.connect(str(db_path)) - conn.row_factory = sqlite3.Row + @pytest_asyncio.fixture + async def make_db(self, data: int = 100): + """Fixture to provide a temporary database with test data.""" + return await make_db(data=data) - # Create the history table - conn.execute( - """ - CREATE TABLE IF NOT EXISTS "history" ( - "id" TEXT PRIMARY KEY, - "type" TEXT NOT NULL, - "url" TEXT NOT NULL, - "data" TEXT NOT NULL, - "created_at" TEXT NOT NULL - ) - """ - ) - - # Insert test data - base_time = datetime.now(UTC) - for i in range(100): - created_at = base_time.replace( - hour=(i // 4) % 24, minute=(i * 15) % 60, second=i % 60 - ).strftime("%Y-%m-%d %H:%M:%S") - - item_data = { - "url": f"https://example.com/video{i}", - "title": f"Test Video {i}", - "id": f"video{i}", - "folder": "/downloads", - "status": "finished", - } - - conn.execute( - 'INSERT INTO "history" ("id", "type", "url", "data", "created_at") VALUES (?, ?, ?, ?, ?)', - ( - f"test-id-{i}", - str(StoreType.HISTORY), - item_data["url"], - json.dumps(item_data), - created_at, - ), - ) - - conn.commit() - yield conn - conn.close() - - def test_get_total_count(self, temp_db): + async def test_get_total_count(self): """Test getting total count of items.""" - datastore = DataStore(type=StoreType.HISTORY, connection=temp_db) - count = datastore.get_total_count() - assert count == 100 + db = await make_db(data=100) + try: + datastore = DataStore(type=StoreType.HISTORY, connection=db) + count = await datastore.get_total_count() + assert count == 100 + finally: + await db.close() - def test_pagination_basic(self, temp_db): + async def test_pagination_basic(self): """Test basic pagination functionality.""" - datastore = DataStore(type=StoreType.HISTORY, connection=temp_db) + db = await make_db(data=100) + try: + datastore = DataStore(type=StoreType.HISTORY, connection=db) + items, total, page, total_pages = await datastore.get_items_paginated(page=1, per_page=10) + assert len(items) == 10 + assert total == 100 + assert page == 1 + assert total_pages == 10 + for _, item in items: + assert isinstance(item, ItemDTO) + assert item._id.startswith("test-id-") + finally: + await db.close() - # Get first page - items, total, page, total_pages = datastore.get_items_paginated(page=1, per_page=10) - - assert len(items) == 10 - assert total == 100 - assert page == 1 - assert total_pages == 10 - - # Verify items are ItemDTO instances - for _item_id, item in items: - assert isinstance(item, ItemDTO) - assert item._id.startswith("test-id-") - - def test_pagination_last_page(self, temp_db): + async def test_pagination_last_page(self): """Test pagination on last page.""" - datastore = DataStore(type=StoreType.HISTORY, connection=temp_db) + db = await make_db(data=100) + try: + datastore = DataStore(type=StoreType.HISTORY, connection=db) + items, total, page, total_pages = await datastore.get_items_paginated(page=10, per_page=10) + assert len(items) == 10 + assert total == 100 + assert page == 10 + assert total_pages == 10 + finally: + await db.close() - # Get last page - items, total, page, total_pages = datastore.get_items_paginated(page=10, per_page=10) - - assert len(items) == 10 - assert total == 100 - assert page == 10 - assert total_pages == 10 - - def test_pagination_partial_page(self, temp_db): + async def test_pagination_partial_page(self): """Test pagination with partial last page.""" - datastore = DataStore(type=StoreType.HISTORY, connection=temp_db) + db = await make_db(data=100) + try: + datastore = DataStore(type=StoreType.HISTORY, connection=db) + items, total, page, total_pages = await datastore.get_items_paginated(page=4, per_page=30) + assert len(items) == 10 # 100 items / 30 per page = 3 full pages + 10 items + assert total == 100 + assert page == 4 + assert total_pages == 4 + finally: + await db.close() - # Get items with per_page that doesn't divide evenly - items, total, page, total_pages = datastore.get_items_paginated(page=4, per_page=30) - - assert len(items) == 10 # 100 items / 30 per page = 3 full pages + 10 items - assert total == 100 - assert page == 4 - assert total_pages == 4 - - def test_pagination_out_of_range(self, temp_db): + async def test_pagination_out_of_range(self): """Test pagination with page number out of range.""" - datastore = DataStore(type=StoreType.HISTORY, connection=temp_db) + db = await make_db(data=100) + try: + datastore = DataStore(type=StoreType.HISTORY, connection=db) + items, total, page, total_pages = await datastore.get_items_paginated(page=999, per_page=10) + assert len(items) == 10 + assert total == 100 + assert page == 10 # Adjusted to last page + assert total_pages == 10 + finally: + await db.close() - # Request page beyond total pages - should return last page - items, total, page, total_pages = datastore.get_items_paginated(page=999, per_page=10) - - assert len(items) == 10 - assert total == 100 - assert page == 10 # Adjusted to last page - assert total_pages == 10 - - def test_pagination_order_desc(self, temp_db): + async def test_pagination_order_desc(self): """Test pagination with descending order (newest first).""" - datastore = DataStore(type=StoreType.HISTORY, connection=temp_db) + db = await make_db(data=100) + try: + datastore = DataStore(type=StoreType.HISTORY, connection=db) + items, _, _, _ = await datastore.get_items_paginated(page=1, per_page=5, order="DESC") + assert len(items) == 5 + finally: + await db.close() - items, _, _, _ = datastore.get_items_paginated(page=1, per_page=5, order="DESC") - - assert len(items) == 5 - # Verify order - should be newest to oldest - # (note: exact order depends on the timestamp generation in fixture) - - def test_pagination_order_asc(self, temp_db): + async def test_pagination_order_asc(self): """Test pagination with ascending order (oldest first).""" - datastore = DataStore(type=StoreType.HISTORY, connection=temp_db) + db = await make_db(data=100) + try: + datastore = DataStore(type=StoreType.HISTORY, connection=db) + items, _, _, _ = await datastore.get_items_paginated(page=1, per_page=5, order="ASC") + assert len(items) == 5 + finally: + await db.close() - items, _, _, _ = datastore.get_items_paginated(page=1, per_page=5, order="ASC") - - assert len(items) == 5 - - def test_pagination_invalid_page(self, temp_db): + async def test_pagination_invalid_page(self): """Test pagination with invalid page number.""" - datastore = DataStore(type=StoreType.HISTORY, connection=temp_db) + db = await make_db(data=100) + try: + datastore = DataStore(type=StoreType.HISTORY, connection=db) + with pytest.raises(ValueError, match="page must be >= 1"): + await datastore.get_items_paginated(page=0, per_page=10) - with pytest.raises(ValueError, match="page must be >= 1"): - datastore.get_items_paginated(page=0, per_page=10) + with pytest.raises(ValueError, match="page must be >= 1"): + await datastore.get_items_paginated(page=-1, per_page=10) + finally: + await db.close() - with pytest.raises(ValueError, match="page must be >= 1"): - datastore.get_items_paginated(page=-1, per_page=10) - - def test_pagination_invalid_per_page(self, temp_db): + async def test_pagination_invalid_per_page(self): """Test pagination with invalid per_page value.""" - datastore = DataStore(type=StoreType.HISTORY, connection=temp_db) + db = await make_db(data=100) + try: + datastore = DataStore(type=StoreType.HISTORY, connection=db) - with pytest.raises(ValueError, match="per_page must be >= 1"): - datastore.get_items_paginated(page=1, per_page=0) + with pytest.raises(ValueError, match="per_page must be >= 1"): + await datastore.get_items_paginated(page=1, per_page=0) - with pytest.raises(ValueError, match="per_page must be >= 1"): - datastore.get_items_paginated(page=1, per_page=-10) + with pytest.raises(ValueError, match="per_page must be >= 1"): + await datastore.get_items_paginated(page=1, per_page=-10) + finally: + await db.close() - def test_pagination_invalid_order(self, temp_db): + async def test_pagination_invalid_order(self): """Test pagination with invalid order parameter.""" - datastore = DataStore(type=StoreType.HISTORY, connection=temp_db) + db = await make_db(data=100) + try: + datastore = DataStore(type=StoreType.HISTORY, connection=db) + with pytest.raises(ValueError, match="order must be 'ASC' or 'DESC'"): + await datastore.get_items_paginated(page=1, per_page=10, order="INVALID") + finally: + await db.close() - with pytest.raises(ValueError, match="order must be 'ASC' or 'DESC'"): - datastore.get_items_paginated(page=1, per_page=10, order="INVALID") - - def test_pagination_empty_store(self): + async def test_pagination_empty_store(self): """Test pagination with empty datastore.""" - with TemporaryDirectory() as tmpdir: - db_path = Path(tmpdir) / "empty.db" - conn = sqlite3.connect(str(db_path)) - conn.row_factory = sqlite3.Row - - # Create empty table - conn.execute( - """ - CREATE TABLE IF NOT EXISTS "history" ( - "id" TEXT PRIMARY KEY, - "type" TEXT NOT NULL, - "url" TEXT NOT NULL, - "data" TEXT NOT NULL, - "created_at" TEXT NOT NULL - ) - """ - ) - conn.commit() - - datastore = DataStore(type=StoreType.HISTORY, connection=conn) - - items, total, page, total_pages = datastore.get_items_paginated(page=1, per_page=10) + db = await make_db(data=0) + try: + datastore = DataStore(type=StoreType.HISTORY, connection=db) + items, total, page, total_pages = await datastore.get_items_paginated(page=1, per_page=10) assert len(items) == 0 assert total == 0 assert page == 1 assert total_pages == 1 + finally: + await db.close() - conn.close() - - def test_pagination_single_item(self): + async def test_pagination_single_item(self): """Test pagination with single item.""" - with TemporaryDirectory() as tmpdir: - db_path = Path(tmpdir) / "single.db" - conn = sqlite3.connect(str(db_path)) - conn.row_factory = sqlite3.Row - - # Create table with single item - conn.execute( - """ - CREATE TABLE IF NOT EXISTS "history" ( - "id" TEXT PRIMARY KEY, - "type" TEXT NOT NULL, - "url" TEXT NOT NULL, - "data" TEXT NOT NULL, - "created_at" TEXT NOT NULL - ) - """ - ) - - item_data = { - "url": "https://example.com/single", - "title": "Single Video", - "id": "single-video", - "folder": "/downloads", - "status": "finished", - } - - conn.execute( - 'INSERT INTO "history" ("id", "type", "url", "data", "created_at") VALUES (?, ?, ?, ?, ?)', - ( - "single-id", - str(StoreType.HISTORY), - item_data["url"], - json.dumps(item_data), - datetime.now(UTC).strftime("%Y-%m-%d %H:%M:%S"), - ), - ) - conn.commit() - - datastore = DataStore(type=StoreType.HISTORY, connection=conn) - - items, total, page, total_pages = datastore.get_items_paginated(page=1, per_page=10) + db = await make_db(data=1) + try: + datastore = DataStore(type=StoreType.HISTORY, connection=db) + items, total, page, total_pages = await datastore.get_items_paginated(page=1, per_page=10) assert len(items) == 1 assert total == 1 assert page == 1 assert total_pages == 1 + finally: + await db.close() - conn.close() - - def test_pagination_status_filter_include(self, temp_db): + async def test_pagination_status_filter_include(self): """Test pagination with status filter (inclusion).""" # Add some items with different status values item_data_pending = { @@ -269,50 +222,51 @@ class TestDataStorePagination: "status": "downloading", } - temp_db.execute( - 'INSERT INTO "history" ("id", "type", "url", "data", "created_at") VALUES (?, ?, ?, ?, ?)', - ( - "pending-id", - str(StoreType.HISTORY), - item_data_pending["url"], - json.dumps(item_data_pending), - datetime.now(UTC).strftime("%Y-%m-%d %H:%M:%S"), - ), - ) - temp_db.execute( - 'INSERT INTO "history" ("id", "type", "url", "data", "created_at") VALUES (?, ?, ?, ?, ?)', - ( - "downloading-id", - str(StoreType.HISTORY), - item_data_downloading["url"], - json.dumps(item_data_downloading), - datetime.now(UTC).strftime("%Y-%m-%d %H:%M:%S"), - ), - ) - temp_db.commit() + db = await make_db(data=100) + try: + await db._conn.execute( + 'INSERT INTO "history" ("id", "type", "url", "data", "created_at") VALUES (?, ?, ?, ?, ?)', + ( + "pending-id", + str(StoreType.HISTORY), + item_data_pending["url"], + json.dumps(item_data_pending), + datetime.now(UTC).strftime("%Y-%m-%d %H:%M:%S"), + ), + ) + await db._conn.execute( + 'INSERT INTO "history" ("id", "type", "url", "data", "created_at") VALUES (?, ?, ?, ?, ?)', + ( + "downloading-id", + str(StoreType.HISTORY), + item_data_downloading["url"], + json.dumps(item_data_downloading), + datetime.now(UTC).strftime("%Y-%m-%d %H:%M:%S"), + ), + ) + await db._conn.commit() + datastore = DataStore(type=StoreType.HISTORY, connection=db) - datastore = DataStore(type=StoreType.HISTORY, connection=temp_db) + # Filter for finished items only + items, total, _, __ = await datastore.get_items_paginated(page=1, per_page=50, status_filter="finished") - # Filter for finished items only - items, total, _page, _total_pages = datastore.get_items_paginated( - page=1, per_page=50, status_filter="finished" - ) + assert len(items) == 50 # First page of finished items + assert total == 100 # Only 100 finished items in fixture + for _item_id, item in items: + assert item.status == "finished" - assert len(items) == 50 # First page of finished items - assert total == 100 # Only 100 finished items in fixture - for _item_id, item in items: - assert item.status == "finished" + # Filter for pending items only + items, total, _page, _total_pages = await datastore.get_items_paginated( + page=1, per_page=50, status_filter="pending" + ) - # Filter for pending items only - items, total, _page, _total_pages = datastore.get_items_paginated( - page=1, per_page=50, status_filter="pending" - ) + assert len(items) == 1 + assert total == 1 + assert items[0][1].status == "pending" + finally: + await db.close() - assert len(items) == 1 - assert total == 1 - assert items[0][1].status == "pending" - - def test_pagination_status_filter_exclude(self, temp_db): + async def test_pagination_status_filter_exclude(self): """Test pagination with status filter (exclusion).""" # Add some items with different status values item_data_pending = { @@ -329,55 +283,62 @@ class TestDataStorePagination: "folder": "/downloads", "status": "error", } + db = await make_db(data=0) + try: + await db._conn.execute( + 'INSERT INTO "history" ("id", "type", "url", "data", "created_at") VALUES (?, ?, ?, ?, ?)', + ( + "pending-id-2", + str(StoreType.HISTORY), + item_data_pending["url"], + json.dumps(item_data_pending), + datetime.now(UTC).strftime("%Y-%m-%d %H:%M:%S"), + ), + ) + await db._conn.execute( + 'INSERT INTO "history" ("id", "type", "url", "data", "created_at") VALUES (?, ?, ?, ?, ?)', + ( + "error-id", + str(StoreType.HISTORY), + item_data_error["url"], + json.dumps(item_data_error), + datetime.now(UTC).strftime("%Y-%m-%d %H:%M:%S"), + ), + ) + await db._conn.commit() - temp_db.execute( - 'INSERT INTO "history" ("id", "type", "url", "data", "created_at") VALUES (?, ?, ?, ?, ?)', - ( - "pending-id-2", - str(StoreType.HISTORY), - item_data_pending["url"], - json.dumps(item_data_pending), - datetime.now(UTC).strftime("%Y-%m-%d %H:%M:%S"), - ), - ) - temp_db.execute( - 'INSERT INTO "history" ("id", "type", "url", "data", "created_at") VALUES (?, ?, ?, ?, ?)', - ( - "error-id", - str(StoreType.HISTORY), - item_data_error["url"], - json.dumps(item_data_error), - datetime.now(UTC).strftime("%Y-%m-%d %H:%M:%S"), - ), - ) - temp_db.commit() + datastore = DataStore(type=StoreType.HISTORY, connection=db) - datastore = DataStore(type=StoreType.HISTORY, connection=temp_db) + # Exclude finished items + items, total, _page, _total_pages = await datastore.get_items_paginated( + page=1, per_page=50, status_filter="!finished" + ) - # Exclude finished items - items, total, _page, _total_pages = datastore.get_items_paginated( - page=1, per_page=50, status_filter="!finished" - ) + assert total == 2 # Only 2 non-finished items + assert len(items) == 2 + for _item_id, item in items: + assert item.status != "finished" - assert total == 2 # Only 2 non-finished items - assert len(items) == 2 - for _item_id, item in items: - assert item.status != "finished" + # Verify we have pending and error + statuses = {item.status for _, item in items} + assert statuses == {"pending", "error"} + finally: + await db.close() - # Verify we have pending and error - statuses = {item.status for _, item in items} - assert statuses == {"pending", "error"} - - def test_pagination_status_filter_none_matching(self, temp_db): + async def test_pagination_status_filter_none_matching(self): """Test pagination with status filter that matches no items.""" - datastore = DataStore(type=StoreType.HISTORY, connection=temp_db) + db = await make_db(data=0) + try: + datastore = DataStore(type=StoreType.HISTORY, connection=db) - # Filter for a status that doesn't exist - items, total, page, total_pages = datastore.get_items_paginated( - page=1, per_page=50, status_filter="nonexistent" - ) + # Filter for a status that doesn't exist + items, total, page, total_pages = await datastore.get_items_paginated( + page=1, per_page=50, status_filter="nonexistent" + ) - assert len(items) == 0 - assert total == 0 - assert page == 1 - assert total_pages == 1 + assert len(items) == 0 + assert total == 0 + assert page == 1 + assert total_pages == 1 + finally: + await db.close() diff --git a/app/tests/test_sqlite_store.py b/app/tests/test_sqlite_store.py new file mode 100644 index 00000000..d7be094b --- /dev/null +++ b/app/tests/test_sqlite_store.py @@ -0,0 +1,183 @@ +from datetime import UTC, datetime, timedelta + +import pytest + +from app.library.ItemDTO import ItemDTO +from app.library.sqlite_store import SqliteStore + + +async def make_store() -> SqliteStore: + """Create an isolated in-memory SqliteStore instance with schema.""" + SqliteStore._reset_singleton() + store = SqliteStore.get_instance(db_path=":memory:") + await store._ensure_conn() + assert store._conn is not None + return store + + +def make_item(idx: int, *, status: str = "finished") -> ItemDTO: + return ItemDTO( + id=f"vid{idx}", + title=f"Video {idx}", + url=f"https://example.com/{idx}", + folder="/downloads", + status=status, + ) + + +@pytest.mark.asyncio +async def test_enqueue_upsert_and_fetch_saved(): + store = await make_store() + item = make_item(1) + + await store.enqueue_upsert("queue", item) + await store.flush() + + saved = await store.fetch_saved("queue") + assert len(saved) == 1 + key, loaded = saved[0] + assert key == item._id + assert loaded.title == item.title + await store.close() + + +@pytest.mark.asyncio +async def test_delete_and_bulk_delete(): + store = await make_store() + items = [make_item(i) for i in range(3)] + for itm in items: + await store.enqueue_upsert("queue", itm) + await store.flush() + + await store.enqueue_delete("queue", items[0]._id) + await store.enqueue_bulk_delete("queue", [items[1]._id]) + await store.flush() + + remaining = await store.fetch_saved("queue") + assert len(remaining) == 1 + assert remaining[0][0] == items[2]._id + await store.close() + + +@pytest.mark.asyncio +async def test_count_with_status_filters(): + store = await make_store() + finished_items = [make_item(i, status="finished") for i in range(2)] + pending_items = [make_item(i + 10, status="pending") for i in range(3)] + + for itm in finished_items + pending_items: + await store.enqueue_upsert("history", itm) + await store.flush() + + assert await store.count("history", status_filter="finished") == 2 + assert await store.count("history", status_filter="pending") == 3 + assert await store.count("history", status_filter="!finished") == 3 + await store.close() + + +@pytest.mark.asyncio +async def test_paginate_order_and_bounds(): + store = await make_store() + base = datetime(2024, 1, 1, tzinfo=UTC) + _list: list[ItemDTO] = [] + for i in range(15): + itm = make_item(i) + encoded = itm.json() + created_at = (base + timedelta(minutes=i)).strftime("%Y-%m-%d %H:%M:%S") + _list.append(itm) + await store._conn.execute( + 'INSERT INTO "history" ("id", "type", "url", "data", "created_at") VALUES (?, ?, ?, ?, ?)', + (itm._id, "history", itm.url, encoded, created_at), + ) + await store._conn.commit() + + items, total, page, total_pages = await store.paginate("history", page=1, per_page=5, order="ASC") + assert total == 15 + assert page == 1 + assert total_pages == 3 + assert items[0][0] == _list[0]._id + + items_desc, *_ = await store.paginate("history", page=1, per_page=3, order="DESC") + assert len(items_desc) == 3 + assert items_desc[0][0] == _list[14]._id + await store.close() + +@pytest.mark.asyncio +async def test_get_by_id(): + store = await make_store() + item = make_item(99) + await store.enqueue_upsert("queue", item) + await store.flush() + assert item._id == (await store.get_by_id("queue", item._id))._id + await store.close() + +@pytest.mark.asyncio +async def test_shutdown_closes_worker_queue(): + store = await make_store() + await store.enqueue_upsert("queue", make_item(99)) + await store.shutdown() + assert store._queue is None + await store.close() + + +@pytest.mark.asyncio +async def test_enqueue_delete_removes_row(): + store = await make_store() + item = make_item(5) + await store.enqueue_upsert("queue", item) + await store.flush() + + await store.enqueue_delete("queue", item._id) + await store.flush() + + rows = await store.fetch_saved("queue") + assert rows == [] + await store.close() + + +@pytest.mark.asyncio +async def test_enqueue_bulk_delete_returns_count_and_bulk_path(): + store = await make_store() + items = [make_item(i) for i in range(3)] + for itm in items: + await store.enqueue_upsert("history", itm) + await store.flush() + + await store.enqueue_bulk_delete("history", [items[0]._id, items[1]._id]) + await store.flush() + + remaining = await store.fetch_saved("history") + assert len(remaining) == 1 + assert remaining[0][0] == items[2]._id + + # direct bulk_delete should report affected rows + count = await store.bulk_delete("history", [items[2]._id]) + assert count == 1 + await store.close() + + +@pytest.mark.asyncio +async def test_paginate_out_of_range_returns_last_page(): + store = await make_store() + for i in range(7): + await store.enqueue_upsert("history", make_item(i)) + await store.flush() + + items, total, page, total_pages = await store.paginate("history", page=10, per_page=3, order="DESC") + assert total == 7 + assert total_pages == 3 + assert page == 3 + assert len(items) == 1 + await store.close() + + +@pytest.mark.asyncio +async def test_on_shutdown_closes_connection(): + store = await make_store() + await store.enqueue_upsert("queue", make_item(1)) + await store.flush() + + await store.on_shutdown(None) + assert store._conn is None + + diff --git a/pyproject.toml b/pyproject.toml index 5fe7565d..39dc5afa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,7 +15,6 @@ dependencies = [ # Cross‐platform "python-socketio>=5.11.1", "aiohttp>=3.9.3", - "caribou>=0.3.0", "coloredlogs>=15.0.1", "aiocron>=1.8", "python-dotenv>=1.0.1", @@ -44,6 +43,7 @@ dependencies = [ "parsel>=1.10.0", "jmespath>=1.0.1", "jsonschema>=4.23.0", + "aiosqlite>=0.22.0", ] [tool.ruff] diff --git a/uv.lock b/uv.lock index e0297e39..7ced9ebd 100644 --- a/uv.lock +++ b/uv.lock @@ -105,6 +105,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/fb/76/641ae371508676492379f16e2fa48f4e2c11741bd63c48be4b12a6b09cba/aiosignal-1.4.0-py3-none-any.whl", hash = "sha256:053243f8b92b990551949e63930a839ff0cf0b0ebbe0597b0f3fb19e1a0fe82e", size = 7490 }, ] +[[package]] +name = "aiosqlite" +version = "0.22.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/3a/0d/449c024bdabd0678ae07d804e60ed3b9786facd3add66f51eee67a0fccea/aiosqlite-0.22.0.tar.gz", hash = "sha256:7e9e52d72b319fcdeac727668975056c49720c995176dc57370935e5ba162bb9", size = 14707 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ef/39/b2181148075272edfbbd6d87e6cd78cc71dca243446fa3b381fd4116950b/aiosqlite-0.22.0-py3-none-any.whl", hash = "sha256:96007fac2ce70eda3ca1bba7a3008c435258a592b8fbf2ee3eeaa36d33971a09", size = 17263 }, +] + [[package]] name = "altgraph" version = "0.17.5" @@ -224,15 +233,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/07/6b/6e92009df3b8b7272f85a0992b306b61c34b7ea1c4776643746e61c380ac/brotlicffi-1.2.0.0-cp38-abi3-win_amd64.whl", hash = "sha256:f139a7cdfe4ae7859513067b736eb44d19fae1186f9e99370092f6915216451b", size = 378586 }, ] -[[package]] -name = "caribou" -version = "0.4.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/6c/74/de2c20c5a24811d69a2a6150d5cd841f2af21ffc65ff964a73f46738bb68/caribou-0.4.1.tar.gz", hash = "sha256:13a66fc9ef9b1c9e9ef220876d59a44ee5eff9e579655252271b7514fc0ad787", size = 14019 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b5/91/795eccdad6abd41b3634502d0971cb696d2e5c9cede67f8b38ebe30d2b1e/caribou-0.4.1-py2.py3-none-any.whl", hash = "sha256:5c7a036584b34021011f1512620152272924e55ca87c7c805073aeef31c5baed", size = 7203 }, -] - [[package]] name = "certifi" version = "2025.11.12" @@ -1823,13 +1823,13 @@ source = { virtual = "." } dependencies = [ { name = "aiocron" }, { name = "aiohttp" }, + { name = "aiosqlite" }, { name = "anyio" }, { name = "apprise" }, { name = "async-timeout" }, { name = "bgutil-ytdlp-pot-provider" }, { name = "brotli" }, { name = "brotlicffi" }, - { name = "caribou" }, { name = "coloredlogs" }, { name = "curl-cffi" }, { name = "dateparser" }, @@ -1874,13 +1874,13 @@ dev = [ requires-dist = [ { name = "aiocron", specifier = ">=1.8" }, { name = "aiohttp", specifier = ">=3.9.3" }, + { name = "aiosqlite", specifier = ">=0.22.0" }, { name = "anyio" }, { name = "apprise", specifier = ">=1.9.3" }, { name = "async-timeout" }, { name = "bgutil-ytdlp-pot-provider", specifier = ">=1.2.1" }, { name = "brotli" }, { name = "brotlicffi" }, - { name = "caribou", specifier = ">=0.3.0" }, { name = "coloredlogs", specifier = ">=15.0.1" }, { name = "curl-cffi", specifier = ">=0.13" }, { name = "dateparser", specifier = ">=1.2.1" }, diff --git a/var/downloads/.gitkeep b/var/downloads/.gitkeep deleted file mode 100644 index e69de29b..00000000