diff --git a/.vscode/settings.json b/.vscode/settings.json index dc24bfef..8c6c6098 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -137,6 +137,7 @@ "noprogress", "onefile", "oneline", + "onupdate", "parsel", "pathex", "pickleable", @@ -164,11 +165,13 @@ "RPAREN", "rtime", "rvfc", + "sessionmaker", "SIGUSR", "smhd", "socketio", "softprops", "solverr", + "SQLA", "sstr", "startswith", "SUPPRESSHELP", diff --git a/app/library/migrate.py b/app/library/migrate.py index 29119344..17cda134 100644 --- a/app/library/migrate.py +++ b/app/library/migrate.py @@ -14,7 +14,8 @@ from importlib.machinery import ModuleSpec, SourceFileLoader from importlib.util import module_from_spec, spec_from_loader from typing import TYPE_CHECKING -import aiosqlite +from sqlalchemy import text +from sqlalchemy.ext.asyncio import AsyncConnection if TYPE_CHECKING: from types import ModuleType @@ -43,19 +44,19 @@ class InvalidNameError(Error): @contextlib.asynccontextmanager -async def execute( - conn: aiosqlite.Connection, sql: str, params: Sequence[object] | None = None -) -> AsyncIterator[aiosqlite.Cursor]: - params = [] if params is None else params - cursor = await conn.execute(sql, params) +async def execute(conn: AsyncConnection, sql: str, params: Sequence[object] | None = None) -> AsyncIterator: + params = {} if params is None else params + result = await conn.execute(text(sql), params) try: - yield cursor + yield result finally: - await cursor.close() + pass @contextlib.asynccontextmanager -async def transaction(conn: aiosqlite.Connection) -> AsyncIterator[None]: +async def transaction(conn: AsyncConnection) -> AsyncIterator[None]: + # SQLAlchemy AsyncConnection manages transactions automatically + # when used with context managers, so we just yield try: yield await conn.commit() @@ -104,10 +105,10 @@ class Migration: raise InvalidNameError(self.filename) return timestamp - async def upgrade(self, conn: aiosqlite.Connection) -> None: + async def upgrade(self, conn: AsyncConnection) -> None: await self.module.upgrade(conn) - async def downgrade(self, conn: aiosqlite.Connection) -> None: + async def downgrade(self, conn: AsyncConnection) -> None: await self.module.downgrade(conn) def has_method(self, name: str) -> bool: @@ -118,7 +119,7 @@ class Migration: class Database: - def __init__(self, db_url: aiosqlite.Connection | str, version_table: str = "migration_version"): + def __init__(self, db_url: AsyncConnection | str, version_table: str = "migration_version"): if not db_url: msg = "Database requires db_url." raise ValueError(msg) @@ -127,16 +128,18 @@ class Database: self._owns_connection = bool(isinstance(db_url, str)) if self._owns_connection: - self.conn: aiosqlite.Connection | None = None + self.conn: AsyncConnection | None = None self.db_url: str = db_url - self._ensure_connection() else: - self.conn: aiosqlite.Connection = db_url + self.conn: AsyncConnection = db_url async def __aenter__(self) -> "Database": - if self._owns_connection: - await self._ensure_connection() + if self._owns_connection and self.conn is None: + # Create connection from string URL + from sqlalchemy.ext.asyncio import create_async_engine + self._engine = create_async_engine(f"sqlite+aiosqlite:///{self.db_url}") + self.conn = await self._engine.connect() return self async def __aexit__(self, exc_type, exc, tb) -> None: @@ -147,23 +150,18 @@ class Database: async def close(self) -> None: if self._owns_connection and self.conn: await self.conn.close() + if hasattr(self, "_engine"): + await self._engine.dispose() self.conn = None - async def _ensure_connection(self) -> None: - if self.conn: - return - self.conn = await aiosqlite.connect(self.db_url) - self.conn.row_factory = aiosqlite.Row - async def is_version_controlled(self) -> bool: - await self._ensure_connection() assert self.conn - sql = """SELECT * FROM sqlite_master WHERE type = 'table' AND name = ?""" - async with execute(self.conn, sql, [self.version_table]) as cursor: - return bool(await cursor.fetchall()) + sql = """SELECT * FROM sqlite_master WHERE type = 'table' AND name = :version_table""" + async with execute(self.conn, sql, {"version_table": self.version_table}) as result: + rows = result.fetchall() + return bool(rows) async def upgrade(self, migrations: list[Migration], target_version: str | None = None) -> None: - await self._ensure_connection() assert self.conn if target_version: _assert_migration_exists(migrations, target_version) @@ -183,7 +181,6 @@ class Database: database_version: str = new_version async def downgrade(self, migrations: list[Migration], target_version: str | int) -> None: - await self._ensure_connection() assert self.conn if target_version not in (0, "0"): _assert_migration_exists(migrations, target_version) @@ -210,29 +207,26 @@ class Database: Return the database's version, or None if it is not under version control. """ - await self._ensure_connection() assert self.conn if not await self.is_version_controlled(): return None sql: str = f"SELECT version FROM {self.version_table}" - async with execute(self.conn, sql) as cursor: - result = await cursor.fetchall() - return result[0][0] if result else "0" + async with execute(self.conn, sql) as result: + rows = result.fetchall() + return rows[0][0] if rows else "0" async def update_version(self, version: str) -> None: - await self._ensure_connection() assert self.conn - sql: str = f"UPDATE {self.version_table} SET version = ?" + sql: str = f"UPDATE {self.version_table} SET version = :version" async with transaction(self.conn): - await self.conn.execute(sql, [version]) + await self.conn.execute(text(sql), {"version": version}) async def initialize_version_control(self) -> None: - await self._ensure_connection() assert self.conn sql: str = f"""CREATE TABLE IF NOT EXISTS {self.version_table} ( version TEXT ) """ async with transaction(self.conn): - await self.conn.execute(sql) - await self.conn.execute(f"INSERT INTO {self.version_table} VALUES (0)") + await self.conn.execute(text(sql)) + await self.conn.execute(text(f"INSERT INTO {self.version_table} VALUES (:version)"), {"version": "0"}) def __repr__(self) -> str: return f'Database("{self.db_url if self._owns_connection else "external_connection"}")' @@ -253,7 +247,7 @@ def load_migrations(directory: str) -> list[Migration]: return [Migration(f) for f in glob.glob(os.path.join(directory, "*.py"))] -async def upgrade(db_url: aiosqlite.Connection | str, migration_dir: str, version: str | None = None) -> None: +async def upgrade(db_url: AsyncConnection | str, migration_dir: str, version: str | None = None) -> None: """ Upgrade the given database with the migrations contained in the migrations directory. If a version is not specified, upgrade @@ -266,7 +260,7 @@ async def upgrade(db_url: aiosqlite.Connection | str, migration_dir: str, versio await db.upgrade(load_migrations(migration_dir), version) -async def downgrade(db_url: str | aiosqlite.Connection, migration_dir: str, version: str) -> None: +async def downgrade(db_url: str | AsyncConnection, migration_dir: str, version: str) -> None: """ Downgrade the database to the given version with the migrations contained in the given migration directory. @@ -279,7 +273,7 @@ async def downgrade(db_url: str | aiosqlite.Connection, migration_dir: str, vers await db.downgrade(migrations, version) -async def get_version(db_url: aiosqlite.Connection | str) -> str | None: +async def get_version(db_url: AsyncConnection | str) -> str | None: """Return the migration version of the given database.""" async with Database(db_url) as db: return await db.get_version() @@ -317,11 +311,13 @@ Migration Name: %(name)s Migration Version: %(version)s \"\"\" +from sqlalchemy import text + async def upgrade(c): # add your upgrade step here - await c.execute("SELECT 1") + await c.execute(text("SELECT 1")) async def downgrade(c): # add your downgrade step here - await c.execute("SELECT 1") + await c.execute(text("SELECT 1")) """ diff --git a/app/library/sqlite_store.py b/app/library/sqlite_store.py index 7a1b7df3..3771042e 100644 --- a/app/library/sqlite_store.py +++ b/app/library/sqlite_store.py @@ -8,8 +8,10 @@ from dataclasses import fields from datetime import UTC, datetime from email.utils import formatdate -import aiosqlite from aiohttp import web +from sqlalchemy import text +from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker, create_async_engine +from sqlalchemy.ext.asyncio.engine import AsyncConnection from .ItemDTO import ItemDTO from .operations import Operation, matches_condition @@ -57,7 +59,9 @@ class SqliteStore(metaclass=ThreadSafe): def __init__(self, db_path: str, *, max_pending: int = 200, flush_interval: float = 0.05): self._db_path: str = db_path - self._conn: aiosqlite.Connection | None = None + self._engine: AsyncEngine | None = None + self._conn = None + self._sessionmaker: async_sessionmaker[AsyncSession] | None = None self._queue: asyncio.Queue[_Op] | None = None self._task: asyncio.Task | None = None self._flush_interval: float = flush_interval @@ -65,28 +69,50 @@ class SqliteStore(metaclass=ThreadSafe): self._lock = asyncio.Lock() async def __aenter__(self) -> "SqliteStore": - await self._ensure_conn() + await self.get_connection() return self async def __aexit__(self, exc_type, exc, tb) -> None: await self.close() + def sessionmaker(self) -> async_sessionmaker[AsyncSession]: + """ + Return the SQLAlchemy async sessionmaker. + + This allows other parts of the system to create SQLAlchemy sessions + that share the same database connection/engine. + + Returns: + async_sessionmaker[AsyncSession]: The sessionmaker instance. + + Raises: + RuntimeError: If called before connection is initialized. + + """ + if not self._sessionmaker: + msg = "Database connection not initialized. Call _ensure_conn() first or use within async context." + raise RuntimeError(msg) + return self._sessionmaker + async def fetch_saved(self, type_value: str) -> list[tuple[str, ItemDTO]]: - await self._ensure_conn() - cursor = await self._conn.execute( - 'SELECT "id", "data", "created_at" FROM "history" WHERE "type" = ? ORDER BY "created_at" ASC', - (type_value,), + await self.get_connection() + result = await self._conn.execute( + text( + 'SELECT "id", "data", "created_at" FROM "history" WHERE "type" = :type_value ORDER BY "created_at" ASC' + ), + {"type_value": type_value}, ) + rows = result.mappings().all() + items: list[tuple[str, ItemDTO]] = [] - async with cursor: - async for row in cursor: - row_date = datetime.strptime(row["created_at"], "%Y-%m-%d %H:%M:%S") # noqa: DTZ007 - data = json.loads(row["data"]) - data.pop("_id", None) - item = init_class(ItemDTO, data, ITEM_DTO_FIELDS) - item._id = row["id"] - item.datetime = formatdate(row_date.replace(tzinfo=UTC).timestamp()) - items.append((row["id"], item)) + for row in rows: + row_date = datetime.strptime(row["created_at"], "%Y-%m-%d %H:%M:%S") # noqa: DTZ007 + data = json.loads(row["data"]) + data.pop("_id", None) + item = init_class(ItemDTO, data, ITEM_DTO_FIELDS) + item._id = row["id"] + item.datetime = formatdate(row_date.replace(tzinfo=UTC).timestamp()) + items.append((row["id"], item)) return items async def exists(self, type_value: str, key: str | None = None, url: str | None = None) -> bool: @@ -97,23 +123,27 @@ class SqliteStore(metaclass=ThreadSafe): msg = "key or url must be provided." raise KeyError(msg) - await self._ensure_conn() + await self.get_connection() clauses: list[str] = [] - params: list[str] = [] + params: dict[str, str] = {"type_value": type_value} if key: - clauses.append('"id" = ?') - params.append(key) + clauses.append('"id" = :key') + params["key"] = key if url: - clauses.append("json_extract(data, '$.url') = ?") - params.append(url) + clauses.append("json_extract(data, '$.url') = :url") + params["url"] = url where_clause = " OR ".join(clauses) - query = f'SELECT "id", "data", "created_at" FROM "history" WHERE "type" = ? AND ({where_clause}) LIMIT 1' # noqa: S608 - cursor = await self._conn.execute(query, (type_value, *params)) - row = await cursor.fetchone() + query = ( + f'SELECT "id", "data", "created_at" FROM "history" WHERE "type" = :type_value AND ({where_clause}) LIMIT 1' # noqa: S608 + ) + + result = await self._conn.execute(text(query), params) + row = result.mappings().first() + if not row: return None @@ -126,12 +156,13 @@ class SqliteStore(metaclass=ThreadSafe): return item async def get_by_id(self, type_value: str, id: str) -> ItemDTO | None: - await self._ensure_conn() - cursor = await self._conn.execute( - 'SELECT "id", "data", "created_at" FROM "history" WHERE "type" = ? AND "id" = ?', - (type_value, id), + await self.get_connection() + result = await self._conn.execute( + text('SELECT "id", "data", "created_at" FROM "history" WHERE "type" = :type_value AND "id" = :id'), + {"type_value": type_value, "id": id}, ) - row = await cursor.fetchone() + row = result.mappings().first() + if not row: return None @@ -154,10 +185,11 @@ class SqliteStore(metaclass=ThreadSafe): if not kwargs: return None - await self._ensure_conn() + await self.get_connection() clauses: list[str] = [] - params: list[str | float | int] = [] + params: dict[str, str | float | int] = {"type_value": type_value} + param_counter = 0 def _safe_key(key: str) -> str | None: return key if key.replace("_", "").isalnum() else None @@ -178,47 +210,51 @@ class SqliteStore(metaclass=ThreadSafe): except ValueError: operation = Operation.EQUAL - path = f"$.{safe_key}" - json_extract = f"json_extract(data, '{path}')" + path: str = f"$.{safe_key}" + json_extract: str = f"json_extract(data, '{path}')" + param_name: str = f"param_{param_counter}" + param_counter += 1 if Operation.EQUAL == operation: - clauses.append(f"{json_extract} = ?") - params.append(value) + clauses.append(f"{json_extract} = :{param_name}") + params[param_name] = value elif Operation.NOT_EQUAL == operation: - clauses.append(f"{json_extract} != ?") - params.append(value) + clauses.append(f"{json_extract} != :{param_name}") + params[param_name] = value elif Operation.CONTAIN == operation: - clauses.append(f"{json_extract} LIKE ? ESCAPE '\\'") - params.append(f"%{value}%") + clauses.append(f"{json_extract} LIKE :{param_name} ESCAPE '\\'") + params[param_name] = f"%{value}%" elif Operation.NOT_CONTAIN == operation: - clauses.append(f"({json_extract} IS NULL OR {json_extract} NOT LIKE ? ESCAPE '\\')") - params.append(f"%{value}%") + clauses.append(f"({json_extract} IS NULL OR {json_extract} NOT LIKE :{param_name} ESCAPE '\\')") + params[param_name] = f"%{value}%" elif Operation.STARTS_WITH == operation: - clauses.append(f"{json_extract} LIKE ? ESCAPE '\\'") - params.append(f"{value}%") + clauses.append(f"{json_extract} LIKE :{param_name} ESCAPE '\\'") + params[param_name] = f"{value}%" elif Operation.ENDS_WITH == operation: - clauses.append(f"{json_extract} LIKE ? ESCAPE '\\'") - params.append(f"%{value}") + clauses.append(f"{json_extract} LIKE :{param_name} ESCAPE '\\'") + params[param_name] = f"%{value}" elif Operation.GREATER_THAN == operation: - clauses.append(f"{json_extract} > ?") - params.append(value) + clauses.append(f"{json_extract} > :{param_name}") + params[param_name] = value elif Operation.LESS_THAN == operation: - clauses.append(f"{json_extract} < ?") - params.append(value) + clauses.append(f"{json_extract} < :{param_name}") + params[param_name] = value elif Operation.GREATER_EQUAL == operation: - clauses.append(f"{json_extract} >= ?") - params.append(value) + clauses.append(f"{json_extract} >= :{param_name}") + params[param_name] = value elif Operation.LESS_EQUAL == operation: - clauses.append(f"{json_extract} <= ?") - params.append(value) + clauses.append(f"{json_extract} <= :{param_name}") + params[param_name] = value if not clauses: return None - where_clause = " OR ".join(f"({clause})" for clause in clauses) - query = f'SELECT "id", "data", "created_at" FROM "history" WHERE "type" = ? AND ({where_clause}) ORDER BY "created_at" ASC LIMIT 1' # noqa: S608 - cursor = await self._conn.execute(query, (type_value, *params)) - row = await cursor.fetchone() + where_clause: str = " OR ".join(f"({clause})" for clause in clauses) + query: str = f'SELECT "id", "data", "created_at" FROM "history" WHERE "type" = :type_value AND ({where_clause}) ORDER BY "created_at" ASC LIMIT 1' # noqa: S608 + + result = await self._conn.execute(text(query), params) + row = result.mappings().first() + if not row: return None @@ -232,23 +268,25 @@ class SqliteStore(metaclass=ThreadSafe): return item if any(matches_condition(k, v, item.__dict__) for k, v in kwargs.items()) else None async def count(self, type_value: str, status_filter: str | None = None) -> int: - await self._ensure_conn() - where_clauses = ['"type" = ?'] - query_params: list[str] = [type_value] + await self.get_connection() + where_clauses: list[str] = ['"type" = :type_value'] + params: dict[str, str] = {"type_value": type_value} if status_filter: if status_filter.startswith("!"): status_value = status_filter[1:] - where_clauses.append("json_extract(data, '$.status') != ?") - query_params.append(status_value) + where_clauses.append("json_extract(data, '$.status') != :status") + params["status"] = status_value else: - where_clauses.append("json_extract(data, '$.status') = ?") - query_params.append(status_filter) + where_clauses.append("json_extract(data, '$.status') = :status") + params["status"] = status_filter + + where_clause: str = " AND ".join(where_clauses) + count_query: str = f'SELECT COUNT(*) as count FROM "history" WHERE {where_clause}' # noqa: S608 + + result = await self._conn.execute(text(count_query), params) + row = result.mappings().first() - where_clause = " AND ".join(where_clauses) - count_query = f'SELECT COUNT(*) as count FROM "history" WHERE {where_clause}' # noqa: S608 - cursor = await self._conn.execute(count_query, tuple(query_params)) - row = await cursor.fetchone() return row["count"] if row else 0 async def paginate( @@ -259,74 +297,79 @@ class SqliteStore(metaclass=ThreadSafe): order: str, status_filter: str | None = None, ) -> tuple[list[tuple[str, ItemDTO]], int, int, int]: - await self._ensure_conn() - where_clauses = ['"type" = ?'] - query_params: list[str | int] = [type_value] + await self.get_connection() + where_clauses: list[str] = ['"type" = :type_value'] + params: dict[str, str | int] = {"type_value": type_value} if status_filter: if status_filter.startswith("!"): status_value = status_filter[1:] - where_clauses.append("json_extract(data, '$.status') != ?") - query_params.append(status_value) + where_clauses.append("json_extract(data, '$.status') != :status") + params["status"] = status_value else: - where_clauses.append("json_extract(data, '$.status') = ?") - query_params.append(status_filter) + where_clauses.append("json_extract(data, '$.status') = :status") + params["status"] = status_filter + + where_clause: str = " AND ".join(where_clauses) + count_query: str = f'SELECT COUNT(*) as count FROM "history" WHERE {where_clause}' # noqa: S608 + + result = await self._conn.execute(text(count_query), params) + count_row = result.mappings().first() - where_clause = " AND ".join(where_clauses) - count_query = f'SELECT COUNT(*) as count FROM "history" WHERE {where_clause}' # noqa: S608 - count_cursor = await self._conn.execute(count_query, tuple(query_params)) - count_row = await count_cursor.fetchone() total_items = count_row["count"] if count_row else 0 total_pages = (total_items + per_page - 1) // per_page if total_items > 0 else 1 if page > total_pages and total_items > 0: page = total_pages - offset = (page - 1) * per_page - query_params.extend([per_page, offset]) + offset: int = (page - 1) * per_page + params["limit"] = per_page + params["offset"] = offset - cursor = await self._conn.execute( - f'SELECT "id", "data", "created_at" FROM "history" WHERE {where_clause} ORDER BY "created_at" {order} LIMIT ? OFFSET ?', # noqa: S608 - tuple(query_params), - ) + query: str = f'SELECT "id", "data", "created_at" FROM "history" WHERE {where_clause} ORDER BY "created_at" {order} LIMIT :limit OFFSET :offset' # noqa: S608 + + result = await self._conn.execute(text(query), params) + rows = result.mappings().all() items: list[tuple[str, ItemDTO]] = [] - async with cursor: - async for row in cursor: - row_date = datetime.strptime(row["created_at"], "%Y-%m-%d %H:%M:%S") # noqa: DTZ007 - data = json.loads(row["data"]) - data.pop("_id", None) - item = init_class(ItemDTO, data, ITEM_DTO_FIELDS) - item._id = row["id"] - item.datetime = formatdate(row_date.replace(tzinfo=UTC).timestamp()) - items.append((row["id"], item)) + for row in rows: + row_date = datetime.strptime(row["created_at"], "%Y-%m-%d %H:%M:%S") # noqa: DTZ007 + data = json.loads(row["data"]) + data.pop("_id", None) + item = init_class(ItemDTO, data, ITEM_DTO_FIELDS) + item._id = row["id"] + item.datetime = formatdate(row_date.replace(tzinfo=UTC).timestamp()) + items.append((row["id"], item)) return items, total_items, page, total_pages async def upsert(self, type_value: str, item: ItemDTO) -> None: - await self._ensure_conn() + await self.get_connection() await self._upsert_now(type_value, item) async def delete(self, type_value: str, key: str) -> None: - await self._ensure_conn() - await self._conn.execute('DELETE FROM "history" WHERE "type" = ? AND "id" = ?', (type_value, key)) + await self.get_connection() + await self._conn.execute( + text('DELETE FROM "history" WHERE "type" = :type_value AND "id" = :key'), + {"type_value": type_value, "key": key}, + ) + await self._conn.commit() async def bulk_delete(self, type_value: str, keys: Iterable[str]) -> int: - await self._ensure_conn() - keys_list = list(keys) + await self.get_connection() + keys_list: list[str] = list(keys) if not keys_list: return 0 - placeholders = ",".join("?" for _ in keys_list) - params = [type_value, *keys_list] - cur = await self._conn.execute( - f'DELETE FROM "history" WHERE "type" = ? AND "id" IN ({placeholders})', # noqa: S608 - tuple(params), - ) - return cur.rowcount if cur else 0 + placeholders = ",".join(f":key_{i}" for i in range(len(keys_list))) + params: dict[str, str] = {"type_value": type_value} + params.update({f"key_{i}": key for i, key in enumerate(keys_list)}) - async def commit(self) -> None: - if self._conn: - await self._conn.commit() + result = await self._conn.execute( + text(f'DELETE FROM "history" WHERE "type" = :type_value AND "id" IN ({placeholders})'), # noqa: S608 + params, + ) + await self._conn.commit() + return result.rowcount if result else 0 async def enqueue_upsert(self, type_value: str, item: ItemDTO) -> None: await self._enqueue(_Op("upsert", type_value, item, None, None)) @@ -340,11 +383,9 @@ class SqliteStore(metaclass=ThreadSafe): async def flush(self) -> None: if self._queue: await self._queue.join() - if self._conn: - await self._conn.commit() async def shutdown(self) -> None: - """Flush pending writes, stop writer task, and commit.""" + """Flush pending writes and stop writer task.""" if self._queue: try: await self._queue.put(_Op(Terminator(), "", None, None, None)) @@ -360,20 +401,21 @@ class SqliteStore(metaclass=ThreadSafe): with contextlib.suppress(asyncio.CancelledError): await asyncio.wait_for(self._task, timeout=2) - if self._conn: - LOG.debug("Committing final changes to SqliteStore...") - await self._conn.commit() - self._task = None self._queue = None async def close(self) -> None: await self.shutdown() + if self._conn: - LOG.debug("Closing SqliteStore connection...") await self._conn.close() self._conn = None + if self._engine: + await self._engine.dispose() + self._engine = None + self._sessionmaker = None + async def _enqueue(self, op: _Op) -> None: self._ensure_worker() await self._queue.put(op) @@ -399,45 +441,101 @@ class SqliteStore(metaclass=ThreadSafe): await asyncio.sleep(self._flush_interval) async def _apply(self, op: _Op) -> None: - await self._ensure_conn() + await self.get_connection() if op.op == "upsert" and op.item: - await self._upsert_now(op.type_value, op.item) + await self._upsert_now_conn(self._conn, op.type_value, op.item) + await self._conn.commit() elif op.op == "delete" and op.key: - await self._conn.execute('DELETE FROM "history" WHERE "type" = ? AND "id" = ?', (op.type_value, op.key)) - elif op.op == "bulk_delete" and op.keys: - placeholders = ",".join("?" for _ in op.keys) - params = [op.type_value, *op.keys] await self._conn.execute( - f'DELETE FROM "history" WHERE "type" = ? AND "id" IN ({placeholders})', # noqa: S608 - tuple(params), + text('DELETE FROM "history" WHERE "type" = :type_value AND "id" = :key'), + {"type_value": op.type_value, "key": op.key}, ) - await self._conn.commit() + await self._conn.commit() + elif op.op == "bulk_delete" and op.keys: + placeholders = ",".join(f":key_{i}" for i in range(len(op.keys))) + params: dict[str, str] = {"type_value": op.type_value} + params.update({f"key_{i}": key for i, key in enumerate(op.keys)}) + await self._conn.execute( + text(f'DELETE FROM "history" WHERE "type" = :type_value AND "id" IN ({placeholders})'), # noqa: S608 + params, + ) + await self._conn.commit() async def _upsert_now(self, type_value: str, item: ItemDTO) -> None: - await self._ensure_conn() + await self.get_connection() + await self._upsert_now_conn(self._conn, type_value, item) + await self._conn.commit() + + async def _upsert_now_conn(self, conn, type_value: str, item: ItemDTO) -> None: + """Helper to upsert using an existing connection.""" sql = """ INSERT INTO "history" ("id", "type", "url", "data") - VALUES (?, ?, ?, ?) - ON CONFLICT DO UPDATE SET "type" = ?, "url" = ?, "data" = ?, created_at = ? + VALUES (:id, :type, :url, :data) + ON CONFLICT DO UPDATE SET "type" = :type2, "url" = :url2, "data" = :data2, created_at = :created_at """ encoded = item.json() - await self._conn.execute( - sql.strip(), - ( - item._id, - type_value, - item.url, - encoded, - type_value, - item.url, - encoded, - datetime.now(UTC).strftime("%Y-%m-%d %H:%M:%S"), - ), + await conn.execute( + text(sql.strip()), + { + "id": item._id, + "type": type_value, + "url": item.url, + "data": encoded, + "type2": type_value, + "url2": item.url, + "data2": encoded, + "created_at": datetime.now(UTC).strftime("%Y-%m-%d %H:%M:%S"), + }, ) - async def _ensure_conn(self) -> None: + async def execute_raw(self, query: str, params: dict | tuple | None = None) -> None: + """Execute raw SQL query (for testing purposes).""" + await self.get_connection() + if isinstance(params, tuple): + # Convert positional params to dict for SQLAlchemy + # Assuming queries use ? placeholders, we need to count them + placeholders = query.count("?") + if placeholders != len(params): + msg = ( + f"Parameter count mismatch: query has {placeholders} placeholders but {len(params)} params provided" + ) + raise ValueError(msg) + # Create numbered parameters + param_dict = {f"p{i}": params[i] for i in range(len(params))} + # Replace ? with :p0, :p1, etc. + for i in range(len(params)): + query = query.replace("?", f":p{i}", 1) + await self._conn.execute(text(query), param_dict) + elif isinstance(params, dict): + await self._conn.execute(text(query), params) + else: + await self._conn.execute(text(query)) + await self._conn.commit() + + async def fetch_raw(self, query: str, params: dict | tuple | None = None): + """Fetch results from raw SQL query (for testing purposes).""" + await self.get_connection() + if isinstance(params, tuple): + # Convert positional params to dict for SQLAlchemy + placeholders = query.count("?") + if placeholders != len(params): + msg = ( + f"Parameter count mismatch: query has {placeholders} placeholders but {len(params)} params provided" + ) + raise ValueError(msg) + param_dict = {f"p{i}": params[i] for i in range(len(params))} + for i in range(len(params)): + query = query.replace("?", f":p{i}", 1) + result = await self._conn.execute(text(query), param_dict) + elif isinstance(params, dict): + result = await self._conn.execute(text(query), params) + else: + result = await self._conn.execute(text(query)) + return result.mappings().all() + + async def get_connection(self) -> AsyncConnection: if self._conn: - return + return self._conn if not self._db_path: msg = "No database path specified for SqliteStore." @@ -446,21 +544,31 @@ class SqliteStore(metaclass=ThreadSafe): from app.library import migrate from app.main import ROOT_PATH - if not self._db_path.startswith(":memory"): + if self._db_path.startswith(":memory"): + db_url: str = "sqlite+aiosqlite:///file::memory:?cache=shared&uri=true" + else: os.makedirs(os.path.dirname(self._db_path) or ".", exist_ok=True) + db_url: str = f"sqlite+aiosqlite:///{self._db_path}" - self._conn = await aiosqlite.connect(database=self._db_path, isolation_level=None) - self._conn.row_factory = aiosqlite.Row - version = await migrate.get_version(self._conn) - if version: + self._engine = create_async_engine( + db_url, + echo=False, + connect_args={"check_same_thread": False, "uri": self._db_path.startswith(":memory")}, + ) + self._conn: AsyncConnection = await self._engine.connect() + + if version := await migrate.get_version(self._conn): LOG.debug(f"DB Version: '{version}'.") await migrate.upgrade(self._conn, ROOT_PATH / "migrations") if not version: - version = await migrate.get_version(self._conn) - LOG.debug(f"DB Version after initial migration: '{version}'.") + LOG.debug(f"DB Version after initial migration: '{await migrate.get_version(self._conn)}'.") - await self._conn.execute("PRAGMA journal_mode=wal") - await self._conn.execute("PRAGMA busy_timeout=5000") - await self._conn.execute("PRAGMA foreign_keys=ON") + await self._conn.execute(text("PRAGMA journal_mode=wal")) + await self._conn.execute(text("PRAGMA busy_timeout=5000")) + await self._conn.execute(text("PRAGMA foreign_keys=ON")) await self._conn.commit() + + self._sessionmaker = async_sessionmaker(bind=self._engine, class_=AsyncSession, expire_on_commit=False) + + return self._conn diff --git a/app/migrations/20231115152938_initial.py b/app/migrations/20231115152938_initial.py index 043327db..c71f2759 100644 --- a/app/migrations/20231115152938_initial.py +++ b/app/migrations/20231115152938_initial.py @@ -5,28 +5,25 @@ Migration Name: initial Migration Version: 20231115152938 """ +from sqlalchemy import text + async def upgrade(c): - sql = """ - CREATE TABLE "history" ( - "id" TEXT PRIMARY KEY UNIQUE NOT NULL, - "type" TEXT NOT NULL, - "url" TEXT NOT NULL, - "data" JSON NOT NULL, - "created_at" TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP - ); - """ - await c.execute(sql) - - sql = """ - CREATE INDEX "history_type" ON "history" ("type"); - """ - await c.execute(sql) - - sql = """ - CREATE UNIQUE INDEX "history_url" ON "history" ("url"); - """ - await c.execute(sql) + sql: list[str] = [ + """ + CREATE TABLE "history" ( + "id" TEXT PRIMARY KEY UNIQUE NOT NULL, + "type" TEXT NOT NULL, + "url" TEXT NOT NULL, + "data" JSON NOT NULL, + "created_at" TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP + ); + """, + 'CREATE INDEX "history_type" ON "history" ("type");', + 'CREATE UNIQUE INDEX "history_url" ON "history" ("url");', + ] + for sql_stmt in sql: + await c.execute(text(sql_stmt)) await c.commit() @@ -36,4 +33,4 @@ async def downgrade(c): DROP TABLE IF EXISTS "history"; """ - await c.execute(sql) + await c.execute(text(sql)) diff --git a/app/migrations/20251116173731_add_status_index.py b/app/migrations/20251116173731_add_status_index.py index 0778b414..588c3a12 100644 --- a/app/migrations/20251116173731_add_status_index.py +++ b/app/migrations/20251116173731_add_status_index.py @@ -5,8 +5,10 @@ Migration Name: add_status_index Migration Version: 20251116173731 """ +from sqlalchemy import text -async def upgrade(connection): + +async def upgrade(c): """ Add index on json_extract(data, '$.status') for better query performance. @@ -16,14 +18,14 @@ async def upgrade(connection): sql = """ CREATE INDEX IF NOT EXISTS "history_status" ON "history" (json_extract("data", '$.status')); """ - await connection.execute(sql) + await c.execute(text(sql)) -async def downgrade(connection): +async def downgrade(c): """ Remove the status index. """ sql = """ DROP INDEX IF EXISTS "history_status"; """ - await connection.execute(sql) + await c.execute(text(sql)) diff --git a/app/scripts/create_migration.py b/app/scripts/create_migration.py index c3ac4447..f9e8fe68 100644 --- a/app/scripts/create_migration.py +++ b/app/scripts/create_migration.py @@ -8,6 +8,7 @@ Provides commands to create, list, and apply database migrations. from __future__ import annotations import argparse +import asyncio import sys import traceback from pathlib import Path @@ -40,32 +41,32 @@ def cmd_create(args: argparse.Namespace) -> None: o_print(f"created migration {path}") -def cmd_version(args: argparse.Namespace) -> None: - version = migrate.get_version(args.database_path) +async def cmd_version(args: argparse.Namespace) -> None: + version = await migrate.get_version(args.database_path) if version: o_print(f"the db [{args.database_path}] is at version {version}") else: o_print(f"the db [{args.database_path}] is not under version control") -def cmd_upgrade(args: argparse.Namespace) -> None: +async def cmd_upgrade(args: argparse.Namespace) -> None: if args.version: o_print(f"upgrading db [{args.database_path}] to version [{args.version}]") else: o_print(f"upgrading db [{args.database_path}] to most recent version") - migrate.upgrade(args.database_path, args.migration_dir, args.version) + await migrate.upgrade(args.database_path, args.migration_dir, args.version) - new_version = migrate.get_version(args.database_path) + new_version = await migrate.get_version(args.database_path) if args.version is not None: assert new_version == args.version o_print(f"upgraded [{args.database_path}] successfully to version [{new_version}]") -def cmd_downgrade(args: argparse.Namespace) -> None: +async def cmd_downgrade(args: argparse.Namespace) -> None: o_print(f"downgrading db [{args.database_path}] to version [{args.version}]") - migrate.downgrade(args.database_path, args.migration_dir, args.version) + await migrate.downgrade(args.database_path, args.migration_dir, args.version) o_print(f"downgraded [{args.database_path}] successfully to version [{args.version}]") @@ -122,7 +123,11 @@ def run(argv: list[str]) -> int: try: func: Callable[[argparse.Namespace], None] = args.func - func(args) + # Check if function is async + if asyncio.iscoroutinefunction(func): + asyncio.run(func(args)) + else: + func(args) return 0 except migrate.InvalidMigrationError: diff --git a/app/scripts/seed_db.py b/app/scripts/seed_db.py index d3539104..29cc9bce 100644 --- a/app/scripts/seed_db.py +++ b/app/scripts/seed_db.py @@ -2,9 +2,9 @@ from __future__ import annotations import argparse +import asyncio import logging import re -import sqlite3 import sys import time import uuid @@ -14,6 +14,9 @@ from itertools import cycle, islice from pathlib import Path from typing import TYPE_CHECKING +from sqlalchemy import text +from sqlalchemy.ext.asyncio import AsyncConnection, create_async_engine + APP_ROOT = str((Path(__file__).parent / ".." / "..").resolve()) if APP_ROOT not in sys.path: sys.path.insert(0, APP_ROOT) @@ -65,11 +68,13 @@ def parse_args() -> argparse.Namespace: return parser.parse_args() -def connect_db(db_file: str) -> sqlite3.Connection: - conn = sqlite3.connect(database=db_file, isolation_level=None) - conn.row_factory = sqlite3.Row - conn.execute("PRAGMA journal_mode=wal") - return conn +async def connect_db(db_file: str) -> AsyncConnection: + """Create async SQLAlchemy connection.""" + engine = create_async_engine(f"sqlite+aiosqlite:///{db_file}") + conn = await engine.connect() + await conn.execute(text("PRAGMA journal_mode=wal")) + await conn.commit() + return conn, engine def find_files(root: Path, extension: str) -> list[Path]: @@ -187,14 +192,15 @@ def generate_rows( yield _build_row(path, root, idx, status, preset, store) -def insert_batches( - conn: sqlite3.Connection, +async def insert_batches( + conn: AsyncConnection, rows: Iterable[tuple[str, str, str, str, str]], batch_size: int, ) -> int: + """Insert rows in batches using SQLAlchemy.""" sql = """ INSERT INTO "history" ("id", "type", "url", "data", "created_at") - VALUES (?, ?, ?, ?, ?) + VALUES (:p0, :p1, :p2, :p3, :p4) ON CONFLICT("id") DO UPDATE SET "type" = excluded.type, "url" = excluded.url, @@ -207,19 +213,24 @@ def insert_batches( for row in rows: batch.append(row) if len(batch) >= batch_size: - conn.executemany(sql, batch) + # Convert tuples to dicts for SQLAlchemy + params = [{"p0": r[0], "p1": r[1], "p2": r[2], "p3": r[3], "p4": r[4]} for r in batch] + await conn.execute(text(sql), params) + await conn.commit() total_written += len(batch) LOG.info("Inserted %d rows...", total_written) batch.clear() if batch: - conn.executemany(sql, batch) + params = [{"p0": r[0], "p1": r[1], "p2": r[2], "p3": r[3], "p4": r[4]} for r in batch] + await conn.execute(text(sql), params) + await conn.commit() total_written += len(batch) return total_written -def main() -> None: +async def main() -> None: args = parse_args() logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s") @@ -234,14 +245,15 @@ def main() -> None: return rows = generate_rows(files, args.root, args.count, args.status, args.preset, store) - conn = connect_db(args.db_file) + conn, engine = await connect_db(args.db_file) try: - written = insert_batches(conn, rows, args.batch) + written = await insert_batches(conn, rows, args.batch) finally: - conn.close() + await conn.close() + await engine.dispose() LOG.info("Done. Inserted %d rows into '%s'.", written, store.value) if __name__ == "__main__": - main() + asyncio.run(main()) diff --git a/app/tests/test_datastore.py b/app/tests/test_datastore.py index 8b34c3e0..336397d7 100644 --- a/app/tests/test_datastore.py +++ b/app/tests/test_datastore.py @@ -13,11 +13,25 @@ from app.library.operations import Operation from app.library.sqlite_store import SqliteStore +async def reset_sqlite_store() -> None: + """Close and reset SqliteStore singleton for testing.""" + if SqliteStore in SqliteStore._instances: + instance = SqliteStore._instances[SqliteStore] + # Only close if there's an active connection to avoid event loop issues + if instance._conn is not None: + try: + await instance.close() + except RuntimeError: + # Event loop issues - just reset without closing + pass + SqliteStore._reset_singleton() + + async def make_db(data: int = 0) -> SqliteStore: """Create a temporary database with test data.""" - SqliteStore._reset_singleton() + await reset_sqlite_store() ins = SqliteStore.get_instance(db_path=":memory:") - await ins._ensure_conn() + await ins.get_connection() base_time = datetime.now(UTC) for i in range(data): @@ -33,7 +47,7 @@ async def make_db(data: int = 0) -> SqliteStore: "status": "finished", } - await ins._conn.execute( + await ins.execute_raw( 'INSERT INTO "history" ("id", "type", "url", "data", "created_at") VALUES (?, ?, ?, ?, ?)', ( f"test-id-{i}", @@ -43,8 +57,6 @@ async def make_db(data: int = 0) -> SqliteStore: created_at, ), ) - if data > 0: - await ins._conn.commit() return ins @@ -91,11 +103,10 @@ class TestDataStore: data = asdict(dto) data.pop("_id", None) created = datetime(2024, 1, 2, 3, 4, 5, tzinfo=UTC) - await db._conn.execute( + await db.execute_raw( "INSERT INTO history (id, type, url, data, created_at) VALUES (?, ?, ?, ?, ?)", ("abc", "queue", "http://x", json.dumps(data), created.strftime("%Y-%m-%d %H:%M:%S")), ) - await db._conn.commit() items = await store.saved_items() assert len(items) == 1 @@ -119,21 +130,20 @@ class TestDataStore: assert ret is store._dict[item._id] # Verify row written; JSON should not contain datetime field - cur = await db._conn.execute("SELECT * FROM history WHERE id=?", (item._id,)) - row = await cur.fetchone() - assert row is not None - assert row["type"] == "queue" - assert row["url"] == item.url - assert '"datetime"' not in row["data"] - assert row["id"] == item._id + rows = await db.fetch_raw("SELECT * FROM history WHERE id=?", (item._id,)) + assert len(rows) == 1, "Should find one row" + row = rows[0] + assert row["type"] == "queue", "Type should be queue" + assert row["url"] == item.url, "URL should match" + assert '"datetime"' not in row["data"], "JSON should not contain datetime field" + assert row["id"] == item._id, "ID should match" # Delete and ensure removal await store.delete(item._id) await asyncio.sleep(0) await db.flush() - cur2 = await db._conn.execute("SELECT * FROM history WHERE id=?", (item._id,)) - row2 = await cur2.fetchone() - assert row2 is None + rows2 = await db.fetch_raw("SELECT * FROM history WHERE id=?", (item._id,)) + assert len(rows2) == 0, "Row should be deleted" await db.close() @pytest.mark.asyncio @@ -368,7 +378,6 @@ class TestDataStore: async def test_load(self) -> None: """Test loading items from database into memory.""" db = await make_db() - conn = db._conn # Insert items directly into database item1_data = asdict(make_item(id="vid1", url="http://example.com/1", title="Video 1")) @@ -377,7 +386,7 @@ class TestDataStore: item2_data.pop("_id", None) created = datetime(2024, 1, 1, 12, 0, 0, tzinfo=UTC) - await conn.execute( + await db.execute_raw( "INSERT INTO history (id, type, url, data, created_at) VALUES (?, ?, ?, ?, ?)", ( "id1", @@ -387,7 +396,7 @@ class TestDataStore: created.strftime("%Y-%m-%d %H:%M:%S"), ), ) - await conn.execute( + await db.execute_raw( "INSERT INTO history (id, type, url, data, created_at) VALUES (?, ?, ?, ?, ?)", ( "id2", @@ -397,7 +406,6 @@ class TestDataStore: created.strftime("%Y-%m-%d %H:%M:%S"), ), ) - await conn.commit() store = DataStore(StoreType.QUEUE, db) assert len(store._dict) == 0 @@ -414,7 +422,7 @@ class TestDataStore: async def test_load_with_different_store_types(self) -> None: """Test that load only loads items matching the store type.""" db = await make_db() - conn = db._conn + # conn = db._conn # No longer needed # Insert items with different types item1_data = asdict(make_item(id="vid1", url="http://example.com/1")) @@ -423,7 +431,7 @@ class TestDataStore: item2_data.pop("_id", None) created = datetime(2024, 1, 1, 12, 0, 0, tzinfo=UTC) - await conn.execute( + await db.execute_raw( "INSERT INTO history (id, type, url, data, created_at) VALUES (?, ?, ?, ?, ?)", ( "id1", @@ -433,7 +441,7 @@ class TestDataStore: created.strftime("%Y-%m-%d %H:%M:%S"), ), ) - await conn.execute( + await db.execute_raw( "INSERT INTO history (id, type, url, data, created_at) VALUES (?, ?, ?, ?, ?)", ( "id2", @@ -443,7 +451,7 @@ class TestDataStore: created.strftime("%Y-%m-%d %H:%M:%S"), ), ) - await conn.commit() + # await conn.commit() # Auto-committed # Load QUEUE store - should only get queue items queue_store = DataStore(StoreType.QUEUE, db) @@ -541,18 +549,18 @@ class TestDataStore: """Test get_total_count with items in database.""" store = await make_store_async(StoreType.QUEUE) db = store._connection - conn = db._conn + # conn = db._conn # No longer needed # Add items directly to database created = datetime.now(UTC).strftime("%Y-%m-%d %H:%M:%S") for i in range(5): item_data = asdict(make_item(id=f"vid{i}")) item_data.pop("_id", None) - await conn.execute( + await db.execute_raw( "INSERT INTO history (id, type, url, data, created_at) VALUES (?, ?, ?, ?, ?)", (f"id{i}", str(StoreType.QUEUE), f"http://example.com/{i}", json.dumps(item_data), created), ) - await conn.commit() + # await conn.commit() # Auto-committed count = await store.get_total_count() assert count == 5 @@ -562,7 +570,7 @@ class TestDataStore: async def test_get_total_count_respects_store_type(self) -> None: """Test that get_total_count only counts items of the correct type.""" db = await make_db() - conn = db._conn + # conn = db._conn # No longer needed created = datetime.now(UTC).strftime("%Y-%m-%d %H:%M:%S") @@ -570,7 +578,7 @@ class TestDataStore: for i in range(3): item_data = asdict(make_item(id=f"vid{i}")) item_data.pop("_id", None) - await conn.execute( + await db.execute_raw( "INSERT INTO history (id, type, url, data, created_at) VALUES (?, ?, ?, ?, ?)", (f"q{i}", str(StoreType.QUEUE), f"http://example.com/queue/{i}", json.dumps(item_data), created), ) @@ -579,11 +587,11 @@ class TestDataStore: for i in range(2): item_data = asdict(make_item(id=f"vid{i}")) item_data.pop("_id", None) - await conn.execute( + await db.execute_raw( "INSERT INTO history (id, type, url, data, created_at) VALUES (?, ?, ?, ?, ?)", (f"h{i}", str(StoreType.HISTORY), f"http://example.com/history/{i}", json.dumps(item_data), created), ) - await conn.commit() + # await conn.commit() # Auto-committed queue_store = DataStore(StoreType.QUEUE, db) assert await queue_store.get_total_count() == 3 @@ -641,9 +649,8 @@ class TestDataStore: # Verify nothing was deleted from database await store._connection.flush() - cursor = await db._conn.execute("SELECT * FROM history WHERE id=?", ("nonexistent_id",)) - row = await cursor.fetchone() - assert row is None + rows = await db.fetch_raw("SELECT * FROM history WHERE id=?", ("nonexistent_id",)) + assert len(rows) == 0, "Should find no rows" await db.close() @pytest.mark.asyncio @@ -720,10 +727,9 @@ class TestDataStore: await store._connection.flush() # Verify datetime field is not in stored JSON - cursor = await db._conn.execute("SELECT data FROM history WHERE id=?", (item._id,)) - row = await cursor.fetchone() - assert row is not None - data = json.loads(row["data"]) + rows = await db.fetch_raw("SELECT data FROM history WHERE id=?", (item._id,)) + assert len(rows) == 1, "Should find one row" + data = json.loads(rows[0]["data"]) assert "datetime" not in data await db.close() @@ -742,10 +748,9 @@ class TestDataStore: await store._connection.flush() # Verify live_in field is not in stored JSON when status is finished - cursor = await conn._conn.execute("SELECT data FROM history WHERE id=?", (item._id,)) - row = await cursor.fetchone() - assert row is not None - data = json.loads(row["data"]) + rows = await conn.fetch_raw("SELECT data FROM history WHERE id=?", (item._id,)) + assert len(rows) == 1, "Should find one row" + data = json.loads(rows[0]["data"]) assert "live_in" not in data await store._connection.close() @@ -764,10 +769,9 @@ class TestDataStore: await store._connection.flush() # Verify live_in field IS in stored JSON when status is not finished - cursor = await conn._conn.execute("SELECT data FROM history WHERE id=?", (item._id,)) - row = await cursor.fetchone() - assert row is not None - data = json.loads(row["data"]) + rows = await conn.fetch_raw("SELECT data FROM history WHERE id=?", (item._id,)) + assert len(rows) == 1, "Should find one row" + data = json.loads(rows[0]["data"]) assert "live_in" in data assert data["live_in"] == "PT5M" await store._connection.close() diff --git a/app/tests/test_datastore_pagination.py b/app/tests/test_datastore_pagination.py index 7c54b286..031fc7fb 100644 --- a/app/tests/test_datastore_pagination.py +++ b/app/tests/test_datastore_pagination.py @@ -10,11 +10,25 @@ from app.library.ItemDTO import ItemDTO from app.library.sqlite_store import SqliteStore +async def reset_sqlite_store() -> None: + """Close and reset SqliteStore singleton for testing.""" + if SqliteStore in SqliteStore._instances: + instance = SqliteStore._instances[SqliteStore] + # Only close if there's an active connection to avoid event loop issues + if instance._conn is not None: + try: + await instance.close() + except RuntimeError: + # Event loop issues - just reset without closing + pass + SqliteStore._reset_singleton() + + async def make_db(data: int = 100) -> SqliteStore: """Create a temporary database with test data.""" - SqliteStore._reset_singleton() + await reset_sqlite_store() ins = SqliteStore.get_instance(db_path=":memory:") - await ins._ensure_conn() + await ins.get_connection() base_time = datetime.now(UTC) for i in range(data): @@ -30,7 +44,7 @@ async def make_db(data: int = 100) -> SqliteStore: "status": "finished", } - await ins._conn.execute( + await ins.execute_raw( 'INSERT INTO "history" ("id", "type", "url", "data", "created_at") VALUES (?, ?, ?, ?, ?)', ( f"test-id-{i}", @@ -40,8 +54,6 @@ async def make_db(data: int = 100) -> SqliteStore: created_at, ), ) - if data > 0: - await ins._conn.commit() return ins @@ -226,7 +238,7 @@ class TestDataStorePagination: db = await make_db(data=100) try: - await db._conn.execute( + await db.execute_raw( 'INSERT INTO "history" ("id", "type", "url", "data", "created_at") VALUES (?, ?, ?, ?, ?)', ( "pending-id", @@ -236,7 +248,7 @@ class TestDataStorePagination: datetime.now(UTC).strftime("%Y-%m-%d %H:%M:%S"), ), ) - await db._conn.execute( + await db.execute_raw( 'INSERT INTO "history" ("id", "type", "url", "data", "created_at") VALUES (?, ?, ?, ?, ?)', ( "downloading-id", @@ -246,7 +258,6 @@ class TestDataStorePagination: datetime.now(UTC).strftime("%Y-%m-%d %H:%M:%S"), ), ) - await db._conn.commit() datastore = DataStore(type=StoreType.HISTORY, connection=db) # Filter for finished items only @@ -287,7 +298,7 @@ class TestDataStorePagination: } db = await make_db(data=0) try: - await db._conn.execute( + await db.execute_raw( 'INSERT INTO "history" ("id", "type", "url", "data", "created_at") VALUES (?, ?, ?, ?, ?)', ( "pending-id-2", @@ -297,7 +308,7 @@ class TestDataStorePagination: datetime.now(UTC).strftime("%Y-%m-%d %H:%M:%S"), ), ) - await db._conn.execute( + await db.execute_raw( 'INSERT INTO "history" ("id", "type", "url", "data", "created_at") VALUES (?, ?, ?, ?, ?)', ( "error-id", @@ -307,7 +318,6 @@ class TestDataStorePagination: datetime.now(UTC).strftime("%Y-%m-%d %H:%M:%S"), ), ) - await db._conn.commit() datastore = DataStore(type=StoreType.HISTORY, connection=db) diff --git a/app/tests/test_sqlite_store.py b/app/tests/test_sqlite_store.py index a03d7e15..3a52c3d8 100644 --- a/app/tests/test_sqlite_store.py +++ b/app/tests/test_sqlite_store.py @@ -1,6 +1,7 @@ from datetime import UTC, datetime, timedelta import pytest +from sqlalchemy import text from app.library.ItemDTO import ItemDTO from app.library.operations import Operation @@ -11,8 +12,8 @@ async def make_store() -> SqliteStore: """Create an isolated in-memory SqliteStore instance with schema.""" SqliteStore._reset_singleton() store = SqliteStore.get_instance(db_path=":memory:") - await store._ensure_conn() - assert store._conn is not None + await store.get_connection() + assert store._engine is not None, "Engine should be initialized after _ensure_conn" return store @@ -26,6 +27,53 @@ def make_item(idx: int, *, status: str = "finished") -> ItemDTO: ) +@pytest.mark.asyncio +async def test_sessionmaker_returns_valid_sessionmaker() -> None: + """Test that sessionmaker() returns a working async_sessionmaker.""" + SqliteStore._reset_singleton() + store = SqliteStore.get_instance(db_path=":memory:") + + # Ensure connection is initialized + await store.get_connection() + + # Get sessionmaker + sessionmaker = store.sessionmaker() + assert sessionmaker is not None, "Sessionmaker should not be None after connection initialization" + + # Verify we can create a session + async with sessionmaker() as session: + assert session is not None, "Session should be created successfully" + + await store.close() + + +@pytest.mark.asyncio +async def test_sessionmaker_raises_before_init() -> None: + """Test that sessionmaker() raises error before connection initialization.""" + SqliteStore._reset_singleton() + store = SqliteStore.get_instance(db_path=":memory:") + + with pytest.raises(RuntimeError, match="Database connection not initialized"): + store.sessionmaker() + + await store.close() + + +@pytest.mark.asyncio +async def test_sqlalchemy_engine_disposed_on_close() -> None: + """Test that SQLAlchemy engine is properly disposed on close.""" + SqliteStore._reset_singleton() + store = SqliteStore.get_instance(db_path=":memory:") + + await store.get_connection() + assert store._engine is not None, "Engine should be created" + assert store._sessionmaker is not None, "Sessionmaker should be created" + + await store.close() + assert store._engine is None, "Engine should be None after close" + assert store._sessionmaker is None, "Sessionmaker should be None after close" + + @pytest.mark.asyncio async def test_enqueue_upsert_and_fetch_saved(): store = await make_store() @@ -87,8 +135,10 @@ async def test_paginate_order_and_bounds(): created_at = (base + timedelta(minutes=i)).strftime("%Y-%m-%d %H:%M:%S") _list.append(itm) await store._conn.execute( - 'INSERT INTO "history" ("id", "type", "url", "data", "created_at") VALUES (?, ?, ?, ?, ?)', - (itm._id, "history", itm.url, encoded, created_at), + text( + 'INSERT INTO "history" ("id", "type", "url", "data", "created_at") VALUES (:id, :type, :url, :data, :created_at)' + ), + {"id": itm._id, "type": "history", "url": itm.url, "data": encoded, "created_at": created_at}, ) await store._conn.commit() @@ -224,6 +274,7 @@ async def test_on_shutdown_closes_connection(): await store.on_shutdown(None) assert store._conn is None + # Note: on_shutdown already closes the connection, so no need to call close() again @pytest.mark.asyncio diff --git a/pyproject.toml b/pyproject.toml index 80549294..a419e71d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,6 +44,8 @@ dependencies = [ "jmespath>=1.0.1", "jsonschema>=4.23.0", "aiosqlite>=0.22.0", + "sqlalchemy>=2.0.45", + "pydantic>=2.12.5", ] [tool.ruff] diff --git a/uv.lock b/uv.lock index 313d7f42..23725cdf 100644 --- a/uv.lock +++ b/uv.lock @@ -123,6 +123,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a9/ba/000a1996d4308bc65120167c21241a3b205464a2e0b58deda26ae8ac21d1/altgraph-0.17.5-py2.py3-none-any.whl", hash = "sha256:f3a22400bce1b0c701683820ac4f3b159cd301acab067c51c653e06961600597", size = 21228 }, ] +[[package]] +name = "annotated-types" +version = "0.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ee/67/531ea369ba64dcff5ec9c3402f9f51bf748cec26dde048a2f973a4eea7f5/annotated_types-0.7.0.tar.gz", hash = "sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89", size = 16081 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl", hash = "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53", size = 13643 }, +] + [[package]] name = "anyio" version = "4.12.1" @@ -513,6 +522,37 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/9a/9a/e35b4a917281c0b8419d4207f4334c8e8c5dbf4f3f5f9ada73958d937dcc/frozenlist-1.8.0-py3-none-any.whl", hash = "sha256:0c18a16eab41e82c295618a77502e17b195883241c563b00f0aa5106fc4eaa0d", size = 13409 }, ] +[[package]] +name = "greenlet" +version = "3.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c7/e5/40dbda2736893e3e53d25838e0f19a2b417dfc122b9989c91918db30b5d3/greenlet-3.3.0.tar.gz", hash = "sha256:a82bb225a4e9e4d653dd2fb7b8b2d36e4fb25bc0165422a11e48b88e9e6f78fb", size = 190651 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/02/2f/28592176381b9ab2cafa12829ba7b472d177f3acc35d8fbcf3673d966fff/greenlet-3.3.0-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:a1e41a81c7e2825822f4e068c48cb2196002362619e2d70b148f20a831c00739", size = 275140 }, + { url = "https://files.pythonhosted.org/packages/2c/80/fbe937bf81e9fca98c981fe499e59a3f45df2a04da0baa5c2be0dca0d329/greenlet-3.3.0-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9f515a47d02da4d30caaa85b69474cec77b7929b2e936ff7fb853d42f4bf8808", size = 599219 }, + { url = "https://files.pythonhosted.org/packages/c2/ff/7c985128f0514271b8268476af89aee6866df5eec04ac17dcfbc676213df/greenlet-3.3.0-cp313-cp313-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7d2d9fd66bfadf230b385fdc90426fcd6eb64db54b40c495b72ac0feb5766c54", size = 610211 }, + { url = "https://files.pythonhosted.org/packages/79/07/c47a82d881319ec18a4510bb30463ed6891f2ad2c1901ed5ec23d3de351f/greenlet-3.3.0-cp313-cp313-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:30a6e28487a790417d036088b3bcb3f3ac7d8babaa7d0139edbaddebf3af9492", size = 624311 }, + { url = "https://files.pythonhosted.org/packages/fd/8e/424b8c6e78bd9837d14ff7df01a9829fc883ba2ab4ea787d4f848435f23f/greenlet-3.3.0-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:087ea5e004437321508a8d6f20efc4cfec5e3c30118e1417ea96ed1d93950527", size = 612833 }, + { url = "https://files.pythonhosted.org/packages/b5/ba/56699ff9b7c76ca12f1cdc27a886d0f81f2189c3455ff9f65246780f713d/greenlet-3.3.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ab97cf74045343f6c60a39913fa59710e4bd26a536ce7ab2397adf8b27e67c39", size = 1567256 }, + { url = "https://files.pythonhosted.org/packages/1e/37/f31136132967982d698c71a281a8901daf1a8fbab935dce7c0cf15f942cc/greenlet-3.3.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:5375d2e23184629112ca1ea89a53389dddbffcf417dad40125713d88eb5f96e8", size = 1636483 }, + { url = "https://files.pythonhosted.org/packages/7e/71/ba21c3fb8c5dce83b8c01f458a42e99ffdb1963aeec08fff5a18588d8fd7/greenlet-3.3.0-cp313-cp313-win_amd64.whl", hash = "sha256:9ee1942ea19550094033c35d25d20726e4f1c40d59545815e1128ac58d416d38", size = 301833 }, + { url = "https://files.pythonhosted.org/packages/d7/7c/f0a6d0ede2c7bf092d00bc83ad5bafb7e6ec9b4aab2fbdfa6f134dc73327/greenlet-3.3.0-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:60c2ef0f578afb3c8d92ea07ad327f9a062547137afe91f38408f08aacab667f", size = 275671 }, + { url = "https://files.pythonhosted.org/packages/44/06/dac639ae1a50f5969d82d2e3dd9767d30d6dbdbab0e1a54010c8fe90263c/greenlet-3.3.0-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0a5d554d0712ba1de0a6c94c640f7aeba3f85b3a6e1f2899c11c2c0428da9365", size = 646360 }, + { url = "https://files.pythonhosted.org/packages/e0/94/0fb76fe6c5369fba9bf98529ada6f4c3a1adf19e406a47332245ef0eb357/greenlet-3.3.0-cp314-cp314-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:3a898b1e9c5f7307ebbde4102908e6cbfcb9ea16284a3abe15cab996bee8b9b3", size = 658160 }, + { url = "https://files.pythonhosted.org/packages/93/79/d2c70cae6e823fac36c3bbc9077962105052b7ef81db2f01ec3b9bf17e2b/greenlet-3.3.0-cp314-cp314-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:dcd2bdbd444ff340e8d6bdf54d2f206ccddbb3ccfdcd3c25bf4afaa7b8f0cf45", size = 671388 }, + { url = "https://files.pythonhosted.org/packages/b8/14/bab308fc2c1b5228c3224ec2bf928ce2e4d21d8046c161e44a2012b5203e/greenlet-3.3.0-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5773edda4dc00e173820722711d043799d3adb4f01731f40619e07ea2750b955", size = 660166 }, + { url = "https://files.pythonhosted.org/packages/4b/d2/91465d39164eaa0085177f61983d80ffe746c5a1860f009811d498e7259c/greenlet-3.3.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:ac0549373982b36d5fd5d30beb8a7a33ee541ff98d2b502714a09f1169f31b55", size = 1615193 }, + { url = "https://files.pythonhosted.org/packages/42/1b/83d110a37044b92423084d52d5d5a3b3a73cafb51b547e6d7366ff62eff1/greenlet-3.3.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:d198d2d977460358c3b3a4dc844f875d1adb33817f0613f663a656f463764ccc", size = 1683653 }, + { url = "https://files.pythonhosted.org/packages/7c/9a/9030e6f9aa8fd7808e9c31ba4c38f87c4f8ec324ee67431d181fe396d705/greenlet-3.3.0-cp314-cp314-win_amd64.whl", hash = "sha256:73f51dd0e0bdb596fb0417e475fa3c5e32d4c83638296e560086b8d7da7c4170", size = 305387 }, + { url = "https://files.pythonhosted.org/packages/a0/66/bd6317bc5932accf351fc19f177ffba53712a202f9df10587da8df257c7e/greenlet-3.3.0-cp314-cp314t-macosx_11_0_universal2.whl", hash = "sha256:d6ed6f85fae6cdfdb9ce04c9bf7a08d666cfcfb914e7d006f44f840b46741931", size = 282638 }, + { url = "https://files.pythonhosted.org/packages/30/cf/cc81cb030b40e738d6e69502ccbd0dd1bced0588e958f9e757945de24404/greenlet-3.3.0-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d9125050fcf24554e69c4cacb086b87b3b55dc395a8b3ebe6487b045b2614388", size = 651145 }, + { url = "https://files.pythonhosted.org/packages/9c/ea/1020037b5ecfe95ca7df8d8549959baceb8186031da83d5ecceff8b08cd2/greenlet-3.3.0-cp314-cp314t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:87e63ccfa13c0a0f6234ed0add552af24cc67dd886731f2261e46e241608bee3", size = 654236 }, + { url = "https://files.pythonhosted.org/packages/69/cc/1e4bae2e45ca2fa55299f4e85854606a78ecc37fead20d69322f96000504/greenlet-3.3.0-cp314-cp314t-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2662433acbca297c9153a4023fe2161c8dcfdcc91f10433171cf7e7d94ba2221", size = 662506 }, + { url = "https://files.pythonhosted.org/packages/57/b9/f8025d71a6085c441a7eaff0fd928bbb275a6633773667023d19179fe815/greenlet-3.3.0-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3c6e9b9c1527a78520357de498b0e709fb9e2f49c3a513afd5a249007261911b", size = 653783 }, + { url = "https://files.pythonhosted.org/packages/f6/c7/876a8c7a7485d5d6b5c6821201d542ef28be645aa024cfe1145b35c120c1/greenlet-3.3.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:286d093f95ec98fdd92fcb955003b8a3d054b4e2cab3e2707a5039e7b50520fd", size = 1614857 }, + { url = "https://files.pythonhosted.org/packages/4f/dc/041be1dff9f23dac5f48a43323cd0789cb798342011c19a248d9c9335536/greenlet-3.3.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:6c10513330af5b8ae16f023e8ddbfb486ab355d04467c4679c5cfe4659975dd9", size = 1676034 }, +] + [[package]] name = "h11" version = "0.16.0" @@ -1020,6 +1060,74 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f9/93/45c1cdcbeb182ccd2e144c693eaa097763b08b38cded279f0053ed53c553/pycryptodomex-3.23.0-cp37-abi3-win_arm64.whl", hash = "sha256:02d87b80778c171445d67e23d1caef279bf4b25c3597050ccd2e13970b57fd51", size = 1707161 }, ] +[[package]] +name = "pydantic" +version = "2.12.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "annotated-types" }, + { name = "pydantic-core" }, + { name = "typing-extensions" }, + { name = "typing-inspection" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/69/44/36f1a6e523abc58ae5f928898e4aca2e0ea509b5aa6f6f392a5d882be928/pydantic-2.12.5.tar.gz", hash = "sha256:4d351024c75c0f085a9febbb665ce8c0c6ec5d30e903bdb6394b7ede26aebb49", size = 821591 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5a/87/b70ad306ebb6f9b585f114d0ac2137d792b48be34d732d60e597c2f8465a/pydantic-2.12.5-py3-none-any.whl", hash = "sha256:e561593fccf61e8a20fc46dfc2dfe075b8be7d0188df33f221ad1f0139180f9d", size = 463580 }, +] + +[[package]] +name = "pydantic-core" +version = "2.41.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/71/70/23b021c950c2addd24ec408e9ab05d59b035b39d97cdc1130e1bce647bb6/pydantic_core-2.41.5.tar.gz", hash = "sha256:08daa51ea16ad373ffd5e7606252cc32f07bc72b28284b6bc9c6df804816476e", size = 460952 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/87/06/8806241ff1f70d9939f9af039c6c35f2360cf16e93c2ca76f184e76b1564/pydantic_core-2.41.5-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:941103c9be18ac8daf7b7adca8228f8ed6bb7a1849020f643b3a14d15b1924d9", size = 2120403 }, + { url = "https://files.pythonhosted.org/packages/94/02/abfa0e0bda67faa65fef1c84971c7e45928e108fe24333c81f3bfe35d5f5/pydantic_core-2.41.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:112e305c3314f40c93998e567879e887a3160bb8689ef3d2c04b6cc62c33ac34", size = 1896206 }, + { url = "https://files.pythonhosted.org/packages/15/df/a4c740c0943e93e6500f9eb23f4ca7ec9bf71b19e608ae5b579678c8d02f/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0cbaad15cb0c90aa221d43c00e77bb33c93e8d36e0bf74760cd00e732d10a6a0", size = 1919307 }, + { url = "https://files.pythonhosted.org/packages/9a/e3/6324802931ae1d123528988e0e86587c2072ac2e5394b4bc2bc34b61ff6e/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:03ca43e12fab6023fc79d28ca6b39b05f794ad08ec2feccc59a339b02f2b3d33", size = 2063258 }, + { url = "https://files.pythonhosted.org/packages/c9/d4/2230d7151d4957dd79c3044ea26346c148c98fbf0ee6ebd41056f2d62ab5/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:dc799088c08fa04e43144b164feb0c13f9a0bc40503f8df3e9fde58a3c0c101e", size = 2214917 }, + { url = "https://files.pythonhosted.org/packages/e6/9f/eaac5df17a3672fef0081b6c1bb0b82b33ee89aa5cec0d7b05f52fd4a1fa/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:97aeba56665b4c3235a0e52b2c2f5ae9cd071b8a8310ad27bddb3f7fb30e9aa2", size = 2332186 }, + { url = "https://files.pythonhosted.org/packages/cf/4e/35a80cae583a37cf15604b44240e45c05e04e86f9cfd766623149297e971/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:406bf18d345822d6c21366031003612b9c77b3e29ffdb0f612367352aab7d586", size = 2073164 }, + { url = "https://files.pythonhosted.org/packages/bf/e3/f6e262673c6140dd3305d144d032f7bd5f7497d3871c1428521f19f9efa2/pydantic_core-2.41.5-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:b93590ae81f7010dbe380cdeab6f515902ebcbefe0b9327cc4804d74e93ae69d", size = 2179146 }, + { url = "https://files.pythonhosted.org/packages/75/c7/20bd7fc05f0c6ea2056a4565c6f36f8968c0924f19b7d97bbfea55780e73/pydantic_core-2.41.5-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:01a3d0ab748ee531f4ea6c3e48ad9dac84ddba4b0d82291f87248f2f9de8d740", size = 2137788 }, + { url = "https://files.pythonhosted.org/packages/3a/8d/34318ef985c45196e004bc46c6eab2eda437e744c124ef0dbe1ff2c9d06b/pydantic_core-2.41.5-cp313-cp313-musllinux_1_1_armv7l.whl", hash = "sha256:6561e94ba9dacc9c61bce40e2d6bdc3bfaa0259d3ff36ace3b1e6901936d2e3e", size = 2340133 }, + { url = "https://files.pythonhosted.org/packages/9c/59/013626bf8c78a5a5d9350d12e7697d3d4de951a75565496abd40ccd46bee/pydantic_core-2.41.5-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:915c3d10f81bec3a74fbd4faebe8391013ba61e5a1a8d48c4455b923bdda7858", size = 2324852 }, + { url = "https://files.pythonhosted.org/packages/1a/d9/c248c103856f807ef70c18a4f986693a46a8ffe1602e5d361485da502d20/pydantic_core-2.41.5-cp313-cp313-win32.whl", hash = "sha256:650ae77860b45cfa6e2cdafc42618ceafab3a2d9a3811fcfbd3bbf8ac3c40d36", size = 1994679 }, + { url = "https://files.pythonhosted.org/packages/9e/8b/341991b158ddab181cff136acd2552c9f35bd30380422a639c0671e99a91/pydantic_core-2.41.5-cp313-cp313-win_amd64.whl", hash = "sha256:79ec52ec461e99e13791ec6508c722742ad745571f234ea6255bed38c6480f11", size = 2019766 }, + { url = "https://files.pythonhosted.org/packages/73/7d/f2f9db34af103bea3e09735bb40b021788a5e834c81eedb541991badf8f5/pydantic_core-2.41.5-cp313-cp313-win_arm64.whl", hash = "sha256:3f84d5c1b4ab906093bdc1ff10484838aca54ef08de4afa9de0f5f14d69639cd", size = 1981005 }, + { url = "https://files.pythonhosted.org/packages/ea/28/46b7c5c9635ae96ea0fbb779e271a38129df2550f763937659ee6c5dbc65/pydantic_core-2.41.5-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:3f37a19d7ebcdd20b96485056ba9e8b304e27d9904d233d7b1015db320e51f0a", size = 2119622 }, + { url = "https://files.pythonhosted.org/packages/74/1a/145646e5687e8d9a1e8d09acb278c8535ebe9e972e1f162ed338a622f193/pydantic_core-2.41.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:1d1d9764366c73f996edd17abb6d9d7649a7eb690006ab6adbda117717099b14", size = 1891725 }, + { url = "https://files.pythonhosted.org/packages/23/04/e89c29e267b8060b40dca97bfc64a19b2a3cf99018167ea1677d96368273/pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:25e1c2af0fce638d5f1988b686f3b3ea8cd7de5f244ca147c777769e798a9cd1", size = 1915040 }, + { url = "https://files.pythonhosted.org/packages/84/a3/15a82ac7bd97992a82257f777b3583d3e84bdb06ba6858f745daa2ec8a85/pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:506d766a8727beef16b7adaeb8ee6217c64fc813646b424d0804d67c16eddb66", size = 2063691 }, + { url = "https://files.pythonhosted.org/packages/74/9b/0046701313c6ef08c0c1cf0e028c67c770a4e1275ca73131563c5f2a310a/pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4819fa52133c9aa3c387b3328f25c1facc356491e6135b459f1de698ff64d869", size = 2213897 }, + { url = "https://files.pythonhosted.org/packages/8a/cd/6bac76ecd1b27e75a95ca3a9a559c643b3afcd2dd62086d4b7a32a18b169/pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2b761d210c9ea91feda40d25b4efe82a1707da2ef62901466a42492c028553a2", size = 2333302 }, + { url = "https://files.pythonhosted.org/packages/4c/d2/ef2074dc020dd6e109611a8be4449b98cd25e1b9b8a303c2f0fca2f2bcf7/pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:22f0fb8c1c583a3b6f24df2470833b40207e907b90c928cc8d3594b76f874375", size = 2064877 }, + { url = "https://files.pythonhosted.org/packages/18/66/e9db17a9a763d72f03de903883c057b2592c09509ccfe468187f2a2eef29/pydantic_core-2.41.5-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2782c870e99878c634505236d81e5443092fba820f0373997ff75f90f68cd553", size = 2180680 }, + { url = "https://files.pythonhosted.org/packages/d3/9e/3ce66cebb929f3ced22be85d4c2399b8e85b622db77dad36b73c5387f8f8/pydantic_core-2.41.5-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:0177272f88ab8312479336e1d777f6b124537d47f2123f89cb37e0accea97f90", size = 2138960 }, + { url = "https://files.pythonhosted.org/packages/a6/62/205a998f4327d2079326b01abee48e502ea739d174f0a89295c481a2272e/pydantic_core-2.41.5-cp314-cp314-musllinux_1_1_armv7l.whl", hash = "sha256:63510af5e38f8955b8ee5687740d6ebf7c2a0886d15a6d65c32814613681bc07", size = 2339102 }, + { url = "https://files.pythonhosted.org/packages/3c/0d/f05e79471e889d74d3d88f5bd20d0ed189ad94c2423d81ff8d0000aab4ff/pydantic_core-2.41.5-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:e56ba91f47764cc14f1daacd723e3e82d1a89d783f0f5afe9c364b8bb491ccdb", size = 2326039 }, + { url = "https://files.pythonhosted.org/packages/ec/e1/e08a6208bb100da7e0c4b288eed624a703f4d129bde2da475721a80cab32/pydantic_core-2.41.5-cp314-cp314-win32.whl", hash = "sha256:aec5cf2fd867b4ff45b9959f8b20ea3993fc93e63c7363fe6851424c8a7e7c23", size = 1995126 }, + { url = "https://files.pythonhosted.org/packages/48/5d/56ba7b24e9557f99c9237e29f5c09913c81eeb2f3217e40e922353668092/pydantic_core-2.41.5-cp314-cp314-win_amd64.whl", hash = "sha256:8e7c86f27c585ef37c35e56a96363ab8de4e549a95512445b85c96d3e2f7c1bf", size = 2015489 }, + { url = "https://files.pythonhosted.org/packages/4e/bb/f7a190991ec9e3e0ba22e4993d8755bbc4a32925c0b5b42775c03e8148f9/pydantic_core-2.41.5-cp314-cp314-win_arm64.whl", hash = "sha256:e672ba74fbc2dc8eea59fb6d4aed6845e6905fc2a8afe93175d94a83ba2a01a0", size = 1977288 }, + { url = "https://files.pythonhosted.org/packages/92/ed/77542d0c51538e32e15afe7899d79efce4b81eee631d99850edc2f5e9349/pydantic_core-2.41.5-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:8566def80554c3faa0e65ac30ab0932b9e3a5cd7f8323764303d468e5c37595a", size = 2120255 }, + { url = "https://files.pythonhosted.org/packages/bb/3d/6913dde84d5be21e284439676168b28d8bbba5600d838b9dca99de0fad71/pydantic_core-2.41.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:b80aa5095cd3109962a298ce14110ae16b8c1aece8b72f9dafe81cf597ad80b3", size = 1863760 }, + { url = "https://files.pythonhosted.org/packages/5a/f0/e5e6b99d4191da102f2b0eb9687aaa7f5bea5d9964071a84effc3e40f997/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3006c3dd9ba34b0c094c544c6006cc79e87d8612999f1a5d43b769b89181f23c", size = 1878092 }, + { url = "https://files.pythonhosted.org/packages/71/48/36fb760642d568925953bcc8116455513d6e34c4beaa37544118c36aba6d/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:72f6c8b11857a856bcfa48c86f5368439f74453563f951e473514579d44aa612", size = 2053385 }, + { url = "https://files.pythonhosted.org/packages/20/25/92dc684dd8eb75a234bc1c764b4210cf2646479d54b47bf46061657292a8/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5cb1b2f9742240e4bb26b652a5aeb840aa4b417c7748b6f8387927bc6e45e40d", size = 2218832 }, + { url = "https://files.pythonhosted.org/packages/e2/09/f53e0b05023d3e30357d82eb35835d0f6340ca344720a4599cd663dca599/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:bd3d54f38609ff308209bd43acea66061494157703364ae40c951f83ba99a1a9", size = 2327585 }, + { url = "https://files.pythonhosted.org/packages/aa/4e/2ae1aa85d6af35a39b236b1b1641de73f5a6ac4d5a7509f77b814885760c/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2ff4321e56e879ee8d2a879501c8e469414d948f4aba74a2d4593184eb326660", size = 2041078 }, + { url = "https://files.pythonhosted.org/packages/cd/13/2e215f17f0ef326fc72afe94776edb77525142c693767fc347ed6288728d/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d0d2568a8c11bf8225044aa94409e21da0cb09dcdafe9ecd10250b2baad531a9", size = 2173914 }, + { url = "https://files.pythonhosted.org/packages/02/7a/f999a6dcbcd0e5660bc348a3991c8915ce6599f4f2c6ac22f01d7a10816c/pydantic_core-2.41.5-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:a39455728aabd58ceabb03c90e12f71fd30fa69615760a075b9fec596456ccc3", size = 2129560 }, + { url = "https://files.pythonhosted.org/packages/3a/b1/6c990ac65e3b4c079a4fb9f5b05f5b013afa0f4ed6780a3dd236d2cbdc64/pydantic_core-2.41.5-cp314-cp314t-musllinux_1_1_armv7l.whl", hash = "sha256:239edca560d05757817c13dc17c50766136d21f7cd0fac50295499ae24f90fdf", size = 2329244 }, + { url = "https://files.pythonhosted.org/packages/d9/02/3c562f3a51afd4d88fff8dffb1771b30cfdfd79befd9883ee094f5b6c0d8/pydantic_core-2.41.5-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:2a5e06546e19f24c6a96a129142a75cee553cc018ffee48a460059b1185f4470", size = 2331955 }, + { url = "https://files.pythonhosted.org/packages/5c/96/5fb7d8c3c17bc8c62fdb031c47d77a1af698f1d7a406b0f79aaa1338f9ad/pydantic_core-2.41.5-cp314-cp314t-win32.whl", hash = "sha256:b4ececa40ac28afa90871c2cc2b9ffd2ff0bf749380fbdf57d165fd23da353aa", size = 1988906 }, + { url = "https://files.pythonhosted.org/packages/22/ed/182129d83032702912c2e2d8bbe33c036f342cc735737064668585dac28f/pydantic_core-2.41.5-cp314-cp314t-win_amd64.whl", hash = "sha256:80aa89cad80b32a912a65332f64a4450ed00966111b6615ca6816153d3585a8c", size = 1981607 }, + { url = "https://files.pythonhosted.org/packages/9f/ed/068e41660b832bb0b1aa5b58011dea2a3fe0ba7861ff38c4d4904c1c1a99/pydantic_core-2.41.5-cp314-cp314t-win_arm64.whl", hash = "sha256:35b44f37a3199f771c3eaa53051bc8a70cd7b54f333531c59e29fd4db5d15008", size = 1974769 }, +] + [[package]] name = "pygments" version = "2.19.2" @@ -1590,6 +1698,35 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/32/46/9cb0e58b2deb7f82b84065f37f3bffeb12413f947f9388e4cac22c4621ce/sortedcontainers-2.4.0-py2.py3-none-any.whl", hash = "sha256:a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0", size = 29575 }, ] +[[package]] +name = "sqlalchemy" +version = "2.0.45" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "greenlet", marker = "platform_machine == 'AMD64' or platform_machine == 'WIN32' or platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'ppc64le' or platform_machine == 'win32' or platform_machine == 'x86_64'" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/be/f9/5e4491e5ccf42f5d9cfc663741d261b3e6e1683ae7812114e7636409fcc6/sqlalchemy-2.0.45.tar.gz", hash = "sha256:1632a4bda8d2d25703fdad6363058d882541bdaaee0e5e3ddfa0cd3229efce88", size = 9869912 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6a/c8/7cc5221b47a54edc72a0140a1efa56e0a2730eefa4058d7ed0b4c4357ff8/sqlalchemy-2.0.45-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fe187fc31a54d7fd90352f34e8c008cf3ad5d064d08fedd3de2e8df83eb4a1cf", size = 3277082 }, + { url = "https://files.pythonhosted.org/packages/0e/50/80a8d080ac7d3d321e5e5d420c9a522b0aa770ec7013ea91f9a8b7d36e4a/sqlalchemy-2.0.45-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:672c45cae53ba88e0dad74b9027dddd09ef6f441e927786b05bec75d949fbb2e", size = 3293131 }, + { url = "https://files.pythonhosted.org/packages/da/4c/13dab31266fc9904f7609a5dc308a2432a066141d65b857760c3bef97e69/sqlalchemy-2.0.45-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:470daea2c1ce73910f08caf10575676a37159a6d16c4da33d0033546bddebc9b", size = 3225389 }, + { url = "https://files.pythonhosted.org/packages/74/04/891b5c2e9f83589de202e7abaf24cd4e4fa59e1837d64d528829ad6cc107/sqlalchemy-2.0.45-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:9c6378449e0940476577047150fd09e242529b761dc887c9808a9a937fe990c8", size = 3266054 }, + { url = "https://files.pythonhosted.org/packages/f1/24/fc59e7f71b0948cdd4cff7a286210e86b0443ef1d18a23b0d83b87e4b1f7/sqlalchemy-2.0.45-cp313-cp313-win32.whl", hash = "sha256:4b6bec67ca45bc166c8729910bd2a87f1c0407ee955df110d78948f5b5827e8a", size = 2110299 }, + { url = "https://files.pythonhosted.org/packages/c0/c5/d17113020b2d43073412aeca09b60d2009442420372123b8d49cc253f8b8/sqlalchemy-2.0.45-cp313-cp313-win_amd64.whl", hash = "sha256:afbf47dc4de31fa38fd491f3705cac5307d21d4bb828a4f020ee59af412744ee", size = 2136264 }, + { url = "https://files.pythonhosted.org/packages/3d/8d/bb40a5d10e7a5f2195f235c0b2f2c79b0bf6e8f00c0c223130a4fbd2db09/sqlalchemy-2.0.45-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:83d7009f40ce619d483d26ac1b757dfe3167b39921379a8bd1b596cf02dab4a6", size = 3521998 }, + { url = "https://files.pythonhosted.org/packages/75/a5/346128b0464886f036c039ea287b7332a410aa2d3fb0bb5d404cb8861635/sqlalchemy-2.0.45-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:d8a2ca754e5415cde2b656c27900b19d50ba076aa05ce66e2207623d3fe41f5a", size = 3473434 }, + { url = "https://files.pythonhosted.org/packages/cc/64/4e1913772646b060b025d3fc52ce91a58967fe58957df32b455de5a12b4f/sqlalchemy-2.0.45-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7f46ec744e7f51275582e6a24326e10c49fbdd3fc99103e01376841213028774", size = 3272404 }, + { url = "https://files.pythonhosted.org/packages/b3/27/caf606ee924282fe4747ee4fd454b335a72a6e018f97eab5ff7f28199e16/sqlalchemy-2.0.45-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:883c600c345123c033c2f6caca18def08f1f7f4c3ebeb591a63b6fceffc95cce", size = 3277057 }, + { url = "https://files.pythonhosted.org/packages/85/d0/3d64218c9724e91f3d1574d12eb7ff8f19f937643815d8daf792046d88ab/sqlalchemy-2.0.45-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:2c0b74aa79e2deade948fe8593654c8ef4228c44ba862bb7c9585c8e0db90f33", size = 3222279 }, + { url = "https://files.pythonhosted.org/packages/24/10/dd7688a81c5bc7690c2a3764d55a238c524cd1a5a19487928844cb247695/sqlalchemy-2.0.45-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:8a420169cef179d4c9064365f42d779f1e5895ad26ca0c8b4c0233920973db74", size = 3244508 }, + { url = "https://files.pythonhosted.org/packages/aa/41/db75756ca49f777e029968d9c9fee338c7907c563267740c6d310a8e3f60/sqlalchemy-2.0.45-cp314-cp314-win32.whl", hash = "sha256:e50dcb81a5dfe4b7b4a4aa8f338116d127cb209559124f3694c70d6cd072b68f", size = 2113204 }, + { url = "https://files.pythonhosted.org/packages/89/a2/0e1590e9adb292b1d576dbcf67ff7df8cf55e56e78d2c927686d01080f4b/sqlalchemy-2.0.45-cp314-cp314-win_amd64.whl", hash = "sha256:4748601c8ea959e37e03d13dcda4a44837afcd1b21338e637f7c935b8da06177", size = 2138785 }, + { url = "https://files.pythonhosted.org/packages/42/39/f05f0ed54d451156bbed0e23eb0516bcad7cbb9f18b3bf219c786371b3f0/sqlalchemy-2.0.45-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cd337d3526ec5298f67d6a30bbbe4ed7e5e68862f0bf6dd21d289f8d37b7d60b", size = 3522029 }, + { url = "https://files.pythonhosted.org/packages/54/0f/d15398b98b65c2bce288d5ee3f7d0a81f77ab89d9456994d5c7cc8b2a9db/sqlalchemy-2.0.45-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:9a62b446b7d86a3909abbcd1cd3cc550a832f99c2bc37c5b22e1925438b9367b", size = 3475142 }, + { url = "https://files.pythonhosted.org/packages/bf/e1/3ccb13c643399d22289c6a9786c1a91e3dcbb68bce4beb44926ac2c557bf/sqlalchemy-2.0.45-py3-none-any.whl", hash = "sha256:5225a288e4c8cc2308dbdd874edad6e7d0fd38eac1e9e5f23503425c8eee20d0", size = 1936672 }, +] + [[package]] name = "trio" version = "0.32.0" @@ -1630,6 +1767,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl", hash = "sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548", size = 44614 }, ] +[[package]] +name = "typing-inspection" +version = "0.4.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/55/e3/70399cb7dd41c10ac53367ae42139cf4b1ca5f36bb3dc6c9d33acdb43655/typing_inspection-0.4.2.tar.gz", hash = "sha256:ba561c48a67c5958007083d386c3295464928b01faa735ab8547c5692e87f464", size = 75949 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl", hash = "sha256:4ed1cacbdc298c220f1bd249ed5287caa16f34d44ef4e9c3d0cbad5b521545e7", size = 14611 }, +] + [[package]] name = "tzdata" version = "2025.3" @@ -1868,6 +2017,7 @@ dependencies = [ { name = "parsel" }, { name = "platformdirs" }, { name = "pycryptodome" }, + { name = "pydantic" }, { name = "pyjson5" }, { name = "pysubs2" }, { name = "python-dotenv" }, @@ -1876,6 +2026,7 @@ dependencies = [ { name = "python-socketio" }, { name = "regex" }, { name = "selenium" }, + { name = "sqlalchemy" }, { name = "tzdata", marker = "sys_platform == 'win32'" }, { name = "yt-dlp", extra = ["default"] }, { name = "zipstream-ng" }, @@ -1919,6 +2070,7 @@ requires-dist = [ { name = "parsel", specifier = ">=1.10.0" }, { name = "platformdirs" }, { name = "pycryptodome", specifier = ">=3.23.0" }, + { name = "pydantic", specifier = ">=2.12.5" }, { name = "pyinstaller", marker = "extra == 'installer'" }, { name = "pyjson5" }, { name = "pysubs2" }, @@ -1928,6 +2080,7 @@ requires-dist = [ { name = "python-socketio", specifier = ">=5.11.1" }, { name = "regex" }, { name = "selenium", specifier = ">=4.35.0" }, + { name = "sqlalchemy", specifier = ">=2.0.45" }, { name = "tzdata", marker = "sys_platform == 'win32'" }, { name = "yt-dlp", extras = ["default"] }, { name = "zipstream-ng", specifier = ">=1.8.0" },