danvics/ytptube
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

Refactor: swap sqlite to aiosqlite and add queued db updates

Browse source
This commit is contained in:
arabcoders 2025-12-18 16:31:15 +03:00
parent 8a62f053a5
commit 7a85bcdb5c
19 changed files with 1790 additions and 875 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
.vscode/settings.json vendored
View file

@ -16,6 +16,7 @@
"ahas",
"ahash",
"aiocron",
"aiosqlite",
"anyio",
"Archiver",
"arrowdown",
@ -73,6 +74,8 @@
"falsey",
"faststart",
"Fetc",
"fetchall",
"fetchone",
"filterwarnings",
"finaldir",
"flac",
@ -91,6 +94,7 @@
"hookspath",
"httpx",
"imagetools",
"isinstance",
"jmespath",
"jsonschema",
"kibibytes",
@ -112,7 +116,10 @@
"mebibytes",
"MEIPASS",
"merch",
"metaclass",
"Metaclasses",
"metadataparser",
"mgmt",
"Microformat",
"microformats",
"mkvtoolsnix",
@ -128,6 +135,7 @@
"noninteractive",
"noprogress",
"onefile",
"oneline",
"parsel",
"pathex",
"pickleable",
@ -178,6 +186,7 @@
"Unraid",
"upgrader",
"urandom",
"urlparse",
"urlsafe",
"usegmt",
"usermod",
@ -190,6 +199,7 @@
"xerror",
"youtu",
"youtubepot",
"zstd",
"русский",
"العربية"
],

321
app/library/DataStore.py
View file

@ -1,27 +1,21 @@
import asyncio
import copy
import json
import logging
from collections import OrderedDict
from dataclasses import fields
from datetime import UTC, datetime
from email.utils import formatdate
from collections.abc import Iterable
from enum import Enum
from sqlite3 import Connection
from .Download import Download
from .ItemDTO import ItemDTO
from .operations import matches_condition
from .Utils import init_class
from .sqlite_store import SqliteStore
LOG: logging.Logger = logging.getLogger("datastore")
ITEM_DTO_FIELDS = {f.name for f in fields(ItemDTO)}
"""Caching ItemDTO fields for performance."""
LOG = logging.getLogger("datastore")
class StoreType(str, Enum):
HISTORY: str = "done"
QUEUE: str = "queue"
HISTORY = "done"
QUEUE = "queue"
@classmethod
def all(cls) -> list[str]:
@ -29,55 +23,57 @@ class StoreType(str, Enum):
@classmethod
def from_value(cls, value: str) -> "StoreType":
"""
Returns the StoreType enum member corresponding to the given value.
Args:
value (str): The value to match against the enum members.
Returns:
StoreType: The enum member that matches the value.
Raises:
ValueError: If the value does not match any member.
"""
for member in cls:
if member.value == value:
return member
msg = f"Invalid StoreType value: {value}"
raise ValueError(msg)
def __str__(self) -> str:
def __str__(self) -> str: # pragma: no cover
return self.value
def _strip_transient_fields(item: ItemDTO) -> ItemDTO:
stored = copy.deepcopy(item)
if hasattr(stored, "datetime"):
try:
delattr(stored, "datetime")
except AttributeError:
pass
if stored.status == "finished" and hasattr(stored, "live_in"):
try:
delattr(stored, "live_in")
except AttributeError:
pass
return stored
class DataStore:
"""
Persistent queue.
In-memory queue cache + optional write-behind to persistence.
History reads go straight to persistence.
"""
def __init__(self, type: StoreType, connection: Connection):
def __init__(self, type: StoreType, connection: SqliteStore):
self._type = type
self._connection: SqliteStore = connection
self._dict: OrderedDict[str, Download] = OrderedDict()
"The dictionary of items."
self._type: StoreType = type
"The type of the datastore."
self._connection: Connection = connection
"The database connection."
def load(self) -> None:
for id, item in self.saved_items():
self._dict.update({id: Download(info=item)})
# cache helpers
async def load(self) -> None:
saved = await self._connection.fetch_saved(str(self._type))
for key, item in saved:
self._dict[key] = Download(info=item)
async def saved_items(self) -> list[tuple[str, ItemDTO]]:
return await self._connection.fetch_saved(str(self._type))
def exists(self, key: str | None = None, url: str | None = None) -> bool:
if not key and not url:
msg = "key or url must be provided."
raise KeyError(msg)
if key and key in self._dict:
return True
return any(
(key and self._dict[i].info._id == key) or (url and self._dict[i].info.url == url) for i in self._dict
)
@ -86,261 +82,88 @@ class DataStore:
if not key and not url:
msg = "key or url must be provided."
raise KeyError(msg)
for i in self._dict:
if (key and self._dict[i].info._id == key) or (url and self._dict[i].info.url == url):
return self._dict[i]
msg: str = f"{key=} or {url=} not found."
raise KeyError(msg)
def get_item(self, **kwargs) -> Download | None:
"""
Get a specific item from the datastore based on provided attributes with optional operations.
Args:
**kwargs: Arbitrary keyword arguments representing attributes of the ItemDTO.
Each value can be either:
- A direct value (defaults to EQUAL operation): {"title": "test"}
- A tuple of (Operation, value): {"title": (Operation.CONTAIN, "test")}
If no attributes are provided, the method returns None.
If any attribute matches, the corresponding Download object is returned.
Returns:
Download | None: The requested item if found, otherwise None.
Examples:
# Direct equality check (default)
store.get_item(title="Video 1")
# Using operations
store.get_item(title=(Operation.CONTAIN, "test"))
store.get_item(id=(Operation.EQUAL, "123"), status=(Operation.NOT_EQUAL, "error"))
# Mixed usage
store.get_item(title=(Operation.CONTAIN, "test"), folder="downloads")
"""
if not kwargs:
return None
for i in self._dict:
if not self._dict[i].info:
continue
info = self._dict[i].info.__dict__
if any(matches_condition(key, value, info) for key, value in kwargs.items()):
return self._dict[i]
return None
async def get_by_id(self, id: str) -> Download | None:
val = self._dict.get(id)
if val or self._type == StoreType.QUEUE:
return val
if item := await self._connection.get_by_id(str(self._type), id):
download = Download(info=item)
self._dict[id] = download
return download
return None
def get_by_id(self, id: str) -> Download | None:
return self._dict.get(id, None)
def items(self) -> OrderedDict[tuple[str, Download]]:
def items(self):
return self._dict.items()
def saved_items(self) -> list[tuple[str, ItemDTO]]:
items: list[tuple[str, ItemDTO]] = []
cursor = self._connection.execute(
'SELECT "id", "data", "created_at" FROM "history" WHERE "type" = ? ORDER BY "created_at" ASC',
(str(self._type),),
)
for row in cursor:
rowDate: datetime = datetime.strptime(row["created_at"], "%Y-%m-%d %H:%M:%S") # noqa: DTZ007
data: dict = json.loads(row["data"])
data.pop("_id", None)
item: ItemDTO = init_class(ItemDTO, data, ITEM_DTO_FIELDS)
item._id = row["id"]
item.datetime = formatdate(rowDate.replace(tzinfo=UTC).timestamp())
items.append((row["id"], item))
return items
def put(self, value: Download, no_notify: bool = False) -> Download:
if "error" == value.info.status and not no_notify:
from app.library.Events import EventBus, Events
EventBus.get_instance().emit(Events.ITEM_ERROR, value.info)
self._dict.update({value.info._id: value})
self._update_store_item(self._type, value.info)
return self._dict[value.info._id]
async def put(self, value: Download, no_notify: bool = False) -> Download:
_ = no_notify
self._dict[value.info._id] = value
asyncio.create_task(self._connection.enqueue_upsert(str(self._type), _strip_transient_fields(value.info)))
return value
def delete(self, key: str) -> None:
self._dict.pop(key, None)
self._delete_store_item(key)
asyncio.create_task(self._connection.enqueue_delete(str(self._type), key))
def next(self) -> tuple[str, Download]:
def next(self):
return next(iter(self._dict.items()))
def empty(self):
def empty(self) -> bool:
return not bool(self._dict)
def has_downloads(self):
if 0 == len(self._dict):
return False
def has_downloads(self) -> bool:
return any(self._dict[key].info.auto_start and self._dict[key].started() is False for key in self._dict)
def get_next_download(self) -> Download:
def get_next_download(self) -> Download | None:
for key in self._dict:
ref = self._dict[key]
if ref.info.auto_start and ref.started() is False and ref.is_cancelled() is False:
if ref.info.auto_start and not ref.started() and not ref.is_cancelled():
return ref
return None
async def test(self) -> bool:
self._connection.execute('SELECT "id" FROM "history" LIMIT 1').fetchone()
return True
async def get_total_count(self, status_filter: str | None = None) -> int:
return await self._connection.count(str(self._type), status_filter=status_filter)
def get_total_count(self) -> int:
"""
Get the total count of items in the datastore.
Returns:
int: The total number of items in the datastore.
"""
cursor = self._connection.execute(
'SELECT COUNT(*) as count FROM "history" WHERE "type" = ?',
(str(self._type),),
)
row = cursor.fetchone()
return row["count"] if row else 0
def get_items_paginated(
self,
page: int = 1,
per_page: int = 50,
order: str = "DESC",
status_filter: str | None = None,
) -> tuple[list[tuple[str, ItemDTO]], int, int, int]:
"""
Get paginated items from the datastore.
Args:
page (int): The page number (1-indexed). Defaults to 1.
per_page (int): Number of items per page. Defaults to 50.
order (str): Sort order - 'ASC' or 'DESC'. Defaults to 'DESC' (newest first).
status_filter (str | None): Optional status filter. Can be a status value (e.g., 'finished')
to include only that status, or prefixed with '!' (e.g., '!finished') to exclude that status.
Returns:
tuple[list[tuple[str, ItemDTO]], int, int, int]: A tuple containing:
- List of (id, ItemDTO) tuples for the requested page
- Total number of items
- Current page number
- Total number of pages
Raises:
ValueError: If page < 1 or per_page < 1
"""
async def get_items_paginated(
self, page: int = 1, per_page: int = 50, order: str = "DESC", status_filter: str | None = None
):
if page < 1:
msg = "page must be >= 1"
raise ValueError(msg)
if per_page < 1:
msg = "per_page must be >= 1"
raise ValueError(msg)
order = order.upper()
if order not in ("ASC", "DESC"):
msg = f"order must be 'ASC' or 'DESC', got '{order}'"
raise ValueError(msg)
return await self._connection.paginate(str(self._type), page, per_page, order, status_filter)
order = "ASC" if order == "ASC" else "DESC"
async def bulk_delete(self, ids: Iterable[str]) -> int:
deleted = await self._connection.bulk_delete(str(self._type), ids)
for _id in ids:
self._dict.pop(_id, None)
return deleted
# Build SQL query with status filter if provided
where_clauses = ['"type" = ?']
query_params: list = [str(self._type)]
if status_filter:
# Check if it's an exclusion filter (starts with !)
if status_filter.startswith("!"):
status_value = status_filter[1:]
where_clauses.append("json_extract(data, '$.status') != ?")
query_params.append(status_value)
else:
where_clauses.append("json_extract(data, '$.status') = ?")
query_params.append(status_filter)
where_clause = " AND ".join(where_clauses)
# Get total count with filter
count_query = f'SELECT COUNT(*) as count FROM "history" WHERE {where_clause}' # noqa: S608
count_result = self._connection.execute(count_query, tuple(query_params)).fetchone()
total_items: int = count_result["count"] if count_result else 0
total_pages: int = (total_items + per_page - 1) // per_page if total_items > 0 else 1
# Ensure page is within valid range.
if page > total_pages and total_items > 0:
page = total_pages
offset = (page - 1) * per_page
items: list[tuple[str, ItemDTO]] = []
query_params.extend([per_page, offset])
cursor = self._connection.execute(
f'SELECT "id", "data", "created_at" FROM "history" WHERE {where_clause} ORDER BY "created_at" {order} LIMIT ? OFFSET ?', # noqa: S608
tuple(query_params),
)
for row in cursor:
rowDate: datetime = datetime.strptime(row["created_at"], "%Y-%m-%d %H:%M:%S") # noqa: DTZ007
data: dict = json.loads(row["data"])
data.pop("_id", None)
item: ItemDTO = init_class(ItemDTO, data, ITEM_DTO_FIELDS)
item._id = row["id"]
item.datetime = formatdate(rowDate.replace(tzinfo=UTC).timestamp())
items.append((row["id"], item))
return items, total_items, page, total_pages
def _update_store_item(self, type: StoreType, item: ItemDTO) -> None:
sqlStatement = """
INSERT INTO "history" ("id", "type", "url", "data")
VALUES (?, ?, ?, ?)
ON CONFLICT DO UPDATE SET "type" = ?, "url" = ?, "data" = ?, created_at = ?
"""
stored: ItemDTO = copy.deepcopy(item)
if hasattr(stored, "datetime"):
try:
delattr(stored, "datetime")
except AttributeError:
pass
if hasattr(stored, "live_in") and stored.status == "finished":
try:
delattr(stored, "live_in")
except AttributeError:
pass
encoded: str = stored.json()
self._connection.execute(
sqlStatement.strip(),
(
stored._id,
str(type),
stored.url,
encoded,
str(type),
stored.url,
encoded,
datetime.now(UTC).strftime("%Y-%m-%d %H:%M:%S"),
),
)
def _delete_store_item(self, key: str) -> None:
self._connection.execute('DELETE FROM "history" WHERE "type" = ? AND "id" = ?', (str(self._type), key))
async def test(self) -> bool:
await self._connection.count(str(self._type))
return True

44
app/library/DownloadQueue.py
View file

@ -9,7 +9,6 @@ import uuid
from datetime import UTC, datetime, timedelta
from email.utils import formatdate
from pathlib import Path
from sqlite3 import Connection
from typing import TYPE_CHECKING, Any
import yt_dlp.utils
@ -25,6 +24,7 @@ from .ItemDTO import Item, ItemDTO
from .Presets import Presets
from .Scheduler import Scheduler
from .Singleton import Singleton
from .sqlite_store import SqliteStore
from .Utils import (
archive_add,
arg_converter,
@ -49,14 +49,14 @@ class DownloadQueue(metaclass=Singleton):
DownloadQueue class is a singleton class that manages the download queue and the download history.
"""
def __init__(self, connection: Connection, config: Config | None = None):
def __init__(self, config: Config | None = None):
self.config: Config = config or Config.get_instance()
"Configuration instance."
self._notify: EventBus = EventBus.get_instance()
"Event bus instance."
self.done = DataStore(type=StoreType.HISTORY, connection=connection)
self.done = DataStore(type=StoreType.HISTORY, connection=SqliteStore.get_instance(config.db_file))
"DataStore for the completed downloads."
self.queue = DataStore(type=StoreType.QUEUE, connection=connection)
self.queue = DataStore(type=StoreType.QUEUE, connection=SqliteStore.get_instance(config.db_file))
"DataStore for the download queue."
self.workers = asyncio.Semaphore(self.config.max_workers)
"Semaphore to limit the number of concurrent downloads."
@ -71,8 +71,7 @@ class DownloadQueue(metaclass=Singleton):
self._active: dict[str, Download] = {}
"""Dictionary of active downloads."""
self.done.load()
self.queue.load()
# Loading happens in async initialize to avoid blocking loop creation.
self.paused.set()
@staticmethod
@ -166,6 +165,9 @@ class DownloadQueue(metaclass=Singleton):
"""
Initialize the download queue.
"""
await self.done.test()
await self.done.load()
await self.queue.load()
LOG.info(
f"Using '{self.config.max_workers}' workers for downloading and '{self.config.max_workers_per_extractor}' per extractor."
)
@ -199,7 +201,7 @@ class DownloadQueue(metaclass=Singleton):
continue
item.info.auto_start = True
updated: Download = self.queue.put(item)
updated: Download = await self.queue.put(item)
self._notify.emit(Events.ITEM_UPDATED, data=updated.info)
self._notify.emit(
Events.ITEM_RESUMED,
@ -246,7 +248,7 @@ class DownloadQueue(metaclass=Singleton):
continue
item.info.auto_start = False
updated: Download = self.queue.put(item)
updated: Download = await self.queue.put(item)
self._notify.emit(Events.ITEM_UPDATED, data=updated.info)
self._notify.emit(
Events.ITEM_PAUSED,
@ -534,7 +536,7 @@ class DownloadQueue(metaclass=Singleton):
message=nMessage,
)
itemDownload: Download = self.done.put(dlInfo)
itemDownload: Download = await self.done.put(dlInfo)
elif not hasFormats:
ava: str = entry.get("availability", "public")
nTitle = "Download Error"
@ -547,7 +549,7 @@ class DownloadQueue(metaclass=Singleton):
dlInfo.info.error = nMessage.replace(f" for '{dl.title}'.", ".") + text_logs
dlInfo.info.status = "error"
itemDownload = self.done.put(dlInfo)
itemDownload = await self.done.put(dlInfo)
self._notify.emit(
Events.LOG_WARNING,
@ -580,12 +582,12 @@ class DownloadQueue(metaclass=Singleton):
if _requeue:
nEvent = Events.ITEM_ADDED
itemDownload = self.queue.put(dlInfo)
itemDownload = await self.queue.put(dlInfo)
if item.auto_start:
self.event.set()
else:
dlInfo.info.status = "not_live"
itemDownload = self.done.put(dlInfo)
itemDownload = await self.done.put(dlInfo)
nStore = "history"
nEvent = Events.ITEM_MOVED
nTitle = "Premiering right now"
@ -596,7 +598,7 @@ class DownloadQueue(metaclass=Singleton):
nEvent = Events.ITEM_ADDED
nTitle = "Item Added"
nMessage = f"Item '{dlInfo.info.title}' has been added to the download queue."
itemDownload = self.queue.put(dlInfo)
itemDownload = await self.queue.put(dlInfo)
if item.auto_start:
self.event.set()
else:
@ -732,7 +734,7 @@ class DownloadQueue(metaclass=Singleton):
if archive_file := dlInfo.info.get_ytdlp_opts().get_all().get("download_archive"):
dlInfo.info.msg += f" Found in archive '{archive_file}'."
self.done.put(dlInfo)
await self.done.put(dlInfo)
self._notify.emit(
Events.ITEM_MOVED,
@ -816,7 +818,7 @@ class DownloadQueue(metaclass=Singleton):
extras=item.extras,
)
)
self.done.put(dlInfo)
await self.done.put(dlInfo)
LOG.info(log_message)
self._notify.emit(Events.LOG_INFO, data={}, title="Ignored Download", message=log_message)
@ -910,7 +912,7 @@ class DownloadQueue(metaclass=Singleton):
message=f"Download '{item.info.title}' has been cancelled.",
)
item.info.status = "cancelled"
self.done.put(item)
await self.done.put(item)
self._notify.emit(
Events.ITEM_MOVED,
data={"to": "history", "preset": item.info.preset, "item": item.info},
@ -1003,7 +1005,7 @@ class DownloadQueue(metaclass=Singleton):
return status
def get(self, mode: str = "all") -> dict[str, list[dict[str, ItemDTO]]]:
async def get(self, mode: str = "all") -> dict[str, list[dict[str, ItemDTO]]]:
"""
Get the download queue and the download history.
@ -1017,11 +1019,11 @@ class DownloadQueue(metaclass=Singleton):
items = {"queue": {}, "done": {}}
if mode in ("all", "queue"):
for k, v in self.queue.saved_items():
for k, v in await self.queue.saved_items():
items["queue"][k] = self._active[k].info if k in self._active else v
if mode in ("all", "done"):
for k, v in self.done.saved_items():
for k, v in await self.done.saved_items():
v.get_file_sidecar()
items["done"][k] = v
@ -1186,7 +1188,7 @@ class DownloadQueue(metaclass=Singleton):
self._notify.emit(Events.ITEM_COMPLETED, data=entry.info, title=nTitle, message=nMessage)
self.done.put(entry)
await self.done.put(entry)
self._notify.emit(
Events.ITEM_MOVED,
data={"to": "history", "preset": entry.info.preset, "item": entry.info},
@ -1288,7 +1290,7 @@ class DownloadQueue(metaclass=Singleton):
)
)
except Exception as e:
self.done.put(item)
await self.done.put(item)
LOG.exception(e)
LOG.error(f"Failed to retry item '{item_ref}'. {e!s}")

1
app/library/config.py
View file

@ -441,6 +441,7 @@ class Config(metaclass=Singleton):
("urllib3.connectionpool", logging.WARNING),
("apprise", logging.WARNING),
("httpcore", logging.INFO),
("aiosqlite", logging.INFO),
)
for _tool, _level in _log_levels:
logging.getLogger(_tool).setLevel(_level)

321
app/library/migrate.py Normal file
View file

@ -0,0 +1,321 @@
# flake8: noqa: S608
import contextlib
import datetime
import glob
import os.path
import traceback
from collections.abc import AsyncIterator, Iterable, Sequence
from importlib.machinery import ModuleSpec, SourceFileLoader
from importlib.util import module_from_spec, spec_from_loader
from typing import TYPE_CHECKING
import aiosqlite
if TYPE_CHECKING:
from types import ModuleType
UTC_LENGTH = 14
__version__ = "1.0.0"
class Error(Exception):
"""Base class for all Caribou errors."""
class InvalidMigrationError(Error):
"""Thrown when a client migration contains an error."""
class InvalidNameError(Error):
"""Thrown when a client migration has an invalid filename."""
def __init__(self, filename: str):
msg: str = (
f"Migration filenames must start with a UTC timestamp. The following file has an invalid name: {filename}"
)
super().__init__(msg)
@contextlib.asynccontextmanager
async def execute(
conn: aiosqlite.Connection, sql: str, params: Sequence[object] | None = None
) -> AsyncIterator[aiosqlite.Cursor]:
params = [] if params is None else params
cursor = await conn.execute(sql, params)
try:
yield cursor
finally:
await cursor.close()
@contextlib.asynccontextmanager
async def transaction(conn: aiosqlite.Connection) -> AsyncIterator[None]:
try:
yield
await conn.commit()
except Exception:
await conn.rollback()
raise
class Migration:
"""This class represents a migration version."""
def __init__(self, path: str):
self.path: str = path
self.filename: str = os.path.basename(path)
self.module_name, _ = os.path.splitext(self.filename)
self.get_version() # will assert the filename is valid
self.name: str = self.module_name[UTC_LENGTH:]
while self.name.startswith("_"):
self.name: str = self.name[1:]
loader = SourceFileLoader(self.module_name, path)
spec: ModuleSpec | None = spec_from_loader(self.module_name, loader)
if spec is None:
msg = f"Cannot create spec for {path}"
raise ImportError(msg)
try:
module: ModuleType = module_from_spec(spec)
loader.exec_module(module)
self.module: ModuleType = module
except Exception as e:
msg: str = f"Invalid migration {path}: {traceback.format_exc()}"
raise InvalidMigrationError(msg) from e
targets: list[str] = ["upgrade", "downgrade"]
missing: list[str] = [m for m in targets if not self.has_method(m)]
if missing:
msg = f"Migration '{self.path}' is missing required methods: {', '.join(missing)}."
raise InvalidMigrationError(msg)
def get_version(self) -> str:
if len(self.filename) < UTC_LENGTH:
raise InvalidNameError(self.filename)
timestamp: str = self.filename[:UTC_LENGTH]
if not timestamp.isdigit():
raise InvalidNameError(self.filename)
return timestamp
async def upgrade(self, conn: aiosqlite.Connection) -> None:
await self.module.upgrade(conn)
async def downgrade(self, conn: aiosqlite.Connection) -> None:
await self.module.downgrade(conn)
def has_method(self, name: str) -> bool:
return callable(getattr(self.module, name, None))
def __repr__(self) -> str:
return f"Migration(path={self.filename})"
class Database:
def __init__(self, db_url: aiosqlite.Connection | str, version_table: str = "migration_version"):
if not db_url:
msg = "Database requires db_url."
raise ValueError(msg)
self.version_table: str = version_table
self._owns_connection = bool(isinstance(db_url, str))
if self._owns_connection:
self.conn: aiosqlite.Connection | None = None
self.db_url: str = db_url
self._ensure_connection()
else:
self.conn: aiosqlite.Connection = db_url
async def __aenter__(self) -> "Database":
if self._owns_connection:
await self._ensure_connection()
return self
async def __aexit__(self, exc_type, exc, tb) -> None:
if not self._owns_connection:
return
await self.close()
async def close(self) -> None:
if self._owns_connection and self.conn:
await self.conn.close()
self.conn = None
async def _ensure_connection(self) -> None:
if self.conn:
return
self.conn = await aiosqlite.connect(self.db_url)
self.conn.row_factory = aiosqlite.Row
async def is_version_controlled(self) -> bool:
await self._ensure_connection()
assert self.conn
sql = """SELECT * FROM sqlite_master WHERE type = 'table' AND name = ?"""
async with execute(self.conn, sql, [self.version_table]) as cursor:
return bool(await cursor.fetchall())
async def upgrade(self, migrations: list[Migration], target_version: str | None = None) -> None:
await self._ensure_connection()
assert self.conn
if target_version:
_assert_migration_exists(migrations, target_version)
migrations.sort(key=lambda x: x.get_version())
database_version = await self.get_version()
for migration in migrations:
current_version: str = migration.get_version()
if database_version is not None and current_version <= database_version:
continue
if target_version and current_version > target_version:
break
await migration.upgrade(self.conn)
new_version: str = migration.get_version()
await self.update_version(new_version)
database_version: str = new_version
async def downgrade(self, migrations: list[Migration], target_version: str | int) -> None:
await self._ensure_connection()
assert self.conn
if target_version not in (0, "0"):
_assert_migration_exists(migrations, target_version)
migrations.sort(key=lambda x: x.get_version(), reverse=True)
database_version: str | None = await self.get_version()
for i, migration in enumerate(migrations):
current_version: str = migration.get_version()
if database_version is not None and current_version > database_version:
continue
if current_version <= target_version:
break
await migration.downgrade(self.conn)
next_version: str | int = 0
if i < len(migrations) - 1:
next_migration: Migration = migrations[i + 1]
next_version = next_migration.get_version()
await self.update_version(str(next_version))
database_version = str(next_version)
async def get_version(self) -> str | None:
"""
Return the database's version, or None if it is not under version
control.
"""
await self._ensure_connection()
assert self.conn
if not await self.is_version_controlled():
return None
sql: str = f"SELECT version FROM {self.version_table}"
async with execute(self.conn, sql) as cursor:
result = await cursor.fetchall()
return result[0][0] if result else "0"
async def update_version(self, version: str) -> None:
await self._ensure_connection()
assert self.conn
sql: str = f"UPDATE {self.version_table} SET version = ?"
async with transaction(self.conn):
await self.conn.execute(sql, [version])
async def initialize_version_control(self) -> None:
await self._ensure_connection()
assert self.conn
sql: str = f"""CREATE TABLE IF NOT EXISTS {self.version_table} ( version TEXT ) """
async with transaction(self.conn):
await self.conn.execute(sql)
await self.conn.execute(f"INSERT INTO {self.version_table} VALUES (0)")
def __repr__(self) -> str:
return f'Database("{self.db_url if self._owns_connection else "external_connection"}")'
def _assert_migration_exists(migrations: Iterable[Migration], version: str | int) -> None:
if version not in (m.get_version() for m in migrations):
msg: str = f"No migration with version {version} exists."
raise Error(msg)
def load_migrations(directory: str) -> list[Migration]:
"""Return the migrations contained in the given directory."""
directory = str(directory)
if not os.path.exists(directory) or not os.path.isdir(directory):
msg: str = f"{directory} is not a directory."
raise Error(msg)
return [Migration(f) for f in glob.glob(os.path.join(directory, "*.py"))]
async def upgrade(db_url: aiosqlite.Connection | str, migration_dir: str, version: str | None = None) -> None:
"""
Upgrade the given database with the migrations contained in the
migrations directory. If a version is not specified, upgrade
to the most recent version.
"""
async with Database(db_url) as db:
if not await db.is_version_controlled():
await db.initialize_version_control()
await db.upgrade(load_migrations(migration_dir), version)
async def downgrade(db_url: str | aiosqlite.Connection, migration_dir: str, version: str) -> None:
"""
Downgrade the database to the given version with the migrations
contained in the given migration directory.
"""
async with Database(db_url) as db:
if not await db.is_version_controlled():
msg = f"The database {db_url} is not version controlled."
raise Error(msg)
migrations = load_migrations(migration_dir)
await db.downgrade(migrations, version)
async def get_version(db_url: aiosqlite.Connection | str) -> str | None:
"""Return the migration version of the given database."""
async with Database(db_url) as db:
return await db.get_version()
def create_migration(name: str, directory: str | None = None) -> str:
"""
Create a migration with the given name. If no directory is specified,
the current working directory will be used.
"""
directory = directory if directory else "."
if not os.path.exists(directory) or not os.path.isdir(directory):
msg: str = f"{directory} is not a directory."
raise Error(msg)
now: datetime.datetime = datetime.datetime.now(tz=datetime.UTC)
version: str = now.strftime("%Y%m%d%H%M%S")
contents: str = MIGRATION_TEMPLATE % {"name": name, "version": version}
sanitized: str = name.replace(" ", "_")
filename: str = f"{version}_{sanitized}.py"
path: str = os.path.join(directory, filename)
with open(path, "w") as migration_file:
migration_file.write(contents)
return path
MIGRATION_TEMPLATE = """\
\"\"\"
This module contains a db migration.
Migration Name: %(name)s
Migration Version: %(version)s
\"\"\"
async def upgrade(c):
# add your upgrade step here
await c.execute("SELECT 1")
async def downgrade(c):
# add your downgrade step here
await c.execute("SELECT 1")
"""

348
app/library/sqlite_store.py Normal file
View file

@ -0,0 +1,348 @@
import asyncio
import contextlib
import json
import logging
import os
from collections.abc import Iterable
from dataclasses import fields
from datetime import UTC, datetime
from email.utils import formatdate
import aiosqlite
from aiohttp import web
from .ItemDTO import ItemDTO
from .Singleton import ThreadSafe
from .Utils import init_class
LOG: logging.Logger = logging.getLogger(__name__)
ITEM_DTO_FIELDS: set[str] = {f.name for f in fields(ItemDTO)}
class Terminator:
pass
class _Op:
"""Queued write operation."""
__slots__ = ("item", "key", "keys", "op", "type_value")
def __init__(
self, op: Terminator | str, type_value: str, item: ItemDTO | None, key: str | None, keys: list[str] | None
):
self.op = op
self.type_value = type_value
self.item = item
self.key = key
self.keys = keys
class SqliteStore(metaclass=ThreadSafe):
"""
Async persistence layer with back-pressure and write-behind queue.
Singleton per process (ThreadSafe). Owns its aiosqlite connection.
"""
@staticmethod
def get_instance(db_path: str | None = None) -> "SqliteStore":
return SqliteStore(db_path)
def attach(self, app: web.Application):
"""Get/create singleton bound to db_path."""
app.on_shutdown.append(self.on_shutdown)
async def on_shutdown(self, _: web.Application):
"""Close singleton on app shutdown."""
LOG.debug("Shutting down SqliteStore...")
await self.close()
LOG.debug("SqliteStore shut down complete.")
def __init__(self, db_path: str | None = None, *, max_pending: int = 200, flush_interval: float = 0.05):
self._db_path = db_path
self._conn: aiosqlite.Connection | None = None
self._queue: asyncio.Queue[_Op] | None = None
self._task: asyncio.Task | None = None
self._flush_interval = flush_interval
self._max_pending = max_pending
self._lock = asyncio.Lock()
async def __aenter__(self) -> "SqliteStore":
await self._ensure_conn()
return self
async def __aexit__(self, exc_type, exc, tb) -> None:
await self.close()
# ---------- public API ----------
async def fetch_saved(self, type_value: str) -> list[tuple[str, ItemDTO]]:
await self._ensure_conn()
cursor = await self._conn.execute(
'SELECT "id", "data", "created_at" FROM "history" WHERE "type" = ? ORDER BY "created_at" ASC',
(type_value,),
)
items: list[tuple[str, ItemDTO]] = []
async with cursor:
async for row in cursor:
row_date = datetime.strptime(row["created_at"], "%Y-%m-%d %H:%M:%S") # noqa: DTZ007
data = json.loads(row["data"])
data.pop("_id", None)
item = init_class(ItemDTO, data, ITEM_DTO_FIELDS)
item._id = row["id"]
item.datetime = formatdate(row_date.replace(tzinfo=UTC).timestamp())
items.append((row["id"], item))
return items
async def get_by_id(self, type_value: str, id: str) -> ItemDTO | None:
await self._ensure_conn()
cursor = await self._conn.execute(
'SELECT "id", "data", "created_at" FROM "history" WHERE "type" = ? AND "id" = ?',
(type_value, id),
)
row = await cursor.fetchone()
if not row:
return None
row_date = datetime.strptime(row["created_at"], "%Y-%m-%d %H:%M:%S") # noqa: DTZ007
data = json.loads(row["data"])
data.pop("_id", None)
item = init_class(ItemDTO, data, ITEM_DTO_FIELDS)
item._id = row["id"]
item.datetime = formatdate(row_date.replace(tzinfo=UTC).timestamp())
return item
async def count(self, type_value: str, status_filter: str | None = None) -> int:
await self._ensure_conn()
where_clauses = ['"type" = ?']
query_params: list[str] = [type_value]
if status_filter:
if status_filter.startswith("!"):
status_value = status_filter[1:]
where_clauses.append("json_extract(data, '$.status') != ?")
query_params.append(status_value)
else:
where_clauses.append("json_extract(data, '$.status') = ?")
query_params.append(status_filter)
where_clause = " AND ".join(where_clauses)
count_query = f'SELECT COUNT(*) as count FROM "history" WHERE {where_clause}' # noqa: S608
cursor = await self._conn.execute(count_query, tuple(query_params))
row = await cursor.fetchone()
return row["count"] if row else 0
async def paginate(
self,
type_value: str,
page: int,
per_page: int,
order: str,
status_filter: str | None = None,
) -> tuple[list[tuple[str, ItemDTO]], int, int, int]:
await self._ensure_conn()
where_clauses = ['"type" = ?']
query_params: list[str | int] = [type_value]
if status_filter:
if status_filter.startswith("!"):
status_value = status_filter[1:]
where_clauses.append("json_extract(data, '$.status') != ?")
query_params.append(status_value)
else:
where_clauses.append("json_extract(data, '$.status') = ?")
query_params.append(status_filter)
where_clause = " AND ".join(where_clauses)
count_query = f'SELECT COUNT(*) as count FROM "history" WHERE {where_clause}' # noqa: S608
count_cursor = await self._conn.execute(count_query, tuple(query_params))
count_row = await count_cursor.fetchone()
total_items = count_row["count"] if count_row else 0
total_pages = (total_items + per_page - 1) // per_page if total_items > 0 else 1
if page > total_pages and total_items > 0:
page = total_pages
offset = (page - 1) * per_page
query_params.extend([per_page, offset])
cursor = await self._conn.execute(
f'SELECT "id", "data", "created_at" FROM "history" WHERE {where_clause} ORDER BY "created_at" {order} LIMIT ? OFFSET ?', # noqa: S608
tuple(query_params),
)
items: list[tuple[str, ItemDTO]] = []
async with cursor:
async for row in cursor:
row_date = datetime.strptime(row["created_at"], "%Y-%m-%d %H:%M:%S") # noqa: DTZ007
data = json.loads(row["data"])
data.pop("_id", None)
item = init_class(ItemDTO, data, ITEM_DTO_FIELDS)
item._id = row["id"]
item.datetime = formatdate(row_date.replace(tzinfo=UTC).timestamp())
items.append((row["id"], item))
return items, total_items, page, total_pages
# direct CRUD helpers (used by DataStore)
async def upsert(self, type_value: str, item: ItemDTO) -> None:
await self._ensure_conn()
await self._upsert_now(type_value, item)
async def delete(self, type_value: str, key: str) -> None:
await self._ensure_conn()
await self._conn.execute('DELETE FROM "history" WHERE "type" = ? AND "id" = ?', (type_value, key))
async def bulk_delete(self, type_value: str, keys: Iterable[str]) -> int:
await self._ensure_conn()
keys_list = list(keys)
if not keys_list:
return 0
placeholders = ",".join("?" for _ in keys_list)
params = [type_value, *keys_list]
cur = await self._conn.execute(
f'DELETE FROM "history" WHERE "type" = ? AND "id" IN ({placeholders})', # noqa: S608
tuple(params),
)
return cur.rowcount if cur else 0
async def commit(self) -> None:
if self._conn:
await self._conn.commit()
async def enqueue_upsert(self, type_value: str, item: ItemDTO) -> None:
await self._enqueue(_Op("upsert", type_value, item, None, None))
async def enqueue_delete(self, type_value: str, key: str) -> None:
await self._enqueue(_Op("delete", type_value, None, key, None))
async def enqueue_bulk_delete(self, type_value: str, keys: Iterable[str]) -> None:
await self._enqueue(_Op("bulk_delete", type_value, None, None, list(keys)))
async def flush(self) -> None:
if self._queue:
await self._queue.join()
if self._conn:
await self._conn.commit()
async def shutdown(self) -> None:
"""Flush pending writes, stop writer task, and commit."""
if self._queue:
try:
await self._queue.put(_Op(Terminator(), "", None, None, None))
LOG.debug("Waiting for SqliteStore queue to drain...")
await asyncio.wait_for(self._queue.join(), timeout=2)
except TimeoutError:
LOG.warning("SqliteStore queue did not drain within timeout; forcing writer shutdown.")
if self._task:
LOG.debug("Waiting for SqliteStore writer task to finish...")
if not self._task.done():
self._task.cancel()
with contextlib.suppress(asyncio.CancelledError):
await asyncio.wait_for(self._task, timeout=2)
if self._conn:
LOG.debug("Committing final changes to SqliteStore...")
await self._conn.commit()
self._task = None
self._queue = None
async def close(self) -> None:
await self.shutdown()
if self._conn:
LOG.debug("Closing SqliteStore connection...")
await self._conn.close()
self._conn = None
async def _enqueue(self, op: _Op) -> None:
self._ensure_worker()
await self._queue.put(op)
def _ensure_worker(self) -> None:
if self._queue is None:
self._queue = asyncio.Queue(maxsize=self._max_pending)
self._task = asyncio.create_task(self._writer(), name="sqlite-store-writer")
async def _writer(self) -> None:
while True:
op = await self._queue.get()
if isinstance(op.op, Terminator):
self._queue.task_done()
break
try:
async with self._lock:
await self._apply(op)
except Exception as ex:
LOG.exception(ex)
finally:
self._queue.task_done()
await asyncio.sleep(self._flush_interval)
async def _apply(self, op: _Op) -> None:
await self._ensure_conn()
if op.op == "upsert" and op.item:
await self._upsert_now(op.type_value, op.item)
elif op.op == "delete" and op.key:
await self._conn.execute('DELETE FROM "history" WHERE "type" = ? AND "id" = ?', (op.type_value, op.key))
elif op.op == "bulk_delete" and op.keys:
placeholders = ",".join("?" for _ in op.keys)
params = [op.type_value, *op.keys]
await self._conn.execute(
f'DELETE FROM "history" WHERE "type" = ? AND "id" IN ({placeholders})', # noqa: S608
tuple(params),
)
await self._conn.commit()
async def _upsert_now(self, type_value: str, item: ItemDTO) -> None:
await self._ensure_conn()
sql = """
INSERT INTO "history" ("id", "type", "url", "data")
VALUES (?, ?, ?, ?)
ON CONFLICT DO UPDATE SET "type" = ?, "url" = ?, "data" = ?, created_at = ?
"""
encoded = item.json()
await self._conn.execute(
sql.strip(),
(
item._id,
type_value,
item.url,
encoded,
type_value,
item.url,
encoded,
datetime.now(UTC).strftime("%Y-%m-%d %H:%M:%S"),
),
)
async def _ensure_conn(self) -> None:
if self._conn:
return
if not self._db_path:
msg = "SqliteStore requires db_path or injected connection."
raise RuntimeError(msg)
from app.library import migrate
from app.main import ROOT_PATH
if not self._db_path.startswith(":memory"):
os.makedirs(os.path.dirname(self._db_path) or ".", exist_ok=True)
self._conn = await aiosqlite.connect(database=self._db_path, isolation_level=None)
self._conn.row_factory = aiosqlite.Row
version = await migrate.get_version(self._conn)
if version:
LOG.debug(f"DB Version: '{version}'.")
await migrate.upgrade(self._conn, ROOT_PATH / "migrations")
if not version:
version = await migrate.get_version(self._conn)
LOG.debug(f"DB Version after initial migration: '{version}'.")
await self._conn.execute("PRAGMA journal_mode=wal")
await self._conn.execute("PRAGMA busy_timeout=5000")
await self._conn.execute("PRAGMA foreign_keys=ON")
await self._conn.commit()

20
app/main.py
View file

@ -9,16 +9,15 @@ if APP_ROOT not in sys.path:
import asyncio
import logging
import sqlite3
from pathlib import Path
import caribou
import magic
from aiohttp import web
from app.library.BackgroundWorker import BackgroundWorker
from app.library.conditions import Conditions
from app.library.config import Config
from app.library.DataStore import SqliteStore
from app.library.dl_fields import DLFields
from app.library.DownloadQueue import DownloadQueue
from app.library.Events import EventBus, Events
@ -50,18 +49,6 @@ class Main:
self._check_folders()
LOG.debug(f"DB Version: '{caribou.get_version(self._config.db_file)}'.")
caribou.upgrade(self._config.db_file, ROOT_PATH / "migrations")
connection = sqlite3.connect(database=self._config.db_file, isolation_level=None)
connection.row_factory = sqlite3.Row
connection.execute("PRAGMA journal_mode=wal")
async def _close_connection(_):
LOG.debug("Closing database connection.")
connection.close()
LOG.debug("Database connection closed.")
try:
db_file = Path(self._config.db_file)
if "600" != oct(db_file.stat().st_mode)[-3:]:
@ -69,12 +56,10 @@ class Main:
except Exception:
pass
self._queue = DownloadQueue(connection=connection)
self._queue = DownloadQueue(config=self._config)
self._http = HttpAPI(root_path=ROOT_PATH, queue=self._queue)
self._socket = HttpSocket(root_path=ROOT_PATH, queue=self._queue)
self._app.on_cleanup.append(_close_connection)
def _check_folders(self):
"""Check if the required folders exist and create them if they do not."""
folders = (self._config.download_path, self._config.temp_path, self._config.config_path)
@ -136,6 +121,7 @@ class Main:
Conditions.get_instance().attach(self._app)
DLFields.get_instance().attach(self._app)
TaskDefinitions.get_instance().attach(self._app)
SqliteStore.get_instance().attach(self._app)
self._background_worker.attach(self._app)
EventBus.get_instance().emit(

16
app/migrations/20231115152938_initial.py
View file

@ -1,12 +1,12 @@
"""
This module contains a Caribou migration.
This module contains a db migration.
Migration Name: initial
Migration Version: 20231115152938
"""
def upgrade(connection):
async def upgrade(c):
sql = """
CREATE TABLE "history" (
"id" TEXT PRIMARY KEY UNIQUE NOT NULL,
@ -16,24 +16,24 @@ def upgrade(connection):
"created_at" TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
);
"""
connection.execute(sql)
await c.execute(sql)
sql = """
CREATE INDEX "history_type" ON "history" ("type");
"""
connection.execute(sql)
await c.execute(sql)
sql = """
CREATE UNIQUE INDEX "history_url" ON "history" ("url");
"""
connection.execute(sql)
await c.execute(sql)
connection.commit()
await c.commit()
def downgrade(connection):
async def downgrade(c):
sql = """
DROP TABLE IF EXISTS "history";
"""
connection.execute(sql)
await c.execute(sql)

10
app/migrations/20251116173731_add_status_index.py
View file

@ -1,11 +1,11 @@
"""
This module contains a Caribou migration.
This module contains a db migration.
Migration Name: add_status_index
Migration Version: 20251116173731
"""
def upgrade(connection):
async def upgrade(connection):
"""
Add index on json_extract(data, '$.status') for better query performance.
@ -15,13 +15,13 @@ def upgrade(connection):
sql = """
CREATE INDEX IF NOT EXISTS "history_status" ON "history" (json_extract("data", '$.status'));
"""
connection.execute(sql)
await connection.execute(sql)
def downgrade(connection):
async def downgrade(connection):
"""
Remove the status index.
"""
sql = """
DROP INDEX IF EXISTS "history_status";
"""
connection.execute(sql)
await connection.execute(sql)

4
app/routes/api/browser.py
View file

@ -402,7 +402,7 @@ async def path_actions(request: Request, config: Config, queue: DownloadQueue, n
if sidecar_renamed:
item.info.get_file_sidecar()
queue.done.put(item)
await queue.done.put(item)
notify.emit(Events.ITEM_UPDATED, data=item.info)
if "delete" == action:
@ -508,7 +508,7 @@ async def path_actions(request: Request, config: Config, queue: DownloadQueue, n
if sidecar_moved:
item.info.get_file_sidecar()
queue.done.put(item)
await queue.done.put(item)
notify.emit(Events.ITEM_UPDATED, data=item.info)
return web.json_response(data=operations_status, status=web.HTTPOk.status_code)

68
app/routes/api/history.py
View file

@ -47,30 +47,21 @@ async def items_list(request: Request, queue: DownloadQueue, encoder: Encoder, c
"""
from app.library.DataStore import StoreType
store_type = request.query.get("type", "all").lower()
stores: list[str] = ["all", StoreType.QUEUE.value, StoreType.HISTORY.value]
if store_type not in stores:
store_type: str = request.query.get("type", "queue").lower()
try:
store_type: StoreType = StoreType.from_value(store_type)
except ValueError:
return web.json_response(
data={"error": f"type must be one of {', '.join(stores)}."},
data={"error": f"type must be one of {', '.join(StoreType.all())}."},
status=web.HTTPBadRequest.status_code,
)
# Legacy behavior: return all items without pagination, will be removed in future.
if "all" == store_type:
data: dict = {"queue": [], "history": []}
q = queue.get()
data["queue"].extend([q.get("queue", {}).get(k) for k in q.get("queue", {})])
data["history"].extend([q.get("done", {}).get(k) for k in q.get("done", {})])
return web.json_response(data=data, status=web.HTTPOk.status_code, dumps=encoder.encode)
ds = queue.queue if store_type == StoreType.QUEUE.value else queue.done
ds = queue.queue if store_type == StoreType.QUEUE else queue.done
try:
page = int(request.query.get("page", 1))
per_page = int(request.query.get("per_page", config.default_pagination))
order = request.query.get("order", "DESC").upper()
order: str = request.query.get("order", "DESC").upper()
except ValueError:
return web.json_response(
data={"error": "page and per_page must be valid integers."},
@ -92,13 +83,15 @@ async def items_list(request: Request, queue: DownloadQueue, encoder: Encoder, c
status=web.HTTPBadRequest.status_code,
)
# Parse status filter
status_filter = request.query.get("status", None)
items, total, current_page, total_pages = ds.get_items_paginated(
items, total, current_page, total_pages = await ds.get_items_paginated(
page=page, per_page=per_page, order=order, status_filter=status_filter
)
return web.json_response(
data={
"type": store_type.value,
"pagination": {
"page": current_page,
"per_page": per_page,
@ -108,9 +101,10 @@ async def items_list(request: Request, queue: DownloadQueue, encoder: Encoder, c
"has_prev": current_page > 1,
},
"items": [item for _, item in items],
}
return web.json_response(data=data, status=web.HTTPOk.status_code, dumps=encoder.encode)
},
status=web.HTTPOk.status_code,
dumps=encoder.encode,
)
@route("DELETE", "api/history/", "items_delete")
@ -137,24 +131,26 @@ async def items_delete(request: Request, queue: DownloadQueue, encoder: Encoder)
ids = data.get("ids")
remove_file: bool = bool(data.get("remove_file", True))
_type = data.get("type", data.get("where"))
storeType = data.get("type", data.get("where"))
if not _type:
if not storeType:
return web.json_response(data={"error": "Type is required."}, status=web.HTTPBadRequest.status_code)
if _type not in [StoreType.QUEUE.value, StoreType.HISTORY.value]:
try:
storeType: StoreType = StoreType.from_value(storeType)
except ValueError:
return web.json_response(
data={"error": f"type must be '{StoreType.QUEUE.value}' or '{StoreType.HISTORY.value}'."},
data={"error": f"type must be one of '{StoreType.all()}'."},
status=web.HTTPBadRequest.status_code,
)
ds = queue.queue if _type == StoreType.QUEUE.value else queue.done
ds = queue.queue if storeType == StoreType.QUEUE else queue.done
if ids:
return web.json_response(
data={
"items": await (
queue.cancel(ids) if _type == StoreType.QUEUE.value else queue.clear(ids, remove_file=remove_file)
queue.cancel(ids) if storeType == StoreType.QUEUE else queue.clear(ids, remove_file=remove_file)
),
"deleted": len(ids),
},
@ -174,7 +170,7 @@ async def items_delete(request: Request, queue: DownloadQueue, encoder: Encoder)
per_page = 1000
while True:
items, _, current_page, total_pages = ds.get_items_paginated(
items, _, current_page, total_pages = await ds.get_items_paginated(
page=page, per_page=per_page, order="DESC", status_filter=status_filter
)
@ -192,7 +188,7 @@ async def items_delete(request: Request, queue: DownloadQueue, encoder: Encoder)
data={
"items": await (
queue.cancel(items_to_delete)
if _type == StoreType.QUEUE.value
if storeType == StoreType.QUEUE
else queue.clear(items_to_delete, remove_file=remove_file)
),
"deleted": len(items_to_delete),
@ -221,7 +217,7 @@ async def item_view(request: Request, queue: DownloadQueue, encoder: Encoder) ->
if not id:
return web.json_response(data={"error": "id is required."}, status=web.HTTPBadRequest.status_code)
item: Download | None = queue.done.get_by_id(id) or queue.queue.get_by_id(id)
item: Download | None = await queue.done.get_by_id(id) or await queue.queue.get_by_id(id)
if not item:
return web.json_response(data={"error": "item not found."}, status=web.HTTPNotFound.status_code)
@ -263,7 +259,7 @@ async def item_update(request: Request, queue: DownloadQueue, encoder: Encoder,
if not id:
return web.json_response(data={"error": "id is required."}, status=web.HTTPBadRequest.status_code)
item: Download | None = queue.done.get_by_id(id)
item: Download | None = await queue.done.get_by_id(id)
if not item:
return web.json_response(data={"error": "item not found."}, status=web.HTTPNotFound.status_code)
@ -285,7 +281,7 @@ async def item_update(request: Request, queue: DownloadQueue, encoder: Encoder,
LOG.debug(f"Updated '{k}' to '{v}' for '{item.info.id}'")
if updated:
queue.done.put(item)
await queue.done.put(item)
notify.emit(Events.ITEM_UPDATED, data=item.info)
return web.json_response(
@ -394,7 +390,7 @@ async def item_archive_add(request: Request, queue: DownloadQueue, notify: Event
return web.json_response(data={"error": "id is required."}, status=web.HTTPBadRequest.status_code)
try:
item: Download | None = queue.done.get_by_id(id)
item: Download | None = await queue.done.get_by_id(id)
if not item:
return web.json_response(data={"error": f"item '{id}' not found."}, status=web.HTTPNotFound.status_code)
except KeyError:
@ -425,7 +421,7 @@ async def item_archive_add(request: Request, queue: DownloadQueue, notify: Event
)
item.info.archive_status(force=True)
queue.done.put(item, no_notify=True)
await queue.done.put(item, no_notify=True)
notify.emit(Events.ITEM_UPDATED, data=item.info)
return web.json_response(
@ -452,7 +448,7 @@ async def item_archive_delete(request: Request, queue: DownloadQueue, notify: Ev
return web.json_response(data={"error": "id is required."}, status=web.HTTPBadRequest.status_code)
try:
item: Download | None = queue.done.get_by_id(id)
item: Download | None = await queue.done.get_by_id(id)
if not item:
return web.json_response(data={"error": f"item '{id}' not found."}, status=web.HTTPNotFound.status_code)
except KeyError:
@ -483,7 +479,7 @@ async def item_archive_delete(request: Request, queue: DownloadQueue, notify: Ev
)
item.info.archive_status(force=True)
queue.done.put(item, no_notify=True)
await queue.done.put(item, no_notify=True)
notify.emit(Events.ITEM_UPDATED, data=item.info)
return web.json_response(

4
app/routes/socket/connection.py
View file

@ -44,8 +44,8 @@ async def connect(config: Config, queue: DownloadQueue, notify: EventBus, sid: s
base=Path(config.download_path),
depth_limit=config.download_path_depth - 1,
),
"history_count": queue.done.get_total_count(),
"queue": queue.get("queue")["queue"],
"history_count": await queue.done.get_total_count(),
"queue": (await queue.get("queue"))["queue"],
},
title="Sending initial download data",
message=f"Sending initial download data to client '{sid}'.",

147
app/scripts/create_migration.py Normal file
View file

@ -0,0 +1,147 @@
#!/usr/bin/env python3
"""
Caribou migrations CLI.
Provides commands to create, list, and apply database migrations.
"""
from __future__ import annotations
import argparse
import sys
import traceback
from pathlib import Path
from typing import TYPE_CHECKING
APP_ROOT: Path = Path(__file__).resolve().parents[2]
if str(APP_ROOT) not in sys.path:
sys.path.insert(0, str(APP_ROOT))
from app.library import migrate
if TYPE_CHECKING:
from collections.abc import Callable
def e_print(msg: str = "") -> None:
sys.stderr.write(f"{msg}\n")
def o_print(msg: str = "") -> None:
sys.stdout.write(f"{msg}\n")
def cmd_info(_: argparse.Namespace) -> None:
o_print(f"Version: {migrate.__version__}")
def cmd_create(args: argparse.Namespace) -> None:
path = migrate.create_migration(args.name, args.migration_dir)
o_print(f"created migration {path}")
def cmd_version(args: argparse.Namespace) -> None:
version = migrate.get_version(args.database_path)
if version:
o_print(f"the db [{args.database_path}] is at version {version}")
else:
o_print(f"the db [{args.database_path}] is not under version control")
def cmd_upgrade(args: argparse.Namespace) -> None:
if args.version:
o_print(f"upgrading db [{args.database_path}] to version [{args.version}]")
else:
o_print(f"upgrading db [{args.database_path}] to most recent version")
migrate.upgrade(args.database_path, args.migration_dir, args.version)
new_version = migrate.get_version(args.database_path)
if args.version is not None:
assert new_version == args.version
o_print(f"upgraded [{args.database_path}] successfully to version [{new_version}]")
def cmd_downgrade(args: argparse.Namespace) -> None:
o_print(f"downgrading db [{args.database_path}] to version [{args.version}]")
migrate.downgrade(args.database_path, args.migration_dir, args.version)
o_print(f"downgraded [{args.database_path}] successfully to version [{args.version}]")
def cmd_list(args: argparse.Namespace) -> None:
o_print(f"Migrations in [{args.migration_dir}]:\n")
for m in migrate.load_migrations(args.migration_dir):
o_print(f"{m.get_version()}\t{m.name}\t{m.path}")
def build_parser() -> argparse.ArgumentParser:
p = argparse.ArgumentParser(prog="caribou")
sub = p.add_subparsers(dest="command", required=True)
info = sub.add_parser("info", help="print library version")
info.set_defaults(func=cmd_info)
create = sub.add_parser("create", help="create a new migration file")
create.add_argument("name", help="the name of migration")
create.add_argument("-d", "--migration-dir", default=".", help="the migration directory")
create.set_defaults(func=cmd_create)
ver = sub.add_parser("version", help="return the migration version of the database")
ver.add_argument("database_path", help="path to the sqlite database")
ver.set_defaults(func=cmd_version)
up = sub.add_parser(
"upgrade",
help="upgrade the db (if version isn't specified, upgrade to the most recent)",
)
up.add_argument("database_path", help="path to the sqlite database")
up.add_argument("migration_dir", help="the migration directory")
up.add_argument("-v", "--version", type=int, default=None, help="the target migration version")
up.set_defaults(func=cmd_upgrade)
down = sub.add_parser(
"downgrade",
help="downgrade the db to a particular version (use 0 to rollback all changes)",
)
down.add_argument("database_path", help="path to the sqlite database")
down.add_argument("migration_dir", help="the migration directory")
down.add_argument("version", type=int, help="the target migration version")
down.set_defaults(func=cmd_downgrade)
lst = sub.add_parser("list", help="list the migration versions")
lst.add_argument("migration_dir", help="the migration directory")
lst.set_defaults(func=cmd_list)
return p
def run(argv: list[str]) -> int:
parser = build_parser()
args = parser.parse_args(argv)
try:
func: Callable[[argparse.Namespace], None] = args.func
func(args)
return 0
except migrate.InvalidMigrationError:
e_print(traceback.format_exc())
return 1
except migrate.Error as err:
e_print(f"Error: {err}")
return 1
except Exception:
e_print("an unexpected error occurred:")
e_print(traceback.format_exc())
return 1
def main() -> int:
return run(sys.argv[1:])
if __name__ == "__main__":
raise SystemExit(main())

591
app/tests/test_datastore.py
View file

File diff suppressed because it is too large Load diff

319
app/tests/test_datastore_pagination.py
View file

@ -1,45 +1,25 @@
import json
import sqlite3
from datetime import UTC, datetime
from pathlib import Path
from tempfile import TemporaryDirectory
import pytest
import pytest_asyncio
from app.library.DataStore import DataStore, StoreType
from app.library.ItemDTO import ItemDTO
from app.library.sqlite_store import SqliteStore
class TestDataStorePagination:
"""Test pagination functionality of DataStore."""
@pytest.fixture
def temp_db(self):
async def make_db(data: int = 100) -> SqliteStore:
"""Create a temporary database with test data."""
with TemporaryDirectory() as tmpdir:
db_path = Path(tmpdir) / "test.db"
conn = sqlite3.connect(str(db_path))
conn.row_factory = sqlite3.Row
SqliteStore._reset_singleton()
ins = SqliteStore.get_instance(db_path=":memory:")
await ins._ensure_conn()
# Create the history table
conn.execute(
"""
CREATE TABLE IF NOT EXISTS "history" (
"id" TEXT PRIMARY KEY,
"type" TEXT NOT NULL,
"url" TEXT NOT NULL,
"data" TEXT NOT NULL,
"created_at" TEXT NOT NULL
)
"""
)
# Insert test data
base_time = datetime.now(UTC)
for i in range(100):
created_at = base_time.replace(
hour=(i // 4) % 24, minute=(i * 15) % 60, second=i % 60
).strftime("%Y-%m-%d %H:%M:%S")
for i in range(data):
created_at = base_time.replace(hour=(i // 4) % 24, minute=(i * 15) % 60, second=i % 60).strftime(
"%Y-%m-%d %H:%M:%S"
)
item_data = {
"url": f"https://example.com/video{i}",
@ -49,7 +29,7 @@ class TestDataStorePagination:
"status": "finished",
}
conn.execute(
await ins._conn.execute(
'INSERT INTO "history" ("id", "type", "url", "data", "created_at") VALUES (?, ?, ?, ?, ?)',
(
f"test-id-{i}",
@ -59,199 +39,172 @@ class TestDataStorePagination:
created_at,
),
)
if data > 0:
await ins._conn.commit()
conn.commit()
yield conn
conn.close()
return ins
def test_get_total_count(self, temp_db):
@pytest.mark.asyncio
class TestDataStorePagination:
"""Test pagination functionality of DataStore."""
@pytest_asyncio.fixture
async def make_db(self, data: int = 100):
"""Fixture to provide a temporary database with test data."""
return await make_db(data=data)
async def test_get_total_count(self):
"""Test getting total count of items."""
datastore = DataStore(type=StoreType.HISTORY, connection=temp_db)
count = datastore.get_total_count()
db = await make_db(data=100)
try:
datastore = DataStore(type=StoreType.HISTORY, connection=db)
count = await datastore.get_total_count()
assert count == 100
finally:
await db.close()
def test_pagination_basic(self, temp_db):
async def test_pagination_basic(self):
"""Test basic pagination functionality."""
datastore = DataStore(type=StoreType.HISTORY, connection=temp_db)
# Get first page
items, total, page, total_pages = datastore.get_items_paginated(page=1, per_page=10)
db = await make_db(data=100)
try:
datastore = DataStore(type=StoreType.HISTORY, connection=db)
items, total, page, total_pages = await datastore.get_items_paginated(page=1, per_page=10)
assert len(items) == 10
assert total == 100
assert page == 1
assert total_pages == 10
# Verify items are ItemDTO instances
for _item_id, item in items:
for _, item in items:
assert isinstance(item, ItemDTO)
assert item._id.startswith("test-id-")
finally:
await db.close()
def test_pagination_last_page(self, temp_db):
async def test_pagination_last_page(self):
"""Test pagination on last page."""
datastore = DataStore(type=StoreType.HISTORY, connection=temp_db)
# Get last page
items, total, page, total_pages = datastore.get_items_paginated(page=10, per_page=10)
db = await make_db(data=100)
try:
datastore = DataStore(type=StoreType.HISTORY, connection=db)
items, total, page, total_pages = await datastore.get_items_paginated(page=10, per_page=10)
assert len(items) == 10
assert total == 100
assert page == 10
assert total_pages == 10
finally:
await db.close()
def test_pagination_partial_page(self, temp_db):
async def test_pagination_partial_page(self):
"""Test pagination with partial last page."""
datastore = DataStore(type=StoreType.HISTORY, connection=temp_db)
# Get items with per_page that doesn't divide evenly
items, total, page, total_pages = datastore.get_items_paginated(page=4, per_page=30)
db = await make_db(data=100)
try:
datastore = DataStore(type=StoreType.HISTORY, connection=db)
items, total, page, total_pages = await datastore.get_items_paginated(page=4, per_page=30)
assert len(items) == 10 # 100 items / 30 per page = 3 full pages + 10 items
assert total == 100
assert page == 4
assert total_pages == 4
finally:
await db.close()
def test_pagination_out_of_range(self, temp_db):
async def test_pagination_out_of_range(self):
"""Test pagination with page number out of range."""
datastore = DataStore(type=StoreType.HISTORY, connection=temp_db)
# Request page beyond total pages - should return last page
items, total, page, total_pages = datastore.get_items_paginated(page=999, per_page=10)
db = await make_db(data=100)
try:
datastore = DataStore(type=StoreType.HISTORY, connection=db)
items, total, page, total_pages = await datastore.get_items_paginated(page=999, per_page=10)
assert len(items) == 10
assert total == 100
assert page == 10 # Adjusted to last page
assert total_pages == 10
finally:
await db.close()
def test_pagination_order_desc(self, temp_db):
async def test_pagination_order_desc(self):
"""Test pagination with descending order (newest first)."""
datastore = DataStore(type=StoreType.HISTORY, connection=temp_db)
items, _, _, _ = datastore.get_items_paginated(page=1, per_page=5, order="DESC")
db = await make_db(data=100)
try:
datastore = DataStore(type=StoreType.HISTORY, connection=db)
items, _, _, _ = await datastore.get_items_paginated(page=1, per_page=5, order="DESC")
assert len(items) == 5
# Verify order - should be newest to oldest
# (note: exact order depends on the timestamp generation in fixture)
finally:
await db.close()
def test_pagination_order_asc(self, temp_db):
async def test_pagination_order_asc(self):
"""Test pagination with ascending order (oldest first)."""
datastore = DataStore(type=StoreType.HISTORY, connection=temp_db)
items, _, _, _ = datastore.get_items_paginated(page=1, per_page=5, order="ASC")
db = await make_db(data=100)
try:
datastore = DataStore(type=StoreType.HISTORY, connection=db)
items, _, _, _ = await datastore.get_items_paginated(page=1, per_page=5, order="ASC")
assert len(items) == 5
finally:
await db.close()
def test_pagination_invalid_page(self, temp_db):
async def test_pagination_invalid_page(self):
"""Test pagination with invalid page number."""
datastore = DataStore(type=StoreType.HISTORY, connection=temp_db)
db = await make_db(data=100)
try:
datastore = DataStore(type=StoreType.HISTORY, connection=db)
with pytest.raises(ValueError, match="page must be >= 1"):
await datastore.get_items_paginated(page=0, per_page=10)
with pytest.raises(ValueError, match="page must be >= 1"):
datastore.get_items_paginated(page=0, per_page=10)
await datastore.get_items_paginated(page=-1, per_page=10)
finally:
await db.close()
with pytest.raises(ValueError, match="page must be >= 1"):
datastore.get_items_paginated(page=-1, per_page=10)
def test_pagination_invalid_per_page(self, temp_db):
async def test_pagination_invalid_per_page(self):
"""Test pagination with invalid per_page value."""
datastore = DataStore(type=StoreType.HISTORY, connection=temp_db)
db = await make_db(data=100)
try:
datastore = DataStore(type=StoreType.HISTORY, connection=db)
with pytest.raises(ValueError, match="per_page must be >= 1"):
datastore.get_items_paginated(page=1, per_page=0)
await datastore.get_items_paginated(page=1, per_page=0)
with pytest.raises(ValueError, match="per_page must be >= 1"):
datastore.get_items_paginated(page=1, per_page=-10)
await datastore.get_items_paginated(page=1, per_page=-10)
finally:
await db.close()
def test_pagination_invalid_order(self, temp_db):
async def test_pagination_invalid_order(self):
"""Test pagination with invalid order parameter."""
datastore = DataStore(type=StoreType.HISTORY, connection=temp_db)
db = await make_db(data=100)
try:
datastore = DataStore(type=StoreType.HISTORY, connection=db)
with pytest.raises(ValueError, match="order must be 'ASC' or 'DESC'"):
datastore.get_items_paginated(page=1, per_page=10, order="INVALID")
await datastore.get_items_paginated(page=1, per_page=10, order="INVALID")
finally:
await db.close()
def test_pagination_empty_store(self):
async def test_pagination_empty_store(self):
"""Test pagination with empty datastore."""
with TemporaryDirectory() as tmpdir:
db_path = Path(tmpdir) / "empty.db"
conn = sqlite3.connect(str(db_path))
conn.row_factory = sqlite3.Row
# Create empty table
conn.execute(
"""
CREATE TABLE IF NOT EXISTS "history" (
"id" TEXT PRIMARY KEY,
"type" TEXT NOT NULL,
"url" TEXT NOT NULL,
"data" TEXT NOT NULL,
"created_at" TEXT NOT NULL
)
"""
)
conn.commit()
datastore = DataStore(type=StoreType.HISTORY, connection=conn)
items, total, page, total_pages = datastore.get_items_paginated(page=1, per_page=10)
db = await make_db(data=0)
try:
datastore = DataStore(type=StoreType.HISTORY, connection=db)
items, total, page, total_pages = await datastore.get_items_paginated(page=1, per_page=10)
assert len(items) == 0
assert total == 0
assert page == 1
assert total_pages == 1
finally:
await db.close()
conn.close()
def test_pagination_single_item(self):
async def test_pagination_single_item(self):
"""Test pagination with single item."""
with TemporaryDirectory() as tmpdir:
db_path = Path(tmpdir) / "single.db"
conn = sqlite3.connect(str(db_path))
conn.row_factory = sqlite3.Row
# Create table with single item
conn.execute(
"""
CREATE TABLE IF NOT EXISTS "history" (
"id" TEXT PRIMARY KEY,
"type" TEXT NOT NULL,
"url" TEXT NOT NULL,
"data" TEXT NOT NULL,
"created_at" TEXT NOT NULL
)
"""
)
item_data = {
"url": "https://example.com/single",
"title": "Single Video",
"id": "single-video",
"folder": "/downloads",
"status": "finished",
}
conn.execute(
'INSERT INTO "history" ("id", "type", "url", "data", "created_at") VALUES (?, ?, ?, ?, ?)',
(
"single-id",
str(StoreType.HISTORY),
item_data["url"],
json.dumps(item_data),
datetime.now(UTC).strftime("%Y-%m-%d %H:%M:%S"),
),
)
conn.commit()
datastore = DataStore(type=StoreType.HISTORY, connection=conn)
items, total, page, total_pages = datastore.get_items_paginated(page=1, per_page=10)
db = await make_db(data=1)
try:
datastore = DataStore(type=StoreType.HISTORY, connection=db)
items, total, page, total_pages = await datastore.get_items_paginated(page=1, per_page=10)
assert len(items) == 1
assert total == 1
assert page == 1
assert total_pages == 1
finally:
await db.close()
conn.close()
def test_pagination_status_filter_include(self, temp_db):
async def test_pagination_status_filter_include(self):
"""Test pagination with status filter (inclusion)."""
# Add some items with different status values
item_data_pending = {
@ -269,7 +222,9 @@ class TestDataStorePagination:
"status": "downloading",
}
temp_db.execute(
db = await make_db(data=100)
try:
await db._conn.execute(
'INSERT INTO "history" ("id", "type", "url", "data", "created_at") VALUES (?, ?, ?, ?, ?)',
(
"pending-id",
@ -279,7 +234,7 @@ class TestDataStorePagination:
datetime.now(UTC).strftime("%Y-%m-%d %H:%M:%S"),
),
)
temp_db.execute(
await db._conn.execute(
'INSERT INTO "history" ("id", "type", "url", "data", "created_at") VALUES (?, ?, ?, ?, ?)',
(
"downloading-id",
@ -289,14 +244,11 @@ class TestDataStorePagination:
datetime.now(UTC).strftime("%Y-%m-%d %H:%M:%S"),
),
)
temp_db.commit()
datastore = DataStore(type=StoreType.HISTORY, connection=temp_db)
await db._conn.commit()
datastore = DataStore(type=StoreType.HISTORY, connection=db)
# Filter for finished items only
items, total, _page, _total_pages = datastore.get_items_paginated(
page=1, per_page=50, status_filter="finished"
)
items, total, _, __ = await datastore.get_items_paginated(page=1, per_page=50, status_filter="finished")
assert len(items) == 50 # First page of finished items
assert total == 100 # Only 100 finished items in fixture
@ -304,15 +256,17 @@ class TestDataStorePagination:
assert item.status == "finished"
# Filter for pending items only
items, total, _page, _total_pages = datastore.get_items_paginated(
items, total, _page, _total_pages = await datastore.get_items_paginated(
page=1, per_page=50, status_filter="pending"
)
assert len(items) == 1
assert total == 1
assert items[0][1].status == "pending"
finally:
await db.close()
def test_pagination_status_filter_exclude(self, temp_db):
async def test_pagination_status_filter_exclude(self):
"""Test pagination with status filter (exclusion)."""
# Add some items with different status values
item_data_pending = {
@ -329,8 +283,9 @@ class TestDataStorePagination:
"folder": "/downloads",
"status": "error",
}
temp_db.execute(
db = await make_db(data=0)
try:
await db._conn.execute(
'INSERT INTO "history" ("id", "type", "url", "data", "created_at") VALUES (?, ?, ?, ?, ?)',
(
"pending-id-2",
@ -340,7 +295,7 @@ class TestDataStorePagination:
datetime.now(UTC).strftime("%Y-%m-%d %H:%M:%S"),
),
)
temp_db.execute(
await db._conn.execute(
'INSERT INTO "history" ("id", "type", "url", "data", "created_at") VALUES (?, ?, ?, ?, ?)',
(
"error-id",
@ -350,12 +305,12 @@ class TestDataStorePagination:
datetime.now(UTC).strftime("%Y-%m-%d %H:%M:%S"),
),
)
temp_db.commit()
await db._conn.commit()
datastore = DataStore(type=StoreType.HISTORY, connection=temp_db)
datastore = DataStore(type=StoreType.HISTORY, connection=db)
# Exclude finished items
items, total, _page, _total_pages = datastore.get_items_paginated(
items, total, _page, _total_pages = await datastore.get_items_paginated(
page=1, per_page=50, status_filter="!finished"
)
@ -367,13 +322,17 @@ class TestDataStorePagination:
# Verify we have pending and error
statuses = {item.status for _, item in items}
assert statuses == {"pending", "error"}
finally:
await db.close()
def test_pagination_status_filter_none_matching(self, temp_db):
async def test_pagination_status_filter_none_matching(self):
"""Test pagination with status filter that matches no items."""
datastore = DataStore(type=StoreType.HISTORY, connection=temp_db)
db = await make_db(data=0)
try:
datastore = DataStore(type=StoreType.HISTORY, connection=db)
# Filter for a status that doesn't exist
items, total, page, total_pages = datastore.get_items_paginated(
items, total, page, total_pages = await datastore.get_items_paginated(
page=1, per_page=50, status_filter="nonexistent"
)
@ -381,3 +340,5 @@ class TestDataStorePagination:
assert total == 0
assert page == 1
assert total_pages == 1
finally:
await db.close()

183
app/tests/test_sqlite_store.py Normal file
View file

@ -0,0 +1,183 @@
from datetime import UTC, datetime, timedelta
import pytest
from app.library.ItemDTO import ItemDTO
from app.library.sqlite_store import SqliteStore
async def make_store() -> SqliteStore:
"""Create an isolated in-memory SqliteStore instance with schema."""
SqliteStore._reset_singleton()
store = SqliteStore.get_instance(db_path=":memory:")
await store._ensure_conn()
assert store._conn is not None
return store
def make_item(idx: int, *, status: str = "finished") -> ItemDTO:
return ItemDTO(
id=f"vid{idx}",
title=f"Video {idx}",
url=f"https://example.com/{idx}",
folder="/downloads",
status=status,
)
@pytest.mark.asyncio
async def test_enqueue_upsert_and_fetch_saved():
store = await make_store()
item = make_item(1)
await store.enqueue_upsert("queue", item)
await store.flush()
saved = await store.fetch_saved("queue")
assert len(saved) == 1
key, loaded = saved[0]
assert key == item._id
assert loaded.title == item.title
await store.close()
@pytest.mark.asyncio
async def test_delete_and_bulk_delete():
store = await make_store()
items = [make_item(i) for i in range(3)]
for itm in items:
await store.enqueue_upsert("queue", itm)
await store.flush()
await store.enqueue_delete("queue", items[0]._id)
await store.enqueue_bulk_delete("queue", [items[1]._id])
await store.flush()
remaining = await store.fetch_saved("queue")
assert len(remaining) == 1
assert remaining[0][0] == items[2]._id
await store.close()
@pytest.mark.asyncio
async def test_count_with_status_filters():
store = await make_store()
finished_items = [make_item(i, status="finished") for i in range(2)]
pending_items = [make_item(i + 10, status="pending") for i in range(3)]
for itm in finished_items + pending_items:
await store.enqueue_upsert("history", itm)
await store.flush()
assert await store.count("history", status_filter="finished") == 2
assert await store.count("history", status_filter="pending") == 3
assert await store.count("history", status_filter="!finished") == 3
await store.close()
@pytest.mark.asyncio
async def test_paginate_order_and_bounds():
store = await make_store()
base = datetime(2024, 1, 1, tzinfo=UTC)
_list: list[ItemDTO] = []
for i in range(15):
itm = make_item(i)
encoded = itm.json()
created_at = (base + timedelta(minutes=i)).strftime("%Y-%m-%d %H:%M:%S")
_list.append(itm)
await store._conn.execute(
'INSERT INTO "history" ("id", "type", "url", "data", "created_at") VALUES (?, ?, ?, ?, ?)',
(itm._id, "history", itm.url, encoded, created_at),
)
await store._conn.commit()
items, total, page, total_pages = await store.paginate("history", page=1, per_page=5, order="ASC")
assert total == 15
assert page == 1
assert total_pages == 3
assert items[0][0] == _list[0]._id
items_desc, *_ = await store.paginate("history", page=1, per_page=3, order="DESC")
assert len(items_desc) == 3
assert items_desc[0][0] == _list[14]._id
await store.close()
@pytest.mark.asyncio
async def test_get_by_id():
store = await make_store()
item = make_item(99)
await store.enqueue_upsert("queue", item)
await store.flush()
assert item._id == (await store.get_by_id("queue", item._id))._id
await store.close()
@pytest.mark.asyncio
async def test_shutdown_closes_worker_queue():
store = await make_store()
await store.enqueue_upsert("queue", make_item(99))
await store.shutdown()
assert store._queue is None
await store.close()
@pytest.mark.asyncio
async def test_enqueue_delete_removes_row():
store = await make_store()
item = make_item(5)
await store.enqueue_upsert("queue", item)
await store.flush()
await store.enqueue_delete("queue", item._id)
await store.flush()
rows = await store.fetch_saved("queue")
assert rows == []
await store.close()
@pytest.mark.asyncio
async def test_enqueue_bulk_delete_returns_count_and_bulk_path():
store = await make_store()
items = [make_item(i) for i in range(3)]
for itm in items:
await store.enqueue_upsert("history", itm)
await store.flush()
await store.enqueue_bulk_delete("history", [items[0]._id, items[1]._id])
await store.flush()
remaining = await store.fetch_saved("history")
assert len(remaining) == 1
assert remaining[0][0] == items[2]._id
# direct bulk_delete should report affected rows
count = await store.bulk_delete("history", [items[2]._id])
assert count == 1
await store.close()
@pytest.mark.asyncio
async def test_paginate_out_of_range_returns_last_page():
store = await make_store()
for i in range(7):
await store.enqueue_upsert("history", make_item(i))
await store.flush()
items, total, page, total_pages = await store.paginate("history", page=10, per_page=3, order="DESC")
assert total == 7
assert total_pages == 3
assert page == 3
assert len(items) == 1
await store.close()
@pytest.mark.asyncio
async def test_on_shutdown_closes_connection():
store = await make_store()
await store.enqueue_upsert("queue", make_item(1))
await store.flush()
await store.on_shutdown(None)
assert store._conn is None

2
pyproject.toml
View file

@ -15,7 +15,6 @@ dependencies = [
# Cross‐platform
"python-socketio>=5.11.1",
"aiohttp>=3.9.3",
"caribou>=0.3.0",
"coloredlogs>=15.0.1",
"aiocron>=1.8",
"python-dotenv>=1.0.1",
@ -44,6 +43,7 @@ dependencies = [
"parsel>=1.10.0",
"jmespath>=1.0.1",
"jsonschema>=4.23.0",
"aiosqlite>=0.22.0",
]
[tool.ruff]

22
uv.lock
View file

@ -105,6 +105,15 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/fb/76/641ae371508676492379f16e2fa48f4e2c11741bd63c48be4b12a6b09cba/aiosignal-1.4.0-py3-none-any.whl", hash = "sha256:053243f8b92b990551949e63930a839ff0cf0b0ebbe0597b0f3fb19e1a0fe82e", size = 7490 },
]
[[package]]
name = "aiosqlite"
version = "0.22.0"
source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/3a/0d/449c024bdabd0678ae07d804e60ed3b9786facd3add66f51eee67a0fccea/aiosqlite-0.22.0.tar.gz", hash = "sha256:7e9e52d72b319fcdeac727668975056c49720c995176dc57370935e5ba162bb9", size = 14707 }
wheels = [
{ url = "https://files.pythonhosted.org/packages/ef/39/b2181148075272edfbbd6d87e6cd78cc71dca243446fa3b381fd4116950b/aiosqlite-0.22.0-py3-none-any.whl", hash = "sha256:96007fac2ce70eda3ca1bba7a3008c435258a592b8fbf2ee3eeaa36d33971a09", size = 17263 },
]
[[package]]
name = "altgraph"
version = "0.17.5"
@ -224,15 +233,6 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/07/6b/6e92009df3b8b7272f85a0992b306b61c34b7ea1c4776643746e61c380ac/brotlicffi-1.2.0.0-cp38-abi3-win_amd64.whl", hash = "sha256:f139a7cdfe4ae7859513067b736eb44d19fae1186f9e99370092f6915216451b", size = 378586 },
]
[[package]]
name = "caribou"
version = "0.4.1"
source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/6c/74/de2c20c5a24811d69a2a6150d5cd841f2af21ffc65ff964a73f46738bb68/caribou-0.4.1.tar.gz", hash = "sha256:13a66fc9ef9b1c9e9ef220876d59a44ee5eff9e579655252271b7514fc0ad787", size = 14019 }
wheels = [
{ url = "https://files.pythonhosted.org/packages/b5/91/795eccdad6abd41b3634502d0971cb696d2e5c9cede67f8b38ebe30d2b1e/caribou-0.4.1-py2.py3-none-any.whl", hash = "sha256:5c7a036584b34021011f1512620152272924e55ca87c7c805073aeef31c5baed", size = 7203 },
]
[[package]]
name = "certifi"
version = "2025.11.12"
@ -1823,13 +1823,13 @@ source = { virtual = "." }
dependencies = [
{ name = "aiocron" },
{ name = "aiohttp" },
{ name = "aiosqlite" },
{ name = "anyio" },
{ name = "apprise" },
{ name = "async-timeout" },
{ name = "bgutil-ytdlp-pot-provider" },
{ name = "brotli" },
{ name = "brotlicffi" },
{ name = "caribou" },
{ name = "coloredlogs" },
{ name = "curl-cffi" },
{ name = "dateparser" },
@ -1874,13 +1874,13 @@ dev = [
requires-dist = [
{ name = "aiocron", specifier = ">=1.8" },
{ name = "aiohttp", specifier = ">=3.9.3" },
{ name = "aiosqlite", specifier = ">=0.22.0" },
{ name = "anyio" },
{ name = "apprise", specifier = ">=1.9.3" },
{ name = "async-timeout" },
{ name = "bgutil-ytdlp-pot-provider", specifier = ">=1.2.1" },
{ name = "brotli" },
{ name = "brotlicffi" },
{ name = "caribou", specifier = ">=0.3.0" },
{ name = "coloredlogs", specifier = ">=15.0.1" },
{ name = "curl-cffi", specifier = ">=0.13" },
{ name = "dateparser", specifier = ">=1.2.1" },

0
var/downloads/.gitkeep
View file