From 44089e5b1fe52e7ee64619614449bf357179f29d Mon Sep 17 00:00:00 2001 From: Yiorgis Gozadinos Date: Wed, 3 Jun 2026 11:57:03 +0300 Subject: [PATCH] Support a dburi for the ingester queue (SQLite + Postgres) Migrate the ingester queue storage from raw aiosqlite to SQLAlchemy Core async. The backend is chosen by ingester.queue.dburi: a SQLAlchemy async URL points the queue at a database server, and SQLite remains the default when unset. The Postgres path claims jobs with FOR UPDATE SKIP LOCKED so multiple ingester processes can share one queue; SQLite caps the pool to a single connection to keep the select-then-update claim atomic. --- CHANGELOG.md | 1 + haiku_rag_slim/haiku/rag/config/models.py | 10 +- haiku_rag_slim/haiku/rag/ingester/app.py | 34 +- haiku_rag_slim/haiku/rag/ingester/cli.py | 34 +- haiku_rag_slim/haiku/rag/ingester/queue/db.py | 95 +++ .../haiku/rag/ingester/queue/migrations.py | 98 +-- .../haiku/rag/ingester/queue/repository.py | 632 +++++++++--------- .../haiku/rag/ingester/queue/schema.py | 80 --- haiku_rag_slim/pyproject.toml | 2 + tests/ingester/conftest.py | 34 + tests/ingester/test_api.py | 29 - tests/ingester/test_pollers.py | 28 - tests/ingester/test_queue.py | 83 +-- tests/ingester/test_serve_integration.py | 28 - tests/ingester/test_workers.py | 28 - uv.lock | 157 +++++ 16 files changed, 711 insertions(+), 662 deletions(-) create mode 100644 haiku_rag_slim/haiku/rag/ingester/queue/db.py delete mode 100644 haiku_rag_slim/haiku/rag/ingester/queue/schema.py create mode 100644 tests/ingester/conftest.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 60dd0743..0bc89306 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ ### Added +- `ingester.queue.dburi`: a SQLAlchemy async URL (e.g. `postgresql+asyncpg://user:pw@host/db`) points the ingester queue at a database server. SQLite remains the default when unset. The Postgres path claims jobs with `FOR UPDATE SKIP LOCKED`, so multiple ingester processes can share one queue. - `ingester.queue.retention_days` (default 30): the reaper deletes succeeded/dead jobs whose `completed_at` is older than the window. `null` disables pruning. ### Fixed diff --git a/haiku_rag_slim/haiku/rag/config/models.py b/haiku_rag_slim/haiku/rag/config/models.py index 8c00a0bf..50a9cc1f 100644 --- a/haiku_rag_slim/haiku/rag/config/models.py +++ b/haiku_rag_slim/haiku/rag/config/models.py @@ -260,11 +260,17 @@ class EvaluationsConfig(BaseModel): class QueueConfig(BaseModel): - """SQLite queue for the production ingester.""" + """Job queue for the production ingester. Defaults to a filesystem SQLite + file; set `dburi` to point it at a database server instead.""" path: Path = Field( default_factory=lambda: get_default_data_dir() / "ingester.db", - description="Location of the ingester's SQLite queue file.", + description="SQLite queue file. Used when dburi is unset.", + ) + dburi: str | None = Field( + default=None, + description="SQLAlchemy async URL for the queue, e.g. " + "postgresql+asyncpg://user:pw@host/db. Overrides path when set.", ) retention_days: int | None = Field( default=30, diff --git a/haiku_rag_slim/haiku/rag/ingester/app.py b/haiku_rag_slim/haiku/rag/ingester/app.py index c6767b14..18d6ad39 100644 --- a/haiku_rag_slim/haiku/rag/ingester/app.py +++ b/haiku_rag_slim/haiku/rag/ingester/app.py @@ -16,7 +16,7 @@ from haiku.rag.ingester.workers.pool import WorkerPool from haiku.rag.ingester.workers.retry import RetryPolicy if TYPE_CHECKING: - import aiosqlite + from sqlalchemy.ext.asyncio import AsyncEngine logger = logging.getLogger(__name__) @@ -33,14 +33,14 @@ class BatchReport(BaseModel): class IngesterApp: """Top-level lifecycle for the production ingester. - Owns: SQLite queue connection, JobRepo/SyncStateRepo, PollerManager, + Owns: queue engine, JobRepo/SyncStateRepo, PollerManager, WorkerPool, and a HaikuRAG client for the worker pool to ingest through. """ def __init__(self, *, config: AppConfig, db_path: Path): self._config = config self._db_path = db_path - self._queue_conn: aiosqlite.Connection | None = None + self._engine: AsyncEngine | None = None self._jobs: JobRepo | None = None self._sync: SyncStateRepo | None = None self._pool: WorkerPool | None = None @@ -48,22 +48,18 @@ class IngesterApp: @asynccontextmanager async def _resources(self): - """Open the queue connection and construct the repos, client, pollers + """Open the queue engine and construct the repos, client, pollers and worker pool. Yields with everything built but nothing started — callers own the start/stop lifecycle. Closes the client and queue - connection on exit.""" + engine on exit.""" from haiku.rag.client import HaikuRAG from haiku.rag.converters import get_converter ingester_cfg = self._config.ingester - self._queue_conn = await open_queue(ingester_cfg.queue.path) + self._engine = await open_queue(ingester_cfg.queue) try: - # Single lock shared by both repos so cross-repo calls on the - # same connection (e.g. worker's mark_succeeded then sync.upsert) - # serialize at the cursor/commit boundary. - queue_lock = asyncio.Lock() - self._jobs = JobRepo(self._queue_conn, lock=queue_lock) - self._sync = SyncStateRepo(self._queue_conn, lock=queue_lock) + self._jobs = JobRepo(self._engine) + self._sync = SyncStateRepo(self._engine) supported_extensions = get_converter(self._config).supported_extensions retry = RetryPolicy( @@ -107,13 +103,13 @@ class IngesterApp: ) yield finally: - # Close the queue connection unconditionally. aiosqlite runs the - # underlying sqlite3 in a background thread; leaving it open holds - # the event loop alive and blocks process exit on early failures - # (e.g. HaikuRAG raising MigrationRequiredError). - if self._queue_conn is not None: - await self._queue_conn.close() - self._queue_conn = None + # Dispose the engine unconditionally. aiosqlite runs the underlying + # sqlite3 in a background thread; leaving the pool open holds the + # event loop alive and blocks process exit on early failures (e.g. + # HaikuRAG raising MigrationRequiredError). + if self._engine is not None: + await self._engine.dispose() + self._engine = None async def _stop_pool(self) -> None: """Stop the worker pool, honouring the shutdown grace, then drain any diff --git a/haiku_rag_slim/haiku/rag/ingester/cli.py b/haiku_rag_slim/haiku/rag/ingester/cli.py index b4a1fff4..1097f0d7 100644 --- a/haiku_rag_slim/haiku/rag/ingester/cli.py +++ b/haiku_rag_slim/haiku/rag/ingester/cli.py @@ -9,6 +9,7 @@ load_dotenv(find_dotenv(usecwd=True)) from haiku.rag.config import ( # noqa: E402 AppConfig, + QueueConfig, find_config_file, get_config, load_yaml_config, @@ -60,7 +61,7 @@ def cli() -> None: queue_cli = typer.Typer( name="queue", no_args_is_help=True, - help="Operate the ingester's SQLite job queue.", + help="Operate the ingester's job queue.", ) _cli.add_typer(queue_cli) @@ -79,13 +80,22 @@ def _load_config_with_override(config_path: Path | None) -> AppConfig: return get_config() -def _resolve_queue_path(config: AppConfig, override: Path | None) -> Path: - return Path(override).expanduser() if override else config.ingester.queue.path +def _resolve_queue_config(config: AppConfig, override: Path | None) -> QueueConfig: + """The configured queue, with `--queue` applied as a path override. The + override is ignored when a dburi is set — the queue lives in a server.""" + queue = config.ingester.queue + if override is not None and queue.dburi is None: + return queue.model_copy(update={"path": Path(override).expanduser()}) + return queue -async def _ensure_schema(path: Path) -> None: - conn = await open_queue(path) - await conn.close() +def _queue_target(queue: QueueConfig) -> str: + return queue.dburi or str(queue.path) + + +async def _ensure_schema(queue: QueueConfig) -> None: + engine = await open_queue(queue) + await engine.dispose() @queue_cli.command("init") @@ -98,9 +108,9 @@ def queue_init( ), ) -> None: """Create the queue DB and apply the current schema. Idempotent.""" - path = _resolve_queue_path(get_config(), queue) - asyncio.run(_ensure_schema(path)) - typer.echo(f"Queue initialized at {path}") + queue_config = _resolve_queue_config(get_config(), queue) + asyncio.run(_ensure_schema(queue_config)) + typer.echo(f"Queue initialized at {_queue_target(queue_config)}") @queue_cli.command("migrate") @@ -113,9 +123,9 @@ def queue_migrate( ), ) -> None: """Apply any pending schema migrations to an existing queue DB. Idempotent.""" - path = _resolve_queue_path(get_config(), queue) - asyncio.run(_ensure_schema(path)) - typer.echo(f"Queue at {path} is up to date") + queue_config = _resolve_queue_config(get_config(), queue) + asyncio.run(_ensure_schema(queue_config)) + typer.echo(f"Queue at {_queue_target(queue_config)} is up to date") def _resolve_db_path(config: AppConfig, override: Path | None) -> Path: diff --git a/haiku_rag_slim/haiku/rag/ingester/queue/db.py b/haiku_rag_slim/haiku/rag/ingester/queue/db.py new file mode 100644 index 00000000..a0ebb872 --- /dev/null +++ b/haiku_rag_slim/haiku/rag/ingester/queue/db.py @@ -0,0 +1,95 @@ +import sqlalchemy as sa +from sqlalchemy import event +from sqlalchemy.ext.asyncio import AsyncEngine + +SCHEMA_VERSION = 1 + +metadata = sa.MetaData() + +jobs = sa.Table( + "jobs", + metadata, + sa.Column("id", sa.Text, primary_key=True), + sa.Column("source_id", sa.Text, nullable=False), + sa.Column("uri", sa.Text, nullable=False), + sa.Column("op", sa.Text, nullable=False), + sa.Column("content_hash", sa.Text), + sa.Column("revision", sa.Text), + sa.Column("status", sa.Text, nullable=False), + sa.Column("attempts", sa.Integer, nullable=False, server_default=sa.text("0")), + sa.Column("max_attempts", sa.Integer, nullable=False, server_default=sa.text("5")), + sa.Column("last_error", sa.Text), + sa.Column("extra", sa.Text), + sa.Column("enqueued_at", sa.Text, nullable=False), + sa.Column("scheduled_at", sa.Text, nullable=False), + sa.Column("claimed_at", sa.Text), + sa.Column("claimed_by", sa.Text), + sa.Column("completed_at", sa.Text), +) + +# A (source_id, uri) pair can only have one live job (queued or claimed) at a +# time, regardless of op. Live UPSERT and DELETE for the same URI can't both +# exist — preventing a DELETE worker from removing a document a sibling UPSERT +# just ingested. Once succeeded or dead, the row no longer satisfies the WHERE +# clause and a re-enqueue is allowed. +_live = jobs.c.status.in_(["queued", "claimed"]) +sa.Index( + "uq_jobs_live", + jobs.c.source_id, + jobs.c.uri, + unique=True, + sqlite_where=_live, + postgresql_where=_live, +) + +_queued = jobs.c.status == "queued" +sa.Index( + "idx_jobs_claimable", + jobs.c.scheduled_at, + sqlite_where=_queued, + postgresql_where=_queued, +) + +_succeeded = jobs.c.status == "succeeded" +sa.Index( + "idx_jobs_succeeded_completed", + jobs.c.completed_at, + sqlite_where=_succeeded, + postgresql_where=_succeeded, +) + +sync_state = sa.Table( + "sync_state", + metadata, + sa.Column("source_id", sa.Text, primary_key=True), + sa.Column("uri", sa.Text, primary_key=True), + sa.Column("revision", sa.Text), + sa.Column("content_hash", sa.Text), + sa.Column("last_seen_at", sa.Text, nullable=False), + sa.Column("last_ingested_at", sa.Text), +) + +schema_version = sa.Table( + "schema_version", + metadata, + sa.Column("version", sa.Integer, primary_key=True), +) + + +def install_sqlite_pragmas(engine: AsyncEngine) -> None: + """Register a connect listener that sets the per-connection pragmas the + queue relies on. SQLite-only — Postgres has no equivalent and asyncpg + rejects PRAGMA, so the listener is never attached for it.""" + if engine.dialect.name != "sqlite": + return + + @event.listens_for(engine.sync_engine, "connect") + def _set_pragmas(dbapi_conn, _record): + cursor = dbapi_conn.cursor() + try: + cursor.execute("PRAGMA journal_mode=WAL") + cursor.execute("PRAGMA synchronous=NORMAL") + cursor.execute("PRAGMA foreign_keys=ON") + cursor.execute("PRAGMA busy_timeout=30000") + finally: + cursor.close() diff --git a/haiku_rag_slim/haiku/rag/ingester/queue/migrations.py b/haiku_rag_slim/haiku/rag/ingester/queue/migrations.py index 239f4d28..77d9abf9 100644 --- a/haiku_rag_slim/haiku/rag/ingester/queue/migrations.py +++ b/haiku_rag_slim/haiku/rag/ingester/queue/migrations.py @@ -1,56 +1,60 @@ -from pathlib import Path +import sqlalchemy as sa +from sqlalchemy.engine import make_url +from sqlalchemy.ext.asyncio import AsyncEngine, create_async_engine -import aiosqlite +from haiku.rag.config.models import QueueConfig +from haiku.rag.ingester.queue.db import ( + SCHEMA_VERSION, + install_sqlite_pragmas, + metadata, + schema_version, +) -from haiku.rag.ingester.queue.schema import ALL_DDL, SCHEMA_VERSION - -__all__ = ["SCHEMA_VERSION", "apply_migrations", "open_queue"] +__all__ = ["SCHEMA_VERSION", "apply_migrations", "make_engine", "open_queue"] -async def _exec(conn: aiosqlite.Connection, sql: str, *params) -> None: - """Execute a statement and finalize the cursor. aiosqlite cursors stay - attached until closed, blocking subsequent commits with 'SQL statements - in progress'.""" - async with conn.execute(sql, params): - pass - - -async def apply_migrations(conn: aiosqlite.Connection) -> int: - """Idempotently create tables/indexes/views and pin schema_version. - - Returns the schema version after the call. Safe to call on a fresh DB - or on one already at the latest version. - """ - await _exec(conn, "PRAGMA journal_mode=WAL") - await _exec(conn, "PRAGMA synchronous=NORMAL") - await _exec(conn, "PRAGMA foreign_keys=ON") - - for stmt in ALL_DDL: - await _exec(conn, stmt) - - async with conn.execute("SELECT version FROM schema_version LIMIT 1") as cursor: - row = await cursor.fetchone() - if row is None: - await _exec( - conn, "INSERT INTO schema_version (version) VALUES (?)", SCHEMA_VERSION - ) +def make_engine(config: QueueConfig) -> AsyncEngine: + """Build the queue's AsyncEngine from config. Uses `dburi` when set, + otherwise a `sqlite+aiosqlite` URL pointing at the resolved `path` + (creating the parent directory). SQLite is capped to a single pooled + connection so the two-statement claim stays atomic without row locks.""" + if config.dburi: + url = make_url(config.dburi) else: - current = row[0] - if current < SCHEMA_VERSION: # pragma: no cover - no migrations yet - # No diff migrations exist yet — future versions add UPDATE/ALTER - # statements between here and the version bump. - await _exec(conn, "UPDATE schema_version SET version = ?", SCHEMA_VERSION) + path = config.path.expanduser().resolve() + path.parent.mkdir(parents=True, exist_ok=True) + url = make_url(f"sqlite+aiosqlite:///{path}") - await conn.commit() + if url.get_backend_name() == "sqlite": + engine = create_async_engine(url, pool_size=1, max_overflow=0) + else: + engine = create_async_engine(url) + install_sqlite_pragmas(engine) + return engine + + +async def apply_migrations(engine: AsyncEngine) -> int: + """Idempotently create tables/indexes and pin schema_version. + + Returns the schema version after the call. Safe on a fresh DB or one + already at the latest version. + """ + async with engine.begin() as conn: + await conn.run_sync(metadata.create_all) + current = ( + await conn.execute(sa.select(schema_version.c.version).limit(1)) + ).scalar_one_or_none() + if current is None: + await conn.execute(sa.insert(schema_version).values(version=SCHEMA_VERSION)) + elif current < SCHEMA_VERSION: # pragma: no cover - no migrations yet + # No diff migrations exist yet — future versions add ALTER/UPDATE + # statements between create_all and the version bump. + await conn.execute(sa.update(schema_version).values(version=SCHEMA_VERSION)) return SCHEMA_VERSION -async def open_queue(path: str | Path) -> aiosqlite.Connection: - """Open the queue database at `path`, creating it (and the parent dir) - if needed, and ensuring the schema is up-to-date.""" - db_path = Path(path).expanduser().resolve() - db_path.parent.mkdir(parents=True, exist_ok=True) - conn = await aiosqlite.connect(str(db_path)) - conn.row_factory = aiosqlite.Row - await apply_migrations(conn) - return conn +async def open_queue(config: QueueConfig) -> AsyncEngine: + """Build the queue engine and ensure its schema is up to date.""" + engine = make_engine(config) + await apply_migrations(engine) + return engine diff --git a/haiku_rag_slim/haiku/rag/ingester/queue/repository.py b/haiku_rag_slim/haiku/rag/ingester/queue/repository.py index 85852b5f..e7aaaafc 100644 --- a/haiku_rag_slim/haiku/rag/ingester/queue/repository.py +++ b/haiku_rag_slim/haiku/rag/ingester/queue/repository.py @@ -1,10 +1,15 @@ import asyncio import json import uuid +from collections.abc import Mapping from datetime import UTC, datetime, timedelta -import aiosqlite +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql +from sqlalchemy.dialects import sqlite as sqlite_dialect +from sqlalchemy.ext.asyncio import AsyncEngine +from haiku.rag.ingester.queue.db import jobs, sync_state from haiku.rag.ingester.queue.models import ( Job, JobOp, @@ -22,7 +27,19 @@ def _parse_dt(value: str | None) -> datetime | None: return datetime.fromisoformat(value) if value else None -def _row_to_job(row: aiosqlite.Row) -> Job: +def _insert(table: sa.Table, dialect: str): + """Dialect-specific INSERT exposing on_conflict_* (and `.excluded`).""" + if dialect == "postgresql": + return postgresql.insert(table) + return sqlite_dialect.insert(table) + + +def _attempts_minus_one() -> sa.ColumnElement[int]: + """attempts - 1, floored at 0. Renders identically on both dialects.""" + return sa.case((jobs.c.attempts - 1 < 0, 0), else_=jobs.c.attempts - 1) + + +def _row_to_job(row: Mapping) -> Job: extra_text = row["extra"] return Job( id=row["id"], @@ -44,7 +61,7 @@ def _row_to_job(row: aiosqlite.Row) -> Job: ) -def _row_to_sync_state(row: aiosqlite.Row) -> SyncStateRow: +def _row_to_sync_state(row: Mapping) -> SyncStateRow: return SyncStateRow( source_id=row["source_id"], uri=row["uri"], @@ -56,24 +73,12 @@ def _row_to_sync_state(row: aiosqlite.Row) -> SyncStateRow: class JobRepo: - def __init__( - self, - conn: aiosqlite.Connection, - lock: asyncio.Lock | None = None, - ): - # Row access by name in helpers below. - conn.row_factory = aiosqlite.Row - self._conn = conn - # Serialize repo calls on the shared connection so cursors from one - # coroutine don't sit "in progress" when another tries to commit. - # aiosqlite executes statements on a single worker thread, but - # individual cursors don't finalize until closed or GC'd — SQLite - # then refuses commit() with "SQL statements in progress". When - # JobRepo and SyncStateRepo share the same connection, callers must - # pass the same lock instance so cross-repo calls also serialize. - self._lock = lock or asyncio.Lock() - # Notified after a successful enqueue so workers can wake - # immediately instead of polling on a fixed sleep interval. + def __init__(self, engine: AsyncEngine): + self._engine = engine + self._dialect = engine.dialect.name + # Notified after a successful enqueue so workers in this process wake + # immediately instead of polling on a fixed sleep interval. Workers in + # other processes (a shared Postgres queue) fall back to polling. self.job_available = asyncio.Condition() async def enqueue( @@ -88,37 +93,33 @@ class JobRepo: extra: dict | None = None, ) -> Job | None: """Enqueue an upsert/delete job. Returns the inserted Job, or None if a - live (queued/claimed) job already exists for the same (source_id, uri, - op). The partial unique index enforces atomicity.""" + live (queued/claimed) job already exists for the same (source_id, uri). + The partial unique index enforces atomicity.""" job_id = str(uuid.uuid4()) now = _utcnow_iso() extra_json = json.dumps(extra) if extra is not None else None - async with self._lock: - async with self._conn.execute( - """ - INSERT INTO jobs ( - id, source_id, uri, op, content_hash, revision, status, - attempts, max_attempts, last_error, extra, - enqueued_at, scheduled_at - ) VALUES (?, ?, ?, ?, ?, ?, 'queued', 0, ?, NULL, ?, ?, ?) - ON CONFLICT DO NOTHING - RETURNING * - """, - ( - job_id, - source_id, - uri, - op.value, - content_hash, - revision, - max_attempts, - extra_json, - now, - now, - ), - ) as cursor: - row = await cursor.fetchone() - await self._conn.commit() + stmt = ( + _insert(jobs, self._dialect) + .values( + id=job_id, + source_id=source_id, + uri=uri, + op=op.value, + content_hash=content_hash, + revision=revision, + status="queued", + attempts=0, + max_attempts=max_attempts, + last_error=None, + extra=extra_json, + enqueued_at=now, + scheduled_at=now, + ) + .on_conflict_do_nothing() + .returning(*jobs.c) + ) + async with self._engine.begin() as conn: + row = (await conn.execute(stmt)).mappings().one_or_none() if row is not None: async with self.job_available: self.job_available.notify_all() @@ -126,36 +127,42 @@ class JobRepo: async def claim_next(self, worker_id: str) -> Job | None: """Atomically claim the oldest queued job whose scheduled_at <= now. - Implemented as a single UPDATE ... RETURNING — no SELECT/UPDATE race.""" + A `SELECT ... FOR UPDATE SKIP LOCKED` picks the row (multi-process safe + on Postgres; the clause is omitted on SQLite, where pool_size=1 keeps + the select-then-update atomic), then a guarded UPDATE claims it.""" now = _utcnow_iso() - async with self._lock: - async with self._conn.execute( - """ - UPDATE jobs - SET status = 'claimed', - claimed_at = ?, - claimed_by = ?, - attempts = attempts + 1 - WHERE id = ( - SELECT id FROM jobs - WHERE status = 'queued' AND scheduled_at <= ? - ORDER BY scheduled_at - LIMIT 1 + select_candidate = ( + sa.select(jobs.c.id) + .where(jobs.c.status == "queued", jobs.c.scheduled_at <= now) + .order_by(jobs.c.scheduled_at) + .limit(1) + .with_for_update(skip_locked=True) + ) + async with self._engine.begin() as conn: + job_id = (await conn.execute(select_candidate)).scalar_one_or_none() + if job_id is None: + return None + claim = ( + sa.update(jobs) + .where(jobs.c.id == job_id) + .values( + status="claimed", + claimed_at=now, + claimed_by=worker_id, + attempts=jobs.c.attempts + 1, ) - RETURNING * - """, - (now, worker_id, now), - ) as cursor: - row = await cursor.fetchone() - await self._conn.commit() - return _row_to_job(row) if row else None + .returning(*jobs.c) + ) + row = (await conn.execute(claim)).mappings().one() + return _row_to_job(row) async def get_job(self, job_id: str) -> Job | None: - async with self._lock: - async with self._conn.execute( - "SELECT * FROM jobs WHERE id = ?", (job_id,) - ) as cursor: - row = await cursor.fetchone() + async with self._engine.connect() as conn: + row = ( + (await conn.execute(sa.select(jobs).where(jobs.c.id == job_id))) + .mappings() + .one_or_none() + ) return _row_to_job(row) if row else None async def mark_succeeded(self, job_id: str, claimed_by: str) -> bool: @@ -163,27 +170,35 @@ class JobRepo: `status='claimed' AND claimed_by=?` so a reaper-resurrected job picked up by a different worker isn't clobbered by the original slow worker. Returns True when the row was updated.""" - async with self._lock: - async with self._conn.execute( - "UPDATE jobs SET status='succeeded', completed_at=? " - "WHERE id=? AND status='claimed' AND claimed_by=? RETURNING id", - (_utcnow_iso(), job_id, claimed_by), - ) as cursor: - row = await cursor.fetchone() - await self._conn.commit() + stmt = ( + sa.update(jobs) + .where( + jobs.c.id == job_id, + jobs.c.status == "claimed", + jobs.c.claimed_by == claimed_by, + ) + .values(status="succeeded", completed_at=_utcnow_iso()) + .returning(jobs.c.id) + ) + async with self._engine.begin() as conn: + row = (await conn.execute(stmt)).first() return row is not None async def mark_dead(self, job_id: str, error: str, claimed_by: str) -> bool: """Transition a still-claimed job to `dead`. See `mark_succeeded` for the guard semantics.""" - async with self._lock: - async with self._conn.execute( - "UPDATE jobs SET status='dead', completed_at=?, last_error=? " - "WHERE id=? AND status='claimed' AND claimed_by=? RETURNING id", - (_utcnow_iso(), error, job_id, claimed_by), - ) as cursor: - row = await cursor.fetchone() - await self._conn.commit() + stmt = ( + sa.update(jobs) + .where( + jobs.c.id == job_id, + jobs.c.status == "claimed", + jobs.c.claimed_by == claimed_by, + ) + .values(status="dead", completed_at=_utcnow_iso(), last_error=error) + .returning(jobs.c.id) + ) + async with self._engine.begin() as conn: + row = (await conn.execute(stmt)).first() return row is not None async def reschedule( @@ -194,22 +209,24 @@ class JobRepo: slow worker can't clobber a re-claim that happened after the reaper reset its claim. Returns True when the row was updated.""" scheduled = (datetime.now(UTC) + timedelta(seconds=delay_seconds)).isoformat() - async with self._lock: - async with self._conn.execute( - """ - UPDATE jobs - SET status='queued', - scheduled_at=?, - claimed_at=NULL, - claimed_by=NULL, - last_error=? - WHERE id=? AND status='claimed' AND claimed_by=? - RETURNING id - """, - (scheduled, error, job_id, claimed_by), - ) as cursor: - row = await cursor.fetchone() - await self._conn.commit() + stmt = ( + sa.update(jobs) + .where( + jobs.c.id == job_id, + jobs.c.status == "claimed", + jobs.c.claimed_by == claimed_by, + ) + .values( + status="queued", + scheduled_at=scheduled, + claimed_at=None, + claimed_by=None, + last_error=error, + ) + .returning(jobs.c.id) + ) + async with self._engine.begin() as conn: + row = (await conn.execute(stmt)).first() return row is not None async def retry(self, job_id: str) -> Job: @@ -219,24 +236,22 @@ class JobRepo: rows (re-ingest via UPSERT instead). Raises KeyError when the row is missing or in a non-retryable state.""" now = _utcnow_iso() - async with self._lock: - async with self._conn.execute( - """ - UPDATE jobs - SET status='queued', - attempts=0, - last_error=NULL, - claimed_at=NULL, - claimed_by=NULL, - completed_at=NULL, - scheduled_at=? - WHERE id=? AND status IN ('dead', 'queued') - RETURNING * - """, - (now, job_id), - ) as cursor: - row = await cursor.fetchone() - await self._conn.commit() + stmt = ( + sa.update(jobs) + .where(jobs.c.id == job_id, jobs.c.status.in_(["dead", "queued"])) + .values( + status="queued", + attempts=0, + last_error=None, + claimed_at=None, + claimed_by=None, + completed_at=None, + scheduled_at=now, + ) + .returning(*jobs.c) + ) + async with self._engine.begin() as conn: + row = (await conn.execute(stmt)).mappings().one_or_none() if not row: raise KeyError(f"Job {job_id!r} not found or not retryable") return _row_to_job(row) @@ -244,13 +259,13 @@ class JobRepo: async def cancel(self, job_id: str) -> bool: """True iff a queued/claimed row was removed; terminal jobs aren't cancellable (succeeded/dead rows are kept for history).""" - async with self._lock: - async with self._conn.execute( - "DELETE FROM jobs WHERE id=? AND status IN ('queued', 'claimed') RETURNING id", - (job_id,), - ) as cursor: - row = await cursor.fetchone() - await self._conn.commit() + stmt = ( + sa.delete(jobs) + .where(jobs.c.id == job_id, jobs.c.status.in_(["queued", "claimed"])) + .returning(jobs.c.id) + ) + async with self._engine.begin() as conn: + row = (await conn.execute(stmt)).first() return row is not None async def list_jobs( @@ -262,25 +277,16 @@ class JobRepo: limit: int = 50, offset: int = 0, ) -> list[Job]: - clauses: list[str] = [] - params: list[object] = [] + query = sa.select(jobs) if status is not None: - clauses.append("status = ?") - params.append(status.value) + query = query.where(jobs.c.status == status.value) if source_id is not None: - clauses.append("source_id = ?") - params.append(source_id) + query = query.where(jobs.c.source_id == source_id) if uri is not None: - clauses.append("uri = ?") - params.append(uri) - where = ("WHERE " + " AND ".join(clauses)) if clauses else "" - params.extend([limit, offset]) - async with self._lock: - async with self._conn.execute( - f"SELECT * FROM jobs {where} ORDER BY enqueued_at DESC LIMIT ? OFFSET ?", - params, - ) as cursor: - rows = await cursor.fetchall() + query = query.where(jobs.c.uri == uri) + query = query.order_by(jobs.c.enqueued_at.desc()).limit(limit).offset(offset) + async with self._engine.connect() as conn: + rows = (await conn.execute(query)).mappings().all() return [_row_to_job(r) for r in rows] async def has_pending(self, source_id: str) -> bool: @@ -290,78 +296,78 @@ class JobRepo: outstanding work — the queue's unique index would dedupe new enqueues anyway, so a sweep into a saturated queue is pure wasted listing work. """ - async with self._lock: - async with self._conn.execute( - "SELECT 1 FROM jobs WHERE source_id=? AND status IN ('queued','claimed') LIMIT 1", - (source_id,), - ) as cursor: - row = await cursor.fetchone() + query = ( + sa.select(jobs.c.id) + .where( + jobs.c.source_id == source_id, + jobs.c.status.in_(["queued", "claimed"]), + ) + .limit(1) + ) + async with self._engine.connect() as conn: + row = (await conn.execute(query)).first() return row is not None async def counts_by_status(self) -> dict[str, int]: - async with self._lock: - async with self._conn.execute( - "SELECT status, COUNT(*) AS n FROM jobs GROUP BY status" - ) as cursor: - rows = await cursor.fetchall() - return {row["status"]: row["n"] for row in rows} + query = sa.select(jobs.c.status, sa.func.count().label("n")).group_by( + jobs.c.status + ) + async with self._engine.connect() as conn: + rows = (await conn.execute(query)).all() + return {status: n for status, n in rows} async def counts_by_status_since(self, since: datetime) -> dict[str, int]: """status -> count of jobs that reached a terminal state at or after `since` (by completed_at). Only succeeded/dead set completed_at, so those are the only keys returned. Lets a one-shot batch report the work it finished, independent of terminal rows from earlier runs.""" - async with self._lock: - async with self._conn.execute( - "SELECT status, COUNT(*) AS n FROM jobs " - "WHERE completed_at >= ? GROUP BY status", - (since.isoformat(),), - ) as cursor: - rows = await cursor.fetchall() - return {row["status"]: row["n"] for row in rows} + query = ( + sa.select(jobs.c.status, sa.func.count().label("n")) + .where(jobs.c.completed_at >= since.isoformat()) + .group_by(jobs.c.status) + ) + async with self._engine.connect() as conn: + rows = (await conn.execute(query)).all() + return {status: n for status, n in rows} async def count_succeeded_since(self, seconds: int) -> int: """How many jobs reached `succeeded` in the last `seconds` seconds. Drives the dashboard's rolling-throughput chips.""" threshold = (datetime.now(UTC) - timedelta(seconds=seconds)).isoformat() - async with self._lock: - async with self._conn.execute( - "SELECT COUNT(*) AS n FROM jobs WHERE status='succeeded' AND completed_at >= ?", - (threshold,), - ) as cursor: - row = await cursor.fetchone() - return int(row["n"]) if row else 0 + query = sa.select(sa.func.count()).where( + jobs.c.status == "succeeded", jobs.c.completed_at >= threshold + ) + async with self._engine.connect() as conn: + count = (await conn.execute(query)).scalar() + return int(count or 0) async def oldest_queued_age_seconds(self) -> float | None: """Age (in seconds) of the oldest job sitting in `queued` whose scheduled_at is in the past. Returns None when nothing is waiting. Tells operators whether work is backing up.""" now = datetime.now(UTC) - async with self._lock: - async with self._conn.execute( - "SELECT MIN(scheduled_at) AS oldest FROM jobs " - "WHERE status='queued' AND scheduled_at <= ?", - (now.isoformat(),), - ) as cursor: - row = await cursor.fetchone() - if not row or row["oldest"] is None: + query = sa.select(sa.func.min(jobs.c.scheduled_at)).where( + jobs.c.status == "queued", jobs.c.scheduled_at <= now.isoformat() + ) + async with self._engine.connect() as conn: + oldest = (await conn.execute(query)).scalar() + if oldest is None: return None - return (now - datetime.fromisoformat(row["oldest"])).total_seconds() + return (now - datetime.fromisoformat(oldest)).total_seconds() async def counts_by_source(self, *statuses: str) -> dict[str, int]: """source_id → count of jobs in any of the given statuses. Drives the dashboard's per-source DLQ and backlog summaries.""" if not statuses: return {} - placeholders = ",".join("?" * len(statuses)) - async with self._lock: - async with self._conn.execute( - f"SELECT source_id, COUNT(*) AS n FROM jobs " - f"WHERE status IN ({placeholders}) GROUP BY source_id", - statuses, - ) as cursor: - rows = await cursor.fetchall() - return {row["source_id"]: row["n"] for row in rows} + query = ( + sa.select(jobs.c.source_id, sa.func.count().label("n")) + .where(jobs.c.status.in_(statuses)) + .group_by(jobs.c.source_id) + ) + async with self._engine.connect() as conn: + rows = (await conn.execute(query)).all() + return {source_id: n for source_id, n in rows} async def release_if_claimed(self, job_id: str, claimed_by: str) -> bool: """Reset a still-claimed job back to queued, immediately reclaimable. @@ -370,23 +376,24 @@ class JobRepo: re-claimed after a reaper reset. Decrements attempts to undo the increment from `claim_next`, since a cancellation isn't a failed attempt. Returns True if the row was released.""" - now = _utcnow_iso() - async with self._lock: - async with self._conn.execute( - """ - UPDATE jobs - SET status='queued', - claimed_at=NULL, - claimed_by=NULL, - scheduled_at=?, - attempts=MAX(0, attempts - 1) - WHERE id=? AND status='claimed' AND claimed_by=? - RETURNING id - """, - (now, job_id, claimed_by), - ) as cursor: - row = await cursor.fetchone() - await self._conn.commit() + stmt = ( + sa.update(jobs) + .where( + jobs.c.id == job_id, + jobs.c.status == "claimed", + jobs.c.claimed_by == claimed_by, + ) + .values( + status="queued", + claimed_at=None, + claimed_by=None, + scheduled_at=_utcnow_iso(), + attempts=_attempts_minus_one(), + ) + .returning(jobs.c.id) + ) + async with self._engine.begin() as conn: + row = (await conn.execute(stmt)).first() return row is not None async def prune_dead(self, source_id: str, uri: str) -> int: @@ -394,31 +401,27 @@ class JobRepo: successful DELETE to clear stale UPSERT failures for the same URI — the document is gone, so a "couldn't ingest this" entry is no longer actionable. Returns the number of rows removed.""" - async with self._lock: - cursor = await self._conn.execute( - "DELETE FROM jobs WHERE source_id=? AND uri=? AND status='dead'", - (source_id, uri), - ) - rowcount = cursor.rowcount or 0 - await cursor.close() - await self._conn.commit() - return rowcount + stmt = sa.delete(jobs).where( + jobs.c.source_id == source_id, + jobs.c.uri == uri, + jobs.c.status == "dead", + ) + async with self._engine.begin() as conn: + result = await conn.execute(stmt) + return result.rowcount or 0 async def prune_terminal(self, max_age_seconds: int) -> int: """Delete terminal jobs (succeeded/dead) whose completed_at is older than max_age_seconds. General housekeeping so the table doesn't grow without bound. Returns the number of rows removed.""" threshold = (datetime.now(UTC) - timedelta(seconds=max_age_seconds)).isoformat() - async with self._lock: - cursor = await self._conn.execute( - "DELETE FROM jobs WHERE status IN ('succeeded','dead') " - "AND completed_at < ?", - (threshold,), - ) - rowcount = cursor.rowcount or 0 - await cursor.close() - await self._conn.commit() - return rowcount + stmt = sa.delete(jobs).where( + jobs.c.status.in_(["succeeded", "dead"]), + jobs.c.completed_at < threshold, + ) + async with self._engine.begin() as conn: + result = await conn.execute(stmt) + return result.rowcount or 0 async def reap_stale(self, claim_timeout_seconds: int) -> int: """Reset claimed jobs whose claimed_at is older than the timeout @@ -427,35 +430,25 @@ class JobRepo: threshold = ( datetime.now(UTC) - timedelta(seconds=claim_timeout_seconds) ).isoformat() - async with self._lock: - cursor = await self._conn.execute( - """ - UPDATE jobs - SET status='queued', - claimed_at=NULL, - claimed_by=NULL, - attempts=MAX(0, attempts - 1) - WHERE status='claimed' AND claimed_at < ? - """, - (threshold,), + stmt = ( + sa.update(jobs) + .where(jobs.c.status == "claimed", jobs.c.claimed_at < threshold) + .values( + status="queued", + claimed_at=None, + claimed_by=None, + attempts=_attempts_minus_one(), ) - rowcount = cursor.rowcount or 0 - await cursor.close() - await self._conn.commit() - return rowcount + ) + async with self._engine.begin() as conn: + result = await conn.execute(stmt) + return result.rowcount or 0 class SyncStateRepo: - def __init__( - self, - conn: aiosqlite.Connection, - lock: asyncio.Lock | None = None, - ): - conn.row_factory = aiosqlite.Row - self._conn = conn - # Pass the same lock instance JobRepo uses when both wrap one - # connection. See JobRepo for the SQLite cursor + commit constraint. - self._lock = lock or asyncio.Lock() + def __init__(self, engine: AsyncEngine): + self._engine = engine + self._dialect = engine.dialect.name async def get_revision_snapshot(self, source_id: str) -> dict[str, str]: """uri -> revision map for URIs that have a stored revision. Sources @@ -464,35 +457,63 @@ class SyncStateRepo: that didn't complete) are excluded — they have no revision to compare against; the closing-loop DELETE diff uses list_known_uris instead.""" - async with self._lock: - async with self._conn.execute( - "SELECT uri, revision FROM sync_state WHERE source_id=? AND revision IS NOT NULL", - (source_id,), - ) as cursor: - rows = await cursor.fetchall() - return {row["uri"]: row["revision"] for row in rows} + query = sa.select(sync_state.c.uri, sync_state.c.revision).where( + sync_state.c.source_id == source_id, + sync_state.c.revision.is_not(None), + ) + async with self._engine.connect() as conn: + rows = (await conn.execute(query)).all() + return {uri: revision for uri, revision in rows} async def list_known_uris(self, source_id: str) -> set[str]: """Every URI the source has ever produced. Used by the closing-loop diff in discover() so a URI previously seen but no longer visible (FS file deleted, HTTP URL removed from config) emits DELETE.""" - async with self._lock: - async with self._conn.execute( - "SELECT uri FROM sync_state WHERE source_id=?", - (source_id,), - ) as cursor: - rows = await cursor.fetchall() - return {row["uri"] for row in rows} + query = sa.select(sync_state.c.uri).where(sync_state.c.source_id == source_id) + async with self._engine.connect() as conn: + rows = (await conn.execute(query)).all() + return {uri for (uri,) in rows} async def get_row(self, source_id: str, uri: str) -> SyncStateRow | None: - async with self._lock: - async with self._conn.execute( - "SELECT * FROM sync_state WHERE source_id=? AND uri=?", - (source_id, uri), - ) as cursor: - row = await cursor.fetchone() + query = sa.select(sync_state).where( + sync_state.c.source_id == source_id, sync_state.c.uri == uri + ) + async with self._engine.connect() as conn: + row = (await conn.execute(query)).mappings().one_or_none() return _row_to_sync_state(row) if row else None + def _upsert_stmt( + self, + source_id: str, + uri: str, + revision: str | None, + content_hash: str | None, + last_seen_at: str, + last_ingested_at: str | None, + ): + ins = _insert(sync_state, self._dialect).values( + source_id=source_id, + uri=uri, + revision=revision, + content_hash=content_hash, + last_seen_at=last_seen_at, + last_ingested_at=last_ingested_at, + ) + excluded = ins.excluded + return ins.on_conflict_do_update( + index_elements=[sync_state.c.source_id, sync_state.c.uri], + set_={ + "revision": sa.func.coalesce(excluded.revision, sync_state.c.revision), + "content_hash": sa.func.coalesce( + excluded.content_hash, sync_state.c.content_hash + ), + "last_seen_at": excluded.last_seen_at, + "last_ingested_at": sa.func.coalesce( + excluded.last_ingested_at, sync_state.c.last_ingested_at + ), + }, + ) + async def upsert( self, source_id: str, @@ -507,57 +528,32 @@ class SyncStateRepo: `revision=None` and `content_hash=None` leave any existing values untouched.""" now = _utcnow_iso() - ingested_at = now if ingested else None - async with self._lock: - async with self._conn.execute( - """ - INSERT INTO sync_state ( - source_id, uri, revision, content_hash, last_seen_at, last_ingested_at - ) - VALUES (?, ?, ?, ?, ?, ?) - ON CONFLICT(source_id, uri) DO UPDATE SET - revision = COALESCE(excluded.revision, revision), - content_hash = COALESCE(excluded.content_hash, content_hash), - last_seen_at = excluded.last_seen_at, - last_ingested_at = COALESCE(excluded.last_ingested_at, last_ingested_at) - """, - (source_id, uri, revision, content_hash, now, ingested_at), - ): - pass - await self._conn.commit() + stmt = self._upsert_stmt( + source_id, uri, revision, content_hash, now, now if ingested else None + ) + async with self._engine.begin() as conn: + await conn.execute(stmt) async def batch_upsert(self, rows: list[SyncRow]) -> None: """Batch insert-or-update sync_state rows in a single transaction.""" if not rows: return now = _utcnow_iso() - async with self._lock: + async with self._engine.begin() as conn: for source_id, uri, revision, content_hash, ingested in rows: - ingested_at = now if ingested else None - await self._conn.execute( - """ - INSERT INTO sync_state ( - source_id, uri, revision, content_hash, - last_seen_at, last_ingested_at - ) - VALUES (?, ?, ?, ?, ?, ?) - ON CONFLICT(source_id, uri) DO UPDATE SET - revision = COALESCE(excluded.revision, revision), - content_hash = COALESCE(excluded.content_hash, content_hash), - last_seen_at = excluded.last_seen_at, - last_ingested_at = COALESCE( - excluded.last_ingested_at, last_ingested_at - ) - """, - (source_id, uri, revision, content_hash, now, ingested_at), + stmt = self._upsert_stmt( + source_id, + uri, + revision, + content_hash, + now, + now if ingested else None, ) - await self._conn.commit() + await conn.execute(stmt) async def delete(self, source_id: str, uri: str) -> None: - async with self._lock: - async with self._conn.execute( - "DELETE FROM sync_state WHERE source_id=? AND uri=?", - (source_id, uri), - ): - pass - await self._conn.commit() + stmt = sa.delete(sync_state).where( + sync_state.c.source_id == source_id, sync_state.c.uri == uri + ) + async with self._engine.begin() as conn: + await conn.execute(stmt) diff --git a/haiku_rag_slim/haiku/rag/ingester/queue/schema.py b/haiku_rag_slim/haiku/rag/ingester/queue/schema.py deleted file mode 100644 index a05ee068..00000000 --- a/haiku_rag_slim/haiku/rag/ingester/queue/schema.py +++ /dev/null @@ -1,80 +0,0 @@ -SCHEMA_VERSION = 1 - - -JOBS_DDL = """ -CREATE TABLE IF NOT EXISTS jobs ( - id TEXT PRIMARY KEY, - source_id TEXT NOT NULL, - uri TEXT NOT NULL, - op TEXT NOT NULL, - content_hash TEXT, - revision TEXT, - status TEXT NOT NULL, - attempts INTEGER NOT NULL DEFAULT 0, - max_attempts INTEGER NOT NULL DEFAULT 5, - last_error TEXT, - extra TEXT, - enqueued_at TEXT NOT NULL, - scheduled_at TEXT NOT NULL, - claimed_at TEXT, - claimed_by TEXT, - completed_at TEXT -) -""" - -# Partial unique index: a (source_id, uri) pair can only have one live job -# (queued or claimed) at a time, regardless of op. Live UPSERT and DELETE -# for the same URI can't both exist — preventing a DELETE worker from -# removing a document a sibling UPSERT just ingested. Once succeeded or -# dead, the row no longer satisfies the WHERE clause and a re-enqueue is -# allowed. -JOBS_LIVE_INDEX = """ -CREATE UNIQUE INDEX IF NOT EXISTS uq_jobs_live -ON jobs(source_id, uri) -WHERE status IN ('queued', 'claimed') -""" - -JOBS_CLAIMABLE_INDEX = """ -CREATE INDEX IF NOT EXISTS idx_jobs_claimable -ON jobs(scheduled_at) -WHERE status = 'queued' -""" - -JOBS_SUCCEEDED_COMPLETED_INDEX = """ -CREATE INDEX IF NOT EXISTS idx_jobs_succeeded_completed -ON jobs(completed_at) -WHERE status = 'succeeded' -""" - -SYNC_STATE_DDL = """ -CREATE TABLE IF NOT EXISTS sync_state ( - source_id TEXT NOT NULL, - uri TEXT NOT NULL, - revision TEXT, - content_hash TEXT, - last_seen_at TEXT NOT NULL, - last_ingested_at TEXT, - PRIMARY KEY (source_id, uri) -) -""" - -SCHEMA_VERSION_DDL = """ -CREATE TABLE IF NOT EXISTS schema_version ( - version INTEGER PRIMARY KEY -) -""" - -DLQ_VIEW = """ -CREATE VIEW IF NOT EXISTS dlq AS -SELECT * FROM jobs WHERE status = 'dead' -""" - -ALL_DDL: tuple[str, ...] = ( - JOBS_DDL, - JOBS_LIVE_INDEX, - JOBS_CLAIMABLE_INDEX, - JOBS_SUCCEEDED_COMPLETED_INDEX, - SYNC_STATE_DDL, - SCHEMA_VERSION_DDL, - DLQ_VIEW, -) diff --git a/haiku_rag_slim/pyproject.toml b/haiku_rag_slim/pyproject.toml index 1b5c41ac..ba365f8f 100644 --- a/haiku_rag_slim/pyproject.toml +++ b/haiku_rag_slim/pyproject.toml @@ -58,7 +58,9 @@ cross-encoder = ["sentence-transformers>=3.0.0"] ingester = [ "fastapi>=0.125", "uvicorn[standard]>=0.32", + "sqlalchemy[asyncio]>=2.0", "aiosqlite>=0.20", + "asyncpg>=0.29", "haiku.rag-slim[s3]", ] # TUI (chat and inspect commands) diff --git a/tests/ingester/conftest.py b/tests/ingester/conftest.py new file mode 100644 index 00000000..01dfe179 --- /dev/null +++ b/tests/ingester/conftest.py @@ -0,0 +1,34 @@ +import aiosqlite +import pytest + +from haiku.rag.config import QueueConfig +from haiku.rag.ingester.queue.migrations import open_queue +from haiku.rag.ingester.queue.repository import JobRepo, SyncStateRepo + + +@pytest.fixture +async def engine(tmp_path): + eng = await open_queue(QueueConfig(path=tmp_path / "queue.db")) + yield eng + await eng.dispose() + + +@pytest.fixture +async def conn(tmp_path, engine): + """Raw connection to the queue file for test-only SQL inspection and + backdating. Shares the WAL database the engine writes through.""" + connection = await aiosqlite.connect(str(tmp_path / "queue.db")) + connection.row_factory = aiosqlite.Row + await connection.execute("PRAGMA busy_timeout=30000") + yield connection + await connection.close() + + +@pytest.fixture +def jobs(engine): + return JobRepo(engine) + + +@pytest.fixture +def sync(engine): + return SyncStateRepo(engine) diff --git a/tests/ingester/test_api.py b/tests/ingester/test_api.py index 9146f73b..d266d8b4 100644 --- a/tests/ingester/test_api.py +++ b/tests/ingester/test_api.py @@ -1,16 +1,12 @@ -import asyncio from datetime import UTC, datetime -import aiosqlite import httpx import pytest from httpx import ASGITransport from haiku.rag.config import AppConfig from haiku.rag.ingester.api.server import APIState, build_app -from haiku.rag.ingester.queue.migrations import apply_migrations from haiku.rag.ingester.queue.models import JobOp, JobStatus -from haiku.rag.ingester.queue.repository import JobRepo, SyncStateRepo from haiku.rag.ingester.sources.base import ( FetchResult, SourceEvent, @@ -18,31 +14,6 @@ from haiku.rag.ingester.sources.base import ( ) -@pytest.fixture -async def conn(tmp_path): - path = tmp_path / "queue.db" - connection = await aiosqlite.connect(str(path)) - connection.row_factory = aiosqlite.Row - await apply_migrations(connection) - yield connection - await connection.close() - - -@pytest.fixture -def queue_lock(): - return asyncio.Lock() - - -@pytest.fixture -def jobs(conn, queue_lock): - return JobRepo(conn, lock=queue_lock) - - -@pytest.fixture -def sync(conn, queue_lock): - return SyncStateRepo(conn, lock=queue_lock) - - @pytest.fixture def state(jobs, sync): return APIState( diff --git a/tests/ingester/test_pollers.py b/tests/ingester/test_pollers.py index de766731..eca4d418 100644 --- a/tests/ingester/test_pollers.py +++ b/tests/ingester/test_pollers.py @@ -2,7 +2,6 @@ import asyncio from datetime import UTC, datetime from pathlib import Path -import aiosqlite import pytest from haiku.rag.config import ( @@ -15,9 +14,7 @@ from haiku.rag.config import ( from haiku.rag.ingester.pollers.circuit_breaker import CircuitBreaker from haiku.rag.ingester.pollers.manager import PollerManager from haiku.rag.ingester.pollers.periodic import PeriodicPoller -from haiku.rag.ingester.queue.migrations import apply_migrations from haiku.rag.ingester.queue.models import JobOp, JobStatus -from haiku.rag.ingester.queue.repository import JobRepo, SyncStateRepo from haiku.rag.ingester.sources.base import ( FetchResult, SourceEvent, @@ -25,31 +22,6 @@ from haiku.rag.ingester.sources.base import ( ) -@pytest.fixture -async def conn(tmp_path): - path = tmp_path / "queue.db" - connection = await aiosqlite.connect(str(path)) - connection.row_factory = aiosqlite.Row - await apply_migrations(connection) - yield connection - await connection.close() - - -@pytest.fixture -def queue_lock(): - return asyncio.Lock() - - -@pytest.fixture -def jobs(conn, queue_lock): - return JobRepo(conn, lock=queue_lock) - - -@pytest.fixture -def sync(conn, queue_lock): - return SyncStateRepo(conn, lock=queue_lock) - - class _StubSource: """Test double that yields a scripted sequence of events on each discover() call. `fetch` and `supports` aren't exercised by pollers.""" diff --git a/tests/ingester/test_queue.py b/tests/ingester/test_queue.py index 524806ab..71575aea 100644 --- a/tests/ingester/test_queue.py +++ b/tests/ingester/test_queue.py @@ -1,38 +1,12 @@ import asyncio from datetime import UTC, datetime, timedelta -import aiosqlite import pytest +import sqlalchemy as sa +from haiku.rag.config import QueueConfig from haiku.rag.ingester.queue.migrations import apply_migrations, open_queue from haiku.rag.ingester.queue.models import JobOp, JobStatus, SyncRow -from haiku.rag.ingester.queue.repository import JobRepo, SyncStateRepo - - -@pytest.fixture -async def conn(tmp_path): - path = tmp_path / "queue.db" - connection = await aiosqlite.connect(str(path)) - connection.row_factory = aiosqlite.Row - await apply_migrations(connection) - yield connection - await connection.close() - - -@pytest.fixture -def queue_lock(): - return asyncio.Lock() - - -@pytest.fixture -def jobs(conn, queue_lock): - return JobRepo(conn, lock=queue_lock) - - -@pytest.fixture -def sync(conn, queue_lock): - return SyncStateRepo(conn, lock=queue_lock) - # --- migrations / schema --- @@ -46,9 +20,9 @@ async def test_apply_migrations_sets_schema_version(conn): @pytest.mark.asyncio -async def test_apply_migrations_is_idempotent(conn): - await apply_migrations(conn) - await apply_migrations(conn) +async def test_apply_migrations_is_idempotent(engine, conn): + await apply_migrations(engine) + await apply_migrations(engine) cursor = await conn.execute("SELECT COUNT(*) AS n FROM schema_version") row = await cursor.fetchone() assert row["n"] == 1 @@ -57,26 +31,16 @@ async def test_apply_migrations_is_idempotent(conn): @pytest.mark.asyncio async def test_open_queue_creates_file_and_schema(tmp_path): path = tmp_path / "subdir" / "queue.db" - connection = await open_queue(path) + eng = await open_queue(QueueConfig(path=path)) try: assert path.exists() - cursor = await connection.execute("SELECT version FROM schema_version") - row = await cursor.fetchone() - assert row is not None + async with eng.connect() as conn: + version = ( + await conn.execute(sa.text("SELECT version FROM schema_version")) + ).scalar() + assert version is not None finally: - await connection.close() - - -@pytest.mark.asyncio -async def test_dlq_view_exposes_dead_jobs(conn, jobs): - job = await jobs.enqueue("s", "u", JobOp.UPSERT) - claimed = await jobs.claim_next("w") - assert claimed is not None - await jobs.mark_dead(job.id, "permanent", "w") - - cursor = await conn.execute("SELECT id FROM dlq") - rows = await cursor.fetchall() - assert [r["id"] for r in rows] == [job.id] + await eng.dispose() # --- enqueue --- @@ -797,29 +761,6 @@ async def test_counts_by_source_no_statuses_returns_empty(jobs): assert await jobs.counts_by_source() == {} -# --- cross-repo lock sharing --- - - -def test_repos_share_lock_when_constructed_with_one(conn): - """JobRepo and SyncStateRepo wrap one shared aiosqlite.Connection in - production. They must accept and share a single asyncio.Lock so cross- - repo calls serialize at the cursor/commit boundary — otherwise a - SyncStateRepo.upsert cursor open while JobRepo.mark_succeeded tries - to commit would trip SQLite's 'SQL statements in progress' error.""" - shared = asyncio.Lock() - j = JobRepo(conn, lock=shared) - s = SyncStateRepo(conn, lock=shared) - assert j._lock is s._lock is shared - - -def test_repos_default_to_independent_locks_when_used_alone(conn): - """Backward-compat: a single-repo caller can still construct without - passing a lock and each repo creates its own.""" - j = JobRepo(conn) - s = SyncStateRepo(conn) - assert j._lock is not s._lock - - # --- sync state --- diff --git a/tests/ingester/test_serve_integration.py b/tests/ingester/test_serve_integration.py index 46c42334..34962d98 100644 --- a/tests/ingester/test_serve_integration.py +++ b/tests/ingester/test_serve_integration.py @@ -3,45 +3,17 @@ import asyncio from unittest.mock import AsyncMock -import aiosqlite import pytest from haiku.rag.client import HaikuRAG from haiku.rag.config import FSSourceConfig, HTTPSourceConfig from haiku.rag.ingester.pollers.manager import PollerManager -from haiku.rag.ingester.queue.migrations import apply_migrations from haiku.rag.ingester.queue.models import JobOp -from haiku.rag.ingester.queue.repository import JobRepo, SyncStateRepo from haiku.rag.ingester.sources.http import HTTPSource from haiku.rag.ingester.workers.pool import WorkerPool from haiku.rag.store.models.document import Document -@pytest.fixture -async def conn(tmp_path): - path = tmp_path / "queue.db" - connection = await aiosqlite.connect(str(path)) - connection.row_factory = aiosqlite.Row - await apply_migrations(connection) - yield connection - await connection.close() - - -@pytest.fixture -def queue_lock(): - return asyncio.Lock() - - -@pytest.fixture -def jobs(conn, queue_lock): - return JobRepo(conn, lock=queue_lock) - - -@pytest.fixture -def sync(conn, queue_lock): - return SyncStateRepo(conn, lock=queue_lock) - - async def _wait_for(predicate, *, timeout: float = 5.0, interval: float = 0.05): """Poll `predicate` until it returns truthy or `timeout` elapses.""" deadline = asyncio.get_running_loop().time() + timeout diff --git a/tests/ingester/test_workers.py b/tests/ingester/test_workers.py index 60596d19..c10ac475 100644 --- a/tests/ingester/test_workers.py +++ b/tests/ingester/test_workers.py @@ -1,44 +1,16 @@ import asyncio from unittest.mock import AsyncMock -import aiosqlite import pytest from haiku.rag.client import HaikuRAG from haiku.rag.ingester.exceptions import PermanentError, TransientError -from haiku.rag.ingester.queue.migrations import apply_migrations from haiku.rag.ingester.queue.models import JobOp, JobStatus -from haiku.rag.ingester.queue.repository import JobRepo, SyncStateRepo from haiku.rag.ingester.workers.pool import WorkerPool from haiku.rag.ingester.workers.retry import RetryPolicy from haiku.rag.store.models.document import Document -@pytest.fixture -async def conn(tmp_path): - path = tmp_path / "queue.db" - connection = await aiosqlite.connect(str(path)) - connection.row_factory = aiosqlite.Row - await apply_migrations(connection) - yield connection - await connection.close() - - -@pytest.fixture -def queue_lock(): - return asyncio.Lock() - - -@pytest.fixture -def jobs(conn, queue_lock): - return JobRepo(conn, lock=queue_lock) - - -@pytest.fixture -def sync(conn, queue_lock): - return SyncStateRepo(conn, lock=queue_lock) - - @pytest.fixture def client(): return AsyncMock(spec=HaikuRAG) diff --git a/uv.lock b/uv.lock index 2d69bb67..28616ab6 100644 --- a/uv.lock +++ b/uv.lock @@ -243,6 +243,46 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/38/0e/27be9fdef66e72d64c0cdc3cc2823101b80585f8119b5c112c2e8f5f7dab/anyio-4.12.1-py3-none-any.whl", hash = "sha256:d405828884fc140aa80a3c667b8beed277f1dfedec42ba031bd6ac3db606ab6c", size = 113592, upload-time = "2026-01-06T11:45:19.497Z" }, ] +[[package]] +name = "asyncpg" +version = "0.31.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/fe/cc/d18065ce2380d80b1bcce927c24a2642efd38918e33fd724bc4bca904877/asyncpg-0.31.0.tar.gz", hash = "sha256:c989386c83940bfbd787180f2b1519415e2d3d6277a70d9d0f0145ac73500735", size = 993667, upload-time = "2025-11-24T23:27:00.812Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2a/a6/59d0a146e61d20e18db7396583242e32e0f120693b67a8de43f1557033e2/asyncpg-0.31.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:b44c31e1efc1c15188ef183f287c728e2046abb1d26af4d20858215d50d91fad", size = 662042, upload-time = "2025-11-24T23:25:49.578Z" }, + { url = "https://files.pythonhosted.org/packages/36/01/ffaa189dcb63a2471720615e60185c3f6327716fdc0fc04334436fbb7c65/asyncpg-0.31.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0c89ccf741c067614c9b5fc7f1fc6f3b61ab05ae4aaa966e6fd6b93097c7d20d", size = 638504, upload-time = "2025-11-24T23:25:51.501Z" }, + { url = "https://files.pythonhosted.org/packages/9f/62/3f699ba45d8bd24c5d65392190d19656d74ff0185f42e19d0bbd973bb371/asyncpg-0.31.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:12b3b2e39dc5470abd5e98c8d3373e4b1d1234d9fbdedf538798b2c13c64460a", size = 3426241, upload-time = "2025-11-24T23:25:53.278Z" }, + { url = "https://files.pythonhosted.org/packages/8c/d1/a867c2150f9c6e7af6462637f613ba67f78a314b00db220cd26ff559d532/asyncpg-0.31.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:aad7a33913fb8bcb5454313377cc330fbb19a0cd5faa7272407d8a0c4257b671", size = 3520321, upload-time = "2025-11-24T23:25:54.982Z" }, + { url = "https://files.pythonhosted.org/packages/7a/1a/cce4c3f246805ecd285a3591222a2611141f1669d002163abef999b60f98/asyncpg-0.31.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3df118d94f46d85b2e434fd62c84cb66d5834d5a890725fe625f498e72e4d5ec", size = 3316685, upload-time = "2025-11-24T23:25:57.43Z" }, + { url = "https://files.pythonhosted.org/packages/40/ae/0fc961179e78cc579e138fad6eb580448ecae64908f95b8cb8ee2f241f67/asyncpg-0.31.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:bd5b6efff3c17c3202d4b37189969acf8927438a238c6257f66be3c426beba20", size = 3471858, upload-time = "2025-11-24T23:25:59.636Z" }, + { url = "https://files.pythonhosted.org/packages/52/b2/b20e09670be031afa4cbfabd645caece7f85ec62d69c312239de568e058e/asyncpg-0.31.0-cp312-cp312-win32.whl", hash = "sha256:027eaa61361ec735926566f995d959ade4796f6a49d3bde17e5134b9964f9ba8", size = 527852, upload-time = "2025-11-24T23:26:01.084Z" }, + { url = "https://files.pythonhosted.org/packages/b5/f0/f2ed1de154e15b107dc692262395b3c17fc34eafe2a78fc2115931561730/asyncpg-0.31.0-cp312-cp312-win_amd64.whl", hash = "sha256:72d6bdcbc93d608a1158f17932de2321f68b1a967a13e014998db87a72ed3186", size = 597175, upload-time = "2025-11-24T23:26:02.564Z" }, + { url = "https://files.pythonhosted.org/packages/95/11/97b5c2af72a5d0b9bc3fa30cd4b9ce22284a9a943a150fdc768763caf035/asyncpg-0.31.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:c204fab1b91e08b0f47e90a75d1b3c62174dab21f670ad6c5d0f243a228f015b", size = 661111, upload-time = "2025-11-24T23:26:04.467Z" }, + { url = "https://files.pythonhosted.org/packages/1b/71/157d611c791a5e2d0423f09f027bd499935f0906e0c2a416ce712ba51ef3/asyncpg-0.31.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:54a64f91839ba59008eccf7aad2e93d6e3de688d796f35803235ea1c4898ae1e", size = 636928, upload-time = "2025-11-24T23:26:05.944Z" }, + { url = "https://files.pythonhosted.org/packages/2e/fc/9e3486fb2bbe69d4a867c0b76d68542650a7ff1574ca40e84c3111bb0c6e/asyncpg-0.31.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c0e0822b1038dc7253b337b0f3f676cadc4ac31b126c5d42691c39691962e403", size = 3424067, upload-time = "2025-11-24T23:26:07.957Z" }, + { url = "https://files.pythonhosted.org/packages/12/c6/8c9d076f73f07f995013c791e018a1cd5f31823c2a3187fc8581706aa00f/asyncpg-0.31.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bef056aa502ee34204c161c72ca1f3c274917596877f825968368b2c33f585f4", size = 3518156, upload-time = "2025-11-24T23:26:09.591Z" }, + { url = "https://files.pythonhosted.org/packages/ae/3b/60683a0baf50fbc546499cfb53132cb6835b92b529a05f6a81471ab60d0c/asyncpg-0.31.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:0bfbcc5b7ffcd9b75ab1558f00db2ae07db9c80637ad1b2469c43df79d7a5ae2", size = 3319636, upload-time = "2025-11-24T23:26:11.168Z" }, + { url = "https://files.pythonhosted.org/packages/50/dc/8487df0f69bd398a61e1792b3cba0e47477f214eff085ba0efa7eac9ce87/asyncpg-0.31.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:22bc525ebbdc24d1261ecbf6f504998244d4e3be1721784b5f64664d61fbe602", size = 3472079, upload-time = "2025-11-24T23:26:13.164Z" }, + { url = "https://files.pythonhosted.org/packages/13/a1/c5bbeeb8531c05c89135cb8b28575ac2fac618bcb60119ee9696c3faf71c/asyncpg-0.31.0-cp313-cp313-win32.whl", hash = "sha256:f890de5e1e4f7e14023619399a471ce4b71f5418cd67a51853b9910fdfa73696", size = 527606, upload-time = "2025-11-24T23:26:14.78Z" }, + { url = "https://files.pythonhosted.org/packages/91/66/b25ccb84a246b470eb943b0107c07edcae51804912b824054b3413995a10/asyncpg-0.31.0-cp313-cp313-win_amd64.whl", hash = "sha256:dc5f2fa9916f292e5c5c8b2ac2813763bcd7f58e130055b4ad8a0531314201ab", size = 596569, upload-time = "2025-11-24T23:26:16.189Z" }, + { url = "https://files.pythonhosted.org/packages/3c/36/e9450d62e84a13aea6580c83a47a437f26c7ca6fa0f0fd40b6670793ea30/asyncpg-0.31.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:f6b56b91bb0ffc328c4e3ed113136cddd9deefdf5f79ab448598b9772831df44", size = 660867, upload-time = "2025-11-24T23:26:17.631Z" }, + { url = "https://files.pythonhosted.org/packages/82/4b/1d0a2b33b3102d210439338e1beea616a6122267c0df459ff0265cd5807a/asyncpg-0.31.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:334dec28cf20d7f5bb9e45b39546ddf247f8042a690bff9b9573d00086e69cb5", size = 638349, upload-time = "2025-11-24T23:26:19.689Z" }, + { url = "https://files.pythonhosted.org/packages/41/aa/e7f7ac9a7974f08eff9183e392b2d62516f90412686532d27e196c0f0eeb/asyncpg-0.31.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:98cc158c53f46de7bb677fd20c417e264fc02b36d901cc2a43bd6cb0dc6dbfd2", size = 3410428, upload-time = "2025-11-24T23:26:21.275Z" }, + { url = "https://files.pythonhosted.org/packages/6f/de/bf1b60de3dede5c2731e6788617a512bc0ebd9693eac297ee74086f101d7/asyncpg-0.31.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9322b563e2661a52e3cdbc93eed3be7748b289f792e0011cb2720d278b366ce2", size = 3471678, upload-time = "2025-11-24T23:26:23.627Z" }, + { url = "https://files.pythonhosted.org/packages/46/78/fc3ade003e22d8bd53aaf8f75f4be48f0b460fa73738f0391b9c856a9147/asyncpg-0.31.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:19857a358fc811d82227449b7ca40afb46e75b33eb8897240c3839dd8b744218", size = 3313505, upload-time = "2025-11-24T23:26:25.235Z" }, + { url = "https://files.pythonhosted.org/packages/bf/e9/73eb8a6789e927816f4705291be21f2225687bfa97321e40cd23055e903a/asyncpg-0.31.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ba5f8886e850882ff2c2ace5732300e99193823e8107e2c53ef01c1ebfa1e85d", size = 3434744, upload-time = "2025-11-24T23:26:26.944Z" }, + { url = "https://files.pythonhosted.org/packages/08/4b/f10b880534413c65c5b5862f79b8e81553a8f364e5238832ad4c0af71b7f/asyncpg-0.31.0-cp314-cp314-win32.whl", hash = "sha256:cea3a0b2a14f95834cee29432e4ddc399b95700eb1d51bbc5bfee8f31fa07b2b", size = 532251, upload-time = "2025-11-24T23:26:28.404Z" }, + { url = "https://files.pythonhosted.org/packages/d3/2d/7aa40750b7a19efa5d66e67fc06008ca0f27ba1bd082e457ad82f59aba49/asyncpg-0.31.0-cp314-cp314-win_amd64.whl", hash = "sha256:04d19392716af6b029411a0264d92093b6e5e8285ae97a39957b9a9c14ea72be", size = 604901, upload-time = "2025-11-24T23:26:30.34Z" }, + { url = "https://files.pythonhosted.org/packages/ce/fe/b9dfe349b83b9dee28cc42360d2c86b2cdce4cb551a2c2d27e156bcac84d/asyncpg-0.31.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:bdb957706da132e982cc6856bb2f7b740603472b54c3ebc77fe60ea3e57e1bd2", size = 702280, upload-time = "2025-11-24T23:26:32Z" }, + { url = "https://files.pythonhosted.org/packages/6a/81/e6be6e37e560bd91e6c23ea8a6138a04fd057b08cf63d3c5055c98e81c1d/asyncpg-0.31.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6d11b198111a72f47154fa03b85799f9be63701e068b43f84ac25da0bda9cb31", size = 682931, upload-time = "2025-11-24T23:26:33.572Z" }, + { url = "https://files.pythonhosted.org/packages/a6/45/6009040da85a1648dd5bc75b3b0a062081c483e75a1a29041ae63a0bf0dc/asyncpg-0.31.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:18c83b03bc0d1b23e6230f5bf8d4f217dc9bc08644ce0502a9d91dc9e634a9c7", size = 3581608, upload-time = "2025-11-24T23:26:35.638Z" }, + { url = "https://files.pythonhosted.org/packages/7e/06/2e3d4d7608b0b2b3adbee0d0bd6a2d29ca0fc4d8a78f8277df04e2d1fd7b/asyncpg-0.31.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e009abc333464ff18b8f6fd146addffd9aaf63e79aa3bb40ab7a4c332d0c5e9e", size = 3498738, upload-time = "2025-11-24T23:26:37.275Z" }, + { url = "https://files.pythonhosted.org/packages/7d/aa/7d75ede780033141c51d83577ea23236ba7d3a23593929b32b49db8ed36e/asyncpg-0.31.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:3b1fbcb0e396a5ca435a8826a87e5c2c2cc0c8c68eb6fadf82168056b0e53a8c", size = 3401026, upload-time = "2025-11-24T23:26:39.423Z" }, + { url = "https://files.pythonhosted.org/packages/ba/7a/15e37d45e7f7c94facc1e9148c0e455e8f33c08f0b8a0b1deb2c5171771b/asyncpg-0.31.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:8df714dba348efcc162d2adf02d213e5fab1bd9f557e1305633e851a61814a7a", size = 3429426, upload-time = "2025-11-24T23:26:41.032Z" }, + { url = "https://files.pythonhosted.org/packages/13/d5/71437c5f6ae5f307828710efbe62163974e71237d5d46ebd2869ea052d10/asyncpg-0.31.0-cp314-cp314t-win32.whl", hash = "sha256:1b41f1afb1033f2b44f3234993b15096ddc9cd71b21a42dbd87fc6a57b43d65d", size = 614495, upload-time = "2025-11-24T23:26:42.659Z" }, + { url = "https://files.pythonhosted.org/packages/3c/d7/8fb3044eaef08a310acfe23dae9a8e2e07d305edc29a53497e52bc76eca7/asyncpg-0.31.0-cp314-cp314t-win_amd64.whl", hash = "sha256:bd4107bb7cdd0e9e65fae66a62afd3a249663b844fa34d479f6d5b3bef9c04c3", size = 706062, upload-time = "2025-11-24T23:26:44.086Z" }, +] + [[package]] name = "attrs" version = "26.1.0" @@ -1420,6 +1460,73 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/69/28/23eea8acd65972bbfe295ce3666b28ac510dfcb115fac089d3edb0feb00a/googleapis_common_protos-1.73.0-py3-none-any.whl", hash = "sha256:dfdaaa2e860f242046be561e6d6cb5c5f1541ae02cfbcb034371aadb2942b4e8", size = 297578, upload-time = "2026-03-06T21:52:33.933Z" }, ] +[[package]] +name = "greenlet" +version = "3.5.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/6d/6e/802acd792aebb2256fbbee8cacf2727faaeb6f240ac11008f09eae4414bc/greenlet-3.5.1.tar.gz", hash = "sha256:5a56aeb7d5d9cc4b3a735efb5095bd4b4f6f0e4f93e5ca876d0e2315137b7829", size = 197356, upload-time = "2026-05-20T15:05:03.917Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c4/37/4549f149c9797c21b32c2683c33522af22522099de128b2406672526d005/greenlet-3.5.1-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:fa4f98af3a528f0c3fd592a26df7f376f93329c8f4d987f6bb979057af8bf5e2", size = 286220, upload-time = "2026-05-20T13:07:28.463Z" }, + { url = "https://files.pythonhosted.org/packages/38/ff/a4f436709716965eaab9f36ea7b906c8a927fbe32fb1372a2071d964f6b1/greenlet-3.5.1-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ffea73584b216150eab159b6d12348fb253e68757974de1e2c40d8a318ac89ed", size = 601585, upload-time = "2026-05-20T14:00:06.141Z" }, + { url = "https://files.pythonhosted.org/packages/65/ad/54bc3fcee3ad368a61b19b67d88117f7a8c29727bf71fffdeda81fbd946e/greenlet-3.5.1-cp312-cp312-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1072b4f9edcc1e192d9283a66a3e68d6b84c561de33a83d7858beb9ba1effe10", size = 614215, upload-time = "2026-05-20T14:05:42.675Z" }, + { url = "https://files.pythonhosted.org/packages/7c/6c/de5b1b388cd2d9fbdfeab324863daba37d54e6e233ddbefd70b385a8c591/greenlet-3.5.1-cp312-cp312-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:89101bfd5011e069be974903cb3a4e4523845e4ece2d62dcd8d358933c0ef249", size = 620094, upload-time = "2026-05-20T14:09:09.18Z" }, + { url = "https://files.pythonhosted.org/packages/40/69/b91cda0647df839483201545913514c2827ebea5e5ccdf931842763bc127/greenlet-3.5.1-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:add5217d68b31130f0beca584d7fef4878327d2e31642b66618a14eef312b63b", size = 611358, upload-time = "2026-05-20T13:14:26.37Z" }, + { url = "https://files.pythonhosted.org/packages/4a/43/1204baffab8a6476464795a7ccf394a3248d4f22c9f87173a15b36b6d971/greenlet-3.5.1-cp312-cp312-manylinux_2_39_riscv64.whl", hash = "sha256:e6cd99ea59dd5d89f0c956606571d79bfe6f68c9eb7f4a4083a41a7f1587edee", size = 422782, upload-time = "2026-05-20T14:01:39.597Z" }, + { url = "https://files.pythonhosted.org/packages/59/90/3cf77e080350cd02fa307bb2abf05df48f4482c240275bbd2c203ba8bb1c/greenlet-3.5.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:a5ea42a752d47a145eae922b605cd1634665ac3d5ec1e72402d5048e8d60d207", size = 1570475, upload-time = "2026-05-20T14:02:25.29Z" }, + { url = "https://files.pythonhosted.org/packages/65/2c/18cece62045e74598c3c393f70dce4a63f56222015ba29a5d4eeb04f764c/greenlet-3.5.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c5551170cf4f5ff5623e9af81323751979fee2c731e2287b61f73cd27257b823", size = 1635625, upload-time = "2026-05-20T13:14:34.027Z" }, + { url = "https://files.pythonhosted.org/packages/30/f5/310d104ddf41eb5a70f4c268d22508dfb0c3c8e86fec152be34d0d2ed819/greenlet-3.5.1-cp312-cp312-win_amd64.whl", hash = "sha256:3c8bb982ad117d29478ef8f5533e97df21f1e2befd17a299257b0c96d1371c0b", size = 238791, upload-time = "2026-05-20T13:10:39.018Z" }, + { url = "https://files.pythonhosted.org/packages/62/90/ceca11f504cd23a8047a3dea31919adc48df9b626dd0c13f0d858734fdfd/greenlet-3.5.1-cp312-cp312-win_arm64.whl", hash = "sha256:80eb4b04dadc4e67df3fae179a32c4706a3f495bc7f22fc8a81115d5f5512188", size = 235580, upload-time = "2026-05-20T13:08:45.056Z" }, + { url = "https://files.pythonhosted.org/packages/27/69/7f7e5372d998b81001899b1c0823c957aa413ba0f2662e65821611cc31e4/greenlet-3.5.1-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:51518ff74664078fc51bffcc6fc529b0df5ae58da192691cee765d45ce944a2b", size = 285060, upload-time = "2026-05-20T13:08:51.899Z" }, + { url = "https://files.pythonhosted.org/packages/b1/bf/387f9b6b865fd2ae0d0be09e0004827295a01b71be76ed350dd1e28a91a4/greenlet-3.5.1-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1ffdb3c0bb002c99cd8f298957e046c3dbf6006b5b7cdf11a4e19194624a0a0a", size = 604370, upload-time = "2026-05-20T14:00:07.492Z" }, + { url = "https://files.pythonhosted.org/packages/32/f5/169ce3d4e4c67291bd18f8cbe0299c9f3e45102c7f1fb3c14780c93e4532/greenlet-3.5.1-cp313-cp313-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7715a5a2c3378ba602c3a440558261e13a820bb53a82693aacd7b7f6d964e283", size = 616987, upload-time = "2026-05-20T14:05:44.237Z" }, + { url = "https://files.pythonhosted.org/packages/19/ba/c24110c55dffa55aa6e1d98b45310da33801aeba7686ff0190fe5d46fd32/greenlet-3.5.1-cp313-cp313-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d40a890035c0058cadbdc4af7569800fd28a0e527a0fdbb7b5f9418f176846ce", size = 622911, upload-time = "2026-05-20T14:09:10.598Z" }, + { url = "https://files.pythonhosted.org/packages/ee/e5/7f2e41d5273be07e77560d61ea4e56485b4d6c316d2a84518c62d1364061/greenlet-3.5.1-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:dc71ff466927a201b08305acac451ebe1aedfcea002f62f1f2f2ac2ac1e6a135", size = 613911, upload-time = "2026-05-20T13:14:27.539Z" }, + { url = "https://files.pythonhosted.org/packages/ec/7b/d20db2e8a5ad6c038702f3179b136f93f0a3d1a21a0c0777f3e470cdf4b2/greenlet-3.5.1-cp313-cp313-manylinux_2_39_riscv64.whl", hash = "sha256:67821bb03e4e98664490edb787ff6af501194c29bbee0f5c1dfdcf1dc3d9d436", size = 425228, upload-time = "2026-05-20T14:01:40.837Z" }, + { url = "https://files.pythonhosted.org/packages/c5/a4/fbdc67579b73615a1f91615e814303cc71e06128f7baaba87be79b8fb90c/greenlet-3.5.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:cd443683db272ebaaca03af98c0b063ab30db70ea8a31a1559f35e3f7b744ccd", size = 1570689, upload-time = "2026-05-20T14:02:27.225Z" }, + { url = "https://files.pythonhosted.org/packages/e6/b4/77abbe35078be39718a46cd49caf16bceb35662f97a34101dca28aa98e47/greenlet-3.5.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:089fff7a6ce8d9316d1f65ebc00273a56be258c1725b32b94de90a3a979557e1", size = 1635602, upload-time = "2026-05-20T13:14:36.344Z" }, + { url = "https://files.pythonhosted.org/packages/37/f7/129f27ca700845b8ee8ca88ce7f43435a1239c2eddb7677fc938822762cf/greenlet-3.5.1-cp313-cp313-win_amd64.whl", hash = "sha256:110a1ca7b49b014b097f6078272c3f4ed31af45b254de5228b79adba879f6af9", size = 238683, upload-time = "2026-05-20T13:11:50.57Z" }, + { url = "https://files.pythonhosted.org/packages/6d/5c/a485a36e87df8d8fd0632ee01511244f5156a20ed3746cc6599340326395/greenlet-3.5.1-cp313-cp313-win_arm64.whl", hash = "sha256:f16ba1efc0715b680a18b8123d90dad887c6112ae3555b4b5c32c149540c6b4e", size = 235499, upload-time = "2026-05-20T13:12:42.028Z" }, + { url = "https://files.pythonhosted.org/packages/8a/cb/c62454606daf5640369c94d8a9dd540599b1bfc090e2d2180cb77f4038d2/greenlet-3.5.1-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:d8ab31c9de8651a2facdd5c5bb0011f2380dd1a7af78ce2adf4b56095294fc07", size = 285579, upload-time = "2026-05-20T13:08:56.396Z" }, + { url = "https://files.pythonhosted.org/packages/ec/71/c4270398c2eba968a6071af1dfbdcaeee6ec1c24bc8b435b8cc452700da6/greenlet-3.5.1-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5e300185139abc337ade480c327183adf42a875ac7181bfe66d7d4efea31fbea", size = 651106, upload-time = "2026-05-20T14:00:09.448Z" }, + { url = "https://files.pythonhosted.org/packages/1a/ab/71e34b78a44ec271fb5f550c17bc46d301ddc5953890d935f270b0dcdb5a/greenlet-3.5.1-cp314-cp314-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7ffdb990dcaa0234cf9845aead5df2e3c3a8b6507d409274dd87e0d5ab05ffc2", size = 663478, upload-time = "2026-05-20T14:05:45.88Z" }, + { url = "https://files.pythonhosted.org/packages/c6/2d/2d80842910da44f78c286532d084b8a5c3717c844ae80ceb3858738ae89a/greenlet-3.5.1-cp314-cp314-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6c09df69dc1712d131332054a858a3e5cca400967fa3a672e2324fbb0971448c", size = 667767, upload-time = "2026-05-20T14:09:12.15Z" }, + { url = "https://files.pythonhosted.org/packages/77/96/4efd6fa5c62c85426a0c19077a586258ebc3a2a146ff2493e4312a697a22/greenlet-3.5.1-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2f82b3597e9d83b63408affed0b48fd0f54935edac4302237b9a837be0dae33c", size = 660800, upload-time = "2026-05-20T13:14:29.129Z" }, + { url = "https://files.pythonhosted.org/packages/e9/d3/dad2eecedfbb1ed7050a20dcfae40c1442b74bc7423608be2c7e03ee7133/greenlet-3.5.1-cp314-cp314-manylinux_2_39_riscv64.whl", hash = "sha256:a4764e0bfc6a4d114c865b32520805c16a990ef5f286a514413b05d5ecd6a23d", size = 470786, upload-time = "2026-05-20T14:01:42.064Z" }, + { url = "https://files.pythonhosted.org/packages/7a/e0/6c71401a25cac7000261304e866a2f2cc04dc74810d40e2f118aa4799495/greenlet-3.5.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c0141e37414c10164e702b8fb1473304221ad98f71600850c6ef7ff4880feba0", size = 1617518, upload-time = "2026-05-20T14:02:28.662Z" }, + { url = "https://files.pythonhosted.org/packages/41/26/c5c06643e8c0af9e7bf18e16cb51d0ab7625155f0392e1c9015d66d556cd/greenlet-3.5.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:50ae25a67bea74ea41fb14b960bc532df73eb713417b2d61892dced82fe8d3bc", size = 1681593, upload-time = "2026-05-20T13:14:39.417Z" }, + { url = "https://files.pythonhosted.org/packages/8a/bd/e11a108317485075e68af9d23039619b86b28130c3b50d227d42edece64b/greenlet-3.5.1-cp314-cp314-win_amd64.whl", hash = "sha256:8a17c42330e261299766b75ac1ea32caa437a9453c8f65d16a13140db378ecd3", size = 239800, upload-time = "2026-05-20T13:09:30.128Z" }, + { url = "https://files.pythonhosted.org/packages/47/f8/8e8e8417b7bf28639a5a56356ef934d0375e1d0c70a57e04d7701e870ffe/greenlet-3.5.1-cp314-cp314-win_arm64.whl", hash = "sha256:7b5f5fae05b8ac6d176a61b60c394a8cbdc2b5b91b81793066e68745cf165e54", size = 236862, upload-time = "2026-05-20T13:09:10.498Z" }, + { url = "https://files.pythonhosted.org/packages/90/12/41bf27fde4d3605d3773ae57751eda182b8be2f5398011c041173b1d9534/greenlet-3.5.1-cp314-cp314t-macosx_11_0_universal2.whl", hash = "sha256:ea8da1e900d758d078810d4255d8c6aa572181896a31ec79d779eb79c3adc9ad", size = 293637, upload-time = "2026-05-20T13:12:35.529Z" }, + { url = "https://files.pythonhosted.org/packages/44/44/ba14b23e9757707050c2f397d305bbcae62e5d7cad122f8b6baec5ae4a1f/greenlet-3.5.1-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a19570c52a21420dcbc94e661994bc325c0b5b11304540fed514586da5dc8f2e", size = 650840, upload-time = "2026-05-20T14:00:11.079Z" }, + { url = "https://files.pythonhosted.org/packages/a8/37/5ddc2b686a6844f91abecef43411842426da2e1573f60b49ecf2547f4ae1/greenlet-3.5.1-cp314-cp314t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:3d955c89b75eeca4723d7cc14135f393cd47c32e2a6cb4a8e4c6e760a26b0986", size = 656416, upload-time = "2026-05-20T14:05:47.118Z" }, + { url = "https://files.pythonhosted.org/packages/8c/46/5987dcd1a2570ba84f3b187536b2ca3ae97613387e57f5cfa99df068fe5e/greenlet-3.5.1-cp314-cp314t-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:ea37d5a157eb9493820d3792ac4ece28619a394391d2b9f2f78057d396ff0f0f", size = 656607, upload-time = "2026-05-20T14:09:13.949Z" }, + { url = "https://files.pythonhosted.org/packages/e1/f0/d17510297c35a2992712f0bf84de3779749999f7d3d63aa1f09db7c62dbe/greenlet-3.5.1-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:de2daaaebd1a5aa88c49045b6baf9310b3263796bd88db713edf37cf53e7bb4e", size = 654397, upload-time = "2026-05-20T13:14:30.696Z" }, + { url = "https://files.pythonhosted.org/packages/2c/c1/6da0a9ddcc29d7e51ef14883fa3dc1e53b3f4ffba00582106c7bf55da1d8/greenlet-3.5.1-cp314-cp314t-manylinux_2_39_riscv64.whl", hash = "sha256:8d8a23250ea3ec7b36de8fa4b541e9e2db3ee82915cc060ab0631609ad8b28de", size = 488287, upload-time = "2026-05-20T14:01:43.143Z" }, + { url = "https://files.pythonhosted.org/packages/37/eb/147387705bb89092645b012586e7273cb5ed3c90ef7eaf3a69173eaf0209/greenlet-3.5.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:3bfbd69cc349e43bf3a8ae1c85548ff0718efc887615c2db16c3833d7b0b072d", size = 1614469, upload-time = "2026-05-20T14:02:30.192Z" }, + { url = "https://files.pythonhosted.org/packages/a6/4e/37ee0da7732b7aa9896f17e15579a9df34b9fcb9dd494f0adfa749af6623/greenlet-3.5.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:4378720dd888136c27215a0214d32a4d37c3852765d45bc37aad0623423cfd78", size = 1675115, upload-time = "2026-05-20T13:14:40.972Z" }, + { url = "https://files.pythonhosted.org/packages/57/f3/97dfcf4a6eb5077f8a672234216fb5923eb89f2cab7081cb10b2cf75b605/greenlet-3.5.1-cp314-cp314t-win_amd64.whl", hash = "sha256:45718441607f9325d948db98cbc691276059316d0358c188c246da4e1d4d23d2", size = 245246, upload-time = "2026-05-20T13:12:22.646Z" }, + { url = "https://files.pythonhosted.org/packages/5d/73/d7f72e34b582f694f4a9b248162db7b09cc458a259ba8f0c0bfa1a34ea7d/greenlet-3.5.1-cp315-cp315-macosx_11_0_universal2.whl", hash = "sha256:2baee5ca02031757ffe8cc3d69f0cc0aec7065ce362622da74f32d3bcab1c541", size = 285575, upload-time = "2026-05-20T13:12:07.043Z" }, + { url = "https://files.pythonhosted.org/packages/df/59/fa9c6e87dc8ad27a95dabe2f29f372b733d05a8a67470f6c901ed9975655/greenlet-3.5.1-cp315-cp315-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9b1ec3274918a81d3ea778b9e75b56b72b33f300edb6cf7f3a7fe1dae56683de", size = 656428, upload-time = "2026-05-20T14:00:12.556Z" }, + { url = "https://files.pythonhosted.org/packages/f6/f9/e753408871eaa61dfe35e619cfc67512b036fde99893685d50eea9e07146/greenlet-3.5.1-cp315-cp315-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:111e2390ffffc47d5840b01711dd7fac07d4c09283d0283e7f3264b14e284c64", size = 667064, upload-time = "2026-05-20T14:05:48.662Z" }, + { url = "https://files.pythonhosted.org/packages/dc/74/807a047255bf1e09303627c46dc043dca596b6958a354d904f32ab382005/greenlet-3.5.1-cp315-cp315-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:10a9a1c0bfbc93d41156ffcb90c75fbc05544054faf15dcc1fdf9765f8b607f0", size = 672962, upload-time = "2026-05-20T14:09:15.532Z" }, + { url = "https://files.pythonhosted.org/packages/96/27/5565b5b40389f1c7753003a07e21892fda8660926787036d5bc0308b8113/greenlet-3.5.1-cp315-cp315-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e630136e905fe5ff43e86945ae41220b6d1470956a39220e708110ac48d01ea5", size = 665697, upload-time = "2026-05-20T13:14:32.943Z" }, + { url = "https://files.pythonhosted.org/packages/76/32/19d4e13225193c29b13e308015223f7d75fd3d8623d49dd19040d2ce8ec1/greenlet-3.5.1-cp315-cp315-manylinux_2_39_riscv64.whl", hash = "sha256:ef08c1567c78074b22d1a200183d52d04a14df447bf70bcbb6a3507a48e776fc", size = 476047, upload-time = "2026-05-20T14:01:44.39Z" }, + { url = "https://files.pythonhosted.org/packages/cf/82/e7de4178c0c2d1c9a5a3be3cc0b33e46a85b3ee4a77c071bf7ad8600e079/greenlet-3.5.1-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:975eac34b44a7077ca4d421348455b94f0f518246a7f14bc6d2fdcfe5b584368", size = 1621256, upload-time = "2026-05-20T14:02:31.91Z" }, + { url = "https://files.pythonhosted.org/packages/00/10/f2dddcf7dacac17dfc68691809589adad06135eb28930429cf58a6467a2f/greenlet-3.5.1-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:9ab3c3a0b2ae6198e67c898dad5215a49f9ae0d0081b3c3ec59f333e39eeca26", size = 1685956, upload-time = "2026-05-20T13:14:42.55Z" }, + { url = "https://files.pythonhosted.org/packages/22/17/4a232b32133230ada52f70e9d7f5b65b0caef8772f01849bd8d149e7e4ca/greenlet-3.5.1-cp315-cp315-win_amd64.whl", hash = "sha256:cbfc69be86e10dcfef5b1e6269d1d6926552aa89ee39e1de3353360c1b6989ab", size = 239802, upload-time = "2026-05-20T13:13:15.481Z" }, + { url = "https://files.pythonhosted.org/packages/c2/ae/4e623a7e6d4d2a5f4cb8e4c82de4169fc637942caae68d6e676b8a128ac5/greenlet-3.5.1-cp315-cp315-win_arm64.whl", hash = "sha256:92fd6d44ac5e5a887c8a5dc4a8ba0ba908527c31c12f78c6bc7dcfe8aab279f6", size = 236853, upload-time = "2026-05-20T13:15:37.301Z" }, + { url = "https://files.pythonhosted.org/packages/7a/57/816d9cff29119da3505b3d6a5e14a8af89006ac36f47f891ff293ee05af1/greenlet-3.5.1-cp315-cp315t-macosx_11_0_universal2.whl", hash = "sha256:a6fdf2433a5441ef9a95464f7c3e674775da1c8c1177fff311cee1acad4626ed", size = 293877, upload-time = "2026-05-20T13:10:19.078Z" }, + { url = "https://files.pythonhosted.org/packages/23/a1/59b0a7c7d140ff1a75626680b9a9899b79a9176cab298b394968fb023295/greenlet-3.5.1-cp315-cp315t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7546556f0d649f99f6a361098a55f761181bb2ea12ff150bb16d26092ad88244", size = 655333, upload-time = "2026-05-20T14:00:14.758Z" }, + { url = "https://files.pythonhosted.org/packages/72/1b/5efe127597625042218939d01855109f352779050768b670b52edcc16a6c/greenlet-3.5.1-cp315-cp315t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d5ee3ea898009fa898f85f9982255d35278c477bebe185beca249cab42d4526c", size = 659443, upload-time = "2026-05-20T14:05:50.159Z" }, + { url = "https://files.pythonhosted.org/packages/c9/9d/1dcdf7b95ab3cf8c7b6d7277c18a5e167312f2b362ddfcc5d5e6d8d84b43/greenlet-3.5.1-cp315-cp315t-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a57b0d05a0448eed231d59c0ceb287dde984551e54cbc51ac2d4865712838e9c", size = 659998, upload-time = "2026-05-20T14:09:16.912Z" }, + { url = "https://files.pythonhosted.org/packages/6c/6d/c404246ea4d22d097a7426d0efb5b781bd7eb67715f09e79001bd552ab18/greenlet-3.5.1-cp315-cp315t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a5c81f74d204d3edd136ebfd50dce53acbb776995d721a0fe801626cfc93b8cd", size = 658356, upload-time = "2026-05-20T13:14:35.091Z" }, + { url = "https://files.pythonhosted.org/packages/05/7e/c4959664fc231d587d66d8e81f2095e98056ba1954beafdcbe635e251052/greenlet-3.5.1-cp315-cp315t-manylinux_2_39_riscv64.whl", hash = "sha256:b0703c2cef53e01baec47f7a3868009913ad71ec678bbecb42a6f40895e4ce62", size = 494470, upload-time = "2026-05-20T14:01:45.611Z" }, + { url = "https://files.pythonhosted.org/packages/51/02/f8ee37fb6d2219329f350af241c27fcf12df57e723d11f6fc6d3bacdadaa/greenlet-3.5.1-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:2c18ef16bf6d4dd410e4dd52996888ea1497be26892fe5bbc73580aba4287b8e", size = 1619216, upload-time = "2026-05-20T14:02:33.403Z" }, + { url = "https://files.pythonhosted.org/packages/93/c5/3dc9475ace2c7a3680da12372cddd7f1ac874eb410a1ac48d3e9dab83782/greenlet-3.5.1-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:17d86354f0ae6b61bf9be5148d0dd34e06c3cb7c602c671f79f29ac3b150e659", size = 1678427, upload-time = "2026-05-20T13:14:43.71Z" }, + { url = "https://files.pythonhosted.org/packages/df/4e/750c15c317a41ffb36f0bf40b933e3d744a7dede61889f74443ea69690cf/greenlet-3.5.1-cp315-cp315t-win_amd64.whl", hash = "sha256:e7516cf6ae6b8a582c2770a0caed47b8a48373ed732c33d69a72913ae6ac923e", size = 245225, upload-time = "2026-05-20T13:13:59.366Z" }, + { url = "https://files.pythonhosted.org/packages/4f/fd/d3baea2eeb7b617efd47e87ca06e2ec2c6118d303aa9e918e0ce16eadc10/greenlet-3.5.1-cp315-cp315t-win_arm64.whl", hash = "sha256:5028648bf2253ec4745add746129d3904121fa7fe871a76bed23c5720573ce0a", size = 239590, upload-time = "2026-05-20T13:13:37.382Z" }, +] + [[package]] name = "griffelib" version = "2.0.1" @@ -1594,8 +1701,10 @@ groq = [ ] ingester = [ { name = "aiosqlite" }, + { name = "asyncpg" }, { name = "fastapi" }, { name = "obstore" }, + { name = "sqlalchemy", extra = ["asyncio"] }, { name = "uvicorn", extra = ["standard"] }, ] jina = [ @@ -1631,6 +1740,7 @@ zeroentropy = [ [package.metadata] requires-dist = [ { name = "aiosqlite", marker = "extra == 'ingester'", specifier = ">=0.20" }, + { name = "asyncpg", marker = "extra == 'ingester'", specifier = ">=0.29" }, { name = "cohere", marker = "extra == 'cohere'", specifier = ">=5.21.1" }, { name = "docling", marker = "extra == 'docling'", specifier = ">=2.93.0" }, { name = "docling-core", specifier = ">=2.75.0" }, @@ -1660,6 +1770,7 @@ requires-dist = [ { name = "pyyaml", specifier = ">=6.0.3" }, { name = "rich", specifier = ">=14.3.3" }, { name = "sentence-transformers", marker = "extra == 'cross-encoder'", specifier = ">=3.0.0" }, + { name = "sqlalchemy", extras = ["asyncio"], marker = "extra == 'ingester'", specifier = ">=2.0" }, { name = "textual", marker = "extra == 'tui'", specifier = ">=8.2.4" }, { name = "textual-image", marker = "extra == 'tui'", specifier = ">=0.8.5" }, { name = "torch", marker = "extra == 'jina'", specifier = ">=2.0.0" }, @@ -4805,6 +4916,52 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/46/2c/1462b1d0a634697ae9e55b3cecdcb64788e8b7d63f54d923fcd0bb140aed/soupsieve-2.8.3-py3-none-any.whl", hash = "sha256:ed64f2ba4eebeab06cc4962affce381647455978ffc1e36bb79a545b91f45a95", size = 37016, upload-time = "2026-01-20T04:27:01.012Z" }, ] +[[package]] +name = "sqlalchemy" +version = "2.0.50" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "greenlet", marker = "platform_machine == 'AMD64' or platform_machine == 'WIN32' or platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'ppc64le' or platform_machine == 'win32' or platform_machine == 'x86_64'" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/57/da/6fbf010c8ebb347679d0d100b22fe9ba5e13fd04046c5df7280d2f0bf706/sqlalchemy-2.0.50.tar.gz", hash = "sha256:af5607d11ef90fd6a5c0549fe0045dce1663d427426bcfb506dcb5346a85a3b9", size = 9907424, upload-time = "2026-05-24T19:20:04.018Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/be/b0/a9d19b43f38f878b1278bca5b00b909f7540d41494396dd2561f9ad0956d/sqlalchemy-2.0.50-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:23ae23d8b9d344d30d0a92f06d45825024a5790f1c1dd4cf452636a50d3e58cb", size = 2159807, upload-time = "2026-05-24T19:27:53.086Z" }, + { url = "https://files.pythonhosted.org/packages/f5/2c/191dd58a248fd2cfd4780fa82c375c505e4ad98c8b522fa69ec492130d77/sqlalchemy-2.0.50-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:47b71b933e7b4ebad407c8fdfd70d2c4f08b78b3238bb30eebdd6eb32ca51b89", size = 3343358, upload-time = "2026-05-24T20:09:29.279Z" }, + { url = "https://files.pythonhosted.org/packages/8a/2b/514fce8a7df81cf5bad7ff7865de7ac0c5776a38cc043475c4703eb7fe8b/sqlalchemy-2.0.50-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:110fdac56ace278949f00de805edacbd6141e382d992f9ba28238b3a0827a600", size = 3357994, upload-time = "2026-05-24T20:17:13.495Z" }, + { url = "https://files.pythonhosted.org/packages/35/a6/a0e283f5494f92b0d77e319ff77e437b1ffe4a051ba67c81d53234825475/sqlalchemy-2.0.50-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:0f5e4ac70e9e757f6b3e87c0491ff034442ecd8dfd36d041a50564c322dafc0e", size = 3289399, upload-time = "2026-05-24T20:09:32.239Z" }, + { url = "https://files.pythonhosted.org/packages/b7/96/1b07325ba71752d6a028b77d07bed1483ad545f794e8b1dc89b3ba3b3c68/sqlalchemy-2.0.50-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:724f3dcbe53dd0151e3cb5e7ec4ba4c620bede579caacd16275dc35ce06e8615", size = 3321216, upload-time = "2026-05-24T20:17:15.581Z" }, + { url = "https://files.pythonhosted.org/packages/ed/8e/bad6ed253e8a99edfc99af02f7173ec48a1d3ed1b9b35a1b8bc1700900cc/sqlalchemy-2.0.50-cp312-cp312-win32.whl", hash = "sha256:1208050441471d003b7c8cb4054fb084f185cf35ac3f0ea270803865bca9939a", size = 2119194, upload-time = "2026-05-24T19:50:04.943Z" }, + { url = "https://files.pythonhosted.org/packages/b6/2d/314a6690dda4b9cfc571eab1a63cf6fe6e1470aa3759ccda6aa016ee0f5a/sqlalchemy-2.0.50-cp312-cp312-win_amd64.whl", hash = "sha256:9d1af51558029a156a70986b7df88f042b3d158d7c8d8fb5072912d4b32d89c7", size = 2146186, upload-time = "2026-05-24T19:50:06.74Z" }, + { url = "https://files.pythonhosted.org/packages/0b/c4/c42356b527296e9862f67990efce31ef78b4cf69cd3f80873a528a060320/sqlalchemy-2.0.50-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:06a9210bdc5f4298cff0781087e2ff45683922252dacc452846373a58761f093", size = 2156697, upload-time = "2026-05-24T19:27:54.764Z" }, + { url = "https://files.pythonhosted.org/packages/60/a1/b1a70e3c4365ac7fe9e347f3710f19b562c866fb96d45e3c891588789a7b/sqlalchemy-2.0.50-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8b53784972ade4f8174b9aa661f31a06f8a936d2cfdd602913ff3c6dd40ae873", size = 3284260, upload-time = "2026-05-24T20:09:34.195Z" }, + { url = "https://files.pythonhosted.org/packages/3f/4a/f3ac3caa19f263d57b0a47f8c91bbf56583dc2d3fc63acfbf644abb24fe0/sqlalchemy-2.0.50-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:31648fa14460537e768a7303b078e4344d208e0d23e06867c1f376a227ed82db", size = 3302280, upload-time = "2026-05-24T20:17:17.825Z" }, + { url = "https://files.pythonhosted.org/packages/66/55/ccada3e3d62254587819749a0bc69f41173eb48a6e385d10e66d32a9c88e/sqlalchemy-2.0.50-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:03f4323c980ad0e918cc9e5369b015f759f4e534db5bbaf4dc36832c10d05064", size = 3231580, upload-time = "2026-05-24T20:09:36.406Z" }, + { url = "https://files.pythonhosted.org/packages/05/f6/6809349130a2de0e109e7f00fd7d431da9565b9b2868b32ee684754f672b/sqlalchemy-2.0.50-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:2b9dcc43afef8ac157cd92fce96985d6b8b0cfbd3df4d666f66b4d55a75d202f", size = 3269375, upload-time = "2026-05-24T20:17:20.34Z" }, + { url = "https://files.pythonhosted.org/packages/48/84/278a811ef4e07be9c89dc5cdd7be833268509a66a68c4897cf585e67428f/sqlalchemy-2.0.50-cp313-cp313-win32.whl", hash = "sha256:60922d6599065ddca2c6f376b9aa2f41a6b85a271725e0909490bbc50b1998a5", size = 2117229, upload-time = "2026-05-24T19:50:08.215Z" }, + { url = "https://files.pythonhosted.org/packages/f6/1c/067cc6187ed32d2ec222fe6d2643acc1659a6d0659f8a7cbc5ad3ae83280/sqlalchemy-2.0.50-cp313-cp313-win_amd64.whl", hash = "sha256:287086e67275a212c4582d166a6fb03a65ccc5551d80866270ce0dd9f34eccd3", size = 2143126, upload-time = "2026-05-24T19:50:09.691Z" }, + { url = "https://files.pythonhosted.org/packages/df/32/10ac51b4be7cdecd7e93d069251c86dfbf70b7adbd7c67b48ccea6c49e1c/sqlalchemy-2.0.50-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:c966932507a4d7d0a37314927dbfcd89720e3f37d2a1e3352e7ae7939fa8e8a0", size = 2158519, upload-time = "2026-05-24T19:27:56.472Z" }, + { url = "https://files.pythonhosted.org/packages/5a/76/e703d2f7681d7d66c4c891af3f07c7ccf4c76ad7f18351de035b5eda007a/sqlalchemy-2.0.50-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:faffef4bcc20a1892e65e155293d99d60855bbbc79250ab712819cfd56a8e6bb", size = 3282063, upload-time = "2026-05-24T20:09:38.57Z" }, + { url = "https://files.pythonhosted.org/packages/31/26/ef168b184a25701f9995e8fb7e503fafd7a99c1c77cda1bc1a26ea2ed486/sqlalchemy-2.0.50-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6c206aec519a2e7bd08abbfb33436e325fd22c632d9c21a9047e376ce241646e", size = 3287069, upload-time = "2026-05-24T20:17:21.942Z" }, + { url = "https://files.pythonhosted.org/packages/c2/15/765acc2bc693bccc43ca4a95d5b69750da8aaf6db1b5c616536e087f8920/sqlalchemy-2.0.50-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:bef4ac756363227ef6402a75fee025a4bc690f92328e825868939b3b3a446a6d", size = 3230453, upload-time = "2026-05-24T20:09:40.398Z" }, + { url = "https://files.pythonhosted.org/packages/63/61/08e03c3adbf5db0087a0b6816746fec8f3032fb2f7fc899a9bb9b2a48ce4/sqlalchemy-2.0.50-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:96fbee6b19c19cd1556c8bf9419447cf2ec149ffcab7ab64348c23e54ef8547f", size = 3252413, upload-time = "2026-05-24T20:17:24.067Z" }, + { url = "https://files.pythonhosted.org/packages/03/0c/370a1f2db38436c615e10134c8a37de3688e74084792380695f3f5083860/sqlalchemy-2.0.50-cp314-cp314-win32.whl", hash = "sha256:8f00e3eb43ba30eb1b238ee03a8a62309486d1321eda3328bb611e0340033ad8", size = 2120063, upload-time = "2026-05-24T19:50:11.08Z" }, + { url = "https://files.pythonhosted.org/packages/7f/a0/fe92bb9817863bc13ba093bda931979a26cc2ca69f8e8f26d07add3d7c6f/sqlalchemy-2.0.50-cp314-cp314-win_amd64.whl", hash = "sha256:15708c613cd5005b7dffe1f66ee6a63ee8f5e46799f71c70ebad74178c676a39", size = 2145830, upload-time = "2026-05-24T19:50:12.452Z" }, + { url = "https://files.pythonhosted.org/packages/cc/ff/e5640a98a0b2f491eb8fde10fb6c773621a2e44340de231fafcc9370f4a9/sqlalchemy-2.0.50-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:3699dac4be410e97049a1658e9480da9cde956594aa0f3aebc60b88f21c5ba70", size = 2178435, upload-time = "2026-05-24T19:42:58.889Z" }, + { url = "https://files.pythonhosted.org/packages/b7/85/337116e186f1236375b5fb70c21cfac98e8e8ab0d3a47be838dc47a59e08/sqlalchemy-2.0.50-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f96233858e3df43932ac11589e22520da6e8aeb624b03fedfeebb0e8ea213086", size = 3566059, upload-time = "2026-05-24T20:01:20.848Z" }, + { url = "https://files.pythonhosted.org/packages/96/34/bb0e190e161c3c2c24314a65add57218be14a4a9486886b7f5047c1ff7c8/sqlalchemy-2.0.50-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c4e70c46fad30c3bcc6a4708bc0130a3173e11a5b25f0ea4a9d8911b450f1f52", size = 3535366, upload-time = "2026-05-24T20:03:56.768Z" }, + { url = "https://files.pythonhosted.org/packages/df/5a/a7f759f97e4fd499c5d4e4488c760d5a7fbecf3028b465a04274fcd52384/sqlalchemy-2.0.50-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:1918a3cf564d16d95bca7301005f41ab2ad50b07cd3b9da50d3ed986db148d6a", size = 3474879, upload-time = "2026-05-24T20:01:23.058Z" }, + { url = "https://files.pythonhosted.org/packages/9d/d9/2907ea38eb60687d297bf9c39e5ee58053c87b57fe8a9cae97090cecbf10/sqlalchemy-2.0.50-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:b00098cdbdbd38c7be3d568b0c9c3122b8c0ec62b911b57cd5e6e0254d60a76d", size = 3486117, upload-time = "2026-05-24T20:03:59.052Z" }, + { url = "https://files.pythonhosted.org/packages/f2/e3/5aa06f167559f8c0bdae487e297d23ba548150ab016a3418265d617a4985/sqlalchemy-2.0.50-cp314-cp314t-win32.whl", hash = "sha256:1fbd55a969d7ac44a98e3dec75016074f809fa08f871585ace58dde110d1bf3e", size = 2150823, upload-time = "2026-05-24T20:08:58.644Z" }, + { url = "https://files.pythonhosted.org/packages/65/9b/112fb8f977582d7489d036e409e3723948bcf5320b3ac465f3c481bbe8f9/sqlalchemy-2.0.50-cp314-cp314t-win_amd64.whl", hash = "sha256:c5c3cdb753a9004183e1ccb634b41611654c989e61bc68617ce878e46d6f1e51", size = 2185794, upload-time = "2026-05-24T20:09:00.319Z" }, + { url = "https://files.pythonhosted.org/packages/d0/10/f7220e9b784d295d241c86ed99aeb537f92afcd469a64861f2717e9bb077/sqlalchemy-2.0.50-py3-none-any.whl", hash = "sha256:92064363517a3ff8212b5a93b8c62876579d8dfd1ca5b561335f30152d884fa9", size = 1943861, upload-time = "2026-05-24T19:59:01.119Z" }, +] + +[package.optional-dependencies] +asyncio = [ + { name = "greenlet" }, +] + [[package]] name = "sse-starlette" version = "3.3.3"