From 4b3641244ca1fbb38cba4c5c7fd12ac750c79778 Mon Sep 17 00:00:00 2001 From: Yiorgis Gozadinos Date: Thu, 21 May 2026 17:34:47 +0300 Subject: [PATCH] SQLite queue + haiku-ingester CLI skeleton --- haiku_rag_slim/haiku/rag/config/__init__.py | 4 + haiku_rag_slim/haiku/rag/config/models.py | 16 + haiku_rag_slim/haiku/rag/ingester/cli.py | 90 ++++ .../haiku/rag/ingester/queue/__init__.py | 19 + .../haiku/rag/ingester/queue/migrations.py | 50 +++ .../haiku/rag/ingester/queue/models.py | 46 ++ .../haiku/rag/ingester/queue/repository.py | 307 +++++++++++++ .../haiku/rag/ingester/queue/schema.py | 70 +++ haiku_rag_slim/pyproject.toml | 8 + pyproject.toml | 1 + tests/ingester/test_queue.py | 406 ++++++++++++++++++ uv.lock | 115 ++++- 12 files changed, 1130 insertions(+), 2 deletions(-) create mode 100644 haiku_rag_slim/haiku/rag/ingester/cli.py create mode 100644 haiku_rag_slim/haiku/rag/ingester/queue/__init__.py create mode 100644 haiku_rag_slim/haiku/rag/ingester/queue/migrations.py create mode 100644 haiku_rag_slim/haiku/rag/ingester/queue/models.py create mode 100644 haiku_rag_slim/haiku/rag/ingester/queue/repository.py create mode 100644 haiku_rag_slim/haiku/rag/ingester/queue/schema.py create mode 100644 tests/ingester/test_queue.py diff --git a/haiku_rag_slim/haiku/rag/config/__init__.py b/haiku_rag_slim/haiku/rag/config/__init__.py index 4ba286de..8a7225ac 100644 --- a/haiku_rag_slim/haiku/rag/config/__init__.py +++ b/haiku_rag_slim/haiku/rag/config/__init__.py @@ -8,6 +8,7 @@ from haiku.rag.config.models import ( ConversionOptions, EmbeddingModelConfig, EmbeddingsConfig, + IngesterConfig, LanceDBConfig, ModelConfig, MonitorConfig, @@ -16,6 +17,7 @@ from haiku.rag.config.models import ( PromptsConfig, ProvidersConfig, QAConfig, + QueueConfig, RerankingConfig, S3MonitorEntry, StorageConfig, @@ -27,6 +29,7 @@ __all__ = [ "ConversionOptions", "EmbeddingModelConfig", "EmbeddingsConfig", + "IngesterConfig", "LanceDBConfig", "ModelConfig", "MonitorConfig", @@ -35,6 +38,7 @@ __all__ = [ "PromptsConfig", "ProvidersConfig", "QAConfig", + "QueueConfig", "RerankingConfig", "S3MonitorEntry", "StorageConfig", diff --git a/haiku_rag_slim/haiku/rag/config/models.py b/haiku_rag_slim/haiku/rag/config/models.py index a30c4524..884c53ce 100644 --- a/haiku_rag_slim/haiku/rag/config/models.py +++ b/haiku_rag_slim/haiku/rag/config/models.py @@ -249,6 +249,21 @@ class EvaluationsConfig(BaseModel): ) +class QueueConfig(BaseModel): + """SQLite queue for the production ingester.""" + + path: Path = Field( + default_factory=lambda: get_default_data_dir() / "ingester.db", + description="Location of the ingester's SQLite queue file.", + ) + + +class IngesterConfig(BaseModel): + """Production ingester settings. Expanded across chunks 4-7.""" + + queue: QueueConfig = Field(default_factory=QueueConfig) + + class AppConfig(BaseModel): environment: str = "production" storage: StorageConfig = Field(default_factory=StorageConfig) @@ -262,6 +277,7 @@ class AppConfig(BaseModel): search: SearchConfig = Field(default_factory=SearchConfig) providers: ProvidersConfig = Field(default_factory=ProvidersConfig) prompts: PromptsConfig = Field(default_factory=PromptsConfig) + ingester: IngesterConfig = Field(default_factory=IngesterConfig) evaluations: "EvaluationsConfig" = Field( default_factory=lambda: EvaluationsConfig() ) diff --git a/haiku_rag_slim/haiku/rag/ingester/cli.py b/haiku_rag_slim/haiku/rag/ingester/cli.py new file mode 100644 index 00000000..b5ad5c9c --- /dev/null +++ b/haiku_rag_slim/haiku/rag/ingester/cli.py @@ -0,0 +1,90 @@ +import asyncio +from pathlib import Path + +import typer +from dotenv import find_dotenv, load_dotenv + +load_dotenv(find_dotenv(usecwd=True)) + +from haiku.rag.config import ( # noqa: E402 + AppConfig, + find_config_file, + get_config, + load_yaml_config, + set_config, +) +from haiku.rag.ingester.queue.migrations import open_queue # noqa: E402 + +cli = typer.Typer( + name="haiku-ingester", + no_args_is_help=True, + pretty_exceptions_show_locals=False, + help="Production ingester for haiku.rag.", +) + +queue_cli = typer.Typer( + name="queue", + no_args_is_help=True, + help="Operate the ingester's SQLite job queue.", +) +cli.add_typer(queue_cli) + + +def _load_config_with_override(config_path: Path | None) -> AppConfig: + """Mirror the haiku-rag CLI's config-loading pattern.""" + if config_path: + config = AppConfig.model_validate(load_yaml_config(config_path)) + set_config(config) + return config + if (found := find_config_file(None)) is not None: + config = AppConfig.model_validate(load_yaml_config(found)) + set_config(config) + return config + return get_config() + + +def _resolve_queue_path(config: AppConfig, override: Path | None) -> Path: + return Path(override).expanduser() if override else config.ingester.queue.path + + +async def _ensure_schema(path: Path) -> None: + conn = await open_queue(path) + await conn.close() + + +@queue_cli.command("init") +def queue_init( + config: Path | None = typer.Option( + None, "--config", "-c", help="Path to haiku.rag.yaml." + ), + queue: Path | None = typer.Option( + None, + "--queue", + "-q", + help="Override the queue DB path (defaults to ingester.queue.path).", + ), +) -> None: + """Create the queue DB and apply the current schema. Idempotent.""" + app_config = _load_config_with_override(config) + path = _resolve_queue_path(app_config, queue) + asyncio.run(_ensure_schema(path)) + typer.echo(f"Queue initialized at {path}") + + +@queue_cli.command("migrate") +def queue_migrate( + config: Path | None = typer.Option( + None, "--config", "-c", help="Path to haiku.rag.yaml." + ), + queue: Path | None = typer.Option( + None, + "--queue", + "-q", + help="Override the queue DB path (defaults to ingester.queue.path).", + ), +) -> None: + """Apply any pending schema migrations to an existing queue DB. Idempotent.""" + app_config = _load_config_with_override(config) + path = _resolve_queue_path(app_config, queue) + asyncio.run(_ensure_schema(path)) + typer.echo(f"Queue at {path} is up to date") diff --git a/haiku_rag_slim/haiku/rag/ingester/queue/__init__.py b/haiku_rag_slim/haiku/rag/ingester/queue/__init__.py new file mode 100644 index 00000000..775d8c26 --- /dev/null +++ b/haiku_rag_slim/haiku/rag/ingester/queue/__init__.py @@ -0,0 +1,19 @@ +from haiku.rag.ingester.queue.migrations import ( + SCHEMA_VERSION, + apply_migrations, + open_queue, +) +from haiku.rag.ingester.queue.models import Job, JobOp, JobStatus, SyncStateRow +from haiku.rag.ingester.queue.repository import JobRepo, SyncStateRepo + +__all__ = [ + "Job", + "JobOp", + "JobRepo", + "JobStatus", + "SCHEMA_VERSION", + "SyncStateRepo", + "SyncStateRow", + "apply_migrations", + "open_queue", +] diff --git a/haiku_rag_slim/haiku/rag/ingester/queue/migrations.py b/haiku_rag_slim/haiku/rag/ingester/queue/migrations.py new file mode 100644 index 00000000..b7ff944c --- /dev/null +++ b/haiku_rag_slim/haiku/rag/ingester/queue/migrations.py @@ -0,0 +1,50 @@ +from pathlib import Path + +import aiosqlite + +from haiku.rag.ingester.queue.schema import ALL_DDL, SCHEMA_VERSION + +__all__ = ["SCHEMA_VERSION", "apply_migrations", "open_queue"] + + +async def apply_migrations(conn: aiosqlite.Connection) -> int: + """Idempotently create tables/indexes/views and pin schema_version. + + Returns the schema version after the call. Safe to call on a fresh DB + or on one already at the latest version. + """ + await conn.execute("PRAGMA journal_mode=WAL") + await conn.execute("PRAGMA synchronous=NORMAL") + await conn.execute("PRAGMA foreign_keys=ON") + + for stmt in ALL_DDL: + await conn.execute(stmt) + + cursor = await conn.execute("SELECT version FROM schema_version LIMIT 1") + row = await cursor.fetchone() + if row is None: + await conn.execute( + "INSERT INTO schema_version (version) VALUES (?)", (SCHEMA_VERSION,) + ) + else: + current = row[0] + if current < SCHEMA_VERSION: + # No diff migrations exist yet — future versions add UPDATE/ALTER + # statements between here and the version bump. + await conn.execute( + "UPDATE schema_version SET version = ?", (SCHEMA_VERSION,) + ) + + await conn.commit() + return SCHEMA_VERSION + + +async def open_queue(path: str | Path) -> aiosqlite.Connection: + """Open the queue database at `path`, creating it (and the parent dir) + if needed, and ensuring the schema is up-to-date.""" + db_path = Path(path).expanduser().resolve() + db_path.parent.mkdir(parents=True, exist_ok=True) + conn = await aiosqlite.connect(str(db_path)) + conn.row_factory = aiosqlite.Row + await apply_migrations(conn) + return conn diff --git a/haiku_rag_slim/haiku/rag/ingester/queue/models.py b/haiku_rag_slim/haiku/rag/ingester/queue/models.py new file mode 100644 index 00000000..aa4e7765 --- /dev/null +++ b/haiku_rag_slim/haiku/rag/ingester/queue/models.py @@ -0,0 +1,46 @@ +from datetime import datetime +from enum import StrEnum + +from pydantic import BaseModel + + +class JobStatus(StrEnum): + QUEUED = "queued" + CLAIMED = "claimed" + SUCCEEDED = "succeeded" + DEAD = "dead" + + +class JobOp(StrEnum): + UPSERT = "upsert" + DELETE = "delete" + + +class Job(BaseModel): + id: str + source_id: str + uri: str + op: JobOp + content_hash: str | None = None + revision: str | None = None + status: JobStatus + attempts: int + max_attempts: int + last_error: str | None = None + # Free-form per-job payload (e.g. storage_options snapshot). Serialized + # to a JSON TEXT column. + extra: dict | None = None + enqueued_at: datetime + scheduled_at: datetime + claimed_at: datetime | None = None + claimed_by: str | None = None + completed_at: datetime | None = None + + +class SyncStateRow(BaseModel): + source_id: str + uri: str + revision: str | None = None + content_hash: str | None = None + last_seen_at: datetime + last_ingested_at: datetime | None = None diff --git a/haiku_rag_slim/haiku/rag/ingester/queue/repository.py b/haiku_rag_slim/haiku/rag/ingester/queue/repository.py new file mode 100644 index 00000000..25aead7e --- /dev/null +++ b/haiku_rag_slim/haiku/rag/ingester/queue/repository.py @@ -0,0 +1,307 @@ +import json +import uuid +from datetime import UTC, datetime, timedelta + +import aiosqlite + +from haiku.rag.ingester.queue.models import Job, JobOp, JobStatus, SyncStateRow + + +def _utcnow_iso() -> str: + return datetime.now(UTC).isoformat() + + +def _parse_dt(value: str | None) -> datetime | None: + return datetime.fromisoformat(value) if value else None + + +def _row_to_job(row: aiosqlite.Row) -> Job: + extra_text = row["extra"] + return Job( + id=row["id"], + source_id=row["source_id"], + uri=row["uri"], + op=JobOp(row["op"]), + content_hash=row["content_hash"], + revision=row["revision"], + status=JobStatus(row["status"]), + attempts=row["attempts"], + max_attempts=row["max_attempts"], + last_error=row["last_error"], + extra=json.loads(extra_text) if extra_text else None, + enqueued_at=datetime.fromisoformat(row["enqueued_at"]), + scheduled_at=datetime.fromisoformat(row["scheduled_at"]), + claimed_at=_parse_dt(row["claimed_at"]), + claimed_by=row["claimed_by"], + completed_at=_parse_dt(row["completed_at"]), + ) + + +def _row_to_sync_state(row: aiosqlite.Row) -> SyncStateRow: + return SyncStateRow( + source_id=row["source_id"], + uri=row["uri"], + revision=row["revision"], + content_hash=row["content_hash"], + last_seen_at=datetime.fromisoformat(row["last_seen_at"]), + last_ingested_at=_parse_dt(row["last_ingested_at"]), + ) + + +class JobRepo: + def __init__(self, conn: aiosqlite.Connection): + # Row access by name in helpers below. + conn.row_factory = aiosqlite.Row + self._conn = conn + + async def enqueue( + self, + source_id: str, + uri: str, + op: JobOp = JobOp.UPSERT, + *, + revision: str | None = None, + content_hash: str | None = None, + max_attempts: int = 5, + extra: dict | None = None, + ) -> Job | None: + """Enqueue an upsert/delete job. Returns the inserted Job, or None if a + live (queued/claimed) job already exists for the same (source_id, uri, + op). The partial unique index enforces atomicity.""" + job_id = str(uuid.uuid4()) + now = _utcnow_iso() + extra_json = json.dumps(extra) if extra is not None else None + cursor = await self._conn.execute( + """ + INSERT INTO jobs ( + id, source_id, uri, op, content_hash, revision, status, + attempts, max_attempts, last_error, extra, + enqueued_at, scheduled_at + ) VALUES (?, ?, ?, ?, ?, ?, 'queued', 0, ?, NULL, ?, ?, ?) + ON CONFLICT DO NOTHING + RETURNING * + """, + ( + job_id, + source_id, + uri, + op.value, + content_hash, + revision, + max_attempts, + extra_json, + now, + now, + ), + ) + row = await cursor.fetchone() + await self._conn.commit() + return _row_to_job(row) if row else None + + async def claim_next(self, worker_id: str) -> Job | None: + """Atomically claim the oldest queued job whose scheduled_at <= now. + Implemented as a single UPDATE ... RETURNING — no SELECT/UPDATE race.""" + now = _utcnow_iso() + cursor = await self._conn.execute( + """ + UPDATE jobs + SET status = 'claimed', + claimed_at = ?, + claimed_by = ?, + attempts = attempts + 1 + WHERE id = ( + SELECT id FROM jobs + WHERE status = 'queued' AND scheduled_at <= ? + ORDER BY scheduled_at + LIMIT 1 + ) + RETURNING * + """, + (now, worker_id, now), + ) + row = await cursor.fetchone() + await self._conn.commit() + return _row_to_job(row) if row else None + + async def get_job(self, job_id: str) -> Job | None: + cursor = await self._conn.execute("SELECT * FROM jobs WHERE id = ?", (job_id,)) + row = await cursor.fetchone() + return _row_to_job(row) if row else None + + async def mark_succeeded(self, job_id: str) -> None: + await self._conn.execute( + "UPDATE jobs SET status='succeeded', completed_at=? WHERE id=?", + (_utcnow_iso(), job_id), + ) + await self._conn.commit() + + async def mark_dead(self, job_id: str, error: str) -> None: + await self._conn.execute( + "UPDATE jobs SET status='dead', completed_at=?, last_error=? WHERE id=?", + (_utcnow_iso(), error, job_id), + ) + await self._conn.commit() + + async def reschedule(self, job_id: str, delay_seconds: float, error: str) -> None: + scheduled = (datetime.now(UTC) + timedelta(seconds=delay_seconds)).isoformat() + await self._conn.execute( + """ + UPDATE jobs + SET status='queued', + scheduled_at=?, + claimed_at=NULL, + claimed_by=NULL, + last_error=? + WHERE id=? + """, + (scheduled, error, job_id), + ) + await self._conn.commit() + + async def retry(self, job_id: str) -> Job: + """Rescue a dead job: status='queued', attempts=0, error cleared. + Raises KeyError if the job doesn't exist.""" + now = _utcnow_iso() + cursor = await self._conn.execute( + """ + UPDATE jobs + SET status='queued', + attempts=0, + last_error=NULL, + claimed_at=NULL, + claimed_by=NULL, + completed_at=NULL, + scheduled_at=? + WHERE id=? + RETURNING * + """, + (now, job_id), + ) + row = await cursor.fetchone() + await self._conn.commit() + if not row: + raise KeyError(f"Job {job_id!r} not found") + return _row_to_job(row) + + async def cancel(self, job_id: str) -> bool: + """Delete a queued or claimed job. Returns True if a row was removed.""" + cursor = await self._conn.execute( + "DELETE FROM jobs WHERE id=? AND status IN ('queued', 'claimed') RETURNING id", + (job_id,), + ) + row = await cursor.fetchone() + await self._conn.commit() + return row is not None + + async def list_jobs( + self, + *, + status: JobStatus | None = None, + source_id: str | None = None, + uri: str | None = None, + limit: int = 50, + offset: int = 0, + ) -> list[Job]: + clauses: list[str] = [] + params: list[object] = [] + if status is not None: + clauses.append("status = ?") + params.append(status.value) + if source_id is not None: + clauses.append("source_id = ?") + params.append(source_id) + if uri is not None: + clauses.append("uri = ?") + params.append(uri) + where = ("WHERE " + " AND ".join(clauses)) if clauses else "" + params.extend([limit, offset]) + cursor = await self._conn.execute( + f"SELECT * FROM jobs {where} ORDER BY enqueued_at DESC LIMIT ? OFFSET ?", + params, + ) + rows = await cursor.fetchall() + return [_row_to_job(r) for r in rows] + + async def counts_by_status(self) -> dict[str, int]: + cursor = await self._conn.execute( + "SELECT status, COUNT(*) AS n FROM jobs GROUP BY status" + ) + rows = await cursor.fetchall() + return {row["status"]: row["n"] for row in rows} + + async def reap_stale(self, claim_timeout_seconds: int) -> int: + """Return claimed jobs whose claimed_at is older than the timeout to + the queue. Used by the reaper to recover from crashed workers.""" + threshold = ( + datetime.now(UTC) - timedelta(seconds=claim_timeout_seconds) + ).isoformat() + cursor = await self._conn.execute( + """ + UPDATE jobs + SET status='queued', claimed_at=NULL, claimed_by=NULL + WHERE status='claimed' AND claimed_at < ? + """, + (threshold,), + ) + await self._conn.commit() + return cursor.rowcount or 0 + + +class SyncStateRepo: + def __init__(self, conn: aiosqlite.Connection): + conn.row_factory = aiosqlite.Row + self._conn = conn + + async def get_snapshot(self, source_id: str) -> dict[str, str]: + """uri -> revision map for the source. Drops rows where revision is + NULL (the poller can't compare against an absent revision).""" + cursor = await self._conn.execute( + "SELECT uri, revision FROM sync_state WHERE source_id=? AND revision IS NOT NULL", + (source_id,), + ) + rows = await cursor.fetchall() + return {row["uri"]: row["revision"] for row in rows} + + async def get_row(self, source_id: str, uri: str) -> SyncStateRow | None: + cursor = await self._conn.execute( + "SELECT * FROM sync_state WHERE source_id=? AND uri=?", + (source_id, uri), + ) + row = await cursor.fetchone() + return _row_to_sync_state(row) if row else None + + async def upsert( + self, + source_id: str, + uri: str, + *, + revision: str | None = None, + content_hash: str | None = None, + ingested: bool = False, + ) -> None: + """Insert-or-update the sync_state row. `ingested=True` stamps + last_ingested_at; otherwise only last_seen_at is bumped.""" + now = _utcnow_iso() + ingested_at = now if ingested else None + await self._conn.execute( + """ + INSERT INTO sync_state ( + source_id, uri, revision, content_hash, last_seen_at, last_ingested_at + ) + VALUES (?, ?, ?, ?, ?, ?) + ON CONFLICT(source_id, uri) DO UPDATE SET + revision = excluded.revision, + content_hash = excluded.content_hash, + last_seen_at = excluded.last_seen_at, + last_ingested_at = COALESCE(excluded.last_ingested_at, last_ingested_at) + """, + (source_id, uri, revision, content_hash, now, ingested_at), + ) + await self._conn.commit() + + async def delete(self, source_id: str, uri: str) -> None: + await self._conn.execute( + "DELETE FROM sync_state WHERE source_id=? AND uri=?", + (source_id, uri), + ) + await self._conn.commit() diff --git a/haiku_rag_slim/haiku/rag/ingester/queue/schema.py b/haiku_rag_slim/haiku/rag/ingester/queue/schema.py new file mode 100644 index 00000000..6d818831 --- /dev/null +++ b/haiku_rag_slim/haiku/rag/ingester/queue/schema.py @@ -0,0 +1,70 @@ +SCHEMA_VERSION = 1 + + +JOBS_DDL = """ +CREATE TABLE IF NOT EXISTS jobs ( + id TEXT PRIMARY KEY, + source_id TEXT NOT NULL, + uri TEXT NOT NULL, + op TEXT NOT NULL, + content_hash TEXT, + revision TEXT, + status TEXT NOT NULL, + attempts INTEGER NOT NULL DEFAULT 0, + max_attempts INTEGER NOT NULL DEFAULT 5, + last_error TEXT, + extra TEXT, + enqueued_at TEXT NOT NULL, + scheduled_at TEXT NOT NULL, + claimed_at TEXT, + claimed_by TEXT, + completed_at TEXT +) +""" + +# Partial unique index: a (source_id, uri, op) triple can only have one live +# job (queued or claimed) at a time. Once succeeded or dead, the row no +# longer satisfies the WHERE clause and a re-enqueue is allowed. +JOBS_LIVE_INDEX = """ +CREATE UNIQUE INDEX IF NOT EXISTS uq_jobs_live +ON jobs(source_id, uri, op) +WHERE status IN ('queued', 'claimed') +""" + +JOBS_CLAIMABLE_INDEX = """ +CREATE INDEX IF NOT EXISTS idx_jobs_claimable +ON jobs(scheduled_at) +WHERE status = 'queued' +""" + +SYNC_STATE_DDL = """ +CREATE TABLE IF NOT EXISTS sync_state ( + source_id TEXT NOT NULL, + uri TEXT NOT NULL, + revision TEXT, + content_hash TEXT, + last_seen_at TEXT NOT NULL, + last_ingested_at TEXT, + PRIMARY KEY (source_id, uri) +) +""" + +SCHEMA_VERSION_DDL = """ +CREATE TABLE IF NOT EXISTS schema_version ( + version INTEGER PRIMARY KEY +) +""" + +DLQ_VIEW = """ +CREATE VIEW IF NOT EXISTS dlq AS +SELECT * FROM jobs WHERE status = 'dead' +""" + +ALL_DDL: tuple[str, ...] = ( + JOBS_DDL, + JOBS_LIVE_INDEX, + JOBS_CLAIMABLE_INDEX, + SYNC_STATE_DDL, + SCHEMA_VERSION_DDL, + DLQ_VIEW, +) diff --git a/haiku_rag_slim/pyproject.toml b/haiku_rag_slim/pyproject.toml index efb058c9..56af1a97 100644 --- a/haiku_rag_slim/pyproject.toml +++ b/haiku_rag_slim/pyproject.toml @@ -53,6 +53,13 @@ cohere = ["cohere>=5.21.1"] zeroentropy = ["zeroentropy>=0.1.0a11"] jina = ["transformers>=4.40.0", "torch>=2.0.0"] cross-encoder = ["sentence-transformers>=3.0.0"] +# Production ingester (queue, workers, API) +ingester = [ + "fastapi>=0.125", + "uvicorn[standard]>=0.32", + "aiosqlite>=0.20", + "haiku.rag-slim[s3]", +] # TUI (chat and inspect commands) tui = [ "textual>=8.2.4", @@ -74,6 +81,7 @@ rag-analysis = "haiku.rag.skills.analysis:create_skill" [project.scripts] haiku-rag = "haiku.rag.cli:cli" +haiku-ingester = "haiku.rag.ingester.cli:cli" [build-system] requires = ["hatchling"] diff --git a/pyproject.toml b/pyproject.toml index 5f661393..8c985a1b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,6 +40,7 @@ haiku-rag = "haiku.rag.cli:cli" tui = ["textual>=8.2.4"] s3 = ["haiku.rag-slim[s3]==0.48.1"] cross-encoder = ["haiku.rag-slim[cross-encoder]==0.48.1"] +ingester = ["haiku.rag-slim[ingester]==0.48.1"] [build-system] requires = ["hatchling"] diff --git a/tests/ingester/test_queue.py b/tests/ingester/test_queue.py new file mode 100644 index 00000000..a00157b9 --- /dev/null +++ b/tests/ingester/test_queue.py @@ -0,0 +1,406 @@ +import asyncio +from datetime import UTC, datetime, timedelta + +import aiosqlite +import pytest + +from haiku.rag.ingester.queue.migrations import apply_migrations, open_queue +from haiku.rag.ingester.queue.models import JobOp, JobStatus +from haiku.rag.ingester.queue.repository import JobRepo, SyncStateRepo + + +@pytest.fixture +async def conn(tmp_path): + path = tmp_path / "queue.db" + connection = await aiosqlite.connect(str(path)) + connection.row_factory = aiosqlite.Row + await apply_migrations(connection) + yield connection + await connection.close() + + +@pytest.fixture +def jobs(conn): + return JobRepo(conn) + + +@pytest.fixture +def sync(conn): + return SyncStateRepo(conn) + + +# --- migrations / schema --- + + +@pytest.mark.asyncio +async def test_apply_migrations_sets_schema_version(conn): + cursor = await conn.execute("SELECT version FROM schema_version") + row = await cursor.fetchone() + assert row is not None + assert row["version"] >= 1 + + +@pytest.mark.asyncio +async def test_apply_migrations_is_idempotent(conn): + await apply_migrations(conn) + await apply_migrations(conn) + cursor = await conn.execute("SELECT COUNT(*) AS n FROM schema_version") + row = await cursor.fetchone() + assert row["n"] == 1 + + +@pytest.mark.asyncio +async def test_open_queue_creates_file_and_schema(tmp_path): + path = tmp_path / "subdir" / "queue.db" + connection = await open_queue(path) + try: + assert path.exists() + cursor = await connection.execute("SELECT version FROM schema_version") + row = await cursor.fetchone() + assert row is not None + finally: + await connection.close() + + +@pytest.mark.asyncio +async def test_dlq_view_exposes_dead_jobs(conn, jobs): + job = await jobs.enqueue("s", "u", JobOp.UPSERT) + await jobs.mark_dead(job.id, "permanent") + + cursor = await conn.execute("SELECT id FROM dlq") + rows = await cursor.fetchall() + assert [r["id"] for r in rows] == [job.id] + + +# --- enqueue --- + + +@pytest.mark.asyncio +async def test_enqueue_creates_queued_job(jobs): + job = await jobs.enqueue( + "fs:/tmp", + "file:///tmp/a.md", + JobOp.UPSERT, + revision="abc", + content_hash="md5_value", + extra={"key": "value"}, + ) + assert job is not None + assert job.status is JobStatus.QUEUED + assert job.source_id == "fs:/tmp" + assert job.uri == "file:///tmp/a.md" + assert job.op is JobOp.UPSERT + assert job.revision == "abc" + assert job.content_hash == "md5_value" + assert job.attempts == 0 + assert job.extra == {"key": "value"} + assert job.enqueued_at.tzinfo is not None + + +@pytest.mark.asyncio +async def test_enqueue_returns_none_on_live_conflict(jobs): + first = await jobs.enqueue("s", "u", JobOp.UPSERT) + second = await jobs.enqueue("s", "u", JobOp.UPSERT) + assert first is not None + assert second is None + + +@pytest.mark.asyncio +async def test_enqueue_after_dead_succeeds(jobs): + first = await jobs.enqueue("s", "u", JobOp.UPSERT) + await jobs.mark_dead(first.id, "error") + second = await jobs.enqueue("s", "u", JobOp.UPSERT) + assert second is not None + assert second.id != first.id + + +@pytest.mark.asyncio +async def test_enqueue_after_succeeded_succeeds(jobs): + await jobs.enqueue("s", "u", JobOp.UPSERT) + claimed = await jobs.claim_next("w") + assert claimed is not None + await jobs.mark_succeeded(claimed.id) + second = await jobs.enqueue("s", "u", JobOp.UPSERT) + assert second is not None + + +@pytest.mark.asyncio +async def test_enqueue_different_ops_coexist(jobs): + upsert = await jobs.enqueue("s", "u", JobOp.UPSERT) + delete = await jobs.enqueue("s", "u", JobOp.DELETE) + assert upsert is not None + assert delete is not None + + +# --- claim_next --- + + +@pytest.mark.asyncio +async def test_claim_next_returns_none_when_empty(jobs): + assert await jobs.claim_next("w") is None + + +@pytest.mark.asyncio +async def test_claim_next_increments_attempts_and_records_worker(jobs): + job = await jobs.enqueue("s", "u", JobOp.UPSERT) + assert job.attempts == 0 + claimed = await jobs.claim_next("worker-1") + assert claimed is not None + assert claimed.id == job.id + assert claimed.status is JobStatus.CLAIMED + assert claimed.attempts == 1 + assert claimed.claimed_by == "worker-1" + assert claimed.claimed_at is not None + + +@pytest.mark.asyncio +async def test_claim_next_returns_oldest_first(jobs): + j1 = await jobs.enqueue("s", "u1", JobOp.UPSERT) + j2 = await jobs.enqueue("s", "u2", JobOp.UPSERT) + first = await jobs.claim_next("w") + second = await jobs.claim_next("w") + assert first is not None + assert second is not None + assert first.id == j1.id + assert second.id == j2.id + + +@pytest.mark.asyncio +async def test_claim_next_skips_future_scheduled(conn, jobs): + job = await jobs.enqueue("s", "u", JobOp.UPSERT) + # Push scheduled_at into the future. + future = (datetime.now(UTC) + timedelta(seconds=60)).isoformat() + await conn.execute( + "UPDATE jobs SET scheduled_at = ? WHERE id = ?", (future, job.id) + ) + await conn.commit() + assert await jobs.claim_next("w") is None + + +@pytest.mark.asyncio +async def test_claim_next_atomic_under_concurrency(jobs): + enqueued = [] + for i in range(5): + j = await jobs.enqueue("s", f"u{i}", JobOp.UPSERT) + assert j is not None + enqueued.append(j) + + results = await asyncio.gather(*(jobs.claim_next(f"w{i}") for i in range(10))) + claimed = [r for r in results if r is not None] + assert len(claimed) == 5 + assert len({c.id for c in claimed}) == 5 + assert {c.id for c in claimed} == {j.id for j in enqueued} + + +# --- terminal transitions --- + + +@pytest.mark.asyncio +async def test_mark_succeeded(jobs): + job = await jobs.enqueue("s", "u", JobOp.UPSERT) + claimed = await jobs.claim_next("w") + assert claimed is not None + await jobs.mark_succeeded(claimed.id) + refreshed = await jobs.get_job(job.id) + assert refreshed is not None + assert refreshed.status is JobStatus.SUCCEEDED + assert refreshed.completed_at is not None + + +@pytest.mark.asyncio +async def test_mark_dead_records_error(jobs): + job = await jobs.enqueue("s", "u", JobOp.UPSERT) + claimed = await jobs.claim_next("w") + assert claimed is not None + await jobs.mark_dead(claimed.id, "permanent failure") + refreshed = await jobs.get_job(job.id) + assert refreshed is not None + assert refreshed.status is JobStatus.DEAD + assert refreshed.last_error == "permanent failure" + + +# --- reschedule + retry --- + + +@pytest.mark.asyncio +async def test_reschedule_pushes_scheduled_at_into_future(jobs): + job = await jobs.enqueue("s", "u", JobOp.UPSERT) + claimed = await jobs.claim_next("w") + assert claimed is not None + await jobs.reschedule(claimed.id, delay_seconds=30.0, error="transient") + refreshed = await jobs.get_job(job.id) + assert refreshed is not None + assert refreshed.status is JobStatus.QUEUED + assert refreshed.claimed_at is None + assert refreshed.claimed_by is None + assert refreshed.last_error == "transient" + assert refreshed.scheduled_at > datetime.now(UTC) + + +@pytest.mark.asyncio +async def test_reschedule_then_claim_skips_until_due(conn, jobs): + job = await jobs.enqueue("s", "u", JobOp.UPSERT) + claimed = await jobs.claim_next("w") + assert claimed is not None + await jobs.reschedule(claimed.id, delay_seconds=60.0, error="transient") + assert await jobs.claim_next("w") is None + + # Backdate scheduled_at to simulate the delay elapsing. + past = datetime.now(UTC).isoformat() + await conn.execute("UPDATE jobs SET scheduled_at = ? WHERE id = ?", (past, job.id)) + await conn.commit() + assert await jobs.claim_next("w") is not None + + +@pytest.mark.asyncio +async def test_retry_revives_dead_job(jobs): + job = await jobs.enqueue("s", "u", JobOp.UPSERT) + claimed = await jobs.claim_next("w") + assert claimed is not None + await jobs.mark_dead(claimed.id, "boom") + + revived = await jobs.retry(job.id) + assert revived.status is JobStatus.QUEUED + assert revived.attempts == 0 + assert revived.last_error is None + assert revived.claimed_at is None + assert revived.claimed_by is None + assert revived.completed_at is None + + +@pytest.mark.asyncio +async def test_retry_unknown_raises(jobs): + with pytest.raises(KeyError): + await jobs.retry("not-a-real-id") + + +# --- cancel --- + + +@pytest.mark.asyncio +async def test_cancel_queued_removes_row(jobs): + job = await jobs.enqueue("s", "u", JobOp.UPSERT) + assert await jobs.cancel(job.id) is True + assert await jobs.get_job(job.id) is None + + +@pytest.mark.asyncio +async def test_cancel_succeeded_returns_false(jobs): + job = await jobs.enqueue("s", "u", JobOp.UPSERT) + claimed = await jobs.claim_next("w") + assert claimed is not None + await jobs.mark_succeeded(claimed.id) + assert await jobs.cancel(job.id) is False + + +# --- reap_stale --- + + +@pytest.mark.asyncio +async def test_reap_stale_resets_old_claims(conn, jobs): + job = await jobs.enqueue("s", "u", JobOp.UPSERT) + claimed = await jobs.claim_next("w") + assert claimed is not None + + # Backdate claimed_at to simulate a crashed worker. + long_ago = (datetime.now(UTC) - timedelta(hours=1)).isoformat() + await conn.execute( + "UPDATE jobs SET claimed_at = ? WHERE id = ?", (long_ago, job.id) + ) + await conn.commit() + + reset = await jobs.reap_stale(claim_timeout_seconds=60) + assert reset == 1 + refreshed = await jobs.get_job(job.id) + assert refreshed is not None + assert refreshed.status is JobStatus.QUEUED + assert refreshed.claimed_at is None + assert refreshed.claimed_by is None + + +@pytest.mark.asyncio +async def test_reap_stale_leaves_fresh_claims_alone(jobs): + job = await jobs.enqueue("s", "u", JobOp.UPSERT) + claimed = await jobs.claim_next("w") + assert claimed is not None + + reset = await jobs.reap_stale(claim_timeout_seconds=3600) + assert reset == 0 + refreshed = await jobs.get_job(job.id) + assert refreshed is not None + assert refreshed.status is JobStatus.CLAIMED + + +# --- list / counts --- + + +@pytest.mark.asyncio +async def test_list_jobs_with_filters(jobs): + j1 = await jobs.enqueue("s1", "u1", JobOp.UPSERT) + j2 = await jobs.enqueue("s2", "u2", JobOp.UPSERT) + j3 = await jobs.enqueue("s1", "u3", JobOp.UPSERT) + claimed = await jobs.claim_next("w") + assert claimed is not None + await jobs.mark_succeeded(claimed.id) + + all_jobs = await jobs.list_jobs() + assert len(all_jobs) == 3 + + by_source = await jobs.list_jobs(source_id="s1") + assert {j.id for j in by_source} == {j1.id, j3.id} + + by_status = await jobs.list_jobs(status=JobStatus.QUEUED) + assert {j.id for j in by_status} == {j2.id, j3.id} + + +@pytest.mark.asyncio +async def test_counts_by_status(jobs): + await jobs.enqueue("s", "u1", JobOp.UPSERT) + j2 = await jobs.enqueue("s", "u2", JobOp.UPSERT) + j3 = await jobs.enqueue("s", "u3", JobOp.UPSERT) + + claimed = await jobs.claim_next("w") + assert claimed is not None + await jobs.mark_succeeded(claimed.id) + await jobs.mark_dead(j2.id, "err") + + counts = await jobs.counts_by_status() + assert counts == {"queued": 1, "succeeded": 1, "dead": 1} + assert j3.id # silence unused + + +# --- sync state --- + + +@pytest.mark.asyncio +async def test_sync_state_get_snapshot_empty(sync): + assert await sync.get_snapshot("unknown") == {} + + +@pytest.mark.asyncio +async def test_sync_state_upsert_and_get(sync): + await sync.upsert("s", "u1", revision="abc", content_hash="m1") + await sync.upsert("s", "u2", revision="def", content_hash="m2") + assert await sync.get_snapshot("s") == {"u1": "abc", "u2": "def"} + + +@pytest.mark.asyncio +async def test_sync_state_upsert_overwrites(sync): + await sync.upsert("s", "u1", revision="abc", content_hash="m1") + await sync.upsert("s", "u1", revision="def", content_hash="m2", ingested=True) + assert await sync.get_snapshot("s") == {"u1": "def"} + + +@pytest.mark.asyncio +async def test_sync_state_delete_removes_entry(sync): + await sync.upsert("s", "u1", revision="abc", content_hash="m1") + await sync.delete("s", "u1") + assert await sync.get_snapshot("s") == {} + + +@pytest.mark.asyncio +async def test_sync_state_snapshot_scoped_per_source(sync): + await sync.upsert("s1", "u", revision="abc", content_hash="m") + await sync.upsert("s2", "u", revision="def", content_hash="m") + assert await sync.get_snapshot("s1") == {"u": "abc"} + assert await sync.get_snapshot("s2") == {"u": "def"} diff --git a/uv.lock b/uv.lock index 83ef0576..e1c6df7c 100644 --- a/uv.lock +++ b/uv.lock @@ -178,6 +178,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/fb/76/641ae371508676492379f16e2fa48f4e2c11741bd63c48be4b12a6b09cba/aiosignal-1.4.0-py3-none-any.whl", hash = "sha256:053243f8b92b990551949e63930a839ff0cf0b0ebbe0597b0f3fb19e1a0fe82e", size = 7490, upload-time = "2025-07-03T22:54:42.156Z" }, ] +[[package]] +name = "aiosqlite" +version = "0.22.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/4e/8a/64761f4005f17809769d23e518d915db74e6310474e733e3593cfc854ef1/aiosqlite-0.22.1.tar.gz", hash = "sha256:043e0bd78d32888c0a9ca90fc788b38796843360c855a7262a532813133a0650", size = 14821, upload-time = "2025-12-23T19:25:43.997Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/00/b7/e3bf5133d697a08128598c8d0abc5e16377b51465a33756de24fa7dee953/aiosqlite-0.22.1-py3-none-any.whl", hash = "sha256:21c002eb13823fad740196c5a2e9d8e62f6243bd9e7e4a1f87fb5e44ecb4fceb", size = 17405, upload-time = "2025-12-23T19:25:42.139Z" }, +] + [[package]] name = "annotated-doc" version = "0.0.4" @@ -1094,6 +1103,22 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/fc/ec/3c4b78eb0d2f6a81fb8cc9286745845bff661e6815741eff7a6ac5fcc9ea/faker-40.11.1-py3-none-any.whl", hash = "sha256:3af3a213ba8fb33ce6ba2af7aef2ac91363dae35d0cec0b2b0337d189e5bee2a", size = 1989484, upload-time = "2026-03-23T14:05:48.793Z" }, ] +[[package]] +name = "fastapi" +version = "0.136.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "annotated-doc" }, + { name = "pydantic" }, + { name = "starlette" }, + { name = "typing-extensions" }, + { name = "typing-inspection" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5d/45/c130091c2dfa061bbfe3150f2a5091ef1adf149f2a8d2ae769ecaf6e99a2/fastapi-0.136.1.tar.gz", hash = "sha256:7af665ad7acfa0a3baf8983d393b6b471b9da10ede59c60045f49fbc89a0fa7f", size = 397448, upload-time = "2026-04-23T16:49:44.046Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5a/ff/2e4eca3ade2c22fe1dea7043b8ee9dabe47753349eb1b56a202de8af6349/fastapi-0.136.1-py3-none-any.whl", hash = "sha256:a6e9d7eeada96c93a4d69cb03836b44fa34e2854accb7244a1ece36cd4781c3f", size = 117683, upload-time = "2026-04-23T16:49:42.437Z" }, +] + [[package]] name = "fastavro" version = "1.12.1" @@ -1442,6 +1467,9 @@ dependencies = [ cross-encoder = [ { name = "haiku-rag-slim", extra = ["cross-encoder"] }, ] +ingester = [ + { name = "haiku-rag-slim", extra = ["ingester"] }, +] s3 = [ { name = "haiku-rag-slim", extra = ["s3"] }, ] @@ -1470,10 +1498,11 @@ dev = [ requires-dist = [ { name = "haiku-rag-slim", extras = ["cross-encoder"], marker = "extra == 'cross-encoder'", editable = "haiku_rag_slim" }, { name = "haiku-rag-slim", extras = ["docling", "voyageai", "mxbai", "cohere", "zeroentropy", "tui", "cross-encoder"], editable = "haiku_rag_slim" }, + { name = "haiku-rag-slim", extras = ["ingester"], marker = "extra == 'ingester'", editable = "haiku_rag_slim" }, { name = "haiku-rag-slim", extras = ["s3"], marker = "extra == 's3'", editable = "haiku_rag_slim" }, { name = "textual", marker = "extra == 'tui'", specifier = ">=8.2.4" }, ] -provides-extras = ["tui", "s3", "cross-encoder"] +provides-extras = ["tui", "s3", "cross-encoder", "ingester"] [package.metadata.requires-dev] dev = [ @@ -1564,6 +1593,12 @@ google = [ groq = [ { name = "pydantic-ai-slim", extra = ["groq"] }, ] +ingester = [ + { name = "aiosqlite" }, + { name = "fastapi" }, + { name = "obstore" }, + { name = "uvicorn", extra = ["standard"] }, +] jina = [ { name = "torch" }, { name = "transformers" }, @@ -1596,9 +1631,12 @@ zeroentropy = [ [package.metadata] requires-dist = [ + { name = "aiosqlite", marker = "extra == 'ingester'", specifier = ">=0.20" }, { name = "cohere", marker = "extra == 'cohere'", specifier = ">=5.21.1" }, { name = "docling", marker = "extra == 'docling'", specifier = ">=2.93.0" }, { name = "docling-core", specifier = ">=2.75.0" }, + { name = "fastapi", marker = "extra == 'ingester'", specifier = ">=0.125" }, + { name = "haiku-rag-slim", extras = ["s3"], marker = "extra == 'ingester'", editable = "haiku_rag_slim" }, { name = "haiku-skills", specifier = ">=0.17.1" }, { name = "httpx", specifier = ">=0.28.1" }, { name = "jinja2", specifier = ">=3.1.0" }, @@ -1630,11 +1668,12 @@ requires-dist = [ { name = "tree-sitter", marker = "extra == 'tui'", specifier = ">=0.25.2" }, { name = "tree-sitter-json", marker = "extra == 'tui'", specifier = ">=0.24.8" }, { name = "typer", specifier = ">=0.21.0,<0.22.0" }, + { name = "uvicorn", extras = ["standard"], marker = "extra == 'ingester'", specifier = ">=0.32" }, { name = "watchfiles", specifier = ">=1.1.1" }, { name = "zeroentropy", marker = "extra == 'zeroentropy'", specifier = ">=0.1.0a11" }, { name = "zstandard", marker = "python_full_version < '3.14'", specifier = ">=0.23.0" }, ] -provides-extras = ["docling", "s3", "voyageai", "mxbai", "cohere", "zeroentropy", "jina", "cross-encoder", "tui", "anthropic", "groq", "google", "mistral", "bedrock", "vertexai"] +provides-extras = ["docling", "s3", "voyageai", "mxbai", "cohere", "zeroentropy", "jina", "cross-encoder", "ingester", "tui", "anthropic", "groq", "google", "mistral", "bedrock", "vertexai"] [[package]] name = "haiku-skills" @@ -1698,6 +1737,35 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7e/f5/f66802a942d491edb555dd61e3a9961140fd64c90bce1eafd741609d334d/httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55", size = 78784, upload-time = "2025-04-24T22:06:20.566Z" }, ] +[[package]] +name = "httptools" +version = "0.7.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b5/46/120a669232c7bdedb9d52d4aeae7e6c7dfe151e99dc70802e2fc7a5e1993/httptools-0.7.1.tar.gz", hash = "sha256:abd72556974f8e7c74a259655924a717a2365b236c882c3f6f8a45fe94703ac9", size = 258961, upload-time = "2025-10-10T03:55:08.559Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/53/7f/403e5d787dc4942316e515e949b0c8a013d84078a915910e9f391ba9b3ed/httptools-0.7.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:38e0c83a2ea9746ebbd643bdfb521b9aa4a91703e2cd705c20443405d2fd16a5", size = 206280, upload-time = "2025-10-10T03:54:39.274Z" }, + { url = "https://files.pythonhosted.org/packages/2a/0d/7f3fd28e2ce311ccc998c388dd1c53b18120fda3b70ebb022b135dc9839b/httptools-0.7.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f25bbaf1235e27704f1a7b86cd3304eabc04f569c828101d94a0e605ef7205a5", size = 110004, upload-time = "2025-10-10T03:54:40.403Z" }, + { url = "https://files.pythonhosted.org/packages/84/a6/b3965e1e146ef5762870bbe76117876ceba51a201e18cc31f5703e454596/httptools-0.7.1-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:2c15f37ef679ab9ecc06bfc4e6e8628c32a8e4b305459de7cf6785acd57e4d03", size = 517655, upload-time = "2025-10-10T03:54:41.347Z" }, + { url = "https://files.pythonhosted.org/packages/11/7d/71fee6f1844e6fa378f2eddde6c3e41ce3a1fb4b2d81118dd544e3441ec0/httptools-0.7.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7fe6e96090df46b36ccfaf746f03034e5ab723162bc51b0a4cf58305324036f2", size = 511440, upload-time = "2025-10-10T03:54:42.452Z" }, + { url = "https://files.pythonhosted.org/packages/22/a5/079d216712a4f3ffa24af4a0381b108aa9c45b7a5cc6eb141f81726b1823/httptools-0.7.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:f72fdbae2dbc6e68b8239defb48e6a5937b12218e6ffc2c7846cc37befa84362", size = 495186, upload-time = "2025-10-10T03:54:43.937Z" }, + { url = "https://files.pythonhosted.org/packages/e9/9e/025ad7b65278745dee3bd0ebf9314934c4592560878308a6121f7f812084/httptools-0.7.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e99c7b90a29fd82fea9ef57943d501a16f3404d7b9ee81799d41639bdaae412c", size = 499192, upload-time = "2025-10-10T03:54:45.003Z" }, + { url = "https://files.pythonhosted.org/packages/6d/de/40a8f202b987d43afc4d54689600ff03ce65680ede2f31df348d7f368b8f/httptools-0.7.1-cp312-cp312-win_amd64.whl", hash = "sha256:3e14f530fefa7499334a79b0cf7e7cd2992870eb893526fb097d51b4f2d0f321", size = 86694, upload-time = "2025-10-10T03:54:45.923Z" }, + { url = "https://files.pythonhosted.org/packages/09/8f/c77b1fcbfd262d422f12da02feb0d218fa228d52485b77b953832105bb90/httptools-0.7.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:6babce6cfa2a99545c60bfef8bee0cc0545413cb0018f617c8059a30ad985de3", size = 202889, upload-time = "2025-10-10T03:54:47.089Z" }, + { url = "https://files.pythonhosted.org/packages/0a/1a/22887f53602feaa066354867bc49a68fc295c2293433177ee90870a7d517/httptools-0.7.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:601b7628de7504077dd3dcb3791c6b8694bbd967148a6d1f01806509254fb1ca", size = 108180, upload-time = "2025-10-10T03:54:48.052Z" }, + { url = "https://files.pythonhosted.org/packages/32/6a/6aaa91937f0010d288d3d124ca2946d48d60c3a5ee7ca62afe870e3ea011/httptools-0.7.1-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:04c6c0e6c5fb0739c5b8a9eb046d298650a0ff38cf42537fc372b28dc7e4472c", size = 478596, upload-time = "2025-10-10T03:54:48.919Z" }, + { url = "https://files.pythonhosted.org/packages/6d/70/023d7ce117993107be88d2cbca566a7c1323ccbaf0af7eabf2064fe356f6/httptools-0.7.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:69d4f9705c405ae3ee83d6a12283dc9feba8cc6aaec671b412917e644ab4fa66", size = 473268, upload-time = "2025-10-10T03:54:49.993Z" }, + { url = "https://files.pythonhosted.org/packages/32/4d/9dd616c38da088e3f436e9a616e1d0cc66544b8cdac405cc4e81c8679fc7/httptools-0.7.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:44c8f4347d4b31269c8a9205d8a5ee2df5322b09bbbd30f8f862185bb6b05346", size = 455517, upload-time = "2025-10-10T03:54:51.066Z" }, + { url = "https://files.pythonhosted.org/packages/1d/3a/a6c595c310b7df958e739aae88724e24f9246a514d909547778d776799be/httptools-0.7.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:465275d76db4d554918aba40bf1cbebe324670f3dfc979eaffaa5d108e2ed650", size = 458337, upload-time = "2025-10-10T03:54:52.196Z" }, + { url = "https://files.pythonhosted.org/packages/fd/82/88e8d6d2c51edc1cc391b6e044c6c435b6aebe97b1abc33db1b0b24cd582/httptools-0.7.1-cp313-cp313-win_amd64.whl", hash = "sha256:322d00c2068d125bd570f7bf78b2d367dad02b919d8581d7476d8b75b294e3e6", size = 85743, upload-time = "2025-10-10T03:54:53.448Z" }, + { url = "https://files.pythonhosted.org/packages/34/50/9d095fcbb6de2d523e027a2f304d4551855c2f46e0b82befd718b8b20056/httptools-0.7.1-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:c08fe65728b8d70b6923ce31e3956f859d5e1e8548e6f22ec520a962c6757270", size = 203619, upload-time = "2025-10-10T03:54:54.321Z" }, + { url = "https://files.pythonhosted.org/packages/07/f0/89720dc5139ae54b03f861b5e2c55a37dba9a5da7d51e1e824a1f343627f/httptools-0.7.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:7aea2e3c3953521c3c51106ee11487a910d45586e351202474d45472db7d72d3", size = 108714, upload-time = "2025-10-10T03:54:55.163Z" }, + { url = "https://files.pythonhosted.org/packages/b3/cb/eea88506f191fb552c11787c23f9a405f4c7b0c5799bf73f2249cd4f5228/httptools-0.7.1-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:0e68b8582f4ea9166be62926077a3334064d422cf08ab87d8b74664f8e9058e1", size = 472909, upload-time = "2025-10-10T03:54:56.056Z" }, + { url = "https://files.pythonhosted.org/packages/e0/4a/a548bdfae6369c0d078bab5769f7b66f17f1bfaa6fa28f81d6be6959066b/httptools-0.7.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:df091cf961a3be783d6aebae963cc9b71e00d57fa6f149025075217bc6a55a7b", size = 470831, upload-time = "2025-10-10T03:54:57.219Z" }, + { url = "https://files.pythonhosted.org/packages/4d/31/14df99e1c43bd132eec921c2e7e11cda7852f65619bc0fc5bdc2d0cb126c/httptools-0.7.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:f084813239e1eb403ddacd06a30de3d3e09a9b76e7894dcda2b22f8a726e9c60", size = 452631, upload-time = "2025-10-10T03:54:58.219Z" }, + { url = "https://files.pythonhosted.org/packages/22/d2/b7e131f7be8d854d48cb6d048113c30f9a46dca0c9a8b08fcb3fcd588cdc/httptools-0.7.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:7347714368fb2b335e9063bc2b96f2f87a9ceffcd9758ac295f8bbcd3ffbc0ca", size = 452910, upload-time = "2025-10-10T03:54:59.366Z" }, + { url = "https://files.pythonhosted.org/packages/53/cf/878f3b91e4e6e011eff6d1fa9ca39f7eb17d19c9d7971b04873734112f30/httptools-0.7.1-cp314-cp314-win_amd64.whl", hash = "sha256:cfabda2a5bb85aa2a904ce06d974a3f30fb36cc63d7feaddec05d2050acede96", size = 88205, upload-time = "2025-10-10T03:55:00.389Z" }, +] + [[package]] name = "httpx" version = "0.28.1" @@ -5336,6 +5404,49 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/0a/89/f8827ccff89c1586027a105e5630ff6139a64da2515e24dafe860bd9ae4d/uvicorn-0.42.0-py3-none-any.whl", hash = "sha256:96c30f5c7abe6f74ae8900a70e92b85ad6613b745d4879eb9b16ccad15645359", size = 68830, upload-time = "2026-03-16T06:19:48.325Z" }, ] +[package.optional-dependencies] +standard = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "httptools" }, + { name = "python-dotenv" }, + { name = "pyyaml" }, + { name = "uvloop", marker = "platform_python_implementation != 'PyPy' and sys_platform != 'cygwin' and sys_platform != 'win32'" }, + { name = "watchfiles" }, + { name = "websockets" }, +] + +[[package]] +name = "uvloop" +version = "0.22.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/06/f0/18d39dbd1971d6d62c4629cc7fa67f74821b0dc1f5a77af43719de7936a7/uvloop-0.22.1.tar.gz", hash = "sha256:6c84bae345b9147082b17371e3dd5d42775bddce91f885499017f4607fdaf39f", size = 2443250, upload-time = "2025-10-16T22:17:19.342Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3d/ff/7f72e8170be527b4977b033239a83a68d5c881cc4775fca255c677f7ac5d/uvloop-0.22.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:fe94b4564e865d968414598eea1a6de60adba0c040ba4ed05ac1300de402cd42", size = 1359936, upload-time = "2025-10-16T22:16:29.436Z" }, + { url = "https://files.pythonhosted.org/packages/c3/c6/e5d433f88fd54d81ef4be58b2b7b0cea13c442454a1db703a1eea0db1a59/uvloop-0.22.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:51eb9bd88391483410daad430813d982010f9c9c89512321f5b60e2cddbdddd6", size = 752769, upload-time = "2025-10-16T22:16:30.493Z" }, + { url = "https://files.pythonhosted.org/packages/24/68/a6ac446820273e71aa762fa21cdcc09861edd3536ff47c5cd3b7afb10eeb/uvloop-0.22.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:700e674a166ca5778255e0e1dc4e9d79ab2acc57b9171b79e65feba7184b3370", size = 4317413, upload-time = "2025-10-16T22:16:31.644Z" }, + { url = "https://files.pythonhosted.org/packages/5f/6f/e62b4dfc7ad6518e7eff2516f680d02a0f6eb62c0c212e152ca708a0085e/uvloop-0.22.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7b5b1ac819a3f946d3b2ee07f09149578ae76066d70b44df3fa990add49a82e4", size = 4426307, upload-time = "2025-10-16T22:16:32.917Z" }, + { url = "https://files.pythonhosted.org/packages/90/60/97362554ac21e20e81bcef1150cb2a7e4ffdaf8ea1e5b2e8bf7a053caa18/uvloop-0.22.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e047cc068570bac9866237739607d1313b9253c3051ad84738cbb095be0537b2", size = 4131970, upload-time = "2025-10-16T22:16:34.015Z" }, + { url = "https://files.pythonhosted.org/packages/99/39/6b3f7d234ba3964c428a6e40006340f53ba37993f46ed6e111c6e9141d18/uvloop-0.22.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:512fec6815e2dd45161054592441ef76c830eddaad55c8aa30952e6fe1ed07c0", size = 4296343, upload-time = "2025-10-16T22:16:35.149Z" }, + { url = "https://files.pythonhosted.org/packages/89/8c/182a2a593195bfd39842ea68ebc084e20c850806117213f5a299dfc513d9/uvloop-0.22.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:561577354eb94200d75aca23fbde86ee11be36b00e52a4eaf8f50fb0c86b7705", size = 1358611, upload-time = "2025-10-16T22:16:36.833Z" }, + { url = "https://files.pythonhosted.org/packages/d2/14/e301ee96a6dc95224b6f1162cd3312f6d1217be3907b79173b06785f2fe7/uvloop-0.22.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:1cdf5192ab3e674ca26da2eada35b288d2fa49fdd0f357a19f0e7c4e7d5077c8", size = 751811, upload-time = "2025-10-16T22:16:38.275Z" }, + { url = "https://files.pythonhosted.org/packages/b7/02/654426ce265ac19e2980bfd9ea6590ca96a56f10c76e63801a2df01c0486/uvloop-0.22.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6e2ea3d6190a2968f4a14a23019d3b16870dd2190cd69c8180f7c632d21de68d", size = 4288562, upload-time = "2025-10-16T22:16:39.375Z" }, + { url = "https://files.pythonhosted.org/packages/15/c0/0be24758891ef825f2065cd5db8741aaddabe3e248ee6acc5e8a80f04005/uvloop-0.22.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0530a5fbad9c9e4ee3f2b33b148c6a64d47bbad8000ea63704fa8260f4cf728e", size = 4366890, upload-time = "2025-10-16T22:16:40.547Z" }, + { url = "https://files.pythonhosted.org/packages/d2/53/8369e5219a5855869bcee5f4d317f6da0e2c669aecf0ef7d371e3d084449/uvloop-0.22.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:bc5ef13bbc10b5335792360623cc378d52d7e62c2de64660616478c32cd0598e", size = 4119472, upload-time = "2025-10-16T22:16:41.694Z" }, + { url = "https://files.pythonhosted.org/packages/f8/ba/d69adbe699b768f6b29a5eec7b47dd610bd17a69de51b251126a801369ea/uvloop-0.22.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:1f38ec5e3f18c8a10ded09742f7fb8de0108796eb673f30ce7762ce1b8550cad", size = 4239051, upload-time = "2025-10-16T22:16:43.224Z" }, + { url = "https://files.pythonhosted.org/packages/90/cd/b62bdeaa429758aee8de8b00ac0dd26593a9de93d302bff3d21439e9791d/uvloop-0.22.1-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:3879b88423ec7e97cd4eba2a443aa26ed4e59b45e6b76aabf13fe2f27023a142", size = 1362067, upload-time = "2025-10-16T22:16:44.503Z" }, + { url = "https://files.pythonhosted.org/packages/0d/f8/a132124dfda0777e489ca86732e85e69afcd1ff7686647000050ba670689/uvloop-0.22.1-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:4baa86acedf1d62115c1dc6ad1e17134476688f08c6efd8a2ab076e815665c74", size = 752423, upload-time = "2025-10-16T22:16:45.968Z" }, + { url = "https://files.pythonhosted.org/packages/a3/94/94af78c156f88da4b3a733773ad5ba0b164393e357cc4bd0ab2e2677a7d6/uvloop-0.22.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:297c27d8003520596236bdb2335e6b3f649480bd09e00d1e3a99144b691d2a35", size = 4272437, upload-time = "2025-10-16T22:16:47.451Z" }, + { url = "https://files.pythonhosted.org/packages/b5/35/60249e9fd07b32c665192cec7af29e06c7cd96fa1d08b84f012a56a0b38e/uvloop-0.22.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c1955d5a1dd43198244d47664a5858082a3239766a839b2102a269aaff7a4e25", size = 4292101, upload-time = "2025-10-16T22:16:49.318Z" }, + { url = "https://files.pythonhosted.org/packages/02/62/67d382dfcb25d0a98ce73c11ed1a6fba5037a1a1d533dcbb7cab033a2636/uvloop-0.22.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:b31dc2fccbd42adc73bc4e7cdbae4fc5086cf378979e53ca5d0301838c5682c6", size = 4114158, upload-time = "2025-10-16T22:16:50.517Z" }, + { url = "https://files.pythonhosted.org/packages/f0/7a/f1171b4a882a5d13c8b7576f348acfe6074d72eaf52cccef752f748d4a9f/uvloop-0.22.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:93f617675b2d03af4e72a5333ef89450dfaa5321303ede6e67ba9c9d26878079", size = 4177360, upload-time = "2025-10-16T22:16:52.646Z" }, + { url = "https://files.pythonhosted.org/packages/79/7b/b01414f31546caf0919da80ad57cbfe24c56b151d12af68cee1b04922ca8/uvloop-0.22.1-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:37554f70528f60cad66945b885eb01f1bb514f132d92b6eeed1c90fd54ed6289", size = 1454790, upload-time = "2025-10-16T22:16:54.355Z" }, + { url = "https://files.pythonhosted.org/packages/d4/31/0bb232318dd838cad3fa8fb0c68c8b40e1145b32025581975e18b11fab40/uvloop-0.22.1-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:b76324e2dc033a0b2f435f33eb88ff9913c156ef78e153fb210e03c13da746b3", size = 796783, upload-time = "2025-10-16T22:16:55.906Z" }, + { url = "https://files.pythonhosted.org/packages/42/38/c9b09f3271a7a723a5de69f8e237ab8e7803183131bc57c890db0b6bb872/uvloop-0.22.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:badb4d8e58ee08dad957002027830d5c3b06aea446a6a3744483c2b3b745345c", size = 4647548, upload-time = "2025-10-16T22:16:57.008Z" }, + { url = "https://files.pythonhosted.org/packages/c1/37/945b4ca0ac27e3dc4952642d4c900edd030b3da6c9634875af6e13ae80e5/uvloop-0.22.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b91328c72635f6f9e0282e4a57da7470c7350ab1c9f48546c0f2866205349d21", size = 4467065, upload-time = "2025-10-16T22:16:58.206Z" }, + { url = "https://files.pythonhosted.org/packages/97/cc/48d232f33d60e2e2e0b42f4e73455b146b76ebe216487e862700457fbf3c/uvloop-0.22.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:daf620c2995d193449393d6c62131b3fbd40a63bf7b307a1527856ace637fe88", size = 4328384, upload-time = "2025-10-16T22:16:59.36Z" }, + { url = "https://files.pythonhosted.org/packages/e4/16/c1fd27e9549f3c4baf1dc9c20c456cd2f822dbf8de9f463824b0c0357e06/uvloop-0.22.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:6cde23eeda1a25c75b2e07d39970f3374105d5eafbaab2a4482be82f272d5a5e", size = 4296730, upload-time = "2025-10-16T22:17:00.744Z" }, +] + [[package]] name = "vcrpy" version = "8.1.1"