From 48826031ac292ae645b6d4e1e2a55a2056d9e16f Mon Sep 17 00:00:00 2001 From: Chris McDonough Date: Mon, 1 Jun 2026 06:28:55 -0400 Subject: [PATCH 1/2] Add partial indexes for has_pending() and dashboard throughput queries MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit has_pending() scans jobs WHERE source_id=? AND status IN ('queued','claimed') on every poller sweep — without an index this is a full table scan as the jobs table grows. Similarly, count_succeeded_since() scans by completed_at for the dashboard's rolling-throughput display. Add two partial indexes: - idx_jobs_pending_by_source: covers has_pending() lookups - idx_jobs_succeeded_completed: covers count_succeeded_since() Both use CREATE INDEX IF NOT EXISTS so they're idempotent on existing databases. --- haiku_rag_slim/haiku/rag/ingester/queue/schema.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/haiku_rag_slim/haiku/rag/ingester/queue/schema.py b/haiku_rag_slim/haiku/rag/ingester/queue/schema.py index 04bb67e2..ada3e007 100644 --- a/haiku_rag_slim/haiku/rag/ingester/queue/schema.py +++ b/haiku_rag_slim/haiku/rag/ingester/queue/schema.py @@ -40,6 +40,18 @@ ON jobs(scheduled_at) WHERE status = 'queued' """ +JOBS_PENDING_BY_SOURCE_INDEX = """ +CREATE INDEX IF NOT EXISTS idx_jobs_pending_by_source +ON jobs(source_id) +WHERE status IN ('queued', 'claimed') +""" + +JOBS_SUCCEEDED_COMPLETED_INDEX = """ +CREATE INDEX IF NOT EXISTS idx_jobs_succeeded_completed +ON jobs(completed_at) +WHERE status = 'succeeded' +""" + SYNC_STATE_DDL = """ CREATE TABLE IF NOT EXISTS sync_state ( source_id TEXT NOT NULL, @@ -67,6 +79,8 @@ ALL_DDL: tuple[str, ...] = ( JOBS_DDL, JOBS_LIVE_INDEX, JOBS_CLAIMABLE_INDEX, + JOBS_PENDING_BY_SOURCE_INDEX, + JOBS_SUCCEEDED_COMPLETED_INDEX, SYNC_STATE_DDL, SCHEMA_VERSION_DDL, DLQ_VIEW, From 50100694742fac3dd350209d5a0bb73acfac0fd7 Mon Sep 17 00:00:00 2001 From: Chris McDonough Date: Mon, 1 Jun 2026 09:24:11 -0400 Subject: [PATCH 2/2] Drop redundant idx_jobs_pending_by_source index has_pending() already uses the leading column of uq_jobs_live (source_id) with the same WHERE clause. The extra index just adds write amplification on every insert/claim/complete without improving reads. --- haiku_rag_slim/haiku/rag/ingester/queue/schema.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/haiku_rag_slim/haiku/rag/ingester/queue/schema.py b/haiku_rag_slim/haiku/rag/ingester/queue/schema.py index ada3e007..a05ee068 100644 --- a/haiku_rag_slim/haiku/rag/ingester/queue/schema.py +++ b/haiku_rag_slim/haiku/rag/ingester/queue/schema.py @@ -40,12 +40,6 @@ ON jobs(scheduled_at) WHERE status = 'queued' """ -JOBS_PENDING_BY_SOURCE_INDEX = """ -CREATE INDEX IF NOT EXISTS idx_jobs_pending_by_source -ON jobs(source_id) -WHERE status IN ('queued', 'claimed') -""" - JOBS_SUCCEEDED_COMPLETED_INDEX = """ CREATE INDEX IF NOT EXISTS idx_jobs_succeeded_completed ON jobs(completed_at) @@ -79,7 +73,6 @@ ALL_DDL: tuple[str, ...] = ( JOBS_DDL, JOBS_LIVE_INDEX, JOBS_CLAIMABLE_INDEX, - JOBS_PENDING_BY_SOURCE_INDEX, JOBS_SUCCEEDED_COMPLETED_INDEX, SYNC_STATE_DDL, SCHEMA_VERSION_DDL,