danvics/openreader
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

refactor(documents): implement exponential backoff with jitter for blob lease retries

Browse source 
Replace fixed retry delay with capped exponential backoff and jitter to reduce
contention and thundering herd effect when acquiring document blob leases. This
improves lease acquisition fairness and efficiency under high load.
This commit is contained in:
Richard R 2026-06-07 13:31:16 -06:00
parent a090fba856
commit f0c8dc2b64
1 changed files with 13 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

15
src/lib/server/documents/blob-lease.ts
View file

@ -6,6 +6,7 @@ import { errorToLog, serverLogger } from '@/lib/server/logger';
const DEFAULT_LEASE_MS = 15 * 60 * 1000;
const RETRY_DELAY_MS = 100;
const MAX_RETRY_DELAY_MS = 2_000;
export type DocumentBlobLease = {
owner: string;
@ -56,10 +57,20 @@ export async function withDocumentBlobLease<T>(
): Promise<T> {
const deadline = Date.now() + (options?.waitMs ?? 30_000);
let lease: DocumentBlobLease | null = null;
let backoffMs = RETRY_DELAY_MS;
while (!lease && Date.now() <= deadline) {
while (!lease && Date.now() < deadline) {
lease = await tryAcquireDocumentBlobLease(documentId, options);
if (!lease) await sleep(RETRY_DELAY_MS);
if (lease) break;
const remainingMs = deadline - Date.now();
if (remainingMs <= 0) break;
// Exponential backoff with jitter to avoid thundering-herd retries under
// contention, capped and never sleeping past the deadline.
const jitterMs = Math.random() * backoffMs * 0.1;
await sleep(Math.min(backoffMs + jitterMs, remainingMs));
backoffMs = Math.min(backoffMs * 2, MAX_RETRY_DELAY_MS);
}
if (!lease) {
throw new Error(`Timed out waiting for document blob lease: ${documentId}`);