From 8fb4cb1a33ae5d834bcf38756db53515e3bbdf62 Mon Sep 17 00:00:00 2001 From: Tanner Linsley Date: Wed, 26 Aug 2026 09:30:39 -0600 Subject: [PATCH] Back off Builder sync recovery --- src/utils/builder-project-sync.client.ts | 17 +++- tests/builder-project-sync-client.test.ts | 103 ++++++++++++++++++++++ 2 files changed, 117 insertions(+), 3 deletions(-) diff --git a/src/utils/builder-project-sync.client.ts b/src/utils/builder-project-sync.client.ts index 2095d12de..f1b4a5878 100644 --- a/src/utils/builder-project-sync.client.ts +++ b/src/utils/builder-project-sync.client.ts @@ -38,6 +38,8 @@ const commandRequestTimeoutMs = 20_000 const heartbeatRequestTimeoutMs = 8_000 const outboxRetryInitialDelayMs = 500 const outboxRetryMaxDelayMs = 30_000 +const streamRecoveryRetryInitialDelayMs = 500 +const streamRecoveryRetryMaxDelayMs = 30_000 const uuidPattern = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i @@ -1169,6 +1171,7 @@ function openBuilderProjectStream({ let stopped = false let recovering = false + let recoveryRetryDelayMs = streamRecoveryRetryInitialDelayMs let closeSource: (() => void) | undefined const markCaughtUpIfReady = () => { @@ -1211,6 +1214,7 @@ function openBuilderProjectStream({ project = authoritative.snapshot.project knownKeys = replacement.keys readyCursor = authoritative.headCursor + recoveryRetryDelayMs = streamRecoveryRetryInitialDelayMs recovering = false closeSource?.() openSource() @@ -1218,7 +1222,14 @@ function openBuilderProjectStream({ } catch (error) { if (stopped || context.signal.aborted) return reportStreamError(error) - await waitForBuilderProjectSyncRetry(context.signal) + await waitForBuilderProjectSyncRetry( + context.signal, + recoveryRetryDelayMs, + ) + recoveryRetryDelayMs = Math.min( + recoveryRetryDelayMs * 2, + streamRecoveryRetryMaxDelayMs, + ) } } })() @@ -1419,7 +1430,7 @@ function rememberBuilderProjectSyncChanges( } } -function waitForBuilderProjectSyncRetry(signal: AbortSignal) { +function waitForBuilderProjectSyncRetry(signal: AbortSignal, delayMs: number) { if (signal.aborted) return Promise.resolve() return new Promise((resolve) => { @@ -1428,7 +1439,7 @@ function waitForBuilderProjectSyncRetry(signal: AbortSignal) { signal.removeEventListener('abort', finish) resolve() } - const timeout = setTimeout(finish, 500) + const timeout = setTimeout(finish, delayMs) signal.addEventListener('abort', finish, { once: true }) }) } diff --git a/tests/builder-project-sync-client.test.ts b/tests/builder-project-sync-client.test.ts index 8611daed6..0d8403f58 100644 --- a/tests/builder-project-sync-client.test.ts +++ b/tests/builder-project-sync-client.test.ts @@ -646,6 +646,109 @@ test('a terminal Builder EventSource recovers from an authoritative snapshot', a }) }) +test('Builder stream recovery backs off repeated snapshot failures', async (context) => { + context.mock.timers.enable({ apis: ['setTimeout'] }) + + await withFakeIndexedDb(async () => { + let snapshotRequestCount = 0 + let notifySecondRequest: (() => void) | undefined + let notifyThirdRequest: (() => void) | undefined + let notifyFourthRequest: (() => void) | undefined + let backgroundErrorCount = 0 + let notifyFirstSnapshotFailure: (() => void) | undefined + let notifySecondSnapshotFailure: (() => void) | undefined + let notifyThirdSnapshotFailure: (() => void) | undefined + const secondRequest = new Promise((resolve) => { + notifySecondRequest = resolve + }) + const thirdRequest = new Promise((resolve) => { + notifyThirdRequest = resolve + }) + const fourthRequest = new Promise((resolve) => { + notifyFourthRequest = resolve + }) + const firstSnapshotFailure = new Promise((resolve) => { + notifyFirstSnapshotFailure = resolve + }) + const secondSnapshotFailure = new Promise((resolve) => { + notifySecondSnapshotFailure = resolve + }) + const thirdSnapshotFailure = new Promise((resolve) => { + notifyThirdSnapshotFailure = resolve + }) + + const client = await createBuilderProjectSyncClient({ + projectId, + sessionStorage: memoryStorage(), + createBrowserSessionId: () => sessionId, + fetch: async (_input, init) => { + assert.equal(init?.method ?? 'GET', 'GET') + snapshotRequestCount += 1 + if (snapshotRequestCount === 1) return jsonSnapshotPage(snapshot) + + if (snapshotRequestCount === 2) notifySecondRequest?.() + if (snapshotRequestCount === 3) notifyThirdRequest?.() + if (snapshotRequestCount === 4) notifyFourthRequest?.() + return new Response(JSON.stringify({ error: 'Try again later' }), { + status: 503, + headers: { 'Content-Type': 'application/json' }, + }) + }, + createEventSource: () => { + const listeners = new Map< + string, + (event: MessageEvent) => void + >() + queueMicrotask(() => { + listeners + .get('error') + ?.call(undefined, new MessageEvent('error', { data: '' })) + }) + return { + readyState: 2, + addEventListener: (type, listener) => { + listeners.set(type, listener) + }, + removeEventListener: (type) => { + listeners.delete(type) + }, + close: () => undefined, + } + }, + onBackgroundError: () => { + backgroundErrorCount += 1 + if (backgroundErrorCount === 2) notifyFirstSnapshotFailure?.() + if (backgroundErrorCount === 3) notifySecondSnapshotFailure?.() + if (backgroundErrorCount === 4) notifyThirdSnapshotFailure?.() + }, + }) + + await secondRequest + await firstSnapshotFailure + assert.equal(snapshotRequestCount, 2) + + context.mock.timers.tick(499) + await Promise.resolve() + assert.equal(snapshotRequestCount, 2) + + context.mock.timers.tick(1) + await thirdRequest + await secondSnapshotFailure + assert.equal(snapshotRequestCount, 3) + + context.mock.timers.tick(999) + await Promise.resolve() + assert.equal(snapshotRequestCount, 3) + + context.mock.timers.tick(1) + await fourthRequest + await thirdSnapshotFailure + assert.equal(snapshotRequestCount, 4) + + await client.cleanup() + }) +}) + test('Builder project sync commands are optimistic, durable, and confirmed by replay', async () => { await withFakeIndexedDb(async (indexedDb) => { const eventListeners = new Map<