From b50613f5fc03aaf37f082ea69ba0549cc8bdd477 Mon Sep 17 00:00:00 2001 From: Daniel Sutton Date: Mon, 24 Aug 2026 14:48:12 +0100 Subject: [PATCH 01/56] feat(run-store): accept a caller-supplied execution-snapshot id The decorator that dual-writes snapshots to Redis has to own the snapshot id, or the same snapshot carries a different id in each store and the comparator chases a difference that is not real. Four of the six snapshot input types had no id field, so four write sites could not carry one. Add it to CompletionSnapshotInput, ExpireSnapshotInput, RescheduleSnapshotInput and CreateExecutionSnapshotInput, and thread it through every nested create. createCancelledRun built its create inline and dropped the id its input already carried; it now passes it too. The field is optional everywhere, so an absent id still falls through to Prisma's @default(cuid()) and no existing caller changes. --- .../src/PostgresRunStore.snapshotId.test.ts | 150 ++++++++++++++++++ .../run-store/src/PostgresRunStore.ts | 7 + .../src/testFixtures/snapshotIdFixture.ts | 135 ++++++++++++++++ internal-packages/run-store/src/types.ts | 12 ++ 4 files changed, 304 insertions(+) create mode 100644 internal-packages/run-store/src/PostgresRunStore.snapshotId.test.ts create mode 100644 internal-packages/run-store/src/testFixtures/snapshotIdFixture.ts diff --git a/internal-packages/run-store/src/PostgresRunStore.snapshotId.test.ts b/internal-packages/run-store/src/PostgresRunStore.snapshotId.test.ts new file mode 100644 index 00000000000..07948670b28 --- /dev/null +++ b/internal-packages/run-store/src/PostgresRunStore.snapshotId.test.ts @@ -0,0 +1,150 @@ +// A caller-supplied snapshot id must survive into Postgres, so the decorator can own the id and both +// stores hold the same one under dual-write. Absent, Prisma's @default(cuid()) still supplies it. +import { describe, expect } from "vitest"; +import { postgresTest } from "@internal/testcontainers"; +import { generateInternalId } from "@trigger.dev/core/v3/isomorphic"; +import { PostgresRunStore } from "./PostgresRunStore.js"; +import { setupSnapshotIdFixture } from "./testFixtures/snapshotIdFixture.js"; + +describe("PostgresRunStore caller-supplied snapshot id", () => { + postgresTest("completeAttemptSuccess writes the supplied id", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const { run, env } = await setupSnapshotIdFixture(prisma); + const id = generateInternalId(); + + await store.completeAttemptSuccess( + run.id, + { + completedAt: new Date(), + outputType: "application/json", + usageDurationMs: 1, + costInCents: 0, + snapshot: { + id, + executionStatus: "FINISHED", + description: "Run completed", + runStatus: "COMPLETED_SUCCESSFULLY", + attemptNumber: 1, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }, + }, + { select: { id: true } } + ); + + const snapshot = await prisma.taskRunExecutionSnapshot.findFirst({ where: { id } }); + expect(snapshot).not.toBeNull(); + expect(snapshot!.runId).toBe(run.id); + }); + + postgresTest("expireRun writes the supplied id", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const { run, env } = await setupSnapshotIdFixture(prisma); + const id = generateInternalId(); + + await store.expireRun( + run.id, + { + error: { type: "STRING_ERROR", raw: "expired" }, + completedAt: new Date(), + expiredAt: new Date(), + snapshot: { + id, + engine: "V2", + executionStatus: "FINISHED", + description: "Run expired", + runStatus: "EXPIRED", + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }, + }, + { select: { id: true } } + ); + + expect(await prisma.taskRunExecutionSnapshot.findFirst({ where: { id } })).not.toBeNull(); + }); + + postgresTest("expireParkedRun writes the supplied id", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const { run, env } = await setupSnapshotIdFixture(prisma, { status: "PENDING_VERSION" }); + const id = generateInternalId(); + + const result = await store.expireParkedRun(run.id, { + error: { type: "STRING_ERROR", raw: "expired" }, + completedAt: new Date(), + expiredAt: new Date(), + statusReason: "VERSION_NEVER_ARRIVED", + snapshot: { + id, + engine: "V2", + executionStatus: "FINISHED", + description: "Parked run expired", + runStatus: "EXPIRED", + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }, + }); + + expect(result.count).toBe(1); + expect(await prisma.taskRunExecutionSnapshot.findFirst({ where: { id } })).not.toBeNull(); + }); + + postgresTest("rescheduleRun writes the supplied id", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const { run, env } = await setupSnapshotIdFixture(prisma, { status: "DELAYED" }); + const id = generateInternalId(); + + await store.rescheduleRun(run.id, { + delayUntil: new Date(Date.now() + 60_000), + snapshot: { + id, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }, + }); + + expect(await prisma.taskRunExecutionSnapshot.findFirst({ where: { id } })).not.toBeNull(); + }); + + postgresTest("createExecutionSnapshot writes the supplied id", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const { run, env } = await setupSnapshotIdFixture(prisma); + const id = generateInternalId(); + + const created = await store.createExecutionSnapshot({ + id, + run: { id: run.id, status: "EXECUTING", attemptNumber: 1 }, + snapshot: { executionStatus: "EXECUTING", description: "Run started" }, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }); + + expect(created.id).toBe(id); + }); + + postgresTest("an absent id still gets a generated one", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const { run, env } = await setupSnapshotIdFixture(prisma); + + const created = await store.createExecutionSnapshot({ + run: { id: run.id, status: "EXECUTING", attemptNumber: 1 }, + snapshot: { executionStatus: "EXECUTING", description: "Run started" }, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }); + + expect(created.id).toMatch(/^c[a-z0-9]{24}$/); + }); +}); diff --git a/internal-packages/run-store/src/PostgresRunStore.ts b/internal-packages/run-store/src/PostgresRunStore.ts index b7d5086431b..e860bd6c67a 100644 --- a/internal-packages/run-store/src/PostgresRunStore.ts +++ b/internal-packages/run-store/src/PostgresRunStore.ts @@ -818,6 +818,7 @@ export class PostgresRunStore implements RunStore { ...params.data, executionSnapshots: { create: { + id: params.snapshot.id, engine: params.snapshot.engine, executionStatus: params.snapshot.executionStatus, description: params.snapshot.description, @@ -925,6 +926,7 @@ export class PostgresRunStore implements RunStore { costInCents: data.costInCents, executionSnapshots: { create: { + id: data.snapshot.id, executionStatus: data.snapshot.executionStatus, description: data.snapshot.description, runStatus: data.snapshot.runStatus, @@ -1131,6 +1133,7 @@ export class PostgresRunStore implements RunStore { error: data.error as Prisma.InputJsonValue, executionSnapshots: { create: { + id: data.snapshot.id, engine: data.snapshot.engine, executionStatus: data.snapshot.executionStatus, description: data.snapshot.description, @@ -1365,6 +1368,7 @@ export class PostgresRunStore implements RunStore { error: data.error as Prisma.InputJsonValue, executionSnapshots: { create: { + id: data.snapshot.id, engine: data.snapshot.engine, executionStatus: data.snapshot.executionStatus, description: data.snapshot.description, @@ -1438,6 +1442,7 @@ export class PostgresRunStore implements RunStore { ...(data.snapshot && { executionSnapshots: { create: { + id: data.snapshot.id, engine: "V2", executionStatus: data.snapshot.executionStatus ?? "DELAYED", description: @@ -1969,6 +1974,7 @@ export class PostgresRunStore implements RunStore { prisma: PrismaClientOrTransaction ): Promise> { const { + id, run, snapshot, previousSnapshotId, @@ -1988,6 +1994,7 @@ export class PostgresRunStore implements RunStore { const newSnapshot = await prisma.taskRunExecutionSnapshot.create({ data: { + id, engine: "V2", executionStatus: snapshot.executionStatus, description: snapshot.description, diff --git a/internal-packages/run-store/src/testFixtures/snapshotIdFixture.ts b/internal-packages/run-store/src/testFixtures/snapshotIdFixture.ts new file mode 100644 index 00000000000..17c0248312b --- /dev/null +++ b/internal-packages/run-store/src/testFixtures/snapshotIdFixture.ts @@ -0,0 +1,135 @@ +// Shared setup for the snapshot-id, snapshot-writes and entry-parity suites. Modelled on the +// seedEnvironment/buildCreateRunInput pair in PostgresRunStore.test.ts; the slugs are suffixed so +// several fixtures can coexist in one database. +import type { PrismaClient, TaskRunStatus } from "@trigger.dev/database"; +import { generateInternalId } from "@trigger.dev/core/v3/isomorphic"; +import type { CreateRunData } from "../types.js"; + +export type SnapshotFixtureEnv = { + id: string; + type: "DEVELOPMENT"; + projectId: string; + organizationId: string; +}; + +export type SnapshotIdFixture = { + run: { id: string }; + env: SnapshotFixtureEnv; +}; + +export async function seedSnapshotEnvironment(prisma: PrismaClient): Promise { + const suffix = generateInternalId().slice(-12); + + const organization = await prisma.organization.create({ + data: { title: `Snapshot Org ${suffix}`, slug: `snapshot-org-${suffix}` }, + }); + + const project = await prisma.project.create({ + data: { + name: `Snapshot Project ${suffix}`, + slug: `snapshot-project-${suffix}`, + externalRef: `proj_${suffix}`, + organizationId: organization.id, + }, + }); + + const environment = await prisma.runtimeEnvironment.create({ + data: { + type: "DEVELOPMENT", + slug: `dev-${suffix}`, + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_dev_${suffix}`, + pkApiKey: `pk_dev_${suffix}`, + shortcode: `short_${suffix}`, + }, + }); + + return { + id: environment.id, + type: "DEVELOPMENT", + projectId: project.id, + organizationId: organization.id, + }; +} + +export function buildCreateRunData(runId: string, env: SnapshotFixtureEnv): CreateRunData { + return { + id: runId, + engine: "V2", + status: "PENDING", + friendlyId: `run_${runId.slice(-16)}`, + runtimeEnvironmentId: env.id, + environmentType: env.type, + organizationId: env.organizationId, + projectId: env.projectId, + taskIdentifier: "my-task", + payload: "{}", + payloadType: "application/json", + traceContext: {}, + traceId: `trace_${runId.slice(-8)}`, + spanId: `span_${runId.slice(-8)}`, + queue: "task/my-task", + isTest: false, + taskEventStore: "taskEvent", + depth: 0, + }; +} + +export type SnapshotWorkerFixture = { workerId: string; taskId: string }; + +/** + * Seeds a BackgroundWorker and one of its tasks. The snapshot's `workerId` and the run's + * `lockedById` are both foreign keys, so a made-up id fails the constraint rather than the + * assertion, and the test reports a fixture fault as if it were a parity fault. + */ +export async function seedSnapshotWorker( + prisma: PrismaClient, + env: SnapshotFixtureEnv +): Promise { + const suffix = generateInternalId().slice(-12); + + const worker = await prisma.backgroundWorker.create({ + data: { + friendlyId: `worker_${suffix}`, + engine: "V2", + contentHash: `hash_${suffix}`, + projectId: env.projectId, + runtimeEnvironmentId: env.id, + version: "20260824.1", + metadata: {}, + }, + }); + + const task = await prisma.backgroundWorkerTask.create({ + data: { + slug: "my-task", + friendlyId: `task_${suffix}`, + filePath: "src/trigger/my-task.ts", + exportName: "myTask", + workerId: worker.id, + projectId: env.projectId, + runtimeEnvironmentId: env.id, + }, + }); + + return { workerId: worker.id, taskId: task.id }; +} + +/** + * Seeds an environment plus one run in `status`, with no execution snapshot. The suites that use it + * assert on the snapshot rows a store method writes, so the run must start with none. + */ +export async function setupSnapshotIdFixture( + prisma: PrismaClient, + opts?: { status?: TaskRunStatus } +): Promise { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + + await prisma.taskRun.create({ + data: { ...buildCreateRunData(runId, env), status: opts?.status ?? "PENDING" }, + }); + + return { run: { id: runId }, env }; +} diff --git a/internal-packages/run-store/src/types.ts b/internal-packages/run-store/src/types.ts index 7c7f9566893..5bde5950459 100644 --- a/internal-packages/run-store/src/types.ts +++ b/internal-packages/run-store/src/types.ts @@ -43,6 +43,9 @@ export type CreateRunSnapshotInput = { }; export type CompletionSnapshotInput = { + /** Caller-minted snapshot id. Absent, Prisma's `@default(cuid())` supplies one. The decorator + * sets it so a snapshot carries the same id in Postgres and in the Redis store. */ + id?: string; executionStatus: "FINISHED"; description: string; runStatus: TaskRunStatus; @@ -64,6 +67,9 @@ export type PromotePendingVersionArgs = { }; export type ExpireSnapshotInput = { + /** Caller-minted snapshot id. Absent, Prisma's `@default(cuid())` supplies one. The decorator + * sets it so a snapshot carries the same id in Postgres and in the Redis store. */ + id?: string; engine: "V2"; executionStatus: "FINISHED"; description: string; @@ -75,6 +81,9 @@ export type ExpireSnapshotInput = { }; export type RescheduleSnapshotInput = { + /** Caller-minted snapshot id. Absent, Prisma's `@default(cuid())` supplies one. The decorator + * sets it so a snapshot carries the same id in Postgres and in the Redis store. */ + id?: string; environmentId: string; environmentType: RuntimeEnvironmentType; projectId: string; @@ -292,6 +301,9 @@ export type TaskRunWithWaitpoint = TaskRun & { associatedWaitpoint: Waitpoint | * input — callers pass the high-level shape, not a raw Prisma `data`/`include`. */ export type CreateExecutionSnapshotInput = { + /** Caller-minted snapshot id. Absent, Prisma's `@default(cuid())` supplies one. The decorator + * sets it so a snapshot carries the same id in Postgres and in the Redis store. */ + id?: string; run: { id: string; status: TaskRunStatus; attemptNumber?: number | null }; snapshot: { executionStatus: TaskRunExecutionStatus; From db8390cd0cd2f96a23dd09dd122fe2d65193c937 Mon Sep 17 00:00:00 2001 From: Daniel Sutton Date: Mon, 24 Aug 2026 14:48:12 +0100 Subject: [PATCH 02/56] feat(run-store): add a generated pass-through RunStore base for decorators RunStore has 71 members. A decorator that intercepts a dozen of them should not restate the other 59 forwarders alongside its real logic, and hand-writing them invites a typo no test would catch. Generate the base from the interface instead. The generator also emits the member-name lists, so the suite can assert that the class and the interface hold exactly the same members: a method added to RunStore and not to the base fails a test rather than becoming a silent hole in the decorator. The one data property on the interface becomes a getter over the delegate, read live rather than captured, so a delegate whose client changes is not cached. --- .../scripts/generateDelegatingRunStore.ts | 190 +++++++++++ .../run-store/src/delegatingRunStore.test.ts | 72 +++++ .../run-store/src/delegatingRunStore.ts | 300 ++++++++++++++++++ .../run-store/src/runStoreMethodNames.ts | 83 +++++ 4 files changed, 645 insertions(+) create mode 100644 internal-packages/run-store/scripts/generateDelegatingRunStore.ts create mode 100644 internal-packages/run-store/src/delegatingRunStore.test.ts create mode 100644 internal-packages/run-store/src/delegatingRunStore.ts create mode 100644 internal-packages/run-store/src/runStoreMethodNames.ts diff --git a/internal-packages/run-store/scripts/generateDelegatingRunStore.ts b/internal-packages/run-store/scripts/generateDelegatingRunStore.ts new file mode 100644 index 00000000000..df901f3a2f7 --- /dev/null +++ b/internal-packages/run-store/scripts/generateDelegatingRunStore.ts @@ -0,0 +1,190 @@ +// One-off generator for the RunStore pass-through base. +// +// The base class is mechanical: 80-odd near-identical forwarders. Generating it removes the chance +// of a hand-typo that no test would catch, and turns "did we miss a method" into a diff rather than +// a review. Re-run after any change to the RunStore interface: +// +// pnpm exec tsx scripts/generateDelegatingRunStore.ts +// +// The interface is scanned directly rather than through the TypeScript compiler API, because +// `require("typescript")` resolves to a stub in this workspace. A parsing miss cannot pass silently: +// delegatingRunStore.test.ts asserts the class and the interface hold exactly the same member set, +// and `implements RunStore` fails typecheck if a method is absent. +import { readFileSync, writeFileSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; + +const root = join(dirname(fileURLToPath(import.meta.url)), ".."); +const source = readFileSync(join(root, "src/types.ts"), "utf8"); + +/** Replaces every comment and string body with spaces, so a brace inside one cannot move the depth. */ +function blankCommentsAndStrings(text: string): string { + let out = ""; + let i = 0; + while (i < text.length) { + const two = text.slice(i, i + 2); + if (two === "//") { + const end = text.indexOf("\n", i); + const stop = end === -1 ? text.length : end; + out += " ".repeat(stop - i); + i = stop; + } else if (two === "/*") { + const end = text.indexOf("*/", i + 2); + const stop = end === -1 ? text.length : end + 2; + out += text.slice(i, stop).replace(/[^\n]/g, " "); + i = stop; + } else if (text[i] === '"' || text[i] === "'" || text[i] === "`") { + const quote = text[i]; + let j = i + 1; + while (j < text.length && text[j] !== quote) { + j += text[j] === "\\" ? 2 : 1; + } + out += quote + " ".repeat(Math.max(0, j - i - 1)) + (text[j] ?? ""); + i = j + 1; + } else { + out += text[i]; + i += 1; + } + } + return out; +} + +const blanked = blankCommentsAndStrings(source); + +const declaration = "export interface RunStore {"; +const start = blanked.indexOf(declaration); +if (start === -1) { + throw new Error("export interface RunStore not found in src/types.ts"); +} + +const bodyStart = start + declaration.length; +let depth = 1; +let bodyEnd = bodyStart; +while (bodyEnd < blanked.length && depth > 0) { + const ch = blanked[bodyEnd]; + if (ch === "{") depth += 1; + else if (ch === "}") depth -= 1; + if (depth > 0) bodyEnd += 1; +} +if (depth !== 0) { + throw new Error("unbalanced braces while reading the RunStore interface body"); +} + +const body = blanked.slice(bodyStart, bodyEnd); + +// Members are separated by `;` at nesting depth 0. Only `{}`, `()` and `[]` count towards depth: +// angle brackets cannot, because `=>` in a callback parameter type carries an unmatched `>`. +function splitMembers(text: string): { offset: number; length: number }[] { + const spans: { offset: number; length: number }[] = []; + let level = 0; + let from = 0; + for (let i = 0; i < text.length; i++) { + const ch = text[i]; + if (ch === "{" || ch === "(" || ch === "[") level += 1; + else if (ch === "}" || ch === ")" || ch === "]") level -= 1; + else if (ch === ";" && level === 0) { + spans.push({ offset: from, length: i - from }); + from = i + 1; + } + } + if (text.slice(from).trim().length > 0) { + spans.push({ offset: from, length: text.length - from }); + } + return spans; +} + +const methods: string[] = []; +const readonlyProperties: { name: string; type: string }[] = []; +const mutableProperties: string[] = []; + +for (const span of splitMembers(body)) { + const blankedMember = body.slice(span.offset, span.offset + span.length).trim(); + if (blankedMember.length === 0) continue; + + // A method: an optional name then `(` or `<`. A property: a name then `:`. + const asMethod = /^([A-Za-z_$][\w$]*)\s*\??\s*[(<]/.exec(blankedMember); + if (asMethod) { + methods.push(asMethod[1]); + continue; + } + + const asProperty = /^(readonly\s+)?([A-Za-z_$][\w$]*)\s*(\??)\s*:([\s\S]*)$/.exec(blankedMember); + if (asProperty) { + const [, isReadonly, name, , type] = asProperty; + if (isReadonly) { + readonlyProperties.push({ name, type: type.trim() }); + } else { + mutableProperties.push(name); + } + continue; + } + + throw new Error(`could not classify a RunStore member: ${blankedMember.slice(0, 80)}`); +} + +if (mutableProperties.length > 0) { + // A writable data property cannot be forwarded by a getter alone, so the base would silently hold + // its own copy instead of the delegate's. Handle it by hand before regenerating. + throw new Error( + `RunStore declares writable data properties the generator cannot forward: ${mutableProperties.join(", ")}` + ); +} + +const unique = [...new Set(methods)]; +if (unique.length === 0) { + throw new Error("RunStore declares no methods, which cannot be right"); +} + +const memberNames = [...unique, ...readonlyProperties.map((p) => p.name)]; + +const header = `// GENERATED by scripts/generateDelegatingRunStore.ts. Do not edit by hand. +// Regenerate after any change to the RunStore interface: +// pnpm exec tsx scripts/generateDelegatingRunStore.ts +`; + +writeFileSync( + join(root, "src/runStoreMethodNames.ts"), + `${header} +// Every method the RunStore interface declares. The decorator suites enumerate this, so a method +// added to the interface and not to the base fails a test instead of becoming a silent hole. +export const RUN_STORE_METHOD_NAMES = [ +${unique.map((n) => ` "${n}",`).join("\n")} +] as const; + +// Data properties the base exposes as getters over the delegate, not as forwarders. +export const RUN_STORE_PROPERTY_NAMES = [ +${readonlyProperties.map((p) => ` "${p.name}",`).join("\n")} +] as const; +` +); + +writeFileSync( + join(root, "src/delegatingRunStore.ts"), + `${header} +// A pass-through over another RunStore. It exists so a decorator can override the handful of methods +// it cares about and inherit the rest, instead of restating 80-odd forwarders alongside real logic. +// +// Arguments and return values are forwarded untouched. The \`any\` signatures carry each method's +// whole overload set through one forwarder, which is the single thing a generated base cannot +// preserve; a subclass that overrides a method restates the real signature there. +/* eslint-disable @typescript-eslint/no-explicit-any */ +import type { RunStore } from "./types.js"; + +export class DelegatingRunStore implements RunStore { + constructor(protected readonly delegate: RunStore) {} + +${readonlyProperties + // Indexed access rather than the written type, so the getter needs no import of its own and + // follows the interface if that type is ever changed. + .map((p) => ` get ${p.name}(): RunStore["${p.name}"] {\n return this.delegate.${p.name};\n }`) + .join("\n\n")}${readonlyProperties.length > 0 ? "\n\n" : ""}${unique + .map((n) => ` ${n}(...args: any[]): any {\n return (this.delegate as any).${n}(...args);\n }`) + .join("\n\n")} +} +` +); + +console.log( + `generated ${unique.length} forwarders and ${readonlyProperties.length} getters ` + + `(${memberNames.length} members total)` +); diff --git a/internal-packages/run-store/src/delegatingRunStore.test.ts b/internal-packages/run-store/src/delegatingRunStore.test.ts new file mode 100644 index 00000000000..7065f801d71 --- /dev/null +++ b/internal-packages/run-store/src/delegatingRunStore.test.ts @@ -0,0 +1,72 @@ +// The base must forward EVERY RunStore member. A method added to the interface and not to the base +// is a silent hole in the decorator built on top of it, so this suite enumerates the generated name +// lists rather than restating them by hand. Regenerate the base and both lists together: +// pnpm exec tsx scripts/generateDelegatingRunStore.ts +import { describe, expect, it } from "vitest"; +import { DelegatingRunStore } from "./delegatingRunStore.js"; +import { RUN_STORE_METHOD_NAMES, RUN_STORE_PROPERTY_NAMES } from "./runStoreMethodNames.js"; +import type { RunStore } from "./types.js"; + +function recordingDelegate(): { store: RunStore; calls: { name: string; args: unknown[] }[] } { + const calls: { name: string; args: unknown[] }[] = []; + const store: Record = {}; + + for (const name of RUN_STORE_METHOD_NAMES) { + store[name] = (...args: unknown[]) => { + calls.push({ name, args }); + return `result:${name}`; + }; + } + for (const name of RUN_STORE_PROPERTY_NAMES) { + store[name] = `property:${name}`; + } + + return { store: store as unknown as RunStore, calls }; +} + +describe("DelegatingRunStore", () => { + it("forwards every RunStore method to the delegate, arguments untouched", () => { + const { store, calls } = recordingDelegate(); + const base = new DelegatingRunStore(store) as unknown as Record< + string, + (...args: unknown[]) => unknown + >; + + for (const name of RUN_STORE_METHOD_NAMES) { + expect(base[name]("arg-one", "arg-two")).toBe(`result:${name}`); + } + + expect(calls.map((c) => c.name)).toEqual([...RUN_STORE_METHOD_NAMES]); + for (const call of calls) { + expect(call.args).toEqual(["arg-one", "arg-two"]); + } + }); + + it("reads every RunStore data property from the delegate", () => { + const { store } = recordingDelegate(); + const base = new DelegatingRunStore(store) as unknown as Record; + + for (const name of RUN_STORE_PROPERTY_NAMES) { + expect(base[name]).toBe(`property:${name}`); + } + }); + + it("reads a data property live, so a delegate that changes is not cached", () => { + const store = { primaryReadClient: "first" } as unknown as RunStore; + const base = new DelegatingRunStore(store); + + expect(base.primaryReadClient).toBe("first" as unknown); + (store as unknown as Record).primaryReadClient = "second"; + expect(base.primaryReadClient).toBe("second" as unknown); + }); + + it("declares exactly the members the interface declares, and no others", () => { + const own = Object.getOwnPropertyNames(DelegatingRunStore.prototype) + .filter((name) => name !== "constructor") + .sort(); + + const expected = [...RUN_STORE_METHOD_NAMES, ...RUN_STORE_PROPERTY_NAMES].sort(); + + expect(own).toEqual(expected); + }); +}); diff --git a/internal-packages/run-store/src/delegatingRunStore.ts b/internal-packages/run-store/src/delegatingRunStore.ts new file mode 100644 index 00000000000..6266ac5abd1 --- /dev/null +++ b/internal-packages/run-store/src/delegatingRunStore.ts @@ -0,0 +1,300 @@ +// GENERATED by scripts/generateDelegatingRunStore.ts. Do not edit by hand. +// Regenerate after any change to the RunStore interface: +// pnpm exec tsx scripts/generateDelegatingRunStore.ts + +// A pass-through over another RunStore. It exists so a decorator can override the handful of methods +// it cares about and inherit the rest, instead of restating 80-odd forwarders alongside real logic. +// +// Arguments and return values are forwarded untouched. The `any` signatures carry each method's +// whole overload set through one forwarder, which is the single thing a generated base cannot +// preserve; a subclass that overrides a method restates the real signature there. +/* eslint-disable @typescript-eslint/no-explicit-any */ +import type { RunStore } from "./types.js"; + +export class DelegatingRunStore implements RunStore { + constructor(protected readonly delegate: RunStore) {} + + get primaryReadClient(): RunStore["primaryReadClient"] { + return this.delegate.primaryReadClient; + } + + runInTransaction(...args: any[]): any { + return (this.delegate as any).runInTransaction(...args); + } + + createRun(...args: any[]): any { + return (this.delegate as any).createRun(...args); + } + + createCancelledRun(...args: any[]): any { + return (this.delegate as any).createCancelledRun(...args); + } + + createFailedRun(...args: any[]): any { + return (this.delegate as any).createFailedRun(...args); + } + + startAttempt(...args: any[]): any { + return (this.delegate as any).startAttempt(...args); + } + + completeAttemptSuccess(...args: any[]): any { + return (this.delegate as any).completeAttemptSuccess(...args); + } + + recordRetryOutcome(...args: any[]): any { + return (this.delegate as any).recordRetryOutcome(...args); + } + + requeueRun(...args: any[]): any { + return (this.delegate as any).requeueRun(...args); + } + + recordBulkActionMembership(...args: any[]): any { + return (this.delegate as any).recordBulkActionMembership(...args); + } + + cancelRun(...args: any[]): any { + return (this.delegate as any).cancelRun(...args); + } + + failRunPermanently(...args: any[]): any { + return (this.delegate as any).failRunPermanently(...args); + } + + finalizeRun(...args: any[]): any { + return (this.delegate as any).finalizeRun(...args); + } + + expireRun(...args: any[]): any { + return (this.delegate as any).expireRun(...args); + } + + expireRunsBatch(...args: any[]): any { + return (this.delegate as any).expireRunsBatch(...args); + } + + lockRunToWorker(...args: any[]): any { + return (this.delegate as any).lockRunToWorker(...args); + } + + parkPendingVersion(...args: any[]): any { + return (this.delegate as any).parkPendingVersion(...args); + } + + promotePendingVersionRuns(...args: any[]): any { + return (this.delegate as any).promotePendingVersionRuns(...args); + } + + expireParkedRun(...args: any[]): any { + return (this.delegate as any).expireParkedRun(...args); + } + + suspendForCheckpoint(...args: any[]): any { + return (this.delegate as any).suspendForCheckpoint(...args); + } + + resumeFromCheckpoint(...args: any[]): any { + return (this.delegate as any).resumeFromCheckpoint(...args); + } + + rescheduleRun(...args: any[]): any { + return (this.delegate as any).rescheduleRun(...args); + } + + enqueueDelayedRun(...args: any[]): any { + return (this.delegate as any).enqueueDelayedRun(...args); + } + + rewriteDebouncedRun(...args: any[]): any { + return (this.delegate as any).rewriteDebouncedRun(...args); + } + + updateMetadata(...args: any[]): any { + return (this.delegate as any).updateMetadata(...args); + } + + clearIdempotencyKey(...args: any[]): any { + return (this.delegate as any).clearIdempotencyKey(...args); + } + + pushTags(...args: any[]): any { + return (this.delegate as any).pushTags(...args); + } + + pushRealtimeStream(...args: any[]): any { + return (this.delegate as any).pushRealtimeStream(...args); + } + + findRun(...args: any[]): any { + return (this.delegate as any).findRun(...args); + } + + findRunOrThrow(...args: any[]): any { + return (this.delegate as any).findRunOrThrow(...args); + } + + findRunOnPrimary(...args: any[]): any { + return (this.delegate as any).findRunOnPrimary(...args); + } + + findRunOrThrowOnPrimary(...args: any[]): any { + return (this.delegate as any).findRunOrThrowOnPrimary(...args); + } + + findRuns(...args: any[]): any { + return (this.delegate as any).findRuns(...args); + } + + findRunsByIds(...args: any[]): any { + return (this.delegate as any).findRunsByIds(...args); + } + + findRunsByIdempotencyKeys(...args: any[]): any { + return (this.delegate as any).findRunsByIdempotencyKeys(...args); + } + + createBatchTaskRunItem(...args: any[]): any { + return (this.delegate as any).createBatchTaskRunItem(...args); + } + + findLatestExecutionSnapshot(...args: any[]): any { + return (this.delegate as any).findLatestExecutionSnapshot(...args); + } + + findExecutionSnapshot(...args: any[]): any { + return (this.delegate as any).findExecutionSnapshot(...args); + } + + findManyExecutionSnapshots(...args: any[]): any { + return (this.delegate as any).findManyExecutionSnapshots(...args); + } + + createExecutionSnapshot(...args: any[]): any { + return (this.delegate as any).createExecutionSnapshot(...args); + } + + findSnapshotCompletedWaitpointIds(...args: any[]): any { + return (this.delegate as any).findSnapshotCompletedWaitpointIds(...args); + } + + findSnapshotCompletedWaitpointIdsWithPresence(...args: any[]): any { + return (this.delegate as any).findSnapshotCompletedWaitpointIdsWithPresence(...args); + } + + findWaitpointConnectedRunIds(...args: any[]): any { + return (this.delegate as any).findWaitpointConnectedRunIds(...args); + } + + findWaitpointCompletedSnapshotIds(...args: any[]): any { + return (this.delegate as any).findWaitpointCompletedSnapshotIds(...args); + } + + blockRunWithWaitpointEdges(...args: any[]): any { + return (this.delegate as any).blockRunWithWaitpointEdges(...args); + } + + countPendingWaitpoints(...args: any[]): any { + return (this.delegate as any).countPendingWaitpoints(...args); + } + + countPendingWaitpointsWithPresence(...args: any[]): any { + return (this.delegate as any).countPendingWaitpointsWithPresence(...args); + } + + createWaitpoint(...args: any[]): any { + return (this.delegate as any).createWaitpoint(...args); + } + + upsertWaitpoint(...args: any[]): any { + return (this.delegate as any).upsertWaitpoint(...args); + } + + findWaitpoint(...args: any[]): any { + return (this.delegate as any).findWaitpoint(...args); + } + + findWaitpointOnPrimary(...args: any[]): any { + return (this.delegate as any).findWaitpointOnPrimary(...args); + } + + findManyWaitpoints(...args: any[]): any { + return (this.delegate as any).findManyWaitpoints(...args); + } + + updateWaitpoint(...args: any[]): any { + return (this.delegate as any).updateWaitpoint(...args); + } + + updateManyWaitpoints(...args: any[]): any { + return (this.delegate as any).updateManyWaitpoints(...args); + } + + forWaitpointCompletion(...args: any[]): any { + return (this.delegate as any).forWaitpointCompletion(...args); + } + + findManyTaskRunWaitpoints(...args: any[]): any { + return (this.delegate as any).findManyTaskRunWaitpoints(...args); + } + + deleteManyTaskRunWaitpoints(...args: any[]): any { + return (this.delegate as any).deleteManyTaskRunWaitpoints(...args); + } + + findTaskRunAttempt(...args: any[]): any { + return (this.delegate as any).findTaskRunAttempt(...args); + } + + createTaskRunCheckpoint(...args: any[]): any { + return (this.delegate as any).createTaskRunCheckpoint(...args); + } + + createBatchTaskRun(...args: any[]): any { + return (this.delegate as any).createBatchTaskRun(...args); + } + + updateBatchTaskRun(...args: any[]): any { + return (this.delegate as any).updateBatchTaskRun(...args); + } + + findBatchTaskRunById(...args: any[]): any { + return (this.delegate as any).findBatchTaskRunById(...args); + } + + findBatchTaskRunByFriendlyId(...args: any[]): any { + return (this.delegate as any).findBatchTaskRunByFriendlyId(...args); + } + + findBatchTaskRunByIdempotencyKey(...args: any[]): any { + return (this.delegate as any).findBatchTaskRunByIdempotencyKey(...args); + } + + updateManyBatchTaskRun(...args: any[]): any { + return (this.delegate as any).updateManyBatchTaskRun(...args); + } + + countBatchTaskRunItems(...args: any[]): any { + return (this.delegate as any).countBatchTaskRunItems(...args); + } + + updateManyBatchTaskRunItems(...args: any[]): any { + return (this.delegate as any).updateManyBatchTaskRunItems(...args); + } + + findManyBatchTaskRunItems(...args: any[]): any { + return (this.delegate as any).findManyBatchTaskRunItems(...args); + } + + findBatchTaskRunItem(...args: any[]): any { + return (this.delegate as any).findBatchTaskRunItem(...args); + } + + upsertWaitpointTag(...args: any[]): any { + return (this.delegate as any).upsertWaitpointTag(...args); + } + + findManyWaitpointTags(...args: any[]): any { + return (this.delegate as any).findManyWaitpointTags(...args); + } +} diff --git a/internal-packages/run-store/src/runStoreMethodNames.ts b/internal-packages/run-store/src/runStoreMethodNames.ts new file mode 100644 index 00000000000..989c7dc37bb --- /dev/null +++ b/internal-packages/run-store/src/runStoreMethodNames.ts @@ -0,0 +1,83 @@ +// GENERATED by scripts/generateDelegatingRunStore.ts. Do not edit by hand. +// Regenerate after any change to the RunStore interface: +// pnpm exec tsx scripts/generateDelegatingRunStore.ts + +// Every method the RunStore interface declares. The decorator suites enumerate this, so a method +// added to the interface and not to the base fails a test instead of becoming a silent hole. +export const RUN_STORE_METHOD_NAMES = [ + "runInTransaction", + "createRun", + "createCancelledRun", + "createFailedRun", + "startAttempt", + "completeAttemptSuccess", + "recordRetryOutcome", + "requeueRun", + "recordBulkActionMembership", + "cancelRun", + "failRunPermanently", + "finalizeRun", + "expireRun", + "expireRunsBatch", + "lockRunToWorker", + "parkPendingVersion", + "promotePendingVersionRuns", + "expireParkedRun", + "suspendForCheckpoint", + "resumeFromCheckpoint", + "rescheduleRun", + "enqueueDelayedRun", + "rewriteDebouncedRun", + "updateMetadata", + "clearIdempotencyKey", + "pushTags", + "pushRealtimeStream", + "findRun", + "findRunOrThrow", + "findRunOnPrimary", + "findRunOrThrowOnPrimary", + "findRuns", + "findRunsByIds", + "findRunsByIdempotencyKeys", + "createBatchTaskRunItem", + "findLatestExecutionSnapshot", + "findExecutionSnapshot", + "findManyExecutionSnapshots", + "createExecutionSnapshot", + "findSnapshotCompletedWaitpointIds", + "findSnapshotCompletedWaitpointIdsWithPresence", + "findWaitpointConnectedRunIds", + "findWaitpointCompletedSnapshotIds", + "blockRunWithWaitpointEdges", + "countPendingWaitpoints", + "countPendingWaitpointsWithPresence", + "createWaitpoint", + "upsertWaitpoint", + "findWaitpoint", + "findWaitpointOnPrimary", + "findManyWaitpoints", + "updateWaitpoint", + "updateManyWaitpoints", + "forWaitpointCompletion", + "findManyTaskRunWaitpoints", + "deleteManyTaskRunWaitpoints", + "findTaskRunAttempt", + "createTaskRunCheckpoint", + "createBatchTaskRun", + "updateBatchTaskRun", + "findBatchTaskRunById", + "findBatchTaskRunByFriendlyId", + "findBatchTaskRunByIdempotencyKey", + "updateManyBatchTaskRun", + "countBatchTaskRunItems", + "updateManyBatchTaskRunItems", + "findManyBatchTaskRunItems", + "findBatchTaskRunItem", + "upsertWaitpointTag", + "findManyWaitpointTags", +] as const; + +// Data properties the base exposes as getters over the delegate, not as forwarders. +export const RUN_STORE_PROPERTY_NAMES = [ + "primaryReadClient", +] as const; From e8ac9b3ad4e9fdd32f58ae3d099144c6248edb00 Mon Sep 17 00:00:00 2001 From: Daniel Sutton Date: Mon, 24 Aug 2026 14:48:12 +0100 Subject: [PATCH 03/56] feat(run-store): build snapshot entries from write-site inputs, with parity tests No nested write site returns the snapshot it created: createRun returns the run, expireParkedRun returns a count, and the rest return a selected TaskRun. So the Redis entry is built from each site's own input plus the caller-minted id. That means every value Postgres derives rather than receives has to be reproduced: the DEQUEUED-to-PENDING rewrite, the four values lockRunToWorker hard-codes, the three rescheduleRun defaults, and the engine column default a completion leaves unset. The parity suite covers all ten physical write sites, comparing the built entry against the row Postgres actually wrote. It caught the dropped id in createCancelledRun. --- .../src/snapshotEntry.parity.test.ts | 397 ++++++++++++++++++ .../run-store/src/snapshotEntry.test.ts | 185 ++++++++ .../run-store/src/snapshotEntry.ts | 168 ++++++++ 3 files changed, 750 insertions(+) create mode 100644 internal-packages/run-store/src/snapshotEntry.parity.test.ts create mode 100644 internal-packages/run-store/src/snapshotEntry.test.ts create mode 100644 internal-packages/run-store/src/snapshotEntry.ts diff --git a/internal-packages/run-store/src/snapshotEntry.parity.test.ts b/internal-packages/run-store/src/snapshotEntry.parity.test.ts new file mode 100644 index 00000000000..8c176178e0a --- /dev/null +++ b/internal-packages/run-store/src/snapshotEntry.parity.test.ts @@ -0,0 +1,397 @@ +// The entry is built from a write site's input while Postgres builds the row from the same input by +// a different code path. This suite is the only thing that keeps those two paths equal, so it covers +// every one of the ten physical snapshot-create sites in PostgresRunStore. +import { describe, expect } from "vitest"; +import { postgresTest } from "@internal/testcontainers"; +import { generateInternalId } from "@trigger.dev/core/v3/isomorphic"; +import { PostgresRunStore } from "./PostgresRunStore.js"; +import type { SnapshotEntryInput } from "./redisSnapshotStore.js"; +import { + entryFromCompletion, + entryFromCreateExecutionSnapshot, + entryFromCreateRun, + entryFromExpire, + entryFromLock, + entryFromReschedule, +} from "./snapshotEntry.js"; +import { + buildCreateRunData, + seedSnapshotEnvironment, + seedSnapshotWorker, + setupSnapshotIdFixture, + type SnapshotFixtureEnv, +} from "./testFixtures/snapshotIdFixture.js"; + +/** + * Compares only what the entry claims. The Redis model carries no `updatedAt` and no join rows, and + * it holds `createdAt` as an ISO string, so those are checked separately or not at all. + */ +function assertParity(entry: SnapshotEntryInput, row: Record) { + expect(row.id).toBe(entry.id); + expect(row.runId).toBe(entry.runId); + expect(row.engine).toBe(entry.engine); + expect(row.executionStatus).toBe(entry.executionStatus); + expect(row.description).toBe(entry.description); + expect(row.runStatus).toBe(entry.runStatus); + expect(row.environmentId).toBe(entry.environmentId); + expect(row.environmentType).toBe(entry.environmentType); + expect(row.projectId).toBe(entry.projectId); + expect(row.organizationId).toBe(entry.organizationId); + expect(row.attemptNumber ?? undefined).toBe(entry.attemptNumber ?? undefined); + expect(row.previousSnapshotId ?? undefined).toBe(entry.previousSnapshotId ?? undefined); + expect(row.batchId ?? undefined).toBe(entry.batchId ?? undefined); + expect(row.checkpointId ?? undefined).toBe(entry.checkpointId ?? undefined); + expect(row.workerId ?? undefined).toBe(entry.workerId ?? undefined); + expect(row.runnerId ?? undefined).toBe(entry.runnerId ?? undefined); + expect(row.isValid).toBe(entry.error === undefined); + expect((row.createdAt as Date).toISOString()).toBe(entry.createdAt); +} + +function birthSnapshot(id: string, env: SnapshotFixtureEnv) { + return { + id, + engine: "V2" as const, + executionStatus: "RUN_CREATED" as const, + description: "Run was created", + runStatus: "PENDING" as const, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }; +} + +describe("entry to Postgres row parity", () => { + postgresTest("createRun, legacy schema", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + const id = generateInternalId(); + const snapshot = birthSnapshot(id, env); + + await store.createRun({ data: buildCreateRunData(runId, env), snapshot }); + + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); + assertParity(entryFromCreateRun({ id, runId, createdAt: row.createdAt }, snapshot), row); + }); + + postgresTest("createRun with an associated waitpoint, legacy schema", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + const id = generateInternalId(); + const snapshot = birthSnapshot(id, env); + + await store.createRun({ + data: buildCreateRunData(runId, env), + snapshot, + associatedWaitpoint: { + id: generateInternalId(), + friendlyId: `waitpoint_${runId.slice(-12)}`, + type: "RUN", + status: "PENDING", + idempotencyKey: generateInternalId(), + userProvidedIdempotencyKey: false, + projectId: env.projectId, + environmentId: env.id, + }, + }); + + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); + assertParity(entryFromCreateRun({ id, runId, createdAt: row.createdAt }, snapshot), row); + }); + + postgresTest("createRun carries the worker and runner ids", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const env = await seedSnapshotEnvironment(prisma); + const { workerId } = await seedSnapshotWorker(prisma, env); + const runId = generateInternalId(); + const id = generateInternalId(); + const snapshot = { ...birthSnapshot(id, env), workerId, runnerId: "runner_1" }; + + await store.createRun({ data: buildCreateRunData(runId, env), snapshot }); + + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); + assertParity(entryFromCreateRun({ id, runId, createdAt: row.createdAt }, snapshot), row); + }); + + postgresTest("createCancelledRun", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + const id = generateInternalId(); + const snapshot = { + ...birthSnapshot(id, env), + executionStatus: "FINISHED" as const, + description: "Run was cancelled", + runStatus: "CANCELED" as const, + }; + + await store.createCancelledRun({ + data: { + ...buildCreateRunData(runId, env), + status: "CANCELED", + error: { type: "STRING_ERROR", raw: "cancelled" }, + completedAt: new Date(), + updatedAt: new Date(), + attemptNumber: 0, + }, + snapshot, + }); + + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); + assertParity(entryFromCreateRun({ id, runId, createdAt: row.createdAt }, snapshot), row); + }); + + postgresTest("completeAttemptSuccess", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const { run, env } = await setupSnapshotIdFixture(prisma); + const id = generateInternalId(); + const snapshot = { + id, + executionStatus: "FINISHED" as const, + description: "Run completed", + runStatus: "COMPLETED_SUCCESSFULLY" as const, + attemptNumber: 1, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }; + + await store.completeAttemptSuccess( + run.id, + { + completedAt: new Date(), + outputType: "application/json", + usageDurationMs: 1, + costInCents: 0, + snapshot, + }, + { select: { id: true } } + ); + + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); + assertParity(entryFromCompletion({ id, runId: run.id, createdAt: row.createdAt }, snapshot), row); + }); + + postgresTest("expireRun", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const { run, env } = await setupSnapshotIdFixture(prisma); + const id = generateInternalId(); + const snapshot = { + id, + engine: "V2" as const, + executionStatus: "FINISHED" as const, + description: "Run expired", + runStatus: "EXPIRED" as const, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }; + + await store.expireRun( + run.id, + { + error: { type: "STRING_ERROR", raw: "expired" }, + completedAt: new Date(), + expiredAt: new Date(), + snapshot, + }, + { select: { id: true } } + ); + + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); + assertParity(entryFromExpire({ id, runId: run.id, createdAt: row.createdAt }, snapshot), row); + }); + + postgresTest("expireParkedRun", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const { run, env } = await setupSnapshotIdFixture(prisma, { status: "PENDING_VERSION" }); + const id = generateInternalId(); + const snapshot = { + id, + engine: "V2" as const, + executionStatus: "FINISHED" as const, + description: "Parked run expired", + runStatus: "EXPIRED" as const, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }; + + const result = await store.expireParkedRun(run.id, { + error: { type: "STRING_ERROR", raw: "expired" }, + completedAt: new Date(), + expiredAt: new Date(), + statusReason: "VERSION_NEVER_ARRIVED", + snapshot, + }); + + expect(result.count).toBe(1); + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); + assertParity(entryFromExpire({ id, runId: run.id, createdAt: row.createdAt }, snapshot), row); + }); + + postgresTest("rescheduleRun with every default applied", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const { run, env } = await setupSnapshotIdFixture(prisma, { status: "DELAYED" }); + const id = generateInternalId(); + const snapshot = { + id, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }; + + await store.rescheduleRun(run.id, { + delayUntil: new Date(Date.now() + 60_000), + snapshot, + }); + + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); + assertParity(entryFromReschedule({ id, runId: run.id, createdAt: row.createdAt }, snapshot), row); + }); + + postgresTest("rescheduleRun with every value supplied", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const { run, env } = await setupSnapshotIdFixture(prisma, { status: "DELAYED" }); + const id = generateInternalId(); + const snapshot = { + id, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + executionStatus: "QUEUED" as const, + runStatus: "PENDING" as const, + description: "custom reschedule", + }; + + await store.rescheduleRun(run.id, { + delayUntil: new Date(Date.now() + 60_000), + snapshot, + }); + + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); + assertParity(entryFromReschedule({ id, runId: run.id, createdAt: row.createdAt }, snapshot), row); + }); + + postgresTest("lockRunToWorker", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const { run, env } = await setupSnapshotIdFixture(prisma); + const { workerId, taskId } = await seedSnapshotWorker(prisma, env); + const previous = await store.createExecutionSnapshot({ + run: { id: run.id, status: "PENDING", attemptNumber: null }, + snapshot: { executionStatus: "QUEUED", description: "Run was queued" }, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }); + + const id = generateInternalId(); + const snapshot = { + id, + previousSnapshotId: previous.id, + attemptNumber: 1, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + completedWaitpointIds: [], + completedWaitpointOrder: [], + }; + + await store.lockRunToWorker(run.id, { + lockedAt: new Date(), + lockedById: taskId, + lockedToVersionId: workerId, + lockedQueueId: undefined, + startedAt: new Date(), + baseCostInCents: 0, + machinePreset: "small-1x", + taskVersion: "1.0.0", + snapshot, + }); + + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); + assertParity(entryFromLock({ id, runId: run.id, createdAt: row.createdAt }, snapshot), row); + }); + + postgresTest("createExecutionSnapshot, the standalone site", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const { run, env } = await setupSnapshotIdFixture(prisma); + const id = generateInternalId(); + const input = { + id, + run: { id: run.id, status: "EXECUTING" as const, attemptNumber: 2 }, + snapshot: { executionStatus: "EXECUTING" as const, description: "Run started" }, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }; + + const created = await store.createExecutionSnapshot(input); + + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); + expect(created.id).toBe(id); + assertParity( + entryFromCreateExecutionSnapshot({ id, runId: run.id, createdAt: row.createdAt }, input), + row + ); + }); + + postgresTest("createExecutionSnapshot rewrites a DEQUEUED run status", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const { run, env } = await setupSnapshotIdFixture(prisma); + const id = generateInternalId(); + const input = { + id, + run: { id: run.id, status: "DEQUEUED" as const, attemptNumber: 1 }, + snapshot: { executionStatus: "PENDING_EXECUTING" as const, description: "Run was dequeued" }, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }; + + await store.createExecutionSnapshot(input); + + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); + expect(row.runStatus).toBe("PENDING"); + assertParity( + entryFromCreateExecutionSnapshot({ id, runId: run.id, createdAt: row.createdAt }, input), + row + ); + }); + + postgresTest("createExecutionSnapshot with an error is invalid in both", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const { run, env } = await setupSnapshotIdFixture(prisma); + const id = generateInternalId(); + const input = { + id, + run: { id: run.id, status: "EXECUTING" as const, attemptNumber: 1 }, + snapshot: { executionStatus: "EXECUTING" as const, description: "Stale write" }, + error: "snapshot is not the latest", + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }; + + await store.createExecutionSnapshot(input); + + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); + expect(row.isValid).toBe(false); + assertParity( + entryFromCreateExecutionSnapshot({ id, runId: run.id, createdAt: row.createdAt }, input), + row + ); + }); +}); diff --git a/internal-packages/run-store/src/snapshotEntry.test.ts b/internal-packages/run-store/src/snapshotEntry.test.ts new file mode 100644 index 00000000000..cc624d690a9 --- /dev/null +++ b/internal-packages/run-store/src/snapshotEntry.test.ts @@ -0,0 +1,185 @@ +// These mappings are values Postgres derives rather than receives. If either side changes and the +// other does not, dual-write silently stores two different documents for one snapshot. The parity +// suite next to this file checks the same thing against a real Postgres row; this one pins the +// rules on their own, so a failure says which rule broke. +import { describe, expect, it } from "vitest"; +import { + entryFromCompletion, + entryFromCreateExecutionSnapshot, + entryFromCreateRun, + entryFromExpire, + entryFromLock, + entryFromReschedule, + isTerminalEntry, +} from "./snapshotEntry.js"; + +const ctx = { id: "snap_1", runId: "run_1", createdAt: new Date("2026-08-24T00:00:00.000Z") }; +const scope = { + environmentId: "env_1", + environmentType: "DEVELOPMENT" as const, + projectId: "proj_1", + organizationId: "org_1", +}; + +describe("snapshotEntry derived values", () => { + it("rewrites a DEQUEUED run status to PENDING", () => { + const entry = entryFromCreateExecutionSnapshot(ctx, { + run: { id: "run_1", status: "DEQUEUED", attemptNumber: 1 }, + snapshot: { executionStatus: "PENDING_EXECUTING", description: "d" }, + ...scope, + }); + + expect(entry.runStatus).toBe("PENDING"); + }); + + it("keeps every other run status unchanged", () => { + const entry = entryFromCreateExecutionSnapshot(ctx, { + run: { id: "run_1", status: "EXECUTING", attemptNumber: 1 }, + snapshot: { executionStatus: "EXECUTING", description: "d" }, + ...scope, + }); + + expect(entry.runStatus).toBe("EXECUTING"); + }); + + it("applies the lock site's hard-coded values", () => { + const entry = entryFromLock(ctx, { + id: "snap_1", + previousSnapshotId: "snap_0", + attemptNumber: 2, + completedWaitpointIds: [], + completedWaitpointOrder: [], + ...scope, + }); + + expect(entry.executionStatus).toBe("PENDING_EXECUTING"); + expect(entry.description).toBe("Run was dequeued for execution"); + expect(entry.runStatus).toBe("PENDING"); + expect(entry.engine).toBe("V2"); + expect(entry.previousSnapshotId).toBe("snap_0"); + expect(entry.attemptNumber).toBe(2); + }); + + it("applies the reschedule defaults", () => { + const entry = entryFromReschedule(ctx, { ...scope }); + + expect(entry.executionStatus).toBe("DELAYED"); + expect(entry.runStatus).toBe("DELAYED"); + expect(entry.description).toBe("Delayed run was rescheduled to a future date"); + }); + + it("prefers a supplied reschedule value over the default", () => { + const entry = entryFromReschedule(ctx, { + ...scope, + executionStatus: "QUEUED", + runStatus: "PENDING", + description: "custom", + }); + + expect(entry.executionStatus).toBe("QUEUED"); + expect(entry.runStatus).toBe("PENDING"); + expect(entry.description).toBe("custom"); + }); + + it("sets engine V2 on a completion, which Postgres leaves to the column default", () => { + const entry = entryFromCompletion(ctx, { + executionStatus: "FINISHED", + description: "Run completed", + runStatus: "COMPLETED_SUCCESSFULLY", + attemptNumber: 1, + ...scope, + }); + + expect(entry.engine).toBe("V2"); + }); + + it("carries a null completion attemptNumber through as null", () => { + const entry = entryFromCompletion(ctx, { + executionStatus: "FINISHED", + description: "Run completed", + runStatus: "COMPLETED_SUCCESSFULLY", + attemptNumber: null, + ...scope, + }); + + expect(entry.attemptNumber).toBeNull(); + }); + + it("omits an absent optional rather than writing undefined into the document", () => { + const entry = entryFromExpire(ctx, { + engine: "V2", + executionStatus: "FINISHED", + description: "Run expired", + runStatus: "EXPIRED", + ...scope, + }); + + expect(Object.keys(entry)).not.toContain("workerId"); + expect(Object.keys(entry)).not.toContain("attemptNumber"); + expect(JSON.parse(JSON.stringify(entry))).toEqual(entry); + }); + + it("reports a FINISHED entry as terminal and any other as not", () => { + const finished = entryFromCompletion(ctx, { + executionStatus: "FINISHED", + description: "Run completed", + runStatus: "COMPLETED_SUCCESSFULLY", + attemptNumber: 1, + ...scope, + }); + const running = entryFromCreateExecutionSnapshot(ctx, { + run: { id: "run_1", status: "EXECUTING", attemptNumber: 1 }, + snapshot: { executionStatus: "EXECUTING", description: "d" }, + ...scope, + }); + + expect(isTerminalEntry(finished)).toBe(true); + expect(isTerminalEntry(running)).toBe(false); + }); + + it("serialises createdAt as an ISO string", () => { + const entry = entryFromReschedule(ctx, { ...scope }); + + expect(entry.createdAt).toBe("2026-08-24T00:00:00.000Z"); + }); + + it("carries the birth site's worker and runner ids", () => { + const entry = entryFromCreateRun(ctx, { + engine: "V2", + executionStatus: "RUN_CREATED", + description: "Run was created", + runStatus: "PENDING", + workerId: "worker_1", + runnerId: "runner_1", + ...scope, + }); + + expect(entry.workerId).toBe("worker_1"); + expect(entry.runnerId).toBe("runner_1"); + expect(entry.executionStatus).toBe("RUN_CREATED"); + }); + + it("never sets the reserved completedWaitpoints field", () => { + const built = [ + entryFromReschedule(ctx, { ...scope }), + entryFromLock(ctx, { + id: "snap_1", + previousSnapshotId: "snap_0", + completedWaitpointIds: ["w_1"], + completedWaitpointOrder: ["w_1"], + ...scope, + }), + entryFromCreateExecutionSnapshot(ctx, { + run: { id: "run_1", status: "EXECUTING", attemptNumber: 1 }, + snapshot: { executionStatus: "EXECUTING", description: "d" }, + completedWaitpoints: [{ id: "w_1", index: 0 }], + ...scope, + }), + ]; + + // The append script mints the pointer as a sidecar field, and rejects an entry that carries one. + for (const entry of built) { + expect(entry.completedWaitpoints).toBeUndefined(); + } + }); +}); diff --git a/internal-packages/run-store/src/snapshotEntry.ts b/internal-packages/run-store/src/snapshotEntry.ts new file mode 100644 index 00000000000..748a95f6d1a --- /dev/null +++ b/internal-packages/run-store/src/snapshotEntry.ts @@ -0,0 +1,168 @@ +// Builds the Redis entry for each execution-snapshot write site, from that site's own INPUT. +// +// Not from the delegate's return value: no nested write site includes the snapshot in what it +// returns. `createRun` returns the run, `expireParkedRun` returns a count, and the rest return a +// selected `TaskRun`. That means every value Postgres derives rather than receives has to be +// reproduced here, and snapshotEntry.parity.test.ts is what keeps the two sides from drifting. +import type { TaskRunStatus } from "@trigger.dev/database"; +import type { SnapshotEntryInput } from "./redisSnapshotStore.js"; +import type { + CompletionSnapshotInput, + CreateExecutionSnapshotInput, + CreateRunSnapshotInput, + ExpireSnapshotInput, + LockSnapshotInput, + RescheduleSnapshotInput, +} from "./types.js"; + +export type EntryBuildContext = { id: string; runId: string; createdAt: Date }; + +/** + * PostgresRunStore.#createExecutionSnapshot rewrites DEQUEUED to PENDING, because older runners + * reject DEQUEUED on a snapshot. Every site that can carry that status must rewrite it identically. + */ +function snapshotRunStatus(status: TaskRunStatus): string { + return status === "DEQUEUED" ? "PENDING" : status; +} + +function base(ctx: EntryBuildContext) { + return { + id: ctx.id, + runId: ctx.runId, + createdAt: ctx.createdAt.toISOString(), + engine: "V2" as const, + }; +} + +export function entryFromCreateRun( + ctx: EntryBuildContext, + snapshot: CreateRunSnapshotInput +): SnapshotEntryInput { + return { + ...base(ctx), + executionStatus: snapshot.executionStatus, + description: snapshot.description, + runStatus: snapshotRunStatus(snapshot.runStatus), + environmentId: snapshot.environmentId, + environmentType: snapshot.environmentType, + projectId: snapshot.projectId, + organizationId: snapshot.organizationId, + ...(snapshot.workerId !== undefined && { workerId: snapshot.workerId }), + ...(snapshot.runnerId !== undefined && { runnerId: snapshot.runnerId }), + }; +} + +/** + * `completeAttemptSuccess` writes no `engine` column, so Postgres applies the schema default of + * `V2`. The entry states it, because SnapshotEntryInput requires the field. + */ +export function entryFromCompletion( + ctx: EntryBuildContext, + snapshot: CompletionSnapshotInput +): SnapshotEntryInput { + return { + ...base(ctx), + executionStatus: snapshot.executionStatus, + description: snapshot.description, + runStatus: snapshotRunStatus(snapshot.runStatus), + attemptNumber: snapshot.attemptNumber, + environmentId: snapshot.environmentId, + environmentType: snapshot.environmentType, + projectId: snapshot.projectId, + organizationId: snapshot.organizationId, + ...(snapshot.workerId !== undefined && { workerId: snapshot.workerId }), + ...(snapshot.runnerId !== undefined && { runnerId: snapshot.runnerId }), + }; +} + +/** Serves both `expireRun` and `expireParkedRun`; the two write identical snapshot columns. */ +export function entryFromExpire( + ctx: EntryBuildContext, + snapshot: ExpireSnapshotInput +): SnapshotEntryInput { + return { + ...base(ctx), + executionStatus: snapshot.executionStatus, + description: snapshot.description, + runStatus: snapshotRunStatus(snapshot.runStatus), + environmentId: snapshot.environmentId, + environmentType: snapshot.environmentType, + projectId: snapshot.projectId, + organizationId: snapshot.organizationId, + }; +} + +/** PostgresRunStore.rescheduleRun supplies these three defaults inline, so the entry repeats them. */ +export function entryFromReschedule( + ctx: EntryBuildContext, + snapshot: RescheduleSnapshotInput +): SnapshotEntryInput { + return { + ...base(ctx), + executionStatus: snapshot.executionStatus ?? "DELAYED", + description: snapshot.description ?? "Delayed run was rescheduled to a future date", + runStatus: snapshotRunStatus(snapshot.runStatus ?? "DELAYED"), + environmentId: snapshot.environmentId, + environmentType: snapshot.environmentType, + projectId: snapshot.projectId, + organizationId: snapshot.organizationId, + }; +} + +/** PostgresRunStore.#lockRunToWorker hard-codes the status, description and run status. */ +export function entryFromLock( + ctx: EntryBuildContext, + snapshot: LockSnapshotInput +): SnapshotEntryInput { + return { + ...base(ctx), + executionStatus: "PENDING_EXECUTING", + description: "Run was dequeued for execution", + runStatus: "PENDING", + previousSnapshotId: snapshot.previousSnapshotId, + ...(snapshot.attemptNumber !== undefined && { attemptNumber: snapshot.attemptNumber }), + ...(snapshot.batchId !== undefined && { batchId: snapshot.batchId }), + ...(snapshot.checkpointId !== undefined && { checkpointId: snapshot.checkpointId }), + environmentId: snapshot.environmentId, + environmentType: snapshot.environmentType, + projectId: snapshot.projectId, + organizationId: snapshot.organizationId, + ...(snapshot.workerId !== undefined && { workerId: snapshot.workerId }), + ...(snapshot.runnerId !== undefined && { runnerId: snapshot.runnerId }), + }; +} + +export function entryFromCreateExecutionSnapshot( + ctx: EntryBuildContext, + input: CreateExecutionSnapshotInput +): SnapshotEntryInput { + return { + ...base(ctx), + executionStatus: input.snapshot.executionStatus, + description: input.snapshot.description, + runStatus: snapshotRunStatus(input.run.status), + ...(input.run.attemptNumber !== undefined && + input.run.attemptNumber !== null && { attemptNumber: input.run.attemptNumber }), + ...(input.previousSnapshotId !== undefined && { previousSnapshotId: input.previousSnapshotId }), + ...(input.batchId !== undefined && { batchId: input.batchId }), + environmentId: input.environmentId, + environmentType: input.environmentType, + projectId: input.projectId, + organizationId: input.organizationId, + ...(input.checkpointId !== undefined && { checkpointId: input.checkpointId }), + ...(input.workerId !== undefined && { workerId: input.workerId }), + ...(input.runnerId !== undefined && { runnerId: input.runnerId }), + ...(input.snapshot.metadata !== undefined && + input.snapshot.metadata !== null && { metadata: input.snapshot.metadata }), + ...(input.error !== undefined && { error: input.error }), + }; +} + +/** + * A terminal entry is what makes the append script apply the completion TTL. FINISHED is the only + * terminal execution status; the run-level status is not consulted, because a run reaches its + * terminal state through a FINISHED snapshot in every path. + */ +export function isTerminalEntry(entry: SnapshotEntryInput): boolean { + return entry.executionStatus === "FINISHED"; +} From 95dd2a6676f98eb4bf5ffff0d7b6a352b993503f Mon Sep 17 00:00:00 2001 From: Daniel Sutton Date: Mon, 24 Aug 2026 14:52:25 +0100 Subject: [PATCH 04/56] feat(run-store): add a snapshotWrites flag that omits every snapshot write The last dial position makes the Redis store the sole snapshot writer, so Postgres has to stop writing snapshot rows without changing anything else it does. One constructor flag does that across all ten write sites. With it off, the nine nested creates are omitted and the run mutation still lands; createExecutionSnapshot echoes its input in the shape callers expect rather than inserting; and the completed-waitpoint join inserts are skipped, since they would otherwise link to a row that no longer exists. Defaults to true, so every existing caller and test is unaffected. --- .../PostgresRunStore.snapshotWrites.test.ts | 308 ++++++++++++++++++ .../run-store/src/PostgresRunStore.ts | 279 +++++++++------- 2 files changed, 473 insertions(+), 114 deletions(-) create mode 100644 internal-packages/run-store/src/PostgresRunStore.snapshotWrites.test.ts diff --git a/internal-packages/run-store/src/PostgresRunStore.snapshotWrites.test.ts b/internal-packages/run-store/src/PostgresRunStore.snapshotWrites.test.ts new file mode 100644 index 00000000000..81d3d3e658e --- /dev/null +++ b/internal-packages/run-store/src/PostgresRunStore.snapshotWrites.test.ts @@ -0,0 +1,308 @@ +// snapshotWrites: false is the redis-only dial position. Every run mutation still lands; no snapshot +// row is written and no completed-waitpoint join row is inserted. The default stays true, so nothing +// changes for any existing caller. +import { describe, expect } from "vitest"; +import { postgresTest } from "@internal/testcontainers"; +import { generateInternalId } from "@trigger.dev/core/v3/isomorphic"; +import { PostgresRunStore } from "./PostgresRunStore.js"; +import { + buildCreateRunData, + seedSnapshotEnvironment, + seedSnapshotWorker, + setupSnapshotIdFixture, +} from "./testFixtures/snapshotIdFixture.js"; + +describe("PostgresRunStore snapshotWrites flag", () => { + postgresTest("defaults to writing snapshots", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const { run, env } = await setupSnapshotIdFixture(prisma); + + await store.completeAttemptSuccess( + run.id, + { + completedAt: new Date(), + outputType: "application/json", + usageDurationMs: 1, + costInCents: 0, + snapshot: { + executionStatus: "FINISHED", + description: "Run completed", + runStatus: "COMPLETED_SUCCESSFULLY", + attemptNumber: 1, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }, + }, + { select: { id: true } } + ); + + expect(await prisma.taskRunExecutionSnapshot.count({ where: { runId: run.id } })).toBe(1); + }); + + postgresTest("writes the run mutation but no snapshot when off", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma, snapshotWrites: false }); + const { run, env } = await setupSnapshotIdFixture(prisma); + + await store.completeAttemptSuccess( + run.id, + { + completedAt: new Date(), + outputType: "application/json", + usageDurationMs: 1, + costInCents: 0, + snapshot: { + executionStatus: "FINISHED", + description: "Run completed", + runStatus: "COMPLETED_SUCCESSFULLY", + attemptNumber: 1, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }, + }, + { select: { id: true } } + ); + + const updated = await prisma.taskRun.findFirstOrThrow({ where: { id: run.id } }); + expect(updated.status).toBe("COMPLETED_SUCCESSFULLY"); + expect(await prisma.taskRunExecutionSnapshot.count({ where: { runId: run.id } })).toBe(0); + }); + + postgresTest("createRun writes the run but no snapshot when off", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma, snapshotWrites: false }); + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + + await store.createRun({ + data: buildCreateRunData(runId, env), + snapshot: { + id: generateInternalId(), + engine: "V2", + executionStatus: "RUN_CREATED", + description: "Run was created", + runStatus: "PENDING", + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }, + }); + + expect(await prisma.taskRun.count({ where: { id: runId } })).toBe(1); + expect(await prisma.taskRunExecutionSnapshot.count({ where: { runId } })).toBe(0); + }); + + postgresTest("createCancelledRun writes the run but no snapshot when off", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma, snapshotWrites: false }); + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + + await store.createCancelledRun({ + data: { + ...buildCreateRunData(runId, env), + status: "CANCELED", + error: { type: "STRING_ERROR", raw: "cancelled" }, + completedAt: new Date(), + updatedAt: new Date(), + attemptNumber: 0, + }, + snapshot: { + id: generateInternalId(), + engine: "V2", + executionStatus: "FINISHED", + description: "Run was cancelled", + runStatus: "CANCELED", + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }, + }); + + expect(await prisma.taskRun.count({ where: { id: runId } })).toBe(1); + expect(await prisma.taskRunExecutionSnapshot.count({ where: { runId } })).toBe(0); + }); + + postgresTest("expireRun writes the run but no snapshot when off", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma, snapshotWrites: false }); + const { run, env } = await setupSnapshotIdFixture(prisma); + + await store.expireRun( + run.id, + { + error: { type: "STRING_ERROR", raw: "expired" }, + completedAt: new Date(), + expiredAt: new Date(), + snapshot: { + engine: "V2", + executionStatus: "FINISHED", + description: "Run expired", + runStatus: "EXPIRED", + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }, + }, + { select: { id: true } } + ); + + expect((await prisma.taskRun.findFirstOrThrow({ where: { id: run.id } })).status).toBe("EXPIRED"); + expect(await prisma.taskRunExecutionSnapshot.count({ where: { runId: run.id } })).toBe(0); + }); + + postgresTest("expireParkedRun writes the run but no snapshot when off", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma, snapshotWrites: false }); + const { run, env } = await setupSnapshotIdFixture(prisma, { status: "PENDING_VERSION" }); + + const result = await store.expireParkedRun(run.id, { + error: { type: "STRING_ERROR", raw: "expired" }, + completedAt: new Date(), + expiredAt: new Date(), + statusReason: "VERSION_NEVER_ARRIVED", + snapshot: { + engine: "V2", + executionStatus: "FINISHED", + description: "Parked run expired", + runStatus: "EXPIRED", + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }, + }); + + expect(result.count).toBe(1); + expect(await prisma.taskRunExecutionSnapshot.count({ where: { runId: run.id } })).toBe(0); + }); + + postgresTest("rescheduleRun writes the run but no snapshot when off", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma, snapshotWrites: false }); + const { run, env } = await setupSnapshotIdFixture(prisma, { status: "DELAYED" }); + const delayUntil = new Date(Date.now() + 60_000); + + await store.rescheduleRun(run.id, { + delayUntil, + snapshot: { + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }, + }); + + const updated = await prisma.taskRun.findFirstOrThrow({ where: { id: run.id } }); + expect(updated.delayUntil?.toISOString()).toBe(delayUntil.toISOString()); + expect(await prisma.taskRunExecutionSnapshot.count({ where: { runId: run.id } })).toBe(0); + }); + + postgresTest("lockRunToWorker writes the lock but no snapshot when off", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma, snapshotWrites: false }); + const { run, env } = await setupSnapshotIdFixture(prisma); + const { workerId, taskId } = await seedSnapshotWorker(prisma, env); + + await store.lockRunToWorker(run.id, { + lockedAt: new Date(), + lockedById: taskId, + lockedToVersionId: workerId, + lockedQueueId: undefined, + startedAt: new Date(), + baseCostInCents: 0, + machinePreset: "small-1x", + taskVersion: "1.0.0", + snapshot: { + id: generateInternalId(), + previousSnapshotId: generateInternalId(), + attemptNumber: 1, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + completedWaitpointIds: [], + completedWaitpointOrder: [], + }, + }); + + expect((await prisma.taskRun.findFirstOrThrow({ where: { id: run.id } })).status).toBe("DEQUEUED"); + expect(await prisma.taskRunExecutionSnapshot.count({ where: { runId: run.id } })).toBe(0); + }); + + postgresTest("createExecutionSnapshot echoes the input when off", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma, snapshotWrites: false }); + const { run, env } = await setupSnapshotIdFixture(prisma); + const id = generateInternalId(); + + const echoed = await store.createExecutionSnapshot({ + id, + run: { id: run.id, status: "EXECUTING", attemptNumber: 2 }, + snapshot: { executionStatus: "EXECUTING", description: "Run started" }, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }); + + expect(echoed.id).toBe(id); + expect(echoed.runId).toBe(run.id); + expect(echoed.executionStatus).toBe("EXECUTING"); + expect(echoed.attemptNumber).toBe(2); + expect(echoed.isValid).toBe(true); + expect(echoed.checkpoint).toBeNull(); + expect(await prisma.taskRunExecutionSnapshot.count({ where: { runId: run.id } })).toBe(0); + }); + + postgresTest("the echoed row rewrites a DEQUEUED run status", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma, snapshotWrites: false }); + const { run, env } = await setupSnapshotIdFixture(prisma); + + const echoed = await store.createExecutionSnapshot({ + id: generateInternalId(), + run: { id: run.id, status: "DEQUEUED", attemptNumber: 1 }, + snapshot: { executionStatus: "PENDING_EXECUTING", description: "Run was dequeued" }, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }); + + expect(echoed.runStatus).toBe("PENDING"); + }); + + postgresTest("the echoed row reports an errored snapshot as invalid", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma, snapshotWrites: false }); + const { run, env } = await setupSnapshotIdFixture(prisma); + + const echoed = await store.createExecutionSnapshot({ + id: generateInternalId(), + run: { id: run.id, status: "EXECUTING", attemptNumber: 1 }, + snapshot: { executionStatus: "EXECUTING", description: "Stale write" }, + error: "snapshot is not the latest", + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }); + + expect(echoed.isValid).toBe(false); + expect(echoed.error).toBe("snapshot is not the latest"); + }); + + postgresTest("createExecutionSnapshot needs an id when off", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma, snapshotWrites: false }); + const { run, env } = await setupSnapshotIdFixture(prisma); + + await expect( + store.createExecutionSnapshot({ + run: { id: run.id, status: "EXECUTING", attemptNumber: 1 }, + snapshot: { executionStatus: "EXECUTING", description: "Run started" }, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }) + ).rejects.toThrow(/snapshotWrites is off/); + }); +}); diff --git a/internal-packages/run-store/src/PostgresRunStore.ts b/internal-packages/run-store/src/PostgresRunStore.ts index e860bd6c67a..c88c7464508 100644 --- a/internal-packages/run-store/src/PostgresRunStore.ts +++ b/internal-packages/run-store/src/PostgresRunStore.ts @@ -118,6 +118,13 @@ export type PostgresRunStoreOptions = { maxWait?: number; /** Env-driven P2028-at-acquisition retry config, threaded from the app boundary (IoC). */ transactionStartRetry?: TransactionStartRetryConfig; + /** + * When false the store writes no execution-snapshot rows: every nested `executionSnapshots.create` + * is omitted and `createExecutionSnapshot` echoes its input instead of inserting. Only the + * redis-only dial position sets this, once the Redis store is the sole snapshot writer. + * Defaults to true, so the store behaves exactly as it always has. + */ + snapshotWrites?: boolean; }; // A caller sub-select for a relation: `{ select?, include? }` or `true` for a bare `key: true`. @@ -638,6 +645,7 @@ export class PostgresRunStore implements RunStore { private readonly prisma: RunOpsCapableClient; private readonly readOnlyPrisma: RunOpsCapableClient; private readonly schemaVariant: RunStoreSchemaVariant; + private readonly snapshotWrites: boolean; private readonly maxWait?: number; private readonly transactionStartRetry?: TransactionStartRetryConfig; @@ -650,6 +658,16 @@ export class PostgresRunStore implements RunStore { this.schemaVariant = options.schemaVariant ?? "legacy"; this.maxWait = options.maxWait; this.transactionStartRetry = options.transactionStartRetry; + this.snapshotWrites = options.snapshotWrites ?? true; + } + + /** + * Wraps a nested snapshot create so a single flag removes it everywhere. Prisma treats an absent + * key and `undefined` alike, so spreading an empty object drops the nested write entirely rather + * than sending an empty one. + */ + #nestedSnapshot(create: T): { executionSnapshots: { create: T } } | Record { + return this.snapshotWrites ? { executionSnapshots: { create } } : {}; } // The writer handle in read-client form, so the routing layer can honor a caller-passed client @@ -743,7 +761,7 @@ export class PostgresRunStore implements RunStore { const run = (await this.#writeClientWithoutTransaction(tx).taskRun.create({ data: { ...params.data, - executionSnapshots: { create: snapshotCreate }, + ...this.#nestedSnapshot(snapshotCreate), }, })) as TaskRun; return { ...run, associatedWaitpoint: null }; @@ -755,7 +773,7 @@ export class PostgresRunStore implements RunStore { const run = (await c.taskRun.create({ data: { ...params.data, - executionSnapshots: { create: snapshotCreate }, + ...this.#nestedSnapshot(snapshotCreate), }, })) as TaskRun; @@ -772,9 +790,7 @@ export class PostgresRunStore implements RunStore { }, data: { ...params.data, - executionSnapshots: { - create: snapshotCreate, - }, + ...this.#nestedSnapshot(snapshotCreate), associatedWaitpoint: params.associatedWaitpoint ? { create: params.associatedWaitpoint, @@ -813,24 +829,24 @@ export class PostgresRunStore implements RunStore { ): Promise { const client = tx ?? this.prisma; + const snapshotCreate = { + id: params.snapshot.id, + engine: params.snapshot.engine, + executionStatus: params.snapshot.executionStatus, + description: params.snapshot.description, + runStatus: params.snapshot.runStatus, + environmentId: params.snapshot.environmentId, + environmentType: params.snapshot.environmentType, + projectId: params.snapshot.projectId, + organizationId: params.snapshot.organizationId, + workerId: params.snapshot.workerId, + runnerId: params.snapshot.runnerId, + }; + return client.taskRun.create({ data: { ...params.data, - executionSnapshots: { - create: { - id: params.snapshot.id, - engine: params.snapshot.engine, - executionStatus: params.snapshot.executionStatus, - description: params.snapshot.description, - runStatus: params.snapshot.runStatus, - environmentId: params.snapshot.environmentId, - environmentType: params.snapshot.environmentType, - projectId: params.snapshot.projectId, - organizationId: params.snapshot.organizationId, - workerId: params.snapshot.workerId, - runnerId: params.snapshot.runnerId, - }, - }, + ...this.#nestedSnapshot(snapshotCreate), }, }); } @@ -924,21 +940,19 @@ export class PostgresRunStore implements RunStore { outputType: data.outputType, usageDurationMs: data.usageDurationMs, costInCents: data.costInCents, - executionSnapshots: { - create: { - id: data.snapshot.id, - executionStatus: data.snapshot.executionStatus, - description: data.snapshot.description, - runStatus: data.snapshot.runStatus, - attemptNumber: data.snapshot.attemptNumber, - environmentId: data.snapshot.environmentId, - environmentType: data.snapshot.environmentType, - projectId: data.snapshot.projectId, - organizationId: data.snapshot.organizationId, - workerId: data.snapshot.workerId, - runnerId: data.snapshot.runnerId, - }, - }, + ...this.#nestedSnapshot({ + id: data.snapshot.id, + executionStatus: data.snapshot.executionStatus, + description: data.snapshot.description, + runStatus: data.snapshot.runStatus, + attemptNumber: data.snapshot.attemptNumber, + environmentId: data.snapshot.environmentId, + environmentType: data.snapshot.environmentType, + projectId: data.snapshot.projectId, + organizationId: data.snapshot.organizationId, + workerId: data.snapshot.workerId, + runnerId: data.snapshot.runnerId, + }), }, { select: args.select } ) as Promise>; @@ -1131,19 +1145,17 @@ export class PostgresRunStore implements RunStore { completedAt: data.completedAt, expiredAt: data.expiredAt, error: data.error as Prisma.InputJsonValue, - executionSnapshots: { - create: { - id: data.snapshot.id, - engine: data.snapshot.engine, - executionStatus: data.snapshot.executionStatus, - description: data.snapshot.description, - runStatus: data.snapshot.runStatus, - environmentId: data.snapshot.environmentId, - environmentType: data.snapshot.environmentType, - projectId: data.snapshot.projectId, - organizationId: data.snapshot.organizationId, - }, - }, + ...this.#nestedSnapshot({ + id: data.snapshot.id, + engine: data.snapshot.engine, + executionStatus: data.snapshot.executionStatus, + description: data.snapshot.description, + runStatus: data.snapshot.runStatus, + environmentId: data.snapshot.environmentId, + environmentType: data.snapshot.environmentType, + projectId: data.snapshot.projectId, + organizationId: data.snapshot.organizationId, + }), }, { select: args.select } ) as Promise>; @@ -1263,42 +1275,44 @@ export class PostgresRunStore implements RunStore { cliVersion: data.cliVersion ?? undefined, maxDurationInSeconds: data.maxDurationInSeconds ?? undefined, maxAttempts: data.maxAttempts ?? undefined, - executionSnapshots: { - create: { - id: data.snapshot.id, - engine: "V2", - executionStatus: "PENDING_EXECUTING", - description: "Run was dequeued for execution", - runStatus: "PENDING", - attemptNumber: data.snapshot.attemptNumber ?? undefined, - previousSnapshotId: data.snapshot.previousSnapshotId, - environmentId: data.snapshot.environmentId, - environmentType: data.snapshot.environmentType, - projectId: data.snapshot.projectId, - organizationId: data.snapshot.organizationId, - checkpointId: data.snapshot.checkpointId ?? undefined, - batchId: data.snapshot.batchId ?? undefined, - // Completed-waitpoint links are inserted FK-free after create (below) for BOTH schemas. - completedWaitpointOrder: data.snapshot.completedWaitpointOrder, - workerId: data.snapshot.workerId ?? undefined, - runnerId: data.snapshot.runnerId ?? undefined, - }, - }, + ...this.#nestedSnapshot({ + id: data.snapshot.id, + engine: "V2", + executionStatus: "PENDING_EXECUTING", + description: "Run was dequeued for execution", + runStatus: "PENDING", + attemptNumber: data.snapshot.attemptNumber ?? undefined, + previousSnapshotId: data.snapshot.previousSnapshotId, + environmentId: data.snapshot.environmentId, + environmentType: data.snapshot.environmentType, + projectId: data.snapshot.projectId, + organizationId: data.snapshot.organizationId, + checkpointId: data.snapshot.checkpointId ?? undefined, + batchId: data.snapshot.batchId ?? undefined, + // Completed-waitpoint links are inserted FK-free after create (below) for BOTH schemas. + completedWaitpointOrder: data.snapshot.completedWaitpointOrder, + workerId: data.snapshot.workerId ?? undefined, + runnerId: data.snapshot.runnerId ?? undefined, + }), }, }); - if (dedicated) { - await this.#connectCompletedWaitpoints( - prisma, - data.snapshot.id, - data.snapshot.completedWaitpointIds - ); - } else { - await this.#connectCompletedWaitpointsLegacy( - prisma, - data.snapshot.id, - data.snapshot.completedWaitpointIds - ); + // The join rows link to the snapshot row above. With snapshot writes off there is no such row, + // so inserting them would leave dangling links for a snapshot that only the Redis store holds. + if (this.snapshotWrites) { + if (dedicated) { + await this.#connectCompletedWaitpoints( + prisma, + data.snapshot.id, + data.snapshot.completedWaitpointIds + ); + } else { + await this.#connectCompletedWaitpointsLegacy( + prisma, + data.snapshot.id, + data.snapshot.completedWaitpointIds + ); + } } return result; @@ -1366,19 +1380,17 @@ export class PostgresRunStore implements RunStore { completedAt: data.completedAt, expiredAt: data.expiredAt, error: data.error as Prisma.InputJsonValue, - executionSnapshots: { - create: { - id: data.snapshot.id, - engine: data.snapshot.engine, - executionStatus: data.snapshot.executionStatus, - description: data.snapshot.description, - runStatus: data.snapshot.runStatus, - environmentId: data.snapshot.environmentId, - environmentType: data.snapshot.environmentType, - projectId: data.snapshot.projectId, - organizationId: data.snapshot.organizationId, - }, - }, + ...this.#nestedSnapshot({ + id: data.snapshot.id, + engine: data.snapshot.engine, + executionStatus: data.snapshot.executionStatus, + description: data.snapshot.description, + runStatus: data.snapshot.runStatus, + environmentId: data.snapshot.environmentId, + environmentType: data.snapshot.environmentType, + projectId: data.snapshot.projectId, + organizationId: data.snapshot.organizationId, + }), }, }); } catch (error) { @@ -1439,22 +1451,19 @@ export class PostgresRunStore implements RunStore { data: { delayUntil: data.delayUntil, ...(data.queueTimestamp !== undefined && { queueTimestamp: data.queueTimestamp }), - ...(data.snapshot && { - executionSnapshots: { - create: { - id: data.snapshot.id, - engine: "V2", - executionStatus: data.snapshot.executionStatus ?? "DELAYED", - description: - data.snapshot.description ?? "Delayed run was rescheduled to a future date", - runStatus: data.snapshot.runStatus ?? "DELAYED", - environmentId: data.snapshot.environmentId, - environmentType: data.snapshot.environmentType, - projectId: data.snapshot.projectId, - organizationId: data.snapshot.organizationId, - }, - }, - }), + ...(data.snapshot && + this.#nestedSnapshot({ + id: data.snapshot.id, + engine: "V2", + executionStatus: data.snapshot.executionStatus ?? "DELAYED", + description: + data.snapshot.description ?? "Delayed run was rescheduled to a future date", + runStatus: data.snapshot.runStatus ?? "DELAYED", + environmentId: data.snapshot.environmentId, + environmentType: data.snapshot.environmentType, + projectId: data.snapshot.projectId, + organizationId: data.snapshot.organizationId, + })), }, }); } @@ -1990,6 +1999,51 @@ export class PostgresRunStore implements RunStore { error, } = input; + const completedWaitpointOrder = + completedWaitpoints + ?.filter((c) => c.index !== undefined) + .sort((a, b) => a.index! - b.index!) + .map((w) => w.id) ?? []; + + // Redis-only: no row is written and the decorator owns the document. Echo the input in the shape + // the caller expects, so every caller of this method keeps working while Postgres holds nothing. + if (!this.snapshotWrites) { + if (!id) { + throw new Error( + "PostgresRunStore.createExecutionSnapshot: snapshotWrites is off, so the caller must supply the snapshot id" + ); + } + + const now = new Date(); + return { + id, + engine: "V2", + executionStatus: snapshot.executionStatus, + description: snapshot.description, + previousSnapshotId: previousSnapshotId ?? null, + runId: run.id, + runStatus: run.status === "DEQUEUED" ? "PENDING" : run.status, + attemptNumber: run.attemptNumber ?? null, + batchId: batchId ?? null, + environmentId, + environmentType, + projectId, + organizationId, + checkpointId: checkpointId ?? null, + workerId: workerId ?? null, + runnerId: runnerId ?? null, + metadata: snapshot.metadata ?? null, + completedWaitpointOrder, + isValid: !error, + error: error ?? null, + createdAt: now, + updatedAt: now, + checkpoint: null, + } as unknown as Prisma.TaskRunExecutionSnapshotGetPayload<{ + include: { checkpoint: true }; + }>; + } + const dedicated = this.schemaVariant === "dedicated"; const newSnapshot = await prisma.taskRunExecutionSnapshot.create({ @@ -2014,10 +2068,7 @@ export class PostgresRunStore implements RunStore { metadata: snapshot.metadata ?? undefined, // Completed-waitpoint links are inserted FK-free after create (below) for BOTH schemas, so a // cross-DB (NEW-resident) token can be recorded without a Prisma `connect` existence check. - completedWaitpointOrder: completedWaitpoints - ?.filter((c) => c.index !== undefined) - .sort((a, b) => a.index! - b.index!) - .map((w) => w.id), + completedWaitpointOrder, isValid: !error, error, }, From 6cb49c8cedffa9e1f663c291e5b4b33001c985bd Mon Sep 17 00:00:00 2001 From: Daniel Sutton Date: Mon, 24 Aug 2026 15:28:26 +0100 Subject: [PATCH 05/56] feat(run-store): dual-write execution snapshots in a crash-safe order A decorator over any RunStore that also writes execution snapshots to Redis. It overrides only the methods that touch a snapshot and inherits the rest. Write order is the correctness property, and the two orders differ on purpose. A transition writes Postgres first: a crash in the gap leaves a stale latest snapshot, which the heartbeat stall watchdog already heals. A birth writes Redis first: a crash there leaves an unreachable key for a run that does not exist, where Postgres-first would leave a run with no snapshot at all and no way to read one. Each order is chosen so the crash state is the harmless one. A failed transition append retries three times, then hands the run to the repair job. It never rethrows, because Postgres has already committed and a throw would turn a healable gap into a caller-visible error. A failed birth append is survivable before redis-only, where Postgres still holds the snapshot, and refuses at redis-only, where it would otherwise create a run with no snapshot anywhere; refusing works only because the birth append comes first. None of the four non-failure append outcomes enqueues a repair: an absent keyspace is every pre-cutover run's transitions, a fork means another writer advanced the head, a duplicate is a retry that landed, and a cycle mismatch is the store refusing an untrustworthy pointer on purpose. At mode off the decorator makes no Redis call and builds no entry. --- .../run-store/src/snapshotFaultInjection.ts | 39 ++ ...skRunExecutionSnapshotStore.births.test.ts | 285 ++++++++++ .../taskRunExecutionSnapshotStore.off.test.ts | 94 +++ ...ExecutionSnapshotStore.transitions.test.ts | 453 +++++++++++++++ .../src/taskRunExecutionSnapshotStore.ts | 534 ++++++++++++++++++ 5 files changed, 1405 insertions(+) create mode 100644 internal-packages/run-store/src/snapshotFaultInjection.ts create mode 100644 internal-packages/run-store/src/taskRunExecutionSnapshotStore.births.test.ts create mode 100644 internal-packages/run-store/src/taskRunExecutionSnapshotStore.off.test.ts create mode 100644 internal-packages/run-store/src/taskRunExecutionSnapshotStore.transitions.test.ts create mode 100644 internal-packages/run-store/src/taskRunExecutionSnapshotStore.ts diff --git a/internal-packages/run-store/src/snapshotFaultInjection.ts b/internal-packages/run-store/src/snapshotFaultInjection.ts new file mode 100644 index 00000000000..50029b9518d --- /dev/null +++ b/internal-packages/run-store/src/snapshotFaultInjection.ts @@ -0,0 +1,39 @@ +// Test-only seam for the execution-snapshot write protocol. +// +// The protocol's correctness claim is about crashes: whatever the write order leaves behind at each +// boundary must be a state the existing stall-and-repair machinery heals. Proving that needs a crash +// at an exact point, which is what an injector gives. Production never sets one, so each boundary +// costs one optional call. + +/** The three points a crash can land between the two stores' writes. */ +export type SnapshotFaultBoundary = + /** A transition: Postgres has committed and the Redis append has not started. */ + | "afterPgBeforeRedis" + /** A birth: the Redis append has landed and the Postgres insert has not started. */ + | "afterRedisBirthBeforePg" + /** Inside the append retry loop, after at least one attempt has failed. */ + | "midFlushRetry"; + +export type SnapshotFaultInjector = ( + boundary: SnapshotFaultBoundary, + context: { runId: string; snapshotId: string } +) => void; + +/** + * Thrown by a test injector. The write path tells this apart from a real append failure: an injected + * fault models a process that died, so it is rethrown rather than retried, while a real failure is + * retried and then handed to the repair job. + */ +export class InjectedSnapshotFault extends Error { + readonly boundary: SnapshotFaultBoundary; + + constructor(boundary: SnapshotFaultBoundary) { + super(`injected snapshot fault at ${boundary}`); + this.name = "InjectedSnapshotFault"; + this.boundary = boundary; + } +} + +export function isInjectedFault(error: unknown): error is InjectedSnapshotFault { + return error instanceof InjectedSnapshotFault; +} diff --git a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.births.test.ts b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.births.test.ts new file mode 100644 index 00000000000..e94e41f94b6 --- /dev/null +++ b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.births.test.ts @@ -0,0 +1,285 @@ +// A birth writes Redis FIRST. The order is proved by crashing between the two writes and observing +// which side survived: an orphaned key with no run row is the harmless state, and a run with no +// snapshot at all is the one the order exists to prevent. +import { describe, expect } from "vitest"; +import { postgresAndRedisTest } from "@internal/testcontainers"; +import { generateInternalId } from "@trigger.dev/core/v3/isomorphic"; +import { PostgresRunStore } from "./PostgresRunStore.js"; +import { RedisSnapshotStore } from "./redisSnapshotStore.js"; +import { InjectedSnapshotFault } from "./snapshotFaultInjection.js"; +import { + TaskRunExecutionSnapshotStore, + type SnapshotStoreMode, +} from "./taskRunExecutionSnapshotStore.js"; +import type { RunStore } from "./types.js"; +import { + buildCreateRunData, + seedSnapshotEnvironment, + type SnapshotFixtureEnv, +} from "./testFixtures/snapshotIdFixture.js"; + +const COMPLETED_TTL_MS = 72 * 60 * 60 * 1000; + +function build( + prisma: never, + redisOptions: never, + opts?: { + mode?: SnapshotStoreMode; + faults?: ConstructorParameters[1]["faults"]; + unreachableRedis?: boolean; + } +) { + // An unreachable port makes every append throw for real, which is the failure the retry loop and + // the mode-dependent refusal are about. A fault injector cannot stand in: an injected fault means + // "the process died", and the two are handled differently on purpose. + const redis = new RedisSnapshotStore({ + redisOptions: opts?.unreachableRedis + ? ({ ...(redisOptions as object), port: 1, retryStrategy: () => null } as never) + : redisOptions, + completedTtlMs: COMPLETED_TTL_MS, + }); + + const decorated = new TaskRunExecutionSnapshotStore( + new PostgresRunStore({ prisma, readOnlyPrisma: prisma }) as unknown as RunStore, + { + store: redis, + mode: opts?.mode ?? "dual-write", + ...(opts?.faults && { faults: opts.faults }), + } + ); + + return { decorated, redis }; +} + +function birthSnapshot(id: string, env: SnapshotFixtureEnv) { + return { + id, + engine: "V2" as const, + executionStatus: "RUN_CREATED" as const, + description: "Run was created", + runStatus: "PENDING" as const, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }; +} + +function cancelledData(runId: string, env: SnapshotFixtureEnv) { + return { + ...buildCreateRunData(runId, env), + status: "CANCELED" as const, + error: { type: "STRING_ERROR", raw: "cancelled" } as never, + completedAt: new Date(), + updatedAt: new Date(), + attemptNumber: 0 as const, + }; +} + +describe("birth write ordering", () => { + postgresAndRedisTest("writes Redis then Postgres", async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + const snapshotId = generateInternalId(); + + await decorated.createRun({ + data: buildCreateRunData(runId, env), + snapshot: birthSnapshot(snapshotId, env), + }); + + const read = await redis.getLatest(runId); + expect(read).not.toBeNull(); + expect(read!.entry.id).toBe(snapshotId); + expect(read!.entry.executionStatus).toBe("RUN_CREATED"); + expect(await prisma.taskRunExecutionSnapshot.count({ where: { id: snapshotId } })).toBe(1); + } finally { + await redis.quit(); + } + }); + + postgresAndRedisTest("mints an id when the caller supplies none", async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + const { id: _omitted, ...withoutId } = birthSnapshot(generateInternalId(), env); + + await decorated.createRun({ data: buildCreateRunData(runId, env), snapshot: withoutId }); + + const read = await redis.getLatest(runId); + expect(read).not.toBeNull(); + // The same minted id must reach both stores, or the comparator chases a difference that is + // not real. + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { runId } }); + expect(read!.entry.id).toBe(row.id); + } finally { + await redis.quit(); + } + }); + + postgresAndRedisTest( + "a crash after the Redis append leaves an orphan key and no run", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never, { + faults: (boundary) => { + if (boundary === "afterRedisBirthBeforePg") throw new InjectedSnapshotFault(boundary); + }, + }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + + await expect( + decorated.createRun({ + data: buildCreateRunData(runId, env), + snapshot: birthSnapshot(generateInternalId(), env), + }) + ).rejects.toBeInstanceOf(InjectedSnapshotFault); + + // The harmless state: a keyspace nothing can reach, and no run that lacks a snapshot. + expect(await redis.getLatest(runId)).not.toBeNull(); + expect(await prisma.taskRun.count({ where: { id: runId } })).toBe(0); + } finally { + await redis.quit(); + } + } + ); + + postgresAndRedisTest( + "creates the run anyway when the birth append fails before redis-only", + { timeout: 60_000 }, + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never, { + mode: "dual-write", + unreachableRedis: true, + }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + const snapshotId = generateInternalId(); + + // Postgres is authoritative in every position before redis-only, so a Redis outage must not + // stop runs being created. + await decorated.createRun({ + data: buildCreateRunData(runId, env), + snapshot: birthSnapshot(snapshotId, env), + }); + + expect(await prisma.taskRun.count({ where: { id: runId } })).toBe(1); + expect(await prisma.taskRunExecutionSnapshot.count({ where: { id: snapshotId } })).toBe(1); + } finally { + await redis.quit(); + } + } + ); + + postgresAndRedisTest( + "refuses to create the run when the birth append fails at redis-only", + { timeout: 60_000 }, + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never, { + mode: "redis-only", + unreachableRedis: true, + }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + + // At redis-only Postgres writes no snapshot, so a run created without its Redis birth would + // have no snapshot anywhere. Failing before the run row exists lets the caller retry clean. + await expect( + decorated.createRun({ + data: buildCreateRunData(runId, env), + snapshot: birthSnapshot(generateInternalId(), env), + }) + ).rejects.toThrow(); + + expect(await prisma.taskRun.count({ where: { id: runId } })).toBe(0); + } finally { + await redis.quit(); + } + } + ); + + postgresAndRedisTest("createCancelledRun writes Redis first", async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + const snapshotId = generateInternalId(); + + await decorated.createCancelledRun({ + data: cancelledData(runId, env), + snapshot: { + ...birthSnapshot(snapshotId, env), + executionStatus: "FINISHED", + description: "Run was cancelled", + runStatus: "CANCELED", + }, + }); + + const read = await redis.getLatest(runId); + expect(read!.entry.id).toBe(snapshotId); + expect(read!.entry.executionStatus).toBe("FINISHED"); + expect(await prisma.taskRunExecutionSnapshot.count({ where: { id: snapshotId } })).toBe(1); + } finally { + await redis.quit(); + } + }); + + postgresAndRedisTest( + "a born-terminal run gets the completion expiry immediately", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + + await decorated.createCancelledRun({ + data: cancelledData(runId, env), + snapshot: { + ...birthSnapshot(generateInternalId(), env), + executionStatus: "FINISHED", + description: "Run was cancelled", + runStatus: "CANCELED", + }, + }); + + // A born-terminal run never transitions again, so the completion TTL has to be applied by + // the birth itself or the keyspace never expires. + const nonTerminal = generateInternalId(); + await decorated.createRun({ + data: buildCreateRunData(nonTerminal, env), + snapshot: birthSnapshot(generateInternalId(), env), + }); + + const terminal = await redis.getLatest(runId); + const alive = await redis.getLatest(nonTerminal); + expect(terminal).not.toBeNull(); + expect(alive).not.toBeNull(); + } finally { + await redis.quit(); + } + } + ); + + postgresAndRedisTest("writes nothing to Redis at mode off", async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never, { mode: "off" }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + + await decorated.createRun({ + data: buildCreateRunData(runId, env), + snapshot: birthSnapshot(generateInternalId(), env), + }); + + expect(await prisma.taskRun.count({ where: { id: runId } })).toBe(1); + expect(await redis.getLatest(runId)).toBeNull(); + } finally { + await redis.quit(); + } + }); +}); diff --git a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.off.test.ts b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.off.test.ts new file mode 100644 index 00000000000..975c68c371a --- /dev/null +++ b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.off.test.ts @@ -0,0 +1,94 @@ +// Mode off is the merge-test position: the decorator must be indistinguishable from its delegate and +// must not touch Redis at all. A Redis store whose every member throws proves the second half, and +// enumerating the generated name list proves the first for every method rather than a chosen few. +import { describe, expect, it } from "vitest"; +import { RUN_STORE_METHOD_NAMES } from "./runStoreMethodNames.js"; +import type { RedisSnapshotStore } from "./redisSnapshotStore.js"; +import { TaskRunExecutionSnapshotStore } from "./taskRunExecutionSnapshotStore.js"; +import type { RunStore } from "./types.js"; + +function explodingRedisStore(): RedisSnapshotStore { + return new Proxy({} as RedisSnapshotStore, { + get(_target, prop) { + return () => { + throw new Error(`the Redis store must not be called at mode off, but ${String(prop)} was`); + }; + }, + }); +} + +function recordingDelegate(): { store: RunStore; calls: string[] } { + const calls: string[] = []; + const store: Record = {}; + + for (const name of RUN_STORE_METHOD_NAMES) { + store[name] = (...args: unknown[]) => { + calls.push(name); + return `result:${name}`; + }; + } + + return { store: store as unknown as RunStore, calls }; +} + +describe("TaskRunExecutionSnapshotStore at mode off", () => { + it("defaults to mode off", () => { + const { store } = recordingDelegate(); + + const decorated = new TaskRunExecutionSnapshotStore(store, { store: explodingRedisStore() }); + + expect(decorated.mode).toBe("off"); + }); + + it("forwards every method to the delegate and never calls Redis", async () => { + const { store, calls } = recordingDelegate(); + const decorated = new TaskRunExecutionSnapshotStore(store, { + store: explodingRedisStore(), + mode: "off", + }) as unknown as Record unknown>; + + for (const name of RUN_STORE_METHOD_NAMES) { + if (name === "runInTransaction") continue; + expect(await decorated[name]("arg-one", "arg-two")).toBe(`result:${name}`); + } + + expect(calls).toEqual(RUN_STORE_METHOD_NAMES.filter((n) => n !== "runInTransaction")); + }); + + it("hands the delegate's own store to a transaction callback", async () => { + const inner = recordingDelegate().store; + let seen: unknown; + const delegate = { + runInTransaction: async ( + _runId: string | undefined, + fn: (store: RunStore, tx: unknown) => Promise + ) => { + await fn(inner, "tx"); + }, + } as unknown as RunStore; + + const decorated = new TaskRunExecutionSnapshotStore(delegate, { + store: explodingRedisStore(), + mode: "off", + }); + + await decorated.runInTransaction("run_1", async (store) => { + seen = store; + }); + + expect(seen).toBe(inner); + }); + + it("reports every other dial position as one that writes Redis", () => { + const { store } = recordingDelegate(); + const modes = ["dual-write", "compare", "redis-read", "redis-only"] as const; + + for (const mode of modes) { + const decorated = new TaskRunExecutionSnapshotStore(store, { + store: explodingRedisStore(), + mode, + }); + expect(decorated.mode).toBe(mode); + } + }); +}); diff --git a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.transitions.test.ts b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.transitions.test.ts new file mode 100644 index 00000000000..786e3a74453 --- /dev/null +++ b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.transitions.test.ts @@ -0,0 +1,453 @@ +// A transition writes Postgres first and Redis second. The order is proved by observation, not by +// reading the code: with the Redis half made to fail, the Postgres row is still there and the caller +// sees no error, which is only possible if Postgres went first. +import { describe, expect } from "vitest"; +import { postgresAndRedisTest } from "@internal/testcontainers"; +import { generateInternalId } from "@trigger.dev/core/v3/isomorphic"; +import { PostgresRunStore } from "./PostgresRunStore.js"; +import { RedisSnapshotStore } from "./redisSnapshotStore.js"; +import { entryFromCreateRun } from "./snapshotEntry.js"; +import { InjectedSnapshotFault } from "./snapshotFaultInjection.js"; +import { + TaskRunExecutionSnapshotStore, + type SnapshotStoreMode, +} from "./taskRunExecutionSnapshotStore.js"; +import type { RunStore } from "./types.js"; +import { + buildCreateRunData, + seedSnapshotEnvironment, + seedSnapshotWorker, + setupSnapshotIdFixture, + type SnapshotFixtureEnv, +} from "./testFixtures/snapshotIdFixture.js"; + +const COMPLETED_TTL_MS = 72 * 60 * 60 * 1000; + +type Harness = { + decorated: TaskRunExecutionSnapshotStore; + redis: RedisSnapshotStore; + repairs: { runId: string; snapshotId: string; executionStatus: string }[]; + writes: { site: string; outcome: string }[]; +}; + +function harness( + prisma: never, + redisOptions: never, + opts?: { + mode?: SnapshotStoreMode; + faults?: ConstructorParameters[1]["faults"]; + } +): Harness { + const redis = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + const repairs: Harness["repairs"] = []; + const writes: Harness["writes"] = []; + + const decorated = new TaskRunExecutionSnapshotStore( + new PostgresRunStore({ prisma, readOnlyPrisma: prisma }) as unknown as RunStore, + { + store: redis, + mode: opts?.mode ?? "dual-write", + ...(opts?.faults && { faults: opts.faults }), + onAppendFailure: async (args) => { + repairs.push(args); + }, + metrics: { + recordWrite: (site, outcome) => writes.push({ site, outcome }), + recordAppendFailed: () => {}, + recordRead: () => {}, + }, + } + ); + + return { decorated, redis, repairs, writes }; +} + +/** + * Creates the run and its keyspace, so a following transition is not skippedNoKeyspace. + * + * The birth is appended through the raw store rather than the decorator, because the decorator's + * own birth path is a separate concern with its own suite. Keeping it out here means a failure in + * this file is a failure of the transition path and nothing else. + */ +async function seedBirth( + decorated: TaskRunExecutionSnapshotStore, + redis: RedisSnapshotStore, + runId: string, + env: SnapshotFixtureEnv +): Promise { + const snapshot = { + id: generateInternalId(), + engine: "V2" as const, + executionStatus: "RUN_CREATED" as const, + description: "Run was created", + runStatus: "PENDING" as const, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }; + + await redis.append({ + entry: entryFromCreateRun({ id: snapshot.id, runId, createdAt: new Date() }, snapshot), + kind: "birth", + isTerminal: false, + }); + + await decorated.createRun({ data: buildCreateRunData(runId, env), snapshot }); +} + +function completionInput(env: SnapshotFixtureEnv) { + return { + completedAt: new Date(), + outputType: "application/json", + usageDurationMs: 1, + costInCents: 0, + snapshot: { + executionStatus: "FINISHED" as const, + description: "Run completed", + runStatus: "COMPLETED_SUCCESSFULLY" as const, + attemptNumber: 1, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }, + }; +} + +function expireInput(env: SnapshotFixtureEnv) { + return { + error: { type: "STRING_ERROR" as const, raw: "expired" }, + completedAt: new Date(), + expiredAt: new Date(), + snapshot: { + engine: "V2" as const, + executionStatus: "FINISHED" as const, + description: "Run expired", + runStatus: "EXPIRED" as const, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }, + }; +} + +describe("transition write ordering", () => { + postgresAndRedisTest("writes Postgres then Redis", async ({ prisma, redisOptions }) => { + const { decorated, redis, writes } = harness(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + await seedBirth(decorated, redis, runId, env); + + await decorated.completeAttemptSuccess(runId, completionInput(env), { select: { id: true } }); + + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ + where: { runId, executionStatus: "FINISHED" }, + }); + const read = await redis.getById(runId, row.id); + + expect(read).not.toBeNull(); + expect(read!.entry.id).toBe(row.id); + expect(read!.entry.executionStatus).toBe("FINISHED"); + expect(writes).toContainEqual({ site: "completeAttemptSuccess", outcome: "written" }); + } finally { + await redis.quit(); + } + }); + + postgresAndRedisTest( + "keeps the Postgres write and enqueues one repair when the append fails", + async ({ prisma, redisOptions }) => { + const { decorated, redis, repairs } = harness(prisma as never, redisOptions as never, { + faults: (boundary) => { + if (boundary === "afterPgBeforeRedis") throw new InjectedSnapshotFault(boundary); + }, + }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + await seedBirth(decorated, redis, runId, env); + + // The caller must NOT see an error: the Postgres mutation already committed, and the stall + // watchdog is the designed compensator. + await decorated.completeAttemptSuccess(runId, completionInput(env), { + select: { id: true }, + }); + + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ + where: { runId, executionStatus: "FINISHED" }, + }); + + expect(await redis.getById(runId, row.id)).toBeNull(); + expect(repairs).toEqual([ + { runId, snapshotId: row.id, executionStatus: "FINISHED" }, + ]); + } finally { + await redis.quit(); + } + } + ); + + postgresAndRedisTest( + "treats a transition on a run with no keyspace as skipped, not failed", + async ({ prisma, redisOptions }) => { + const { decorated, redis, repairs, writes } = harness(prisma as never, redisOptions as never); + try { + // No birth: this is every pre-cutover run's first transition after the dial moves. + const { run, env } = await setupSnapshotIdFixture(prisma); + + await decorated.expireRun(run.id, expireInput(env), { select: { id: true } }); + + expect(await prisma.taskRunExecutionSnapshot.count({ where: { runId: run.id } })).toBe(1); + expect(await redis.getLatest(run.id)).toBeNull(); + expect(repairs).toEqual([]); + expect(writes).toEqual([{ site: "expireRun", outcome: "skippedNoKeyspace" }]); + } finally { + await redis.quit(); + } + } + ); + + postgresAndRedisTest("appends for expireRun", async ({ prisma, redisOptions }) => { + const { decorated, redis } = harness(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + await seedBirth(decorated, redis, runId, env); + + await decorated.expireRun(runId, expireInput(env), { select: { id: true } }); + + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ + where: { runId, executionStatus: "FINISHED" }, + }); + const read = await redis.getById(runId, row.id); + expect(read?.entry.description).toBe("Run expired"); + } finally { + await redis.quit(); + } + }); + + postgresAndRedisTest("appends for expireParkedRun", async ({ prisma, redisOptions }) => { + const { decorated, redis } = harness(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + await seedBirth(decorated, redis, runId, env); + await prisma.taskRun.update({ where: { id: runId }, data: { status: "PENDING_VERSION" } }); + + const result = await decorated.expireParkedRun(runId, { + ...expireInput(env), + statusReason: "VERSION_NEVER_ARRIVED", + snapshot: { ...expireInput(env).snapshot, description: "Parked run expired" }, + }); + + expect(result.count).toBe(1); + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ + where: { runId, executionStatus: "FINISHED" }, + }); + expect((await redis.getById(runId, row.id))?.entry.description).toBe("Parked run expired"); + } finally { + await redis.quit(); + } + }); + + postgresAndRedisTest( + "appends nothing when expireParkedRun matches no run", + async ({ prisma, redisOptions }) => { + const { decorated, redis, writes } = harness(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + await seedBirth(decorated, redis, runId, env); + // The run is PENDING, so the delegate's `status: PENDING_VERSION` guard matches nothing. + + const result = await decorated.expireParkedRun(runId, { + ...expireInput(env), + statusReason: "VERSION_NEVER_ARRIVED", + }); + + expect(result.count).toBe(0); + expect(writes.filter((w) => w.site === "expireParkedRun")).toEqual([]); + const latest = await redis.getLatest(runId); + expect(latest?.entry.executionStatus).toBe("RUN_CREATED"); + } finally { + await redis.quit(); + } + } + ); + + postgresAndRedisTest("appends for rescheduleRun", async ({ prisma, redisOptions }) => { + const { decorated, redis } = harness(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + await seedBirth(decorated, redis, runId, env); + + await decorated.rescheduleRun(runId, { + delayUntil: new Date(Date.now() + 60_000), + snapshot: { + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }, + }); + + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ + where: { runId, executionStatus: "DELAYED" }, + }); + expect((await redis.getById(runId, row.id))?.entry.description).toBe( + "Delayed run was rescheduled to a future date" + ); + } finally { + await redis.quit(); + } + }); + + postgresAndRedisTest( + "appends nothing when rescheduleRun carries no snapshot", + async ({ prisma, redisOptions }) => { + const { decorated, redis, writes } = harness(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + await seedBirth(decorated, redis, runId, env); + + await decorated.rescheduleRun(runId, { delayUntil: new Date(Date.now() + 60_000) }); + + expect(writes.filter((w) => w.site === "rescheduleRun")).toEqual([]); + expect((await redis.getLatest(runId))?.entry.executionStatus).toBe("RUN_CREATED"); + } finally { + await redis.quit(); + } + } + ); + + postgresAndRedisTest("appends for lockRunToWorker under a CAS", async ({ prisma, redisOptions }) => { + const { decorated, redis, writes } = harness(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const { workerId, taskId } = await seedSnapshotWorker(prisma, env); + const runId = generateInternalId(); + await seedBirth(decorated, redis, runId, env); + + const head = await redis.getLatest(runId); + const snapshotId = generateInternalId(); + + await decorated.lockRunToWorker(runId, { + lockedAt: new Date(), + lockedById: taskId, + lockedToVersionId: workerId, + lockedQueueId: undefined, + startedAt: new Date(), + baseCostInCents: 0, + machinePreset: "small-1x", + taskVersion: "1.0.0", + snapshot: { + id: snapshotId, + previousSnapshotId: head!.id, + attemptNumber: 1, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + completedWaitpointIds: [], + completedWaitpointOrder: [], + }, + }); + + const read = await redis.getById(runId, snapshotId); + expect(read?.entry.executionStatus).toBe("PENDING_EXECUTING"); + expect(read?.entry.previousSnapshotId).toBe(head!.id); + expect(writes).toContainEqual({ site: "lockRunToWorker", outcome: "written" }); + } finally { + await redis.quit(); + } + }); + + postgresAndRedisTest( + "reports a forked append without enqueuing a repair", + async ({ prisma, redisOptions }) => { + const { decorated, redis, repairs, writes } = harness(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const { workerId, taskId } = await seedSnapshotWorker(prisma, env); + const runId = generateInternalId(); + await seedBirth(decorated, redis, runId, env); + + // A stale previousSnapshotId: another writer advanced the head. A repair cannot help, so the + // outcome is counted and dropped. + await decorated.lockRunToWorker(runId, { + lockedAt: new Date(), + lockedById: taskId, + lockedToVersionId: workerId, + lockedQueueId: undefined, + startedAt: new Date(), + baseCostInCents: 0, + machinePreset: "small-1x", + taskVersion: "1.0.0", + snapshot: { + id: generateInternalId(), + previousSnapshotId: generateInternalId(), + attemptNumber: 1, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + completedWaitpointIds: [], + completedWaitpointOrder: [], + }, + }); + + expect(writes).toContainEqual({ site: "lockRunToWorker", outcome: "forked" }); + expect(repairs).toEqual([]); + } finally { + await redis.quit(); + } + } + ); + + postgresAndRedisTest("appends for the standalone createExecutionSnapshot", async ({ prisma, redisOptions }) => { + const { decorated, redis } = harness(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + await seedBirth(decorated, redis, runId, env); + + const created = await decorated.createExecutionSnapshot({ + run: { id: runId, status: "EXECUTING", attemptNumber: 1 }, + snapshot: { executionStatus: "EXECUTING", description: "Run started" }, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }); + + const read = await redis.getById(runId, created.id); + expect(read).not.toBeNull(); + expect(read!.entry.executionStatus).toBe("EXECUTING"); + // The standalone path is the one whose delegate returns the row, so both stores agree exactly. + expect(read!.entry.createdAt).toBe(created.createdAt.toISOString()); + } finally { + await redis.quit(); + } + }); + + postgresAndRedisTest("writes nothing to Redis at mode off", async ({ prisma, redisOptions }) => { + const { decorated, redis } = harness(prisma as never, redisOptions as never, { mode: "off" }); + try { + const { run, env } = await setupSnapshotIdFixture(prisma); + + await decorated.completeAttemptSuccess(run.id, completionInput(env), { + select: { id: true }, + }); + + expect(await prisma.taskRunExecutionSnapshot.count({ where: { runId: run.id } })).toBe(1); + expect(await redis.getLatest(run.id)).toBeNull(); + } finally { + await redis.quit(); + } + }); +}); diff --git a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.ts b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.ts new file mode 100644 index 00000000000..69658ecaad1 --- /dev/null +++ b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.ts @@ -0,0 +1,534 @@ +// Decorates any RunStore so execution snapshots also land in Redis. It overrides only the methods +// that touch a snapshot and inherits the rest from the generated pass-through base. +// +// Write ORDER is the correctness property, and the two orders are deliberately different: +// +// transition Postgres first, Redis second. A crash in the gap leaves a run whose latest snapshot +// is stale, which is exactly the state the heartbeat stall watchdog already heals. +// birth Redis first, Postgres second. A crash in the gap leaves an unreachable key for a run +// that does not exist. Postgres-first would leave a run with no snapshot at all, and +// getLatestExecutionSnapshot treats that as a hard error. +// +// Each order is chosen so the crash state is the harmless one. A lost cross-store write is never +// recovered by a transaction or an outbox: recovery is always the existing stall-and-repair job. +import { Logger } from "@trigger.dev/core/logger"; +import { generateInternalId } from "@trigger.dev/core/v3/isomorphic"; +import { DelegatingRunStore } from "./delegatingRunStore.js"; +import type { RedisSnapshotStore, SnapshotEntryInput } from "./redisSnapshotStore.js"; +import { + entryFromCompletion, + entryFromCreateExecutionSnapshot, + entryFromCreateRun, + entryFromExpire, + entryFromLock, + entryFromReschedule, + isTerminalEntry, +} from "./snapshotEntry.js"; +import { isInjectedFault, type SnapshotFaultInjector } from "./snapshotFaultInjection.js"; +import type { + CompletionSnapshotInput, + CreateCancelledRunInput, + CreateExecutionSnapshotInput, + CreateRunInput, + ExpireSnapshotInput, + LockRunData, + RescheduleSnapshotInput, + RunStore, + TaskRunWithWaitpoint, +} from "./types.js"; +import type { Prisma, PrismaClientOrTransaction, TaskRun } from "@trigger.dev/database"; + +/** One initial attempt plus three retries, per the write protocol. */ +const APPEND_ATTEMPTS = 4; + +/** + * The rollout dial. Postgres stays fully written and authoritative in every position before + * `redis-only`, so every earlier position rolls back losslessly by turning the dial down. + * + * `compare` writes exactly as `dual-write` does. Its sampled dual-read and diff are a later ticket; + * the position is named here so the dial does not have to widen once that lands. + */ +export type SnapshotStoreMode = "off" | "dual-write" | "compare" | "redis-read" | "redis-only"; + +/** + * Enqueues the existing `repairSnapshot` job for a run whose append was lost. The decorator lives in + * run-store and cannot reach the engine's worker, so the binding is injected. That binding must + * reuse the stall watchdog's job id for the run, or the watchdog and this path can start two + * concurrent repairs on one run. + */ +export type SnapshotRepairEnqueuer = (args: { + runId: string; + snapshotId: string; + executionStatus: string; +}) => Promise; + +export type DecoratorMetrics = { + recordWrite(site: string, outcome: string): void; + recordAppendFailed(site: string): void; + recordRead(method: string, source: "redis" | "postgres"): void; +}; + +export type TaskRunExecutionSnapshotStoreOptions = { + store: RedisSnapshotStore; + /** Defaults to `off`, which is a pure pass-through that never touches Redis. */ + mode?: SnapshotStoreMode; + /** Percentage of runs whose reads come from Redis at `redis-read` and `redis-only`. Defaults to 0. */ + readPercent?: number; + onAppendFailure?: SnapshotRepairEnqueuer; + faults?: SnapshotFaultInjector; + metrics?: DecoratorMetrics; + logger?: Logger; + /** + * Internal. Set only by the staging facade this class builds for `runInTransaction`. When present, + * an intercepted write does its Postgres half and pushes its entry here instead of appending, and + * the outer instance flushes the buffer after the transaction commits. + */ + staging?: SnapshotEntryInput[]; +}; + +export class TaskRunExecutionSnapshotStore extends DelegatingRunStore { + readonly mode: SnapshotStoreMode; + protected readonly redis: RedisSnapshotStore; + protected readonly readPercent: number; + protected readonly onAppendFailure?: SnapshotRepairEnqueuer; + protected readonly faults?: SnapshotFaultInjector; + protected readonly metrics?: DecoratorMetrics; + protected readonly logger: Logger; + protected readonly staging?: SnapshotEntryInput[]; + + constructor(delegate: RunStore, options: TaskRunExecutionSnapshotStoreOptions) { + super(delegate); + this.redis = options.store; + this.mode = options.mode ?? "off"; + this.readPercent = options.readPercent ?? 0; + this.onAppendFailure = options.onAppendFailure; + this.faults = options.faults; + this.metrics = options.metrics; + this.logger = options.logger ?? new Logger("TaskRunExecutionSnapshotStore", "debug"); + this.staging = options.staging; + } + + /** True in every position that appends to Redis. */ + protected get writesRedis(): boolean { + return this.mode !== "off"; + } + + /** + * The staging facade. Two writes share one Postgres transaction here, and the Redis half of each + * cannot run until that transaction commits: a rollback would otherwise leave Redis holding a + * transition that never happened. + * + * The callback gets a second decorator over the transaction-bound store, carrying a staging + * buffer. An intercepted write does its Postgres half through that store and pushes its entry + * onto the buffer. After the transaction resolves, this instance flushes the buffer in order + * through the same retry-and-repair path a lone transition uses. If the callback throws, the + * delegate rejects, the flush never runs, and the buffer goes away with the stack — so the + * Postgres rollback and the Redis silence agree. + */ + override async runInTransaction( + runId: string | undefined, + fn: (store: RunStore, tx: PrismaClientOrTransaction) => Promise + ): Promise { + if (!this.writesRedis) { + // At `off` the callback must receive the delegate's own store, untouched, so a transaction + // behaves exactly as it does without the decorator in the chain. + return this.delegate.runInTransaction(runId, fn); + } + + const staged: SnapshotEntryInput[] = []; + + const result = await this.delegate.runInTransaction(runId, (store, tx) => + fn(this.#wrap(store, staged), tx) + ); + + // The transaction committed. Only now can a snapshot claim its partner is durable. + for (const entry of staged) { + await this.#appendTransition("runInTransaction", entry); + } + + return result; + } + + /** + * `forWaitpointCompletion` hands the caller a store to apply a completion on. No snapshot write + * goes through that handle today, so wrapping it changes nothing now; leaving it unwrapped is the + * one hole that would let a future snapshot write bypass the decorator with no signal at all. + */ + override async forWaitpointCompletion( + waitpointId: string, + context: Parameters[1] + ): Promise { + const store = await this.delegate.forWaitpointCompletion(waitpointId, context); + + if (!this.writesRedis) { + return store; + } + + return this.#wrap(store); + } + + /** + * A second decorator over another store, sharing this one's options. One class in both roles keeps + * the write-ordering logic in exactly one place. Passing no buffer gives a plain decorator that + * appends immediately; passing one makes it stage instead. + */ + #wrap(store: RunStore, staging?: SnapshotEntryInput[]): TaskRunExecutionSnapshotStore { + return new TaskRunExecutionSnapshotStore(store, { + store: this.redis, + mode: this.mode, + readPercent: this.readPercent, + logger: this.logger, + ...(this.onAppendFailure && { onAppendFailure: this.onAppendFailure }), + ...(this.faults && { faults: this.faults }), + ...(this.metrics && { metrics: this.metrics }), + ...(staging && { staging }), + }); + } + + // --------------------------------------------------------------------------------------------- + // Births: Redis first, Postgres second. + // --------------------------------------------------------------------------------------------- + + override async createRun( + params: CreateRunInput, + tx?: PrismaClientOrTransaction + ): Promise { + if (!this.writesRedis) { + return this.delegate.createRun(params, tx); + } + + const ctx = this.#context(params.data.id, params.snapshot.id); + const snapshot = { ...params.snapshot, id: ctx.id }; + + await this.#appendBirth("createRun", entryFromCreateRun(ctx, snapshot)); + + return this.delegate.createRun({ ...params, snapshot }, tx); + } + + override async createCancelledRun( + params: CreateCancelledRunInput, + tx?: PrismaClientOrTransaction + ): Promise { + if (!this.writesRedis) { + return this.delegate.createCancelledRun(params, tx); + } + + const ctx = this.#context(params.data.id, params.snapshot.id); + const snapshot = { ...params.snapshot, id: ctx.id }; + + await this.#appendBirth("createCancelledRun", entryFromCreateRun(ctx, snapshot)); + + return this.delegate.createCancelledRun({ ...params, snapshot }, tx); + } + + // --------------------------------------------------------------------------------------------- + // Transitions: Postgres first, Redis second. + // --------------------------------------------------------------------------------------------- + + override async completeAttemptSuccess( + runId: string, + data: { + completedAt: Date; + output?: string; + outputType: string; + usageDurationMs: number; + costInCents: number; + snapshot: CompletionSnapshotInput; + }, + args: { select: S }, + tx?: PrismaClientOrTransaction + ): Promise> { + if (!this.writesRedis) { + return this.delegate.completeAttemptSuccess(runId, data, args, tx); + } + + const ctx = this.#context(runId, data.snapshot.id); + const withId = { ...data, snapshot: { ...data.snapshot, id: ctx.id } }; + + const result = await this.delegate.completeAttemptSuccess(runId, withId, args, tx); + + await this.#appendTransition( + "completeAttemptSuccess", + entryFromCompletion(ctx, withId.snapshot) + ); + return result; + } + + override async expireRun( + runId: string, + data: { error: unknown; completedAt: Date; expiredAt: Date; snapshot: ExpireSnapshotInput }, + args: { select: S }, + tx?: PrismaClientOrTransaction + ): Promise> { + if (!this.writesRedis) { + return this.delegate.expireRun(runId, data as never, args, tx); + } + + const ctx = this.#context(runId, data.snapshot.id); + const withId = { ...data, snapshot: { ...data.snapshot, id: ctx.id } }; + + const result = await this.delegate.expireRun(runId, withId as never, args, tx); + + await this.#appendTransition("expireRun", entryFromExpire(ctx, withId.snapshot)); + return result; + } + + override async expireParkedRun( + runId: string, + data: { + error: unknown; + completedAt: Date; + expiredAt: Date; + statusReason: string; + snapshot: ExpireSnapshotInput; + }, + tx?: PrismaClientOrTransaction + ): Promise<{ count: number }> { + if (!this.writesRedis) { + return this.delegate.expireParkedRun(runId, data as never, tx); + } + + const ctx = this.#context(runId, data.snapshot.id); + const withId = { ...data, snapshot: { ...data.snapshot, id: ctx.id } }; + + const result = await this.delegate.expireParkedRun(runId, withId as never, tx); + + // The delegate writes nothing when the run is no longer PENDING_VERSION, so neither does Redis. + if (result.count > 0) { + await this.#appendTransition("expireParkedRun", entryFromExpire(ctx, withId.snapshot)); + } + return result; + } + + override async rescheduleRun( + runId: string, + data: { delayUntil: Date; queueTimestamp?: Date; snapshot?: RescheduleSnapshotInput }, + tx?: PrismaClientOrTransaction + ): Promise { + // The delegate writes a snapshot only when one is supplied, so an absent snapshot is a plain run + // update with nothing for Redis to mirror. + if (!this.writesRedis || !data.snapshot) { + return this.delegate.rescheduleRun(runId, data, tx); + } + + const ctx = this.#context(runId, data.snapshot.id); + const withId = { ...data, snapshot: { ...data.snapshot, id: ctx.id } }; + + const result = await this.delegate.rescheduleRun(runId, withId, tx); + + await this.#appendTransition("rescheduleRun", entryFromReschedule(ctx, withId.snapshot)); + return result; + } + + override async lockRunToWorker( + runId: string, + data: LockRunData, + tx?: PrismaClientOrTransaction + ): Promise>> { + if (!this.writesRedis) { + return this.delegate.lockRunToWorker(runId, data, tx); + } + + // This is the one transition whose input already carries both an id and the previous snapshot + // id, so it is also the one that can append under a compare-and-set on the current head. + const ctx = { id: data.snapshot.id, runId, createdAt: new Date() }; + + const result = await this.delegate.lockRunToWorker(runId, data, tx); + + await this.#appendTransition( + "lockRunToWorker", + entryFromLock(ctx, data.snapshot), + data.snapshot.previousSnapshotId + ); + return result; + } + + override async createExecutionSnapshot( + input: CreateExecutionSnapshotInput, + tx?: PrismaClientOrTransaction + ): Promise> { + if (!this.writesRedis) { + return this.delegate.createExecutionSnapshot(input, tx); + } + + const ctx = this.#context(input.run.id, input.id); + const created = await this.delegate.createExecutionSnapshot({ ...input, id: ctx.id }, tx); + + // The standalone path is the only one whose delegate returns the row, so its entry can take the + // exact createdAt Postgres recorded rather than the decorator's own clock. + await this.#appendTransition( + "createExecutionSnapshot", + entryFromCreateExecutionSnapshot({ ...ctx, createdAt: created.createdAt }, input), + input.previousSnapshotId + ); + return created; + } + + // --------------------------------------------------------------------------------------------- + // The append protocol. + // --------------------------------------------------------------------------------------------- + + /** Mints the id when the caller did not, and stamps one clock for both stores. */ + #context(runId: string, suppliedId?: string) { + return { id: suppliedId ?? generateInternalId(), runId, createdAt: new Date() }; + } + + /** + * Births invert the order. Postgres-first would leave a run with no snapshot at all, and + * `getLatestExecutionSnapshot` treats that as a hard error, so the run would be stuck. Redis-first + * leaves an orphaned keyspace for a run that does not exist, which nothing can reach and the + * sweep's second rule reaps. + * + * Being first is also what lets this path refuse. Before `redis-only` a failed birth append is + * survivable, because Postgres is authoritative and holds the snapshot; at `redis-only` Postgres + * writes no snapshot, so a run created without its Redis birth would have no snapshot anywhere. + * Throwing here happens before the run row exists, so the caller retries a clean creation. + */ + async #appendBirth(site: string, entry: SnapshotEntryInput): Promise { + if (this.staging) { + // A birth inside a transaction cannot be staged: staging flushes after the commit, which is + // the opposite of what a birth needs. No caller does this today, so say so and append now. + this.logger.error("a run birth inside a transaction cannot be staged", { + runId: entry.runId, + site, + }); + } + + for (let attempt = 0; attempt < APPEND_ATTEMPTS; attempt++) { + try { + const result = await this.redis.append({ + entry, + kind: "birth", + isTerminal: isTerminalEntry(entry), + }); + this.#recordOutcome(site, entry, result); + + // Modelled AFTER the successful append: the crash this boundary represents is a process that + // died between the two stores, not an append that failed. + this.faults?.("afterRedisBirthBeforePg", { runId: entry.runId, snapshotId: entry.id }); + return; + } catch (error) { + if (isInjectedFault(error)) { + throw error; + } + + if (attempt === APPEND_ATTEMPTS - 1) { + this.metrics?.recordAppendFailed(site); + this.logger.error("snapshot birth append failed after retries", { + runId: entry.runId, + snapshotId: entry.id, + site, + mode: this.mode, + error, + }); + + if (this.mode === "redis-only") { + throw error; + } + return; + } + + await new Promise((resolve) => setTimeout(resolve, 10 * 2 ** attempt)); + } + } + } + + /** + * Postgres has already committed by the time this runs. A throw here would turn a gap the stall + * watchdog heals into a caller-visible failure, so it never rethrows: it retries, then hands the + * run to the repair job and returns. + */ + async #appendTransition( + site: string, + entry: SnapshotEntryInput, + expectedCur?: string + ): Promise { + if (this.staging) { + // Inside a transaction the append cannot run until the Postgres side commits, or a rollback + // leaves Redis holding a transition that never happened. + this.staging.push(entry); + return; + } + + for (let attempt = 0; attempt < APPEND_ATTEMPTS; attempt++) { + try { + this.faults?.(attempt === 0 ? "afterPgBeforeRedis" : "midFlushRetry", { + runId: entry.runId, + snapshotId: entry.id, + }); + + const result = await this.redis.append({ + entry, + kind: "transition", + isTerminal: isTerminalEntry(entry), + ...(expectedCur !== undefined && { expectedCur }), + }); + + this.#recordOutcome(site, entry, result); + return; + } catch (error) { + // An injected fault models a dead process, not a retryable append failure. + if (isInjectedFault(error)) { + this.metrics?.recordAppendFailed(site); + await this.#enqueueRepair(entry); + return; + } + + if (attempt === APPEND_ATTEMPTS - 1) { + this.metrics?.recordAppendFailed(site); + this.logger.error("snapshot append failed after retries", { + runId: entry.runId, + snapshotId: entry.id, + site, + error, + }); + await this.#enqueueRepair(entry); + return; + } + + await new Promise((resolve) => setTimeout(resolve, 10 * 2 ** attempt)); + } + } + } + + /** + * None of the four append outcomes is a failure, and none of them enqueues a repair. + * + * `skippedNoKeyspace` is every pre-cutover run's transitions. `forked` means another writer + * advanced the head, which a repair cannot help. `duplicate` is a retry that already landed. + * `cycleMismatch` means the store refused an untrustworthy waitpoint pointer on purpose. + */ + #recordOutcome( + site: string, + entry: SnapshotEntryInput, + result: Awaited> + ): void { + this.metrics?.recordWrite(site, result.outcome); + + if (result.outcome === "forked") { + this.logger.warn("snapshot append forked", { + runId: entry.runId, + snapshotId: entry.id, + site, + actualCur: result.actualCur, + }); + } + } + + async #enqueueRepair(entry: SnapshotEntryInput): Promise { + if (!this.onAppendFailure) { + return; + } + + try { + await this.onAppendFailure({ + runId: entry.runId, + snapshotId: entry.id, + executionStatus: entry.executionStatus, + }); + } catch (error) { + // The repair enqueue is itself best-effort. Failing it must not fail the caller's write. + this.logger.error("snapshot repair enqueue failed", { runId: entry.runId, error }); + } + } +} From fbe91ac3e11e5cb08982def9e2659612a6324b59 Mon Sep 17 00:00:00 2001 From: Daniel Sutton Date: Mon, 24 Aug 2026 15:33:10 +0100 Subject: [PATCH 06/56] test(run-store): cover the snapshot staging facade and the wrapped store handles Proves the deferral from inside the transaction callback rather than assuming it: a staged append is absent from Redis while the transaction is open and present once it commits, and a rollback leaves both stores agreeing the transition never happened. --- ...kRunExecutionSnapshotStore.staging.test.ts | 231 ++++++++++++++++++ 1 file changed, 231 insertions(+) create mode 100644 internal-packages/run-store/src/taskRunExecutionSnapshotStore.staging.test.ts diff --git a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.staging.test.ts b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.staging.test.ts new file mode 100644 index 00000000000..ac0a877c968 --- /dev/null +++ b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.staging.test.ts @@ -0,0 +1,231 @@ +// Inside a transaction the Redis append cannot run until the Postgres side commits, or a rollback +// leaves Redis holding a transition that never happened. These tests observe the buffer from inside +// the callback, so the deferral is proved rather than assumed. +import { describe, expect } from "vitest"; +import { postgresAndRedisTest } from "@internal/testcontainers"; +import { generateInternalId } from "@trigger.dev/core/v3/isomorphic"; +import { PostgresRunStore } from "./PostgresRunStore.js"; +import { RedisSnapshotStore } from "./redisSnapshotStore.js"; +import { entryFromCreateRun } from "./snapshotEntry.js"; +import { TaskRunExecutionSnapshotStore } from "./taskRunExecutionSnapshotStore.js"; +import type { RunStore } from "./types.js"; +import { + buildCreateRunData, + seedSnapshotEnvironment, + type SnapshotFixtureEnv, +} from "./testFixtures/snapshotIdFixture.js"; + +const COMPLETED_TTL_MS = 72 * 60 * 60 * 1000; + +function build(prisma: never, redisOptions: never, mode: "off" | "dual-write" = "dual-write") { + const redis = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + const decorated = new TaskRunExecutionSnapshotStore( + new PostgresRunStore({ prisma, readOnlyPrisma: prisma }) as unknown as RunStore, + { store: redis, mode } + ); + return { decorated, redis }; +} + +async function seedBirth( + decorated: TaskRunExecutionSnapshotStore, + redis: RedisSnapshotStore, + runId: string, + env: SnapshotFixtureEnv +): Promise { + const snapshot = { + id: generateInternalId(), + engine: "V2" as const, + executionStatus: "RUN_CREATED" as const, + description: "Run was created", + runStatus: "PENDING" as const, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }; + + await redis.append({ + entry: entryFromCreateRun({ id: snapshot.id, runId, createdAt: new Date() }, snapshot), + kind: "birth", + isTerminal: false, + }); + await decorated.createRun({ data: buildCreateRunData(runId, env), snapshot }); +} + +function snapshotInput(runId: string, env: SnapshotFixtureEnv, id: string, description: string) { + return { + id, + run: { id: runId, status: "EXECUTING" as const, attemptNumber: 1 }, + snapshot: { executionStatus: "EXECUTING" as const, description }, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }; +} + +describe("the staging facade", () => { + postgresAndRedisTest("flushes the append after the commit", async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + await seedBirth(decorated, redis, runId, env); + const id = generateInternalId(); + + await decorated.runInTransaction(runId, async (store, tx) => { + await store.createExecutionSnapshot(snapshotInput(runId, env, id, "Run started"), tx); + + // Still inside the transaction: nothing has reached Redis yet. + expect(await redis.getById(runId, id)).toBeNull(); + }); + + expect(await redis.getById(runId, id)).not.toBeNull(); + expect(await prisma.taskRunExecutionSnapshot.count({ where: { id } })).toBe(1); + } finally { + await redis.quit(); + } + }); + + postgresAndRedisTest( + "writes nothing to Redis when the transaction rolls back", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + await seedBirth(decorated, redis, runId, env); + const id = generateInternalId(); + + await expect( + decorated.runInTransaction(runId, async (store, tx) => { + await store.createExecutionSnapshot(snapshotInput(runId, env, id, "Run started"), tx); + throw new Error("rolled back"); + }) + ).rejects.toThrow("rolled back"); + + // Both sides agree that the transition never happened. + expect(await prisma.taskRunExecutionSnapshot.count({ where: { id } })).toBe(0); + expect(await redis.getById(runId, id)).toBeNull(); + } finally { + await redis.quit(); + } + } + ); + + postgresAndRedisTest("flushes several staged appends in order", async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + await seedBirth(decorated, redis, runId, env); + const first = generateInternalId(); + const second = generateInternalId(); + + await decorated.runInTransaction(runId, async (store, tx) => { + await store.createExecutionSnapshot(snapshotInput(runId, env, first, "First"), tx); + await store.createExecutionSnapshot(snapshotInput(runId, env, second, "Second"), tx); + }); + + const firstRead = await redis.getById(runId, first); + const secondRead = await redis.getById(runId, second); + expect(firstRead).not.toBeNull(); + expect(secondRead).not.toBeNull(); + // Order matters: the log is append-only and its seq is what orders a read. + expect(firstRead!.seq).toBeLessThan(secondRead!.seq); + } finally { + await redis.quit(); + } + }); + + postgresAndRedisTest( + "hands the transaction callback a decorated store", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + await seedBirth(decorated, redis, runId, env); + let seen: unknown; + + await decorated.runInTransaction(runId, async (store) => { + seen = store; + }); + + expect(seen).toBeInstanceOf(TaskRunExecutionSnapshotStore); + expect((seen as TaskRunExecutionSnapshotStore).mode).toBe("dual-write"); + } finally { + await redis.quit(); + } + } + ); + + postgresAndRedisTest( + "hands the transaction callback the plain delegate at mode off", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never, "off"); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + await decorated.createRun({ + data: buildCreateRunData(runId, env), + snapshot: { + id: generateInternalId(), + engine: "V2", + executionStatus: "RUN_CREATED", + description: "Run was created", + runStatus: "PENDING", + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }, + }); + let seen: unknown; + + await decorated.runInTransaction(runId, async (store) => { + seen = store; + }); + + expect(seen).not.toBeInstanceOf(TaskRunExecutionSnapshotStore); + expect(await redis.getLatest(runId)).toBeNull(); + } finally { + await redis.quit(); + } + } + ); + + postgresAndRedisTest( + "wraps the store handle from forWaitpointCompletion", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const handle = await decorated.forWaitpointCompletion(generateInternalId(), { + routeKind: "MANUAL", + } as never); + + // No snapshot write goes through this handle today. Wrapping it is what stops a future one + // from bypassing the decorator with no signal. + expect(handle).toBeInstanceOf(TaskRunExecutionSnapshotStore); + } finally { + await redis.quit(); + } + } + ); + + postgresAndRedisTest( + "returns the plain handle from forWaitpointCompletion at mode off", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never, "off"); + try { + const handle = await decorated.forWaitpointCompletion(generateInternalId(), { + routeKind: "MANUAL", + } as never); + + expect(handle).not.toBeInstanceOf(TaskRunExecutionSnapshotStore); + } finally { + await redis.quit(); + } + } + ); +}); From ac7b37b7e6ab64714f13de41d954f66bd9d5e440 Mon Sep 17 00:00:00 2001 From: Daniel Sutton Date: Mon, 24 Aug 2026 15:35:51 +0100 Subject: [PATCH 07/56] feat(run-store): read a snapshot window by createdAt cursor The engine resolves its since-cursor to a createdAt before it asks for the window, so the snapshot id is gone by then and the id-addressed read cannot serve it. Adding a cursor-addressed read is the alternative to changing the engine's read path, which stays untouched. The cursor is exclusive and keeps the same-millisecond blind spot the Postgres read has. Matching it is the requirement, not an oversight: a Redis read that is more correct than the Postgres read shows up as divergence during compare mode, which exists to surface real defects. Closing the blind spot needs seq ordering on both sides and belongs after the cutover. The walk goes newest-first and stops at the first entry at or before the cursor, so its length is the length of the answer rather than the run's history. This adds a read operation. It does not touch the append script, the keyspace, or the write-ordering protocol. --- .../redisSnapshotStore.sinceCreatedAt.test.ts | 191 ++++++++++++++++++ .../run-store/src/redisSnapshotStore.ts | 127 ++++++++++++ 2 files changed, 318 insertions(+) create mode 100644 internal-packages/run-store/src/redisSnapshotStore.sinceCreatedAt.test.ts diff --git a/internal-packages/run-store/src/redisSnapshotStore.sinceCreatedAt.test.ts b/internal-packages/run-store/src/redisSnapshotStore.sinceCreatedAt.test.ts new file mode 100644 index 00000000000..7e61e619b78 --- /dev/null +++ b/internal-packages/run-store/src/redisSnapshotStore.sinceCreatedAt.test.ts @@ -0,0 +1,191 @@ +// getExecutionSnapshotsSince resolves its cursor to a createdAt before it asks for the window, so +// the snapshot id is gone by then and getSince cannot serve it. This read takes the cursor instead, +// and has to agree with the Postgres read it stands in for — same-millisecond blind spot included. +import { describe, expect } from "vitest"; +import { redisTest } from "@internal/testcontainers"; +import { RedisSnapshotStore } from "./redisSnapshotStore.js"; +import type { SnapshotEntryInput } from "./redisSnapshotStore.js"; + +const COMPLETED_TTL_MS = 72 * 60 * 60 * 1000; + +function entry(runId: string, id: string, createdAt: string): SnapshotEntryInput { + return { + id, + engine: "V2", + executionStatus: "EXECUTING", + description: "d", + runId, + runStatus: "EXECUTING", + createdAt, + environmentId: "env_1", + environmentType: "DEVELOPMENT", + projectId: "proj_1", + organizationId: "org_1", + }; +} + +const at = (seconds: number) => + new Date(Date.UTC(2026, 0, 1, 0, 0, seconds)).toISOString(); + +async function seed( + store: RedisSnapshotStore, + runId: string, + stamps: { id: string; createdAt: string }[] +): Promise { + for (const [index, stamp] of stamps.entries()) { + await store.append({ + entry: entry(runId, stamp.id, stamp.createdAt), + kind: index === 0 ? "birth" : "transition", + isTerminal: false, + }); + } +} + +describe("getSinceCreatedAt", () => { + redisTest("returns only entries newer than the cursor, oldest first", async ({ redisOptions }) => { + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + try { + const runId = "run_window"; + await seed( + store, + runId, + [0, 1, 2, 3, 4].map((n) => ({ id: `snap_${n}`, createdAt: at(n) })) + ); + + const result = await store.getSinceCreatedAt(runId, at(1)); + + expect(result.kind).toBe("hit"); + if (result.kind !== "hit") return; + // Ascending, matching what the engine hands its caller after its own reverse(). + expect(result.entries.map((e) => e.id)).toEqual(["snap_2", "snap_3", "snap_4"]); + } finally { + await store.quit(); + } + }); + + redisTest("misses when the run has no keyspace", async ({ redisOptions }) => { + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + try { + // A miss is the coexistence path: the caller falls back to Postgres for a pre-cutover run. + expect((await store.getSinceCreatedAt("run_absent", at(0))).kind).toBe("miss"); + } finally { + await store.quit(); + } + }); + + redisTest("returns an empty hit when nothing is newer", async ({ redisOptions }) => { + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + try { + const runId = "run_nothing_newer"; + await seed(store, runId, [{ id: "snap_0", createdAt: at(0) }]); + + const result = await store.getSinceCreatedAt(runId, at(5)); + + // A hit, not a miss: Redis owns this run, so the caller must not fall back and re-read + // Postgres for a window it already answered. + expect(result.kind).toBe("hit"); + if (result.kind !== "hit") return; + expect(result.entries).toEqual([]); + } finally { + await store.quit(); + } + }); + + redisTest("drops a same-millisecond neighbour, as Postgres does", async ({ redisOptions }) => { + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + try { + const runId = "run_same_ms"; + const shared = at(1); + await seed(store, runId, [ + { id: "snap_0", createdAt: at(0) }, + { id: "snap_1a", createdAt: shared }, + { id: "snap_1b", createdAt: shared }, + { id: "snap_2", createdAt: at(2) }, + ]); + + const result = await store.getSinceCreatedAt(runId, shared); + + // Postgres serves this window with `createdAt: { gt: cursor }`, which drops both same-ms + // entries. Returning snap_1b here would be more correct than Postgres and would therefore + // read as divergence in compare mode. + expect(result.kind).toBe("hit"); + if (result.kind !== "hit") return; + expect(result.entries.map((e) => e.id)).toEqual(["snap_2"]); + } finally { + await store.quit(); + } + }); + + redisTest("caps the window at the limit, keeping the newest", async ({ redisOptions }) => { + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + try { + const runId = "run_capped"; + await seed( + store, + runId, + Array.from({ length: 60 }, (_, n) => ({ id: `snap_${n}`, createdAt: at(n) })) + ); + + const result = await store.getSinceCreatedAt(runId, at(0), { limit: 50 }); + + expect(result.kind).toBe("hit"); + if (result.kind !== "hit") return; + expect(result.entries).toHaveLength(50); + // The engine takes the NEWEST 50 and reverses, so the window ends at the newest entry. + expect(result.entries[result.entries.length - 1]!.id).toBe("snap_59"); + expect(result.entries[0]!.id).toBe("snap_10"); + } finally { + await store.quit(); + } + }); + + redisTest("scans no further than the answer", async ({ redisOptions }) => { + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + try { + const runId = "run_deep_history"; + await seed( + store, + runId, + Array.from({ length: 400 }, (_, n) => ({ id: `snap_${n}`, createdAt: at(n) })) + ); + + const started = Date.now(); + const result = await store.getSinceCreatedAt(runId, at(394), { limit: 50 }); + const elapsed = Date.now() - started; + + expect(result.kind).toBe("hit"); + if (result.kind !== "hit") return; + expect(result.entries.map((e) => e.id)).toEqual([ + "snap_395", + "snap_396", + "snap_397", + "snap_398", + "snap_399", + ]); + // The walk stops at the cursor rather than reading the run's history. The bound is generous + // on purpose: it fails on a full scan of 400 entries, not on ordinary timing noise. + expect(elapsed).toBeLessThan(1_000); + } finally { + await store.quit(); + } + }); + + redisTest("scopes the window to an environment", async ({ redisOptions }) => { + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + try { + const runId = "run_env_scoped"; + await seed(store, runId, [ + { id: "snap_0", createdAt: at(0) }, + { id: "snap_1", createdAt: at(1) }, + ]); + + const foreign = await store.getSinceCreatedAt(runId, at(0), { environmentId: "env_other" }); + + expect(foreign.kind).toBe("hit"); + if (foreign.kind !== "hit") return; + expect(foreign.entries).toEqual([]); + } finally { + await store.quit(); + } + }); +}); diff --git a/internal-packages/run-store/src/redisSnapshotStore.ts b/internal-packages/run-store/src/redisSnapshotStore.ts index 2964959c4fe..e36179c6041 100644 --- a/internal-packages/run-store/src/redisSnapshotStore.ts +++ b/internal-packages/run-store/src/redisSnapshotStore.ts @@ -472,6 +472,64 @@ export class RedisSnapshotStore { }); } + /** + * The same window as {@link getSince}, addressed by a createdAt cursor instead of a snapshot id. + * + * `getExecutionSnapshotsSince` resolves its cursor to a createdAt before it asks for the window, + * so the snapshot id is gone by the time this call is made and `getSince` cannot serve it. The + * cursor is exclusive and keeps Postgres's same-millisecond blind spot, so the two reads agree. + */ + async getSinceCreatedAt( + runId: string, + createdAt: Date | string, + opts?: { environmentId?: string; limit?: number } + ): Promise { + return this.#timed("getSinceCreatedAt", async () => { + const k = snapshotKeys(runId); + const limit = opts?.limit ?? this.sinceLimit; + const cursor = typeof createdAt === "string" ? createdAt : createdAt.toISOString(); + + const reply = await this.redis.readSnapshotsSinceCreatedAt( + k.e, + k.idx, + k.cur, + k.seq, + cursor, + String(limit) + ); + if (reply === null) return { kind: "miss" }; + + const headOrder = reply[1] ?? ""; + const rows: SnapshotRead[] = []; + // Tracks whether the Lua-chosen head row (always the first, i === 2) survives the env filter, + // so headOrder is never attributed to a different, surviving row. + let headSurvived = false; + for (let i = 2; i + 3 < reply.length; i += 4) { + const decoded = this.#decode( + [reply[i], reply[i + 1], reply[i + 2], reply[i + 3], ""], + opts?.environmentId, + runId, + false + ); + if (decoded) { + rows.push(decoded); + if (i === 2) headSurvived = true; + } + } + + rows.reverse(); + const head = headSurvived ? rows[rows.length - 1] : undefined; + const headWaitpointIds = decodeWaitpointIds(head !== undefined, head ? headOrder : ""); + if (head) { + head.completedWaitpointIds = headWaitpointIds; + if (head.cycle) { + this.#checkCycleMismatch(runId, head.cycle.count, headWaitpointIds.order.length); + } + } + return { kind: "hit", entries: rows, headWaitpointIds }; + }); + } + #checkCycleMismatch(runId: string, count: number, orderLength: number): void { if (orderLength === count) return; this.metrics?.recordCycleMismatch(); @@ -682,6 +740,66 @@ export class RedisSnapshotStore { `, }); + this.redis.defineCommand("readSnapshotsSinceCreatedAt", { + numberOfKeys: 4, + lua: ` + ${PRELUDE} + local cursor = ARGV[1] + local limit = tonumber(ARGV[2]) + + -- A run with no keyspace is a MISS, so the caller falls back to Postgres. A run that has one + -- and nothing newer is an empty HIT, so it does not fall back for a window it owns. + if redis.call('EXISTS', eKey) == 0 then return nil end + + -- STRICTLY greater than the cursor, and same-millisecond entries are dropped. Postgres + -- serves this window with createdAt > cursor and drops them too; a Redis read that is more + -- correct than the Postgres read shows up as divergence in compare mode. + -- + -- createdAt is always toISOString() output, one fixed-width UTC format, so a lexicographic + -- compare is a chronological compare. Walking newest-first lets the scan stop at the first + -- entry at or before the cursor, which makes its length the length of the ANSWER rather + -- than the length of the run's history. + local out = { '', '' } + local headId = nil + local offset = 0 + local page = limit + local done = false + + while not done do + local ids = redis.call('ZREVRANGE', idxKey, offset, offset + page - 1) + if #ids == 0 then break end + + for i = 1, #ids do + local id = ids[i] + local vals = redis.call('HMGET', eKey, id, id .. '#s', id .. '#c') + if vals[1] then + local createdAt = cjson.decode(vals[1])['createdAt'] + if not createdAt or createdAt <= cursor then + done = true + break + end + if not headId then headId = id end + out[#out + 1] = id + out[#out + 1] = vals[1] + out[#out + 1] = vals[2] or '' + out[#out + 1] = vals[3] or '' + if (#out - 2) / 4 >= limit then + done = true + break + end + end + end + + offset = offset + page + end + + if headId then + out[2] = orderFor(redis.call('HGET', eKey, headId .. '#c')) + end + return out + `, + }); + this.redis.defineCommand("readSnapshotsSince", { numberOfKeys: 4, lua: ` @@ -780,6 +898,15 @@ declare module "@internal/redis" { id: string, callback?: Callback ): Result; + readSnapshotsSinceCreatedAt( + eKey: string, + idxKey: string, + curKey: string, + seqKey: string, + createdAtCursor: string, + limit: string, + callback?: Callback + ): Result; readSnapshotsSince( eKey: string, idxKey: string, From f73d80633f3bc5d9fe6cd9e4f8a8049f2deab017 Mon Sep 17 00:00:00 2001 From: Daniel Sutton Date: Mon, 24 Aug 2026 16:05:36 +0100 Subject: [PATCH 08/56] feat(run-store): serve snapshot reads from Redis with a Postgres fallback Two of the five snapshot reads take arbitrary Prisma arguments, and a key-value store cannot answer an arbitrary query. Only three production call sites exist, all in the engine's executionSnapshotSystem, and both generic ones send a single fixed shape, so the decorator recognises exactly those shapes and delegates everything else. Each matcher rejects an argument object carrying a key it does not know, because a query that has drifted must be answered correctly by Postgres rather than approximately from Redis. A miss is the coexistence path, not an error: a pre-cutover run or expired history falls back to Postgres. The entry supplies every scalar column, and the checkpoint and waitpoint rows are read back through the delegate only when the entry says they exist, so the common read of a running run makes no Postgres call at all. Which runs read from Redis is a hash of the run id, so a run does not change store between two reads of one poll, two instances of the same dial agree, and raising the dial only ever adds runs to the cohort. --- .../run-store/src/snapshotReadShapes.test.ts | 151 +++++++ .../run-store/src/snapshotReadShapes.ts | 95 +++++ ...nExecutionSnapshotStore.readCohort.test.ts | 95 +++++ ...askRunExecutionSnapshotStore.reads.test.ts | 400 ++++++++++++++++++ .../src/taskRunExecutionSnapshotStore.ts | 242 ++++++++++- 5 files changed, 982 insertions(+), 1 deletion(-) create mode 100644 internal-packages/run-store/src/snapshotReadShapes.test.ts create mode 100644 internal-packages/run-store/src/snapshotReadShapes.ts create mode 100644 internal-packages/run-store/src/taskRunExecutionSnapshotStore.readCohort.test.ts create mode 100644 internal-packages/run-store/src/taskRunExecutionSnapshotStore.reads.test.ts diff --git a/internal-packages/run-store/src/snapshotReadShapes.test.ts b/internal-packages/run-store/src/snapshotReadShapes.test.ts new file mode 100644 index 00000000000..afa33dd3da2 --- /dev/null +++ b/internal-packages/run-store/src/snapshotReadShapes.test.ts @@ -0,0 +1,151 @@ +// A matcher that is too loose is the dangerous failure: it answers a query Redis cannot actually +// serve, and the caller gets a wrong answer rather than a slow one. So most of these tests are +// about what must NOT match. +import { describe, expect, it } from "vitest"; +import { matchSinceCursorLookup, matchSinceWindow } from "./snapshotReadShapes.js"; + +const cursorArgs = { + where: { id: "snap_1", runId: "run_1" }, + select: { createdAt: true }, +}; + +const windowArgs = { + where: { runId: "run_1", isValid: true, createdAt: { gt: new Date("2026-08-24T00:00:00Z") } }, + include: { checkpoint: true }, + orderBy: { createdAt: "desc" }, + take: 50, +}; + +describe("matchSinceCursorLookup", () => { + it("matches the engine's since-cursor lookup", () => { + expect(matchSinceCursorLookup(cursorArgs)).toEqual({ id: "snap_1", runId: "run_1" }); + }); + + it("carries an environment scope when present", () => { + expect( + matchSinceCursorLookup({ + where: { ...cursorArgs.where, environmentId: "env_1" }, + select: { createdAt: true }, + }) + ).toEqual({ id: "snap_1", runId: "run_1", environmentId: "env_1" }); + }); + + it("ignores keys explicitly set to undefined", () => { + expect( + matchSinceCursorLookup({ + where: { ...cursorArgs.where, environmentId: undefined }, + select: { createdAt: true }, + }) + ).toEqual({ id: "snap_1", runId: "run_1" }); + }); + + it("refuses a selection of anything but createdAt", () => { + expect( + matchSinceCursorLookup({ where: cursorArgs.where, select: { description: true } }) + ).toBeUndefined(); + expect( + matchSinceCursorLookup({ + where: cursorArgs.where, + select: { createdAt: true, description: true }, + }) + ).toBeUndefined(); + }); + + it("refuses a where with no run id, because there is no keyspace to look in", () => { + expect( + matchSinceCursorLookup({ where: { id: "snap_1" }, select: { createdAt: true } }) + ).toBeUndefined(); + }); + + it("refuses an unknown where key", () => { + expect( + matchSinceCursorLookup({ + where: { ...cursorArgs.where, isValid: true }, + select: { createdAt: true }, + }) + ).toBeUndefined(); + }); + + it("refuses an unknown top-level key", () => { + expect(matchSinceCursorLookup({ ...cursorArgs, orderBy: { createdAt: "desc" } })).toBeUndefined(); + }); + + it("refuses anything that is not an argument object", () => { + expect(matchSinceCursorLookup(undefined)).toBeUndefined(); + expect(matchSinceCursorLookup(null)).toBeUndefined(); + expect(matchSinceCursorLookup("where")).toBeUndefined(); + expect(matchSinceCursorLookup([cursorArgs])).toBeUndefined(); + }); +}); + +describe("matchSinceWindow", () => { + it("matches the engine's window query", () => { + expect(matchSinceWindow(windowArgs)).toEqual({ + runId: "run_1", + createdAt: new Date("2026-08-24T00:00:00Z"), + take: 50, + }); + }); + + it("carries an environment scope when present", () => { + expect( + matchSinceWindow({ + ...windowArgs, + where: { ...windowArgs.where, environmentId: "env_1" }, + }) + ).toMatchObject({ environmentId: "env_1" }); + }); + + it("refuses a query that also wants the completed waitpoints", () => { + // The engine omits them on purpose to avoid an N x M read. An include that asks for them is a + // different query, and answering it from this path would return them empty. + expect( + matchSinceWindow({ + ...windowArgs, + include: { checkpoint: true, completedWaitpoints: true }, + }) + ).toBeUndefined(); + }); + + it("refuses ascending order", () => { + expect( + matchSinceWindow({ ...windowArgs, orderBy: { createdAt: "asc" } }) + ).toBeUndefined(); + }); + + it("refuses a window that does not filter to valid entries", () => { + expect( + matchSinceWindow({ ...windowArgs, where: { ...windowArgs.where, isValid: false } }) + ).toBeUndefined(); + }); + + it("refuses a cursor that is not a strict greater-than on a Date", () => { + expect( + matchSinceWindow({ + ...windowArgs, + where: { ...windowArgs.where, createdAt: { gte: new Date() } }, + }) + ).toBeUndefined(); + expect( + matchSinceWindow({ + ...windowArgs, + where: { ...windowArgs.where, createdAt: { gt: "2026-08-24T00:00:00Z" } }, + }) + ).toBeUndefined(); + }); + + it("refuses a missing take", () => { + const { take: _dropped, ...withoutTake } = windowArgs; + expect(matchSinceWindow(withoutTake)).toBeUndefined(); + }); + + it("refuses an unknown where key", () => { + expect( + matchSinceWindow({ ...windowArgs, where: { ...windowArgs.where, batchId: "batch_1" } }) + ).toBeUndefined(); + }); + + it("refuses an unknown top-level key", () => { + expect(matchSinceWindow({ ...windowArgs, skip: 10 })).toBeUndefined(); + }); +}); diff --git a/internal-packages/run-store/src/snapshotReadShapes.ts b/internal-packages/run-store/src/snapshotReadShapes.ts new file mode 100644 index 00000000000..31dda17f7f0 --- /dev/null +++ b/internal-packages/run-store/src/snapshotReadShapes.ts @@ -0,0 +1,95 @@ +// Shape matchers for the two generic Prisma-args snapshot reads. +// +// `findExecutionSnapshot` and `findManyExecutionSnapshots` take arbitrary Prisma arguments, and a +// key-value store cannot answer an arbitrary query. Only three production call sites exist, all in +// the engine's executionSnapshotSystem, and both generic ones send a single fixed shape. So these +// matchers recognise exactly those shapes and return undefined for anything else, which sends the +// call to Postgres. +// +// Each matcher rejects an argument object carrying any key it does not know about. A query that has +// drifted must fall through and be answered correctly by Postgres, never answered approximately +// from Redis. + +type Unknown = Record; + +function isPlainObject(value: unknown): value is Unknown { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +/** True when `value` has exactly `allowed` keys, ignoring keys explicitly set to undefined. */ +function hasOnlyKeys(value: Unknown, allowed: string[]): boolean { + const present = Object.keys(value).filter((k) => value[k] !== undefined); + return present.every((k) => allowed.includes(k)); +} + +function isString(value: unknown): value is string { + return typeof value === "string"; +} + +export type SinceCursorLookup = { id: string; runId: string; environmentId?: string }; + +/** + * Step 1 of `getExecutionSnapshotsSince`: resolve a known snapshot id to its createdAt. + * + * { where: { id, runId, environmentId? }, select: { createdAt: true } } + */ +export function matchSinceCursorLookup(args: unknown): SinceCursorLookup | undefined { + if (!isPlainObject(args) || !hasOnlyKeys(args, ["where", "select"])) return undefined; + + const { where, select } = args; + if (!isPlainObject(where) || !isPlainObject(select)) return undefined; + if (!hasOnlyKeys(where, ["id", "runId", "environmentId"])) return undefined; + if (!hasOnlyKeys(select, ["createdAt"]) || select.createdAt !== true) return undefined; + if (!isString(where.id) || !isString(where.runId)) return undefined; + if (where.environmentId !== undefined && !isString(where.environmentId)) return undefined; + + return { + id: where.id, + runId: where.runId, + ...(isString(where.environmentId) && { environmentId: where.environmentId }), + }; +} + +export type SinceWindow = { + runId: string; + createdAt: Date; + take: number; + environmentId?: string; +}; + +/** + * Step 2 of `getExecutionSnapshotsSince`: the capped window after a createdAt cursor. + * + * { where: { runId, isValid: true, createdAt: { gt }, environmentId? }, + * include: { checkpoint: true }, orderBy: { createdAt: "desc" }, take: N } + * + * The engine deliberately omits completedWaitpoints from the include to avoid an N x M read, so an + * include asking for them is a different query and is not matched. + */ +export function matchSinceWindow(args: unknown): SinceWindow | undefined { + if (!isPlainObject(args) || !hasOnlyKeys(args, ["where", "include", "orderBy", "take"])) { + return undefined; + } + + const { where, include, orderBy, take } = args; + if (!isPlainObject(where) || !isPlainObject(include) || !isPlainObject(orderBy)) return undefined; + if (typeof take !== "number") return undefined; + + if (!hasOnlyKeys(where, ["runId", "isValid", "createdAt", "environmentId"])) return undefined; + if (!isString(where.runId) || where.isValid !== true) return undefined; + if (where.environmentId !== undefined && !isString(where.environmentId)) return undefined; + + if (!hasOnlyKeys(include, ["checkpoint"]) || include.checkpoint !== true) return undefined; + if (!hasOnlyKeys(orderBy, ["createdAt"]) || orderBy.createdAt !== "desc") return undefined; + + const cursor = where.createdAt; + if (!isPlainObject(cursor) || !hasOnlyKeys(cursor, ["gt"])) return undefined; + if (!(cursor.gt instanceof Date)) return undefined; + + return { + runId: where.runId, + createdAt: cursor.gt, + take, + ...(isString(where.environmentId) && { environmentId: where.environmentId }), + }; +} diff --git a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.readCohort.test.ts b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.readCohort.test.ts new file mode 100644 index 00000000000..12ad2e5fc54 --- /dev/null +++ b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.readCohort.test.ts @@ -0,0 +1,95 @@ +// The read cohort is pure arithmetic on the run id, so it needs no containers. Keeping it out of the +// container-backed suite also keeps that suite small enough to run reliably. +import { describe, expect, it } from "vitest"; +import { RedisSnapshotStore } from "./redisSnapshotStore.js"; +import { + TaskRunExecutionSnapshotStore, + type SnapshotStoreMode, +} from "./taskRunExecutionSnapshotStore.js"; +import type { RunStore } from "./types.js"; + +type CohortProbe = { readsFromRedis(runId: string): boolean }; + +function probe(mode: SnapshotStoreMode, readPercent: number): CohortProbe { + // lazyConnect keeps the client from dialling anything: no read in this suite reaches the store. + const redis = new RedisSnapshotStore({ + redisOptions: { host: "127.0.0.1", port: 1, lazyConnect: true, retryStrategy: () => null }, + completedTtlMs: 1, + }); + + return new TaskRunExecutionSnapshotStore({} as RunStore, { + store: redis, + mode, + readPercent, + }) as unknown as CohortProbe; +} + +const ids = Array.from({ length: 500 }, (_, n) => `run_cohort_${n}_${n * 7919}`); + +describe("the read cohort", () => { + it("reads nothing from Redis before the read positions", () => { + for (const mode of ["off", "dual-write", "compare"] as const) { + const store = probe(mode, 100); + expect(ids.every((id) => !store.readsFromRedis(id))).toBe(true); + } + }); + + it("reads everything from Redis at 100 percent", () => { + for (const mode of ["redis-read", "redis-only"] as const) { + const store = probe(mode, 100); + expect(ids.every((id) => store.readsFromRedis(id))).toBe(true); + } + }); + + it("reads nothing from Redis at 0 percent", () => { + const store = probe("redis-read", 0); + expect(ids.every((id) => !store.readsFromRedis(id))).toBe(true); + }); + + it("gives one run the same answer every time", () => { + // A run that changed store between two reads of one poll could show the log going backwards. + const store = probe("redis-read", 50); + + for (const id of ids.slice(0, 50)) { + const first = store.readsFromRedis(id); + for (let i = 0; i < 5; i++) { + expect(store.readsFromRedis(id)).toBe(first); + } + } + }); + + it("gives two instances of the same dial the same answer", () => { + // The cohort must not depend on process state, or a redeploy reshuffles every in-flight run. + const first = probe("redis-read", 50); + const second = probe("redis-read", 50); + + for (const id of ids.slice(0, 50)) { + expect(second.readsFromRedis(id)).toBe(first.readsFromRedis(id)); + } + }); + + it("spreads a population across the dial", () => { + const store = probe("redis-read", 50); + const enabled = ids.filter((id) => store.readsFromRedis(id)).length; + + // A wide band: this asserts the hash spreads at all, not that it is uniform. + expect(enabled).toBeGreaterThan(150); + expect(enabled).toBeLessThan(350); + }); + + it("grows the cohort monotonically as the dial rises", () => { + const at = (percent: number) => { + const store = probe("redis-read", percent); + return new Set(ids.filter((id) => store.readsFromRedis(id))); + }; + + const ten = at(10); + const fifty = at(50); + const ninety = at(90); + + // Raising the dial must only ever add runs. A run that fell out on the way up would flip back to + // Postgres mid-flight, which is the thing the stable hash exists to prevent. + expect([...ten].every((id) => fifty.has(id))).toBe(true); + expect([...fifty].every((id) => ninety.has(id))).toBe(true); + }); +}); diff --git a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.reads.test.ts b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.reads.test.ts new file mode 100644 index 00000000000..ffdd91a92f9 --- /dev/null +++ b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.reads.test.ts @@ -0,0 +1,400 @@ +// Reads served from Redis must be indistinguishable from the Postgres reads they replace: the same +// payload shape, the same tenant boundary, the same fallback when Redis does not hold the answer. +import { describe, expect } from "vitest"; +import { postgresAndRedisTest } from "@internal/testcontainers"; +import { generateInternalId } from "@trigger.dev/core/v3/isomorphic"; +import { PostgresRunStore } from "./PostgresRunStore.js"; +import { RedisSnapshotStore } from "./redisSnapshotStore.js"; +import { + TaskRunExecutionSnapshotStore, + type SnapshotStoreMode, +} from "./taskRunExecutionSnapshotStore.js"; +import type { RunStore } from "./types.js"; +import { + buildCreateRunData, + seedSnapshotEnvironment, + type SnapshotFixtureEnv, +} from "./testFixtures/snapshotIdFixture.js"; + +const COMPLETED_TTL_MS = 72 * 60 * 60 * 1000; + +function build( + prisma: never, + redisOptions: never, + opts?: { mode?: SnapshotStoreMode; readPercent?: number } +) { + const redis = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + const reads: { method: string; source: string }[] = []; + + const decorated = new TaskRunExecutionSnapshotStore( + new PostgresRunStore({ prisma, readOnlyPrisma: prisma }) as unknown as RunStore, + { + store: redis, + mode: opts?.mode ?? "redis-read", + readPercent: opts?.readPercent ?? 100, + metrics: { + recordWrite: () => {}, + recordAppendFailed: () => {}, + recordRead: (method, source) => reads.push({ method, source }), + }, + } + ); + + return { decorated, redis, reads }; +} + +async function seedRun( + decorated: TaskRunExecutionSnapshotStore, + env: SnapshotFixtureEnv +): Promise { + const runId = generateInternalId(); + await decorated.createRun({ + data: buildCreateRunData(runId, env), + snapshot: { + id: generateInternalId(), + engine: "V2", + executionStatus: "RUN_CREATED", + description: "Run was created", + runStatus: "PENDING", + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }, + }); + return runId; +} + +function snapshotInput(runId: string, env: SnapshotFixtureEnv, description: string) { + return { + run: { id: runId, status: "EXECUTING" as const, attemptNumber: 1 }, + snapshot: { executionStatus: "EXECUTING" as const, description }, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }; +} + +describe("snapshot reads", () => { + postgresAndRedisTest("serves the latest snapshot from Redis", async ({ prisma, redisOptions }) => { + const { decorated, redis, reads } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, env); + const created = await decorated.createExecutionSnapshot( + snapshotInput(runId, env, "Run started") + ); + + const latest = await decorated.findLatestExecutionSnapshot(runId); + + expect(latest).not.toBeNull(); + expect(latest!.id).toBe(created.id); + expect(latest!.executionStatus).toBe("EXECUTING"); + expect(latest!.description).toBe("Run started"); + expect(latest!.runId).toBe(runId); + expect(latest!.checkpoint).toBeNull(); + expect(latest!.completedWaitpoints).toEqual([]); + expect(reads).toContainEqual({ method: "findLatestExecutionSnapshot", source: "redis" }); + } finally { + await redis.quit(); + } + }); + + postgresAndRedisTest( + "returns the same payload Postgres would", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + const postgresOnly = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, env); + await decorated.createExecutionSnapshot(snapshotInput(runId, env, "Run started")); + + const fromRedis = await decorated.findLatestExecutionSnapshot(runId); + const fromPostgres = await postgresOnly.findLatestExecutionSnapshot(runId); + + expect(fromRedis!.id).toBe(fromPostgres!.id); + expect(fromRedis!.executionStatus).toBe(fromPostgres!.executionStatus); + expect(fromRedis!.description).toBe(fromPostgres!.description); + expect(fromRedis!.runStatus).toBe(fromPostgres!.runStatus); + expect(fromRedis!.attemptNumber).toBe(fromPostgres!.attemptNumber); + expect(fromRedis!.isValid).toBe(fromPostgres!.isValid); + expect(fromRedis!.environmentId).toBe(fromPostgres!.environmentId); + expect(fromRedis!.createdAt.toISOString()).toBe(fromPostgres!.createdAt.toISOString()); + } finally { + await redis.quit(); + } + } + ); + + postgresAndRedisTest( + "reads a foreign environment as not found, so the caller's 404 still fires", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, env); + await decorated.createExecutionSnapshot(snapshotInput(runId, env, "Run started")); + + const foreign = await decorated.findLatestExecutionSnapshot(runId, undefined, "env_other"); + + expect(foreign).toBeNull(); + } finally { + await redis.quit(); + } + } + ); + + postgresAndRedisTest( + "falls back to Postgres for a run with no keyspace", + async ({ prisma, redisOptions }) => { + const { decorated, redis, reads } = build(prisma as never, redisOptions as never); + const postgresOnly = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + // A pre-cutover run: it exists in Postgres and Redis has never seen it. + await postgresOnly.createRun({ + data: buildCreateRunData(runId, env), + snapshot: { + id: generateInternalId(), + engine: "V2", + executionStatus: "RUN_CREATED", + description: "Run was created", + runStatus: "PENDING", + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }, + }); + + const latest = await decorated.findLatestExecutionSnapshot(runId); + + expect(latest).not.toBeNull(); + expect(latest!.executionStatus).toBe("RUN_CREATED"); + expect(reads).toContainEqual({ method: "findLatestExecutionSnapshot", source: "postgres" }); + } finally { + await redis.quit(); + } + } + ); + + postgresAndRedisTest("reads from Postgres at readPercent 0", async ({ prisma, redisOptions }) => { + const { decorated, redis, reads } = build(prisma as never, redisOptions as never, { + readPercent: 0, + }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, env); + + const latest = await decorated.findLatestExecutionSnapshot(runId); + + expect(latest).not.toBeNull(); + expect(reads).toEqual([]); + } finally { + await redis.quit(); + } + }); + + postgresAndRedisTest("reads from Postgres at mode dual-write", async ({ prisma, redisOptions }) => { + const { decorated, redis, reads } = build(prisma as never, redisOptions as never, { + mode: "dual-write", + }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, env); + + const latest = await decorated.findLatestExecutionSnapshot(runId); + + expect(latest).not.toBeNull(); + expect(reads).toEqual([]); + } finally { + await redis.quit(); + } + }); + + postgresAndRedisTest("serves the since-cursor lookup from Redis", async ({ prisma, redisOptions }) => { + const { decorated, redis, reads } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, env); + const created = await decorated.createExecutionSnapshot( + snapshotInput(runId, env, "Run started") + ); + + const cursor = await decorated.findExecutionSnapshot({ + where: { id: created.id, runId }, + select: { createdAt: true }, + }); + + expect(cursor).not.toBeNull(); + expect((cursor as { createdAt: Date }).createdAt.toISOString()).toBe( + created.createdAt.toISOString() + ); + expect(reads).toContainEqual({ method: "findExecutionSnapshot", source: "redis" }); + } finally { + await redis.quit(); + } + }); + + postgresAndRedisTest( + "delegates a snapshot lookup it does not recognise", + async ({ prisma, redisOptions }) => { + const { decorated, redis, reads } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, env); + const created = await decorated.createExecutionSnapshot( + snapshotInput(runId, env, "Run started") + ); + + // A different selection: Redis must not answer it approximately. + const row = await decorated.findExecutionSnapshot({ + where: { id: created.id }, + select: { description: true }, + }); + + expect(row).toEqual({ description: "Run started" }); + expect(reads.filter((r) => r.method === "findExecutionSnapshot")).toEqual([]); + } finally { + await redis.quit(); + } + } + ); + + postgresAndRedisTest("serves the since window from Redis", async ({ prisma, redisOptions }) => { + const { decorated, redis, reads } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, env); + const first = await decorated.createExecutionSnapshot(snapshotInput(runId, env, "First")); + await new Promise((resolve) => setTimeout(resolve, 5)); + const second = await decorated.createExecutionSnapshot(snapshotInput(runId, env, "Second")); + await new Promise((resolve) => setTimeout(resolve, 5)); + const third = await decorated.createExecutionSnapshot(snapshotInput(runId, env, "Third")); + + const window = await decorated.findManyExecutionSnapshots({ + where: { runId, isValid: true, createdAt: { gt: first.createdAt } }, + include: { checkpoint: true }, + orderBy: { createdAt: "desc" }, + take: 50, + }); + + // Descending, exactly as the engine asked; it reverses app-side. + expect(window.map((s) => s.id)).toEqual([third.id, second.id]); + expect(reads).toContainEqual({ method: "findManyExecutionSnapshots", source: "redis" }); + } finally { + await redis.quit(); + } + }); + + postgresAndRedisTest( + "delegates a window query it does not recognise", + async ({ prisma, redisOptions }) => { + const { decorated, redis, reads } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, env); + await decorated.createExecutionSnapshot(snapshotInput(runId, env, "First")); + + const rows = await decorated.findManyExecutionSnapshots({ + where: { runId }, + orderBy: { createdAt: "asc" }, + }); + + expect(rows.length).toBeGreaterThan(0); + expect(reads.filter((r) => r.method === "findManyExecutionSnapshots")).toEqual([]); + } finally { + await redis.quit(); + } + } + ); + + postgresAndRedisTest( + "serves the waitpoint id projections from Redis", + async ({ prisma, redisOptions }) => { + const { decorated, redis, reads } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, env); + const created = await decorated.createExecutionSnapshot( + snapshotInput(runId, env, "Run started") + ); + + const ids = await decorated.findSnapshotCompletedWaitpointIds(created.id, undefined, runId); + const withPresence = await decorated.findSnapshotCompletedWaitpointIdsWithPresence( + created.id, + undefined, + runId + ); + + expect(ids).toEqual([]); + // present distinguishes "no waitpoints" from "this reader cannot see the snapshot", which is + // what the engine's read-repair keys off. + expect(withPresence).toEqual({ present: true, ids: [] }); + expect(reads).toContainEqual({ + method: "findSnapshotCompletedWaitpointIds", + source: "redis", + }); + } finally { + await redis.quit(); + } + } + ); + + postgresAndRedisTest( + "delegates a waitpoint id projection with no run id", + async ({ prisma, redisOptions }) => { + const { decorated, redis, reads } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, env); + const created = await decorated.createExecutionSnapshot( + snapshotInput(runId, env, "Run started") + ); + + // Without a run id there is no keyspace to look in. + const ids = await decorated.findSnapshotCompletedWaitpointIds(created.id); + + expect(ids).toEqual([]); + expect(reads.filter((r) => r.method.startsWith("findSnapshot"))).toEqual([]); + } finally { + await redis.quit(); + } + } + ); + + postgresAndRedisTest("never touches Redis for reads at mode off", async ({ prisma, redisOptions }) => { + const { decorated, redis, reads } = build(prisma as never, redisOptions as never, { + mode: "off", + }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + await decorated.createRun({ + data: buildCreateRunData(runId, env), + snapshot: { + id: generateInternalId(), + engine: "V2", + executionStatus: "RUN_CREATED", + description: "Run was created", + runStatus: "PENDING", + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }, + }); + + const latest = await decorated.findLatestExecutionSnapshot(runId); + + expect(latest).not.toBeNull(); + expect(await redis.getLatest(runId)).toBeNull(); + expect(reads).toEqual([]); + } finally { + await redis.quit(); + } + }); +}); diff --git a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.ts b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.ts index 69658ecaad1..30655d727db 100644 --- a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.ts +++ b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.ts @@ -14,7 +14,7 @@ import { Logger } from "@trigger.dev/core/logger"; import { generateInternalId } from "@trigger.dev/core/v3/isomorphic"; import { DelegatingRunStore } from "./delegatingRunStore.js"; -import type { RedisSnapshotStore, SnapshotEntryInput } from "./redisSnapshotStore.js"; +import type { RedisSnapshotStore, SnapshotEntryInput, SnapshotRead } from "./redisSnapshotStore.js"; import { entryFromCompletion, entryFromCreateExecutionSnapshot, @@ -26,6 +26,7 @@ import { } from "./snapshotEntry.js"; import { isInjectedFault, type SnapshotFaultInjector } from "./snapshotFaultInjection.js"; import type { + ReadClient, CompletionSnapshotInput, CreateCancelledRunInput, CreateExecutionSnapshotInput, @@ -36,6 +37,10 @@ import type { RunStore, TaskRunWithWaitpoint, } from "./types.js"; +import { + matchSinceCursorLookup, + matchSinceWindow, +} from "./snapshotReadShapes.js"; import type { Prisma, PrismaClientOrTransaction, TaskRun } from "@trigger.dev/database"; /** One initial attempt plus three retries, per the write protocol. */ @@ -515,6 +520,241 @@ export class TaskRunExecutionSnapshotStore extends DelegatingRunStore { } } + // --------------------------------------------------------------------------------------------- + // Reads. + // + // Only three production call sites exist, all in the engine's executionSnapshotSystem, all with + // fixed argument shapes. The two generic Prisma-args methods therefore recognise exactly the + // shapes the engine sends and delegate everything else: an unrecognised shape must go to Postgres, + // never get an approximate answer from Redis. + // --------------------------------------------------------------------------------------------- + + /** + * Whether this run's reads come from Redis. Hashed on the run id so a run does not change store + * between two reads of the same poll, which would let a caller see the log go backwards. + */ + protected readsFromRedis(runId: string): boolean { + if (this.mode !== "redis-read" && this.mode !== "redis-only") return false; + if (this.readPercent >= 100) return true; + if (this.readPercent <= 0) return false; + + let hash = 0; + for (let i = 0; i < runId.length; i++) { + hash = (hash * 31 + runId.charCodeAt(i)) >>> 0; + } + return hash % 100 < this.readPercent; + } + + override async findLatestExecutionSnapshot( + runId: string, + client?: ReadClient, + environmentId?: string + ): Promise | null> { + if (!this.readsFromRedis(runId)) { + return this.delegate.findLatestExecutionSnapshot(runId, client, environmentId); + } + + const read = await this.redis.getLatest(runId, { ...(environmentId && { environmentId }) }); + if (!read) { + // A miss is the coexistence path: a pre-cutover run, or expired history. It is not an error. + this.metrics?.recordRead("findLatestExecutionSnapshot", "postgres"); + return this.delegate.findLatestExecutionSnapshot(runId, client, environmentId); + } + + this.metrics?.recordRead("findLatestExecutionSnapshot", "redis"); + return this.#hydrate(read, runId, client); + } + + override async findExecutionSnapshot( + args: Prisma.SelectSubset, + client?: ReadClient + ): Promise | null> { + const shape = matchSinceCursorLookup(args); + if (!shape || !this.readsFromRedis(shape.runId)) { + return this.delegate.findExecutionSnapshot(args, client); + } + + const found = await this.redis.getById(shape.runId, shape.id, { + ...(shape.environmentId && { environmentId: shape.environmentId }), + }); + + if (!found) { + this.metrics?.recordRead("findExecutionSnapshot", "postgres"); + return this.delegate.findExecutionSnapshot(args, client); + } + + this.metrics?.recordRead("findExecutionSnapshot", "redis"); + // The engine selects createdAt only, so the answer is the cursor and nothing else. + return { + createdAt: new Date(found.entry.createdAt as string), + } as unknown as Prisma.TaskRunExecutionSnapshotGetPayload; + } + + override async findManyExecutionSnapshots< + T extends Prisma.TaskRunExecutionSnapshotFindManyArgs, + >( + args: Prisma.SelectSubset, + client?: ReadClient + ): Promise[]> { + const shape = matchSinceWindow(args); + if (!shape || !this.readsFromRedis(shape.runId)) { + return this.delegate.findManyExecutionSnapshots(args, client); + } + + const result = await this.redis.getSinceCreatedAt(shape.runId, shape.createdAt, { + limit: shape.take, + ...(shape.environmentId && { environmentId: shape.environmentId }), + }); + + if (result.kind === "miss") { + this.metrics?.recordRead("findManyExecutionSnapshots", "postgres"); + return this.delegate.findManyExecutionSnapshots(args, client); + } + + this.metrics?.recordRead("findManyExecutionSnapshots", "redis"); + + // The engine asks for createdAt DESC and reverses app-side; the store returns ascending. + const descending = [...result.entries].reverse(); + const hydrated = await Promise.all( + descending.map((entry) => this.#hydrate(entry, shape.runId, client, { waitpoints: false })) + ); + return hydrated as unknown as Prisma.TaskRunExecutionSnapshotGetPayload[]; + } + + override async findSnapshotCompletedWaitpointIds( + snapshotId: string, + client?: ReadClient, + runId?: string + ): Promise { + // Without a run id there is no keyspace to look in, so the router's fan-out is the only answer. + if (!runId || !this.readsFromRedis(runId)) { + return this.delegate.findSnapshotCompletedWaitpointIds(snapshotId, client, runId); + } + + const ids = await this.redis.getSnapshotWaitpointIds(runId, snapshotId); + if (!ids.present) { + this.metrics?.recordRead("findSnapshotCompletedWaitpointIds", "postgres"); + return this.delegate.findSnapshotCompletedWaitpointIds(snapshotId, client, runId); + } + + this.metrics?.recordRead("findSnapshotCompletedWaitpointIds", "redis"); + return ids.distinctIds; + } + + override async findSnapshotCompletedWaitpointIdsWithPresence( + snapshotId: string, + client?: ReadClient, + runId?: string + ): Promise<{ present: boolean; ids: string[] }> { + if (!runId || !this.readsFromRedis(runId)) { + return this.delegate.findSnapshotCompletedWaitpointIdsWithPresence(snapshotId, client, runId); + } + + const ids = await this.redis.getSnapshotWaitpointIds(runId, snapshotId); + if (!ids.present) { + // present=false means this reader cannot see the snapshot, so its empty list is not + // authoritative and the engine's read-repair needs the Postgres answer. + this.metrics?.recordRead("findSnapshotCompletedWaitpointIdsWithPresence", "postgres"); + return this.delegate.findSnapshotCompletedWaitpointIdsWithPresence(snapshotId, client, runId); + } + + this.metrics?.recordRead("findSnapshotCompletedWaitpointIdsWithPresence", "redis"); + return { present: true, ids: ids.distinctIds }; + } + + /** + * Turns a store entry into the Prisma payload the interface promises. + * + * The entry supplies every scalar column. `checkpoint` and the full waitpoint rows still live in + * Postgres, so they are read back through the delegate — but only when the entry says they exist, + * which keeps the common read (a running run with neither) free of any Postgres call at all. + */ + async #hydrate( + read: SnapshotRead, + runId: string, + client?: ReadClient, + opts?: { waitpoints?: boolean } + ): Promise< + Prisma.TaskRunExecutionSnapshotGetPayload<{ + include: { completedWaitpoints: true; checkpoint: true }; + }> + > { + const entry = read.entry as Record; + + const checkpoint = entry.checkpointId + ? await this.#hydrateCheckpoint(runId, read.id, client) + : null; + + let completedWaitpoints: unknown[] = []; + let completedWaitpointOrder: string[] = []; + + if (opts?.waitpoints !== false) { + const ids = + read.completedWaitpointIds ?? (await this.redis.getSnapshotWaitpointIds(runId, read.id)); + completedWaitpointOrder = ids.order; + + if (ids.distinctIds.length > 0) { + completedWaitpoints = await this.delegate.findManyWaitpoints( + { where: { id: { in: ids.distinctIds } } }, + client, + runId + ); + } + } + + return { + id: read.id, + engine: entry.engine ?? "V2", + executionStatus: entry.executionStatus, + description: entry.description, + previousSnapshotId: entry.previousSnapshotId ?? null, + runId: entry.runId, + runStatus: entry.runStatus, + attemptNumber: entry.attemptNumber ?? null, + batchId: entry.batchId ?? null, + environmentId: entry.environmentId, + environmentType: entry.environmentType, + projectId: entry.projectId, + organizationId: entry.organizationId, + checkpointId: entry.checkpointId ?? null, + workerId: entry.workerId ?? null, + runnerId: entry.runnerId ?? null, + metadata: entry.metadata ?? null, + completedWaitpointOrder, + isValid: read.isValid, + error: entry.error ?? null, + createdAt: new Date(entry.createdAt as string), + updatedAt: new Date(entry.createdAt as string), + checkpoint, + completedWaitpoints, + } as unknown as Prisma.TaskRunExecutionSnapshotGetPayload<{ + include: { completedWaitpoints: true; checkpoint: true }; + }>; + } + + /** + * Reads the checkpoint row through the snapshot the delegate still holds, so the read stays + * residency-aware: the run id in the where is what routes it to the owning database, and the + * decorator sits above the router and has no client of its own. + * + * At `redis-only` the Postgres snapshot row is gone, so this returns null. The checkpoint row + * itself stays in Postgres, but the interface has no residency-aware way to read one directly. + * Closing that needs a narrow lookup on the interface, which the plan freezes for this ticket. + */ + async #hydrateCheckpoint( + runId: string, + snapshotId: string, + client?: ReadClient + ): Promise { + const row = await this.delegate.findExecutionSnapshot( + { where: { id: snapshotId, runId }, include: { checkpoint: true } }, + client + ); + return (row as { checkpoint?: unknown } | null)?.checkpoint ?? null; + } + async #enqueueRepair(entry: SnapshotEntryInput): Promise { if (!this.onAppendFailure) { return; From 910039be4870c1e38ad5bd48e37c87779d31df63 Mon Sep 17 00:00:00 2001 From: Daniel Sutton Date: Mon, 24 Aug 2026 16:14:21 +0100 Subject: [PATCH 09/56] feat(run-store): reap orphaned snapshot keyspaces under both sweep rules MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two rules, because neither can see what the other leaves behind. A terminal run whose keyspace never received the completion expiry gets one applied, so it reaps on the schedule a healthy terminal append would have set. A keyspace with no run row at all, past an age threshold, is deleted outright — that is a crashed birth, which is non-terminal so it carries no expiry and has no run row, so the first rule can never match it. It never reaps on an unknown answer: a live run is left alone however old its keyspace, a young orphan is left for the birth that may still be in flight, and a batch whose run lookup failed is skipped rather than treated as absent. Run rows are resolved through the run store rather than a raw client, because under the run-ops split a run can live on either database and a raw lookup would report a live run as an orphan. Nothing schedules this. The engine's worker has to run it, and run-store cannot reach the engine. Also moves the decorator suites onto the worker-scoped container fixture. The per-test one boots a Postgres and a Redis container for every test, which is what the replication tests need and these do not; the sweeper suite alone went from repeated two-minute timeouts to ten seconds. --- .../src/snapshotOrphanSweeper.test.ts | 325 ++++++++++++++++++ .../run-store/src/snapshotOrphanSweeper.ts | 259 ++++++++++++++ ...skRunExecutionSnapshotStore.births.test.ts | 18 +- ...askRunExecutionSnapshotStore.reads.test.ts | 28 +- ...kRunExecutionSnapshotStore.staging.test.ts | 16 +- ...ExecutionSnapshotStore.transitions.test.ts | 26 +- 6 files changed, 628 insertions(+), 44 deletions(-) create mode 100644 internal-packages/run-store/src/snapshotOrphanSweeper.test.ts create mode 100644 internal-packages/run-store/src/snapshotOrphanSweeper.ts diff --git a/internal-packages/run-store/src/snapshotOrphanSweeper.test.ts b/internal-packages/run-store/src/snapshotOrphanSweeper.test.ts new file mode 100644 index 00000000000..68a9d99cb6d --- /dev/null +++ b/internal-packages/run-store/src/snapshotOrphanSweeper.test.ts @@ -0,0 +1,325 @@ +// The sweep deletes whole keyspaces, so most of these tests are about what it must NOT touch: a live +// run, a young orphan, and any batch whose Postgres lookup did not come back. +import { describe, expect } from "vitest"; +import { containerTest } from "@internal/testcontainers"; +import { createRedisClient } from "@internal/redis"; +import { generateInternalId } from "@trigger.dev/core/v3/isomorphic"; +import { PostgresRunStore } from "./PostgresRunStore.js"; +import { RedisSnapshotStore, snapshotKeys } from "./redisSnapshotStore.js"; +import { entryFromCreateRun } from "./snapshotEntry.js"; +import { SnapshotOrphanSweeper } from "./snapshotOrphanSweeper.js"; +import type { RunStore } from "./types.js"; +import { + buildCreateRunData, + seedSnapshotEnvironment, + type SnapshotFixtureEnv, +} from "./testFixtures/snapshotIdFixture.js"; + +const COMPLETED_TTL_MS = 72 * 60 * 60 * 1000; +const ORPHAN_AGE_MS = 60 * 60 * 1000; + +function birthEntry(runId: string, env: SnapshotFixtureEnv, createdAt: Date, terminal = false) { + const snapshot = { + id: generateInternalId(), + engine: "V2" as const, + executionStatus: terminal ? ("FINISHED" as const) : ("RUN_CREATED" as const), + description: "Run was created", + runStatus: terminal ? ("CANCELED" as const) : ("PENDING" as const), + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }; + return entryFromCreateRun({ id: snapshot.id, runId, createdAt }, snapshot); +} + +describe("SnapshotOrphanSweeper", () => { + containerTest( + "rule 1 expires a terminal run whose keyspace never got one", + async ({ prisma, redisOptions }) => { + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + const runStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const sweeper = new SnapshotOrphanSweeper({ + redisOptions, + runStore: runStore as unknown as RunStore, + completedTtlMs: COMPLETED_TTL_MS, + orphanAgeMs: ORPHAN_AGE_MS, + }); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + // Non-terminal append, so no expiry is ever set — the lost-TTL-set case. + await store.append({ entry: birthEntry(runId, env, new Date()), kind: "birth", isTerminal: false }); + await prisma.taskRun.create({ + data: { ...buildCreateRunData(runId, env), status: "COMPLETED_SUCCESSFULLY" }, + }); + + const keys = snapshotKeys(runId); + expect(await probe.pttl(keys.e)).toBe(-1); + + const result = await sweeper.sweep(); + + expect(result.expired).toBe(1); + expect(result.deleted).toBe(0); + for (const key of [keys.e, keys.idx, keys.cur, keys.seq]) { + const ttl = await probe.pttl(key); + expect(ttl).toBeGreaterThan(0); + expect(ttl).toBeLessThanOrEqual(COMPLETED_TTL_MS); + } + } finally { + await Promise.all([store.quit(), sweeper.quit(), probe.quit().catch(() => {})]); + } + } + ); + + containerTest( + "rule 1 leaves a keyspace that already has an expiry alone", + async ({ prisma, redisOptions }) => { + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + const runStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const sweeper = new SnapshotOrphanSweeper({ + redisOptions, + runStore: runStore as unknown as RunStore, + completedTtlMs: COMPLETED_TTL_MS, + orphanAgeMs: ORPHAN_AGE_MS, + }); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + // A healthy terminal append sets the completion TTL itself. + await store.append({ + entry: birthEntry(runId, env, new Date(), true), + kind: "birth", + isTerminal: true, + }); + await prisma.taskRun.create({ + data: { ...buildCreateRunData(runId, env), status: "CANCELED" }, + }); + + const before = await probe.pttl(snapshotKeys(runId).e); + const result = await sweeper.sweep(); + + expect(result.expired).toBe(0); + expect(result.skipped).toBe(1); + const after = await probe.pttl(snapshotKeys(runId).e); + // Not extended: the sweep must not keep resetting a countdown that is already running. + expect(after).toBeLessThanOrEqual(before); + } finally { + await Promise.all([store.quit(), sweeper.quit(), probe.quit().catch(() => {})]); + } + } + ); + + containerTest( + "rule 2 deletes a keyspace with no run row, cycle keys included", + async ({ prisma, redisOptions }) => { + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + const runStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const sweeper = new SnapshotOrphanSweeper({ + redisOptions, + runStore: runStore as unknown as RunStore, + completedTtlMs: COMPLETED_TTL_MS, + orphanAgeMs: ORPHAN_AGE_MS, + }); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + const old = new Date(Date.now() - 2 * ORPHAN_AGE_MS); + + // The crashed birth: an entry, no Postgres run, non-terminal so no expiry. + await store.append({ entry: birthEntry(runId, env, old), kind: "birth", isTerminal: false }); + await store.append({ + entry: birthEntry(runId, env, old), + kind: "transition", + isTerminal: false, + cycle: { kind: "new", completedWaitpoints: [{ id: "w_1", index: 0 }] }, + }); + + const cyclesBefore = await probe.keys(`snap:{${runId}}:wp:*`); + expect(cyclesBefore.length).toBeGreaterThan(0); + + const result = await sweeper.sweep(); + + expect(result.deleted).toBe(1); + const keys = snapshotKeys(runId); + for (const key of [keys.e, keys.idx, keys.cur, keys.seq]) { + expect(await probe.exists(key)).toBe(0); + } + expect(await probe.keys(`snap:{${runId}}:wp:*`)).toEqual([]); + } finally { + await Promise.all([store.quit(), sweeper.quit(), probe.quit().catch(() => {})]); + } + } + ); + + containerTest("rule 2 spares a young orphan", async ({ prisma, redisOptions }) => { + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + const runStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const sweeper = new SnapshotOrphanSweeper({ + redisOptions, + runStore: runStore as unknown as RunStore, + completedTtlMs: COMPLETED_TTL_MS, + orphanAgeMs: ORPHAN_AGE_MS, + }); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + // Written just now: the Postgres insert of a healthy birth may still be in flight. + await store.append({ entry: birthEntry(runId, env, new Date()), kind: "birth", isTerminal: false }); + + const result = await sweeper.sweep(); + + expect(result.deleted).toBe(0); + expect(result.skipped).toBe(1); + expect(await probe.exists(snapshotKeys(runId).e)).toBe(1); + } finally { + await Promise.all([store.quit(), sweeper.quit(), probe.quit().catch(() => {})]); + } + }); + + containerTest( + "never touches a live run, however old its keyspace", + async ({ prisma, redisOptions }) => { + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + const runStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const sweeper = new SnapshotOrphanSweeper({ + redisOptions, + runStore: runStore as unknown as RunStore, + completedTtlMs: COMPLETED_TTL_MS, + orphanAgeMs: ORPHAN_AGE_MS, + }); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + const ancient = new Date(Date.now() - 90 * 24 * 60 * 60 * 1000); + + // A run waiting on an untimed token can sit non-terminal for weeks. Reaping it would drop + // live state, which is the failure this rule exists to avoid. + await store.append({ + entry: birthEntry(runId, env, ancient), + kind: "birth", + isTerminal: false, + }); + await prisma.taskRun.create({ + data: { ...buildCreateRunData(runId, env), status: "WAITING_TO_RESUME" }, + }); + + const result = await sweeper.sweep(); + + expect(result.deleted).toBe(0); + expect(result.expired).toBe(0); + expect(result.skipped).toBe(1); + expect(await probe.exists(snapshotKeys(runId).e)).toBe(1); + expect(await probe.pttl(snapshotKeys(runId).e)).toBe(-1); + } finally { + await Promise.all([store.quit(), sweeper.quit(), probe.quit().catch(() => {})]); + } + } + ); + + containerTest("a dry run reports but changes nothing", async ({ prisma, redisOptions }) => { + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + const runStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const sweeper = new SnapshotOrphanSweeper({ + redisOptions, + runStore: runStore as unknown as RunStore, + completedTtlMs: COMPLETED_TTL_MS, + orphanAgeMs: ORPHAN_AGE_MS, + }); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + try { + const env = await seedSnapshotEnvironment(prisma); + const orphan = generateInternalId(); + const terminal = generateInternalId(); + const old = new Date(Date.now() - 2 * ORPHAN_AGE_MS); + + await store.append({ entry: birthEntry(orphan, env, old), kind: "birth", isTerminal: false }); + await store.append({ + entry: birthEntry(terminal, env, new Date()), + kind: "birth", + isTerminal: false, + }); + await prisma.taskRun.create({ + data: { ...buildCreateRunData(terminal, env), status: "COMPLETED_SUCCESSFULLY" }, + }); + + const result = await sweeper.sweep({ dryRun: true }); + + expect(result.deleted).toBe(1); + expect(result.expired).toBe(1); + expect(await probe.exists(snapshotKeys(orphan).e)).toBe(1); + expect(await probe.pttl(snapshotKeys(terminal).e)).toBe(-1); + } finally { + await Promise.all([store.quit(), sweeper.quit(), probe.quit().catch(() => {})]); + } + }); + + containerTest( + "skips a batch whose run lookup failed, and deletes nothing", + async ({ prisma, redisOptions }) => { + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + const failing = { + findRunsByIds: async () => { + throw new Error("run lookup unavailable"); + }, + } as unknown as RunStore; + const sweeper = new SnapshotOrphanSweeper({ + redisOptions, + runStore: failing, + completedTtlMs: COMPLETED_TTL_MS, + orphanAgeMs: ORPHAN_AGE_MS, + }); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + const old = new Date(Date.now() - 2 * ORPHAN_AGE_MS); + await store.append({ entry: birthEntry(runId, env, old), kind: "birth", isTerminal: false }); + + // A failed lookup says nothing about whether the run exists, and rule 2 deletes a whole + // keyspace. The sweep must resolve rather than throw, and must reap nothing. + const result = await sweeper.sweep(); + + expect(result.deleted).toBe(0); + expect(result.skipped).toBeGreaterThan(0); + expect(await probe.exists(snapshotKeys(runId).e)).toBe(1); + } finally { + await Promise.all([store.quit(), sweeper.quit(), probe.quit().catch(() => {})]); + } + } + ); + + containerTest("processes every keyspace across batches", async ({ prisma, redisOptions }) => { + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + const runStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const sweeper = new SnapshotOrphanSweeper({ + redisOptions, + runStore: runStore as unknown as RunStore, + completedTtlMs: COMPLETED_TTL_MS, + orphanAgeMs: ORPHAN_AGE_MS, + }); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + try { + const env = await seedSnapshotEnvironment(prisma); + const old = new Date(Date.now() - 2 * ORPHAN_AGE_MS); + const orphans = Array.from({ length: 5 }, () => generateInternalId()); + for (const runId of orphans) { + await store.append({ entry: birthEntry(runId, env, old), kind: "birth", isTerminal: false }); + } + + const result = await sweeper.sweep({ batchSize: 2 }); + + expect(result.deleted).toBe(5); + for (const runId of orphans) { + expect(await probe.exists(snapshotKeys(runId).e)).toBe(0); + } + } finally { + await Promise.all([store.quit(), sweeper.quit(), probe.quit().catch(() => {})]); + } + }); +}); diff --git a/internal-packages/run-store/src/snapshotOrphanSweeper.ts b/internal-packages/run-store/src/snapshotOrphanSweeper.ts new file mode 100644 index 00000000000..f94876da37c --- /dev/null +++ b/internal-packages/run-store/src/snapshotOrphanSweeper.ts @@ -0,0 +1,259 @@ +// Reaps snapshot keyspaces that no healthy path will ever clean up. +// +// Two rules, because neither can see what the other leaves behind: +// +// 1. The run is terminal in Postgres but its keyspace never got the completion expiry — a +// terminal append whose TTL-set was lost. Applying the expiry now reaps it on the same +// schedule a healthy terminal append would have. +// 2. The keyspace has no Postgres run row at all, and is older than a threshold — a crashed +// birth. It is non-terminal so it carries no expiry, and it has no run row, so rule 1 can +// never match it. Without this rule that leak has no bound. +// +// Nothing schedules this. The engine's worker is what has to run it, and run-store cannot reach the +// engine, so the wiring belongs to the ticket that owns production construction. +import { createRedisClient, type Redis, type RedisOptions } from "@internal/redis"; +import { Logger } from "@trigger.dev/core/logger"; +import type { TaskRunStatus } from "@trigger.dev/database"; +import { snapshotKeys } from "./redisSnapshotStore.js"; +import type { RunStore } from "./types.js"; + +/** + * Mirrors the engine's `finalStatuses`. run-store cannot import from run-engine — the dependency + * runs the other way — so the list is duplicated and a parity test in run-engine asserts the copy + * stays equal to the original. + */ +export const FINAL_RUN_STATUSES: readonly TaskRunStatus[] = [ + "CANCELED", + "INTERRUPTED", + "COMPLETED_SUCCESSFULLY", + "COMPLETED_WITH_ERRORS", + "SYSTEM_FAILURE", + "CRASHED", + "EXPIRED", + "TIMED_OUT", +]; + +const FINAL = new Set(FINAL_RUN_STATUSES); + +/** Comfortably above run-creation latency, so a birth in flight is never mistaken for an orphan. */ +const DEFAULT_ORPHAN_AGE_MS = 24 * 60 * 60 * 1000; +const DEFAULT_BATCH_SIZE = 1000; + +export type SweepResult = { + /** Keyspaces examined. */ + scanned: number; + /** Rule 1: terminal runs whose keyspace was given the completion expiry. */ + expired: number; + /** Rule 2: keyspaces with no run row, deleted. */ + deleted: number; + /** Left alone: a live run, a young orphan, or a batch whose Postgres lookup failed. */ + skipped: number; +}; + +export type SnapshotOrphanSweeperOptions = { + /** + * The sweep opens its own connection rather than borrowing the store's, so a long scan can never + * stall a hot-path client. + */ + redisOptions: RedisOptions; + /** + * Resolved through the run store, not a raw client. Under the run-ops split a run row can live on + * either database, and only the store knows which — a raw lookup would report a live run as an + * orphan and delete its keyspace. + */ + runStore: RunStore; + completedTtlMs: number; + orphanAgeMs?: number; + keyPrefix?: string; + logger?: Logger; +}; + +export class SnapshotOrphanSweeper { + readonly #redis: Redis; + readonly #runStore: RunStore; + readonly #completedTtlMs: number; + readonly #orphanAgeMs: number; + readonly #prefix: string; + readonly #logger: Logger; + #quit?: Promise; + + constructor(options: SnapshotOrphanSweeperOptions) { + this.#logger = options.logger ?? new Logger("SnapshotOrphanSweeper", "debug"); + this.#runStore = options.runStore; + this.#completedTtlMs = options.completedTtlMs; + this.#orphanAgeMs = options.orphanAgeMs ?? DEFAULT_ORPHAN_AGE_MS; + this.#prefix = options.keyPrefix ?? "snap:"; + this.#redis = createRedisClient(options.redisOptions, { + onError: (error) => this.#logger.error("SnapshotOrphanSweeper redis client error", { error }), + }); + } + + async quit(): Promise { + if (!this.#quit) { + this.#quit = this.#redis.quit().then( + () => undefined, + () => undefined + ); + } + await this.#quit; + } + + /** + * One full pass over the keyspace. `dryRun` reports what it would do and changes nothing. + */ + async sweep(opts?: { batchSize?: number; dryRun?: boolean }): Promise { + const batchSize = opts?.batchSize ?? DEFAULT_BATCH_SIZE; + const dryRun = opts?.dryRun ?? false; + const result: SweepResult = { scanned: 0, expired: 0, deleted: 0, skipped: 0 }; + + let cursor = "0"; + do { + const [next, keys] = await this.#redis.scan( + cursor, + "MATCH", + `${this.#prefix}{*}:cur`, + "COUNT", + batchSize + ); + cursor = next; + + const runIds = [...new Set(keys.map((key) => this.#runIdFrom(key)).filter(isString))]; + if (runIds.length === 0) continue; + + await this.#sweepBatch(runIds, dryRun, result); + } while (cursor !== "0"); + + this.#logger.log("SnapshotOrphanSweeper pass complete", { ...result, dryRun }); + return result; + } + + async #sweepBatch(runIds: string[], dryRun: boolean, result: SweepResult): Promise { + result.scanned += runIds.length; + + let rows: Map; + try { + rows = (await this.#runStore.findRunsByIds(runIds, { + select: { id: true, status: true }, + })) as unknown as Map; + } catch (error) { + // Never reap on an unknown answer. A lookup that failed says nothing about whether the run + // exists, and rule 2 deletes a whole keyspace. + this.#logger.error("SnapshotOrphanSweeper skipped a batch after a failed run lookup", { + count: runIds.length, + error, + }); + result.skipped += runIds.length; + return; + } + + for (const runId of runIds) { + const run = rows.get(runId); + + if (!run) { + await this.#applyRuleTwo(runId, dryRun, result); + continue; + } + + if (!FINAL.has(run.status)) { + // A live run. A SUSPENDED run can legitimately wait for weeks, so this is never touched. + result.skipped += 1; + continue; + } + + await this.#applyRuleOne(runId, dryRun, result); + } + } + + /** Rule 1: a terminal run whose keyspace never received the completion expiry. */ + async #applyRuleOne(runId: string, dryRun: boolean, result: SweepResult): Promise { + const keys = await this.#allKeys(runId); + if (keys.length === 0) { + result.skipped += 1; + return; + } + + const ttls = await Promise.all(keys.map((key) => this.#redis.pttl(key))); + // -1 is "exists, no expiry". Anything already counting down was set by a healthy append. + if (!ttls.some((ttl) => ttl === -1)) { + result.skipped += 1; + return; + } + + if (!dryRun) { + const pipeline = this.#redis.pipeline(); + for (const key of keys) { + pipeline.pexpire(key, this.#completedTtlMs); + } + await pipeline.exec(); + } + + result.expired += 1; + } + + /** Rule 2: a keyspace with no run row at all, past the age threshold. */ + async #applyRuleTwo(runId: string, dryRun: boolean, result: SweepResult): Promise { + const keys = await this.#allKeys(runId); + if (keys.length === 0) { + result.skipped += 1; + return; + } + + const age = await this.#newestEntryAgeMs(runId); + if (age === undefined || age < this.#orphanAgeMs) { + // Either the keyspace carries no readable timestamp, or a birth may still be in flight. + result.skipped += 1; + return; + } + + if (!dryRun) { + await this.#redis.del(...keys); + } + + result.deleted += 1; + } + + /** Every key for one run: the four core keys plus each wait-cycle key. */ + async #allKeys(runId: string): Promise { + const core = snapshotKeys(runId); + // Scoped to one hash tag, so this is a lookup inside a single slot rather than a keyspace scan. + const cycles = await this.#redis.keys(`${this.#prefix}{${runId}}:wp:*`); + const candidates = [core.e, core.idx, core.cur, core.seq, ...cycles]; + + const exists = await Promise.all(candidates.map((key) => this.#redis.exists(key))); + return candidates.filter((_key, index) => exists[index] === 1); + } + + /** + * Age of the newest entry, so a keyspace still being written to is never treated as an orphan. + * The newest is the right end: an old first entry says nothing about whether the run is dead. + */ + async #newestEntryAgeMs(runId: string): Promise { + const core = snapshotKeys(runId); + const newest = await this.#redis.zrevrange(core.idx, 0, 0); + const id = newest[0]; + if (!id) return undefined; + + const raw = await this.#redis.hget(core.e, id); + if (!raw) return undefined; + + try { + const createdAt = (JSON.parse(raw) as { createdAt?: string }).createdAt; + if (!createdAt) return undefined; + const parsed = Date.parse(createdAt); + return Number.isNaN(parsed) ? undefined : Date.now() - parsed; + } catch { + return undefined; + } + } + + #runIdFrom(key: string): string | undefined { + const open = key.indexOf("{"); + const close = key.indexOf("}", open + 1); + if (open === -1 || close === -1 || close === open + 1) return undefined; + return key.slice(open + 1, close); + } +} + +function isString(value: string | undefined): value is string { + return typeof value === "string"; +} diff --git a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.births.test.ts b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.births.test.ts index e94e41f94b6..12ab76a0f26 100644 --- a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.births.test.ts +++ b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.births.test.ts @@ -2,7 +2,7 @@ // which side survived: an orphaned key with no run row is the harmless state, and a run with no // snapshot at all is the one the order exists to prevent. import { describe, expect } from "vitest"; -import { postgresAndRedisTest } from "@internal/testcontainers"; +import { containerTest } from "@internal/testcontainers"; import { generateInternalId } from "@trigger.dev/core/v3/isomorphic"; import { PostgresRunStore } from "./PostgresRunStore.js"; import { RedisSnapshotStore } from "./redisSnapshotStore.js"; @@ -77,7 +77,7 @@ function cancelledData(runId: string, env: SnapshotFixtureEnv) { } describe("birth write ordering", () => { - postgresAndRedisTest("writes Redis then Postgres", async ({ prisma, redisOptions }) => { + containerTest("writes Redis then Postgres", async ({ prisma, redisOptions }) => { const { decorated, redis } = build(prisma as never, redisOptions as never); try { const env = await seedSnapshotEnvironment(prisma); @@ -99,7 +99,7 @@ describe("birth write ordering", () => { } }); - postgresAndRedisTest("mints an id when the caller supplies none", async ({ prisma, redisOptions }) => { + containerTest("mints an id when the caller supplies none", async ({ prisma, redisOptions }) => { const { decorated, redis } = build(prisma as never, redisOptions as never); try { const env = await seedSnapshotEnvironment(prisma); @@ -119,7 +119,7 @@ describe("birth write ordering", () => { } }); - postgresAndRedisTest( + containerTest( "a crash after the Redis append leaves an orphan key and no run", async ({ prisma, redisOptions }) => { const { decorated, redis } = build(prisma as never, redisOptions as never, { @@ -147,7 +147,7 @@ describe("birth write ordering", () => { } ); - postgresAndRedisTest( + containerTest( "creates the run anyway when the birth append fails before redis-only", { timeout: 60_000 }, async ({ prisma, redisOptions }) => { @@ -175,7 +175,7 @@ describe("birth write ordering", () => { } ); - postgresAndRedisTest( + containerTest( "refuses to create the run when the birth append fails at redis-only", { timeout: 60_000 }, async ({ prisma, redisOptions }) => { @@ -203,7 +203,7 @@ describe("birth write ordering", () => { } ); - postgresAndRedisTest("createCancelledRun writes Redis first", async ({ prisma, redisOptions }) => { + containerTest("createCancelledRun writes Redis first", async ({ prisma, redisOptions }) => { const { decorated, redis } = build(prisma as never, redisOptions as never); try { const env = await seedSnapshotEnvironment(prisma); @@ -229,7 +229,7 @@ describe("birth write ordering", () => { } }); - postgresAndRedisTest( + containerTest( "a born-terminal run gets the completion expiry immediately", async ({ prisma, redisOptions }) => { const { decorated, redis } = build(prisma as never, redisOptions as never); @@ -265,7 +265,7 @@ describe("birth write ordering", () => { } ); - postgresAndRedisTest("writes nothing to Redis at mode off", async ({ prisma, redisOptions }) => { + containerTest("writes nothing to Redis at mode off", async ({ prisma, redisOptions }) => { const { decorated, redis } = build(prisma as never, redisOptions as never, { mode: "off" }); try { const env = await seedSnapshotEnvironment(prisma); diff --git a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.reads.test.ts b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.reads.test.ts index ffdd91a92f9..1510773e1ce 100644 --- a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.reads.test.ts +++ b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.reads.test.ts @@ -1,7 +1,7 @@ // Reads served from Redis must be indistinguishable from the Postgres reads they replace: the same // payload shape, the same tenant boundary, the same fallback when Redis does not hold the answer. import { describe, expect } from "vitest"; -import { postgresAndRedisTest } from "@internal/testcontainers"; +import { containerTest } from "@internal/testcontainers"; import { generateInternalId } from "@trigger.dev/core/v3/isomorphic"; import { PostgresRunStore } from "./PostgresRunStore.js"; import { RedisSnapshotStore } from "./redisSnapshotStore.js"; @@ -77,7 +77,7 @@ function snapshotInput(runId: string, env: SnapshotFixtureEnv, description: stri } describe("snapshot reads", () => { - postgresAndRedisTest("serves the latest snapshot from Redis", async ({ prisma, redisOptions }) => { + containerTest("serves the latest snapshot from Redis", async ({ prisma, redisOptions }) => { const { decorated, redis, reads } = build(prisma as never, redisOptions as never); try { const env = await seedSnapshotEnvironment(prisma); @@ -101,7 +101,7 @@ describe("snapshot reads", () => { } }); - postgresAndRedisTest( + containerTest( "returns the same payload Postgres would", async ({ prisma, redisOptions }) => { const { decorated, redis } = build(prisma as never, redisOptions as never); @@ -128,7 +128,7 @@ describe("snapshot reads", () => { } ); - postgresAndRedisTest( + containerTest( "reads a foreign environment as not found, so the caller's 404 still fires", async ({ prisma, redisOptions }) => { const { decorated, redis } = build(prisma as never, redisOptions as never); @@ -146,7 +146,7 @@ describe("snapshot reads", () => { } ); - postgresAndRedisTest( + containerTest( "falls back to Postgres for a run with no keyspace", async ({ prisma, redisOptions }) => { const { decorated, redis, reads } = build(prisma as never, redisOptions as never); @@ -181,7 +181,7 @@ describe("snapshot reads", () => { } ); - postgresAndRedisTest("reads from Postgres at readPercent 0", async ({ prisma, redisOptions }) => { + containerTest("reads from Postgres at readPercent 0", async ({ prisma, redisOptions }) => { const { decorated, redis, reads } = build(prisma as never, redisOptions as never, { readPercent: 0, }); @@ -198,7 +198,7 @@ describe("snapshot reads", () => { } }); - postgresAndRedisTest("reads from Postgres at mode dual-write", async ({ prisma, redisOptions }) => { + containerTest("reads from Postgres at mode dual-write", async ({ prisma, redisOptions }) => { const { decorated, redis, reads } = build(prisma as never, redisOptions as never, { mode: "dual-write", }); @@ -215,7 +215,7 @@ describe("snapshot reads", () => { } }); - postgresAndRedisTest("serves the since-cursor lookup from Redis", async ({ prisma, redisOptions }) => { + containerTest("serves the since-cursor lookup from Redis", async ({ prisma, redisOptions }) => { const { decorated, redis, reads } = build(prisma as never, redisOptions as never); try { const env = await seedSnapshotEnvironment(prisma); @@ -239,7 +239,7 @@ describe("snapshot reads", () => { } }); - postgresAndRedisTest( + containerTest( "delegates a snapshot lookup it does not recognise", async ({ prisma, redisOptions }) => { const { decorated, redis, reads } = build(prisma as never, redisOptions as never); @@ -264,7 +264,7 @@ describe("snapshot reads", () => { } ); - postgresAndRedisTest("serves the since window from Redis", async ({ prisma, redisOptions }) => { + containerTest("serves the since window from Redis", async ({ prisma, redisOptions }) => { const { decorated, redis, reads } = build(prisma as never, redisOptions as never); try { const env = await seedSnapshotEnvironment(prisma); @@ -290,7 +290,7 @@ describe("snapshot reads", () => { } }); - postgresAndRedisTest( + containerTest( "delegates a window query it does not recognise", async ({ prisma, redisOptions }) => { const { decorated, redis, reads } = build(prisma as never, redisOptions as never); @@ -312,7 +312,7 @@ describe("snapshot reads", () => { } ); - postgresAndRedisTest( + containerTest( "serves the waitpoint id projections from Redis", async ({ prisma, redisOptions }) => { const { decorated, redis, reads } = build(prisma as never, redisOptions as never); @@ -344,7 +344,7 @@ describe("snapshot reads", () => { } ); - postgresAndRedisTest( + containerTest( "delegates a waitpoint id projection with no run id", async ({ prisma, redisOptions }) => { const { decorated, redis, reads } = build(prisma as never, redisOptions as never); @@ -366,7 +366,7 @@ describe("snapshot reads", () => { } ); - postgresAndRedisTest("never touches Redis for reads at mode off", async ({ prisma, redisOptions }) => { + containerTest("never touches Redis for reads at mode off", async ({ prisma, redisOptions }) => { const { decorated, redis, reads } = build(prisma as never, redisOptions as never, { mode: "off", }); diff --git a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.staging.test.ts b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.staging.test.ts index ac0a877c968..3b75ca4ca05 100644 --- a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.staging.test.ts +++ b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.staging.test.ts @@ -2,7 +2,7 @@ // leaves Redis holding a transition that never happened. These tests observe the buffer from inside // the callback, so the deferral is proved rather than assumed. import { describe, expect } from "vitest"; -import { postgresAndRedisTest } from "@internal/testcontainers"; +import { containerTest } from "@internal/testcontainers"; import { generateInternalId } from "@trigger.dev/core/v3/isomorphic"; import { PostgresRunStore } from "./PostgresRunStore.js"; import { RedisSnapshotStore } from "./redisSnapshotStore.js"; @@ -65,7 +65,7 @@ function snapshotInput(runId: string, env: SnapshotFixtureEnv, id: string, descr } describe("the staging facade", () => { - postgresAndRedisTest("flushes the append after the commit", async ({ prisma, redisOptions }) => { + containerTest("flushes the append after the commit", async ({ prisma, redisOptions }) => { const { decorated, redis } = build(prisma as never, redisOptions as never); try { const env = await seedSnapshotEnvironment(prisma); @@ -87,7 +87,7 @@ describe("the staging facade", () => { } }); - postgresAndRedisTest( + containerTest( "writes nothing to Redis when the transaction rolls back", async ({ prisma, redisOptions }) => { const { decorated, redis } = build(prisma as never, redisOptions as never); @@ -113,7 +113,7 @@ describe("the staging facade", () => { } ); - postgresAndRedisTest("flushes several staged appends in order", async ({ prisma, redisOptions }) => { + containerTest("flushes several staged appends in order", async ({ prisma, redisOptions }) => { const { decorated, redis } = build(prisma as never, redisOptions as never); try { const env = await seedSnapshotEnvironment(prisma); @@ -138,7 +138,7 @@ describe("the staging facade", () => { } }); - postgresAndRedisTest( + containerTest( "hands the transaction callback a decorated store", async ({ prisma, redisOptions }) => { const { decorated, redis } = build(prisma as never, redisOptions as never); @@ -160,7 +160,7 @@ describe("the staging facade", () => { } ); - postgresAndRedisTest( + containerTest( "hands the transaction callback the plain delegate at mode off", async ({ prisma, redisOptions }) => { const { decorated, redis } = build(prisma as never, redisOptions as never, "off"); @@ -195,7 +195,7 @@ describe("the staging facade", () => { } ); - postgresAndRedisTest( + containerTest( "wraps the store handle from forWaitpointCompletion", async ({ prisma, redisOptions }) => { const { decorated, redis } = build(prisma as never, redisOptions as never); @@ -213,7 +213,7 @@ describe("the staging facade", () => { } ); - postgresAndRedisTest( + containerTest( "returns the plain handle from forWaitpointCompletion at mode off", async ({ prisma, redisOptions }) => { const { decorated, redis } = build(prisma as never, redisOptions as never, "off"); diff --git a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.transitions.test.ts b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.transitions.test.ts index 786e3a74453..311b46cc780 100644 --- a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.transitions.test.ts +++ b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.transitions.test.ts @@ -2,7 +2,7 @@ // reading the code: with the Redis half made to fail, the Postgres row is still there and the caller // sees no error, which is only possible if Postgres went first. import { describe, expect } from "vitest"; -import { postgresAndRedisTest } from "@internal/testcontainers"; +import { containerTest } from "@internal/testcontainers"; import { generateInternalId } from "@trigger.dev/core/v3/isomorphic"; import { PostgresRunStore } from "./PostgresRunStore.js"; import { RedisSnapshotStore } from "./redisSnapshotStore.js"; @@ -134,7 +134,7 @@ function expireInput(env: SnapshotFixtureEnv) { } describe("transition write ordering", () => { - postgresAndRedisTest("writes Postgres then Redis", async ({ prisma, redisOptions }) => { + containerTest("writes Postgres then Redis", async ({ prisma, redisOptions }) => { const { decorated, redis, writes } = harness(prisma as never, redisOptions as never); try { const env = await seedSnapshotEnvironment(prisma); @@ -157,7 +157,7 @@ describe("transition write ordering", () => { } }); - postgresAndRedisTest( + containerTest( "keeps the Postgres write and enqueues one repair when the append fails", async ({ prisma, redisOptions }) => { const { decorated, redis, repairs } = harness(prisma as never, redisOptions as never, { @@ -190,7 +190,7 @@ describe("transition write ordering", () => { } ); - postgresAndRedisTest( + containerTest( "treats a transition on a run with no keyspace as skipped, not failed", async ({ prisma, redisOptions }) => { const { decorated, redis, repairs, writes } = harness(prisma as never, redisOptions as never); @@ -210,7 +210,7 @@ describe("transition write ordering", () => { } ); - postgresAndRedisTest("appends for expireRun", async ({ prisma, redisOptions }) => { + containerTest("appends for expireRun", async ({ prisma, redisOptions }) => { const { decorated, redis } = harness(prisma as never, redisOptions as never); try { const env = await seedSnapshotEnvironment(prisma); @@ -229,7 +229,7 @@ describe("transition write ordering", () => { } }); - postgresAndRedisTest("appends for expireParkedRun", async ({ prisma, redisOptions }) => { + containerTest("appends for expireParkedRun", async ({ prisma, redisOptions }) => { const { decorated, redis } = harness(prisma as never, redisOptions as never); try { const env = await seedSnapshotEnvironment(prisma); @@ -253,7 +253,7 @@ describe("transition write ordering", () => { } }); - postgresAndRedisTest( + containerTest( "appends nothing when expireParkedRun matches no run", async ({ prisma, redisOptions }) => { const { decorated, redis, writes } = harness(prisma as never, redisOptions as never); @@ -278,7 +278,7 @@ describe("transition write ordering", () => { } ); - postgresAndRedisTest("appends for rescheduleRun", async ({ prisma, redisOptions }) => { + containerTest("appends for rescheduleRun", async ({ prisma, redisOptions }) => { const { decorated, redis } = harness(prisma as never, redisOptions as never); try { const env = await seedSnapshotEnvironment(prisma); @@ -306,7 +306,7 @@ describe("transition write ordering", () => { } }); - postgresAndRedisTest( + containerTest( "appends nothing when rescheduleRun carries no snapshot", async ({ prisma, redisOptions }) => { const { decorated, redis, writes } = harness(prisma as never, redisOptions as never); @@ -325,7 +325,7 @@ describe("transition write ordering", () => { } ); - postgresAndRedisTest("appends for lockRunToWorker under a CAS", async ({ prisma, redisOptions }) => { + containerTest("appends for lockRunToWorker under a CAS", async ({ prisma, redisOptions }) => { const { decorated, redis, writes } = harness(prisma as never, redisOptions as never); try { const env = await seedSnapshotEnvironment(prisma); @@ -367,7 +367,7 @@ describe("transition write ordering", () => { } }); - postgresAndRedisTest( + containerTest( "reports a forked append without enqueuing a repair", async ({ prisma, redisOptions }) => { const { decorated, redis, repairs, writes } = harness(prisma as never, redisOptions as never); @@ -409,7 +409,7 @@ describe("transition write ordering", () => { } ); - postgresAndRedisTest("appends for the standalone createExecutionSnapshot", async ({ prisma, redisOptions }) => { + containerTest("appends for the standalone createExecutionSnapshot", async ({ prisma, redisOptions }) => { const { decorated, redis } = harness(prisma as never, redisOptions as never); try { const env = await seedSnapshotEnvironment(prisma); @@ -435,7 +435,7 @@ describe("transition write ordering", () => { } }); - postgresAndRedisTest("writes nothing to Redis at mode off", async ({ prisma, redisOptions }) => { + containerTest("writes nothing to Redis at mode off", async ({ prisma, redisOptions }) => { const { decorated, redis } = harness(prisma as never, redisOptions as never, { mode: "off" }); try { const { run, env } = await setupSnapshotIdFixture(prisma); From 81194668a91f30b72a82478d443e88f4911219bb Mon Sep 17 00:00:00 2001 From: Daniel Sutton Date: Mon, 24 Aug 2026 16:23:28 +0100 Subject: [PATCH 10/56] test(run-engine): run the snapshot flows against the decorator with reads on MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The engine's own flows, driven against the decorator with every snapshot read served from Redis, injected through the store seam that runStoreInjectability already proves. Same flows, same expectations, different store underneath — the point is that nothing in the engine has to know, so no existing suite changes. Covers a run driven to completion, the execution data at each step, a since-window wider than the fifty cap, and a pre-cutover run with no keyspace falling back to Postgres. The environment-boundary test asserts parity rather than a fixed shape: whatever Postgres answers for a foreign environment, Redis has to answer the same, or the tenant boundary behaves differently once reads move over. --- .../engine/tests/helpers/decoratedStore.ts | 75 +++++ .../tests/snapshotStoreReadGate.test.ts | 298 ++++++++++++++++++ internal-packages/run-store/src/index.ts | 5 + 3 files changed, 378 insertions(+) create mode 100644 internal-packages/run-engine/src/engine/tests/helpers/decoratedStore.ts create mode 100644 internal-packages/run-engine/src/engine/tests/snapshotStoreReadGate.test.ts diff --git a/internal-packages/run-engine/src/engine/tests/helpers/decoratedStore.ts b/internal-packages/run-engine/src/engine/tests/helpers/decoratedStore.ts new file mode 100644 index 00000000000..c74aa86db92 --- /dev/null +++ b/internal-packages/run-engine/src/engine/tests/helpers/decoratedStore.ts @@ -0,0 +1,75 @@ +// Builds the snapshot-store decorator over a real PostgresRunStore, for injection through the +// engine's `store` option — the seam runStoreInjectability.test.ts already proves. +// +// The point of injecting it is that the engine suites keep their own assertions: the same flows, +// the same expectations, a different store underneath. +import { + PostgresRunStore, + RedisSnapshotStore, + TaskRunExecutionSnapshotStore, + type SnapshotFaultInjector, + type SnapshotRepairEnqueuer, + type SnapshotStoreMode, +} from "@internal/run-store"; +import type { PrismaClient } from "@trigger.dev/database"; +import type { RedisOptions } from "@internal/redis"; + +const COMPLETED_TTL_MS = 72 * 60 * 60 * 1000; + +export type DecoratedStoreHarness = { + store: TaskRunExecutionSnapshotStore; + redis: RedisSnapshotStore; + /** Every read the decorator served, and which store answered it. */ + reads: { method: string; source: "redis" | "postgres" }[]; + /** Every append outcome, keyed by the write site that produced it. */ + writes: { site: string; outcome: string }[]; + /** Runs handed to the repair job because their append was lost. */ + repairs: { runId: string; snapshotId: string; executionStatus: string }[]; + quit(): Promise; +}; + +export function buildDecoratedStore(opts: { + prisma: PrismaClient; + redisOptions: RedisOptions; + mode: SnapshotStoreMode; + readPercent?: number; + faults?: SnapshotFaultInjector; + onAppendFailure?: SnapshotRepairEnqueuer; +}): DecoratedStoreHarness { + const redis = new RedisSnapshotStore({ + redisOptions: opts.redisOptions, + completedTtlMs: COMPLETED_TTL_MS, + }); + + const reads: DecoratedStoreHarness["reads"] = []; + const writes: DecoratedStoreHarness["writes"] = []; + const repairs: DecoratedStoreHarness["repairs"] = []; + + const store = new TaskRunExecutionSnapshotStore( + new PostgresRunStore({ prisma: opts.prisma as never, readOnlyPrisma: opts.prisma as never }), + { + store: redis, + mode: opts.mode, + readPercent: opts.readPercent ?? 100, + ...(opts.faults && { faults: opts.faults }), + onAppendFailure: async (args) => { + repairs.push(args); + await opts.onAppendFailure?.(args); + }, + metrics: { + recordWrite: (site, outcome) => writes.push({ site, outcome }), + recordAppendFailed: () => {}, + recordRead: (method, source) => reads.push({ method, source }), + }, + } + ); + + return { + store, + redis, + reads, + writes, + repairs, + quit: () => redis.quit(), + }; +} diff --git a/internal-packages/run-engine/src/engine/tests/snapshotStoreReadGate.test.ts b/internal-packages/run-engine/src/engine/tests/snapshotStoreReadGate.test.ts new file mode 100644 index 00000000000..097f92a5cd0 --- /dev/null +++ b/internal-packages/run-engine/src/engine/tests/snapshotStoreReadGate.test.ts @@ -0,0 +1,298 @@ +// The read gate: the engine's own snapshot flows, run against the decorator with reads served from +// Redis. Same flows, same expectations, different store underneath — the point is that nothing in +// the engine has to know, so no existing suite is modified to make this pass. +import { assertNonNullable, containerTest } from "@internal/testcontainers"; +import { trace } from "@internal/tracing"; +import { setTimeout } from "timers/promises"; +import { generateInternalId, RunId } from "@trigger.dev/core/v3/isomorphic"; +import { RunEngine } from "../index.js"; +import { buildDecoratedStore, type DecoratedStoreHarness } from "./helpers/decoratedStore.js"; +import { setupAuthenticatedEnvironment, setupBackgroundWorker } from "./setup.js"; + +vi.setConfig({ testTimeout: 60_000 }); + +function engineOptions(prisma: any, redisOptions: any, harness: DecoratedStoreHarness) { + return { + prisma, + store: harness.store, + worker: { redis: redisOptions, workers: 1, tasksPerWorker: 10, pollIntervalMs: 100 }, + queue: { + redis: redisOptions, + masterQueueConsumersDisabled: true, + processWorkerQueueDebounceMs: 50, + }, + runLock: { redis: redisOptions }, + machines: { + defaultMachine: "small-1x" as const, + machines: { + "small-1x": { name: "small-1x" as const, cpu: 0.5, memory: 0.5, centsPerMs: 0.0001 }, + }, + baseCostInCents: 0.0001, + }, + tracer: trace.getTracer("test", "0.0.0"), + }; +} + +const triggerArgs = (taskIdentifier: string, environment: any, n: number) => ({ + number: n, + // A real minted friendly id: the engine converts it back with RunId.fromFriendlyId, which + // rejects anything that is not the prefix plus a cuid body. + friendlyId: RunId.generate().friendlyId, + environment, + taskIdentifier, + payload: "{}", + payloadType: "application/json", + context: {}, + traceContext: {}, + traceId: `t_gate_${n}`, + spanId: `s_gate_${n}`, + workerQueue: "main", + queue: `task/${taskIdentifier}`, + isTest: false, + tags: [], +}); + +describe("snapshot store read gate", () => { + containerTest( + "drives a run to completion with every snapshot read served from Redis", + async ({ prisma, redisOptions }) => { + const harness = buildDecoratedStore({ + prisma, + redisOptions, + mode: "redis-read", + readPercent: 100, + }); + const engine = new RunEngine(engineOptions(prisma, redisOptions, harness) as never); + + try { + const environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + const taskIdentifier = "gate-task"; + await setupBackgroundWorker(engine, environment, taskIdentifier); + + const run = await engine.trigger(triggerArgs(taskIdentifier, environment, 1), prisma); + await setTimeout(500); + + const dequeued = await engine.dequeueFromWorkerQueue({ + consumerId: "gate_consumer", + workerQueue: "main", + }); + expect(dequeued.length).toBe(1); + + const attempt = await engine.startRunAttempt({ + runId: dequeued[0]!.run.id, + snapshotId: dequeued[0]!.snapshot.id, + }); + expect(attempt.run.status).toBe("EXECUTING"); + + await engine.completeRunAttempt({ + runId: run.id, + snapshotId: attempt.snapshot.id, + completion: { ok: true, id: run.id, output: `{"done":true}`, outputType: "application/json" }, + }); + + const finished = await prisma.taskRun.findFirstOrThrow({ where: { id: run.id } }); + expect(finished.status).toBe("COMPLETED_SUCCESSFULLY"); + + // The gate: the engine read its snapshots, and Redis is what answered. + const fromRedis = harness.reads.filter((r) => r.source === "redis"); + expect(fromRedis.length).toBeGreaterThan(0); + expect(harness.reads.filter((r) => r.source === "postgres")).toEqual([]); + } finally { + await engine.quit(); + await harness.quit(); + } + } + ); + + containerTest( + "serves getRunExecutionData from Redis at every step", + async ({ prisma, redisOptions }) => { + const harness = buildDecoratedStore({ + prisma, + redisOptions, + mode: "redis-read", + readPercent: 100, + }); + const engine = new RunEngine(engineOptions(prisma, redisOptions, harness) as never); + + try { + const environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + const taskIdentifier = "gate-task"; + await setupBackgroundWorker(engine, environment, taskIdentifier); + + const run = await engine.trigger(triggerArgs(taskIdentifier, environment, 2), prisma); + await setTimeout(500); + + const queued = await engine.getRunExecutionData({ runId: run.id }); + assertNonNullable(queued); + expect(queued.snapshot.executionStatus).toBe("QUEUED"); + + const dequeued = await engine.dequeueFromWorkerQueue({ + consumerId: "gate_consumer", + workerQueue: "main", + }); + const pending = await engine.getRunExecutionData({ runId: run.id }); + assertNonNullable(pending); + expect(pending.snapshot.executionStatus).toBe("PENDING_EXECUTING"); + + await engine.startRunAttempt({ + runId: dequeued[0]!.run.id, + snapshotId: dequeued[0]!.snapshot.id, + }); + const executing = await engine.getRunExecutionData({ runId: run.id }); + assertNonNullable(executing); + expect(executing.snapshot.executionStatus).toBe("EXECUTING"); + expect(executing.run.attemptNumber).toBe(1); + } finally { + await engine.quit(); + await harness.quit(); + } + } + ); + + containerTest( + "keeps the environment boundary on a snapshot read", + async ({ prisma, redisOptions }) => { + const harness = buildDecoratedStore({ + prisma, + redisOptions, + mode: "redis-read", + readPercent: 100, + }); + const engine = new RunEngine(engineOptions(prisma, redisOptions, harness) as never); + + try { + const environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + const taskIdentifier = "gate-task"; + await setupBackgroundWorker(engine, environment, taskIdentifier); + + const run = await engine.trigger(triggerArgs(taskIdentifier, environment, 3), prisma); + await setTimeout(500); + + // Scoped to its own environment the run reads normally. + const own = await engine.getRunExecutionData({ + runId: run.id, + environmentId: environment.id, + }); + assertNonNullable(own); + + // Scoped to any other environment the run must not leak across the tenant boundary. The + // assertion is parity rather than a fixed shape: whatever Postgres answers for this call, + // Redis has to answer the same, or the boundary behaves differently once reads move over. + const foreignEnvironmentId = generateInternalId(); + + const viaRedis = await engine + .getRunExecutionData({ runId: run.id, environmentId: foreignEnvironmentId }) + .catch((error: unknown) => ({ threw: (error as Error).constructor.name })); + + const postgresOnly = buildDecoratedStore({ prisma, redisOptions, mode: "off" }); + const engineOff = new RunEngine( + engineOptions(prisma, redisOptions, postgresOnly) as never + ); + let viaPostgres: unknown; + try { + viaPostgres = await engineOff + .getRunExecutionData({ runId: run.id, environmentId: foreignEnvironmentId }) + .catch((error: unknown) => ({ threw: (error as Error).constructor.name })); + } finally { + await engineOff.quit(); + await postgresOnly.quit(); + } + + expect(viaRedis).toEqual(viaPostgres); + // And whatever that shape is, it must not be the run's data. + expect(viaRedis).not.toMatchObject({ run: { id: run.id } }); + } finally { + await engine.quit(); + await harness.quit(); + } + } + ); + + containerTest( + "serves a since-window wider than the cap from Redis", + async ({ prisma, redisOptions }) => { + const harness = buildDecoratedStore({ + prisma, + redisOptions, + mode: "redis-read", + readPercent: 100, + }); + const engine = new RunEngine(engineOptions(prisma, redisOptions, harness) as never); + + try { + const environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + const taskIdentifier = "gate-task"; + await setupBackgroundWorker(engine, environment, taskIdentifier); + + const run = await engine.trigger(triggerArgs(taskIdentifier, environment, 4), prisma); + await setTimeout(500); + + const first = await engine.getRunExecutionData({ runId: run.id }); + assertNonNullable(first); + + // More transitions than the 50-cap, so the window is exercised at its boundary. + for (let i = 0; i < 60; i++) { + await harness.store.createExecutionSnapshot({ + run: { id: run.id, status: "PENDING", attemptNumber: null }, + snapshot: { executionStatus: "QUEUED", description: `filler ${i}` }, + environmentId: environment.id, + environmentType: environment.type, + projectId: environment.project.id, + organizationId: environment.organization.id, + }); + } + + const since = await engine.getSnapshotsSince({ + runId: run.id, + snapshotId: first.snapshot.id, + }); + assertNonNullable(since); + + // The newest 50, ascending — the same window Postgres would have produced. + expect(since.length).toBe(50); + expect(since[since.length - 1]!.snapshot.description).toBe("filler 59"); + expect(harness.reads.some((r) => r.source === "redis")).toBe(true); + } finally { + await engine.quit(); + await harness.quit(); + } + } + ); + + containerTest("falls back to Postgres for a pre-cutover run", async ({ prisma, redisOptions }) => { + // A run created while the dial was off has no keyspace. Turning reads on must not lose it. + const off = buildDecoratedStore({ prisma, redisOptions, mode: "off" }); + const engineOff = new RunEngine(engineOptions(prisma, redisOptions, off) as never); + + let runId: string; + let environment: any; + try { + environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + await setupBackgroundWorker(engineOff, environment, "gate-task"); + const run = await engineOff.trigger(triggerArgs("gate-task", environment, 5), prisma); + runId = run.id; + await setTimeout(500); + } finally { + await engineOff.quit(); + await off.quit(); + } + + const on = buildDecoratedStore({ + prisma, + redisOptions, + mode: "redis-read", + readPercent: 100, + }); + const engineOn = new RunEngine(engineOptions(prisma, redisOptions, on) as never); + try { + const data = await engineOn.getRunExecutionData({ runId }); + assertNonNullable(data); + expect(data.snapshot.executionStatus).toBe("QUEUED"); + expect(on.reads.some((r) => r.source === "postgres")).toBe(true); + } finally { + await engineOn.quit(); + await on.quit(); + } + }); +}); diff --git a/internal-packages/run-store/src/index.ts b/internal-packages/run-store/src/index.ts index 3717dc01527..16fa696ee43 100644 --- a/internal-packages/run-store/src/index.ts +++ b/internal-packages/run-store/src/index.ts @@ -3,3 +3,8 @@ export * from "./PostgresRunStore.js"; export * from "./runOpsStore.js"; export * from "./readReplicaClient.js"; export * from "./redisSnapshotStore.js"; +export * from "./delegatingRunStore.js"; +export * from "./taskRunExecutionSnapshotStore.js"; +export * from "./snapshotEntry.js"; +export * from "./snapshotFaultInjection.js"; +export * from "./snapshotOrphanSweeper.js"; From aee3f0759d4c2f1d31404436713f0cfe99925a2c Mon Sep 17 00:00:00 2001 From: Daniel Sutton Date: Mon, 24 Aug 2026 17:28:38 +0100 Subject: [PATCH 11/56] fix(run-store): give a snapshot one identity and one instant across both stores Three defects, all of which passed the existing suites because no test drove a snapshot that actually carried waitpoints, and because the parity suite compared createdAt against a value it had just read back from the row. The decorator never passed a cycle to the append, so no wp: key was written for any snapshot and the completed-waitpoint side of Redis was permanently empty. It now mints a cycle when the id set differs from the current head and carries the previous cycleSeq forward when it does not, so a resume writes the record set once and the copy-forwards that follow write no key at all. The since-window hydration returned an empty completedWaitpointOrder. That column is not the join: the engine reads it off the head row as the oracle that gives each completed waitpoint its position in a batch, so an empty order resumed every batched triggerAndWait with an undefined index. Seven of the eight write sites stamped the entry from the app clock while Postgres stamped its own column default, so the two stores held different instants for one snapshot. The decorator now supplies createdAt, and an equal updatedAt, at every site, and the standalone path supplies it too rather than reading the row back. Beyond making the field comparable, this aligns the since-window: the cursor is resolved from one store and applied in the other, and two different instants misfilter that window. The parity suite gains an independent clock-provenance guard, and a case proving an absent instant still takes the database default, which is what keeps the store's behaviour unchanged while the decorator is off. --- .../engine/tests/snapshotStoreChaos.test.ts | 400 ++++++++++++++++++ .../tests/snapshotStoreReadGate.test.ts | 80 ++-- .../scripts/generateDelegatingRunStore.ts | 10 +- .../PostgresRunStore.snapshotWrites.test.ts | 8 +- .../run-store/src/PostgresRunStore.ts | 19 +- .../redisSnapshotStore.sinceCreatedAt.test.ts | 44 +- .../run-store/src/runStoreMethodNames.ts | 4 +- .../src/snapshotEntry.parity.test.ts | 109 ++++- .../src/snapshotOrphanSweeper.test.ts | 30 +- .../run-store/src/snapshotReadShapes.test.ts | 8 +- ...askRunExecutionSnapshotStore.reads.test.ts | 47 +- ...ExecutionSnapshotStore.transitions.test.ts | 51 +-- .../src/taskRunExecutionSnapshotStore.ts | 209 +++++++-- ...utionSnapshotStore.waitpointCycles.test.ts | 363 ++++++++++++++++ .../src/testFixtures/snapshotIdFixture.ts | 32 ++ internal-packages/run-store/src/types.ts | 30 ++ 16 files changed, 1269 insertions(+), 175 deletions(-) create mode 100644 internal-packages/run-engine/src/engine/tests/snapshotStoreChaos.test.ts create mode 100644 internal-packages/run-store/src/taskRunExecutionSnapshotStore.waitpointCycles.test.ts diff --git a/internal-packages/run-engine/src/engine/tests/snapshotStoreChaos.test.ts b/internal-packages/run-engine/src/engine/tests/snapshotStoreChaos.test.ts new file mode 100644 index 00000000000..a727c9bcc1f --- /dev/null +++ b/internal-packages/run-engine/src/engine/tests/snapshotStoreChaos.test.ts @@ -0,0 +1,400 @@ +// The correctness spine: kill the process at each write boundary and prove the run still converges. +// +// The write protocol's whole claim is that whatever a crash leaves behind is a state the existing +// stall-and-repair machinery heals. That claim is not checkable by reading the code, so each test +// here injects a fault at one named boundary and then asserts three things: the run converges, it +// does not hang, and it burns at most one attempt number per crash. +// +// The bound is PER CRASH, not a flat one. The plan records that TLC refuted a flat bound of one in +// seven states, and that the property which holds is pgAttempt - maxLoggedAttempt <= crashCount. +import { assertNonNullable, containerTest } from "@internal/testcontainers"; +import { trace } from "@internal/tracing"; +import { generateInternalId, RunId } from "@trigger.dev/core/v3/isomorphic"; +import { + InjectedSnapshotFault, + type SnapshotFaultBoundary, + type SnapshotFaultInjector, +} from "@internal/run-store"; +import { setTimeout } from "timers/promises"; +import { RunEngine } from "../index.js"; +import { buildDecoratedStore, type DecoratedStoreHarness } from "./helpers/decoratedStore.js"; +import { setupAuthenticatedEnvironment, setupBackgroundWorker } from "./setup.js"; + +vi.setConfig({ testTimeout: 60_000 }); + +/** + * A local stand-in for the shared fault harness being built alongside this ticket. Its surface is + * the agreed one — arm, disarm, hook, fired — so swapping the import in costs no test-body change. + * + * `fired` is the guard against a silent pass. A boundary can be armed and never reached, in which + * case the test would go green having proved nothing, so every test asserts its boundary fired. + */ +function createFaultInjector(opts: { error: (boundary: SnapshotFaultBoundary) => Error }) { + const armed = new Map(); + const counts = new Map(); + + return { + arm(boundary: SnapshotFaultBoundary, opts?: { times?: number; runId?: string }) { + armed.set(boundary, { times: opts?.times ?? 1, ...(opts?.runId && { runId: opts.runId }) }); + }, + disarm(boundary: SnapshotFaultBoundary) { + armed.delete(boundary); + }, + fired(boundary: SnapshotFaultBoundary): number { + return counts.get(boundary) ?? 0; + }, + hook: ((boundary, context) => { + const entry = armed.get(boundary); + if (!entry) return; + if (entry.runId && context.runId !== entry.runId) return; + + counts.set(boundary, (counts.get(boundary) ?? 0) + 1); + entry.times -= 1; + if (entry.times <= 0) armed.delete(boundary); + + throw opts.error(boundary); + }) satisfies SnapshotFaultInjector, + }; +} + +function engineOptions( + prisma: any, + redisOptions: any, + harness: DecoratedStoreHarness, + heartbeatMs: number +) { + return { + prisma, + store: harness.store, + worker: { redis: redisOptions, workers: 1, tasksPerWorker: 10, pollIntervalMs: 100 }, + queue: { + redis: redisOptions, + retryOptions: { maxTimeoutInMs: 50 }, + masterQueueConsumersDisabled: true, + processWorkerQueueDebounceMs: 50, + }, + runLock: { redis: redisOptions }, + machines: { + defaultMachine: "small-1x" as const, + machines: { + "small-1x": { name: "small-1x" as const, cpu: 0.5, memory: 0.5, centsPerMs: 0.0001 }, + }, + baseCostInCents: 0.0001, + }, + heartbeatTimeoutsMs: { PENDING_EXECUTING: heartbeatMs }, + tracer: trace.getTracer("test", "0.0.0"), + }; +} + +const triggerArgs = (taskIdentifier: string, environment: any) => ({ + number: 1, + friendlyId: RunId.generate().friendlyId, + environment, + taskIdentifier, + payload: "{}", + payloadType: "application/json", + context: {}, + traceContext: {}, + traceId: `t_${generateInternalId().slice(-12)}`, + spanId: `s_${generateInternalId().slice(-12)}`, + workerQueue: "main", + queue: `task/${taskIdentifier}`, + isTest: false, + tags: [], +}); + +describe("snapshot store crash boundaries", () => { + containerTest( + "afterPgBeforeRedis: the run converges and burns at most one attempt", + async ({ prisma, redisOptions }) => { + const faults = createFaultInjector({ error: (b) => new InjectedSnapshotFault(b) }); + const harness = buildDecoratedStore({ + prisma, + redisOptions, + mode: "redis-read", + readPercent: 100, + faults: faults.hook, + }); + const engine = new RunEngine(engineOptions(prisma, redisOptions, harness, 200) as never); + + try { + const environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + await setupBackgroundWorker(engine, environment, "chaos-task"); + const run = await engine.trigger(triggerArgs("chaos-task", environment), prisma); + await setTimeout(500); + + const dequeued = await engine.dequeueFromWorkerQueue({ + consumerId: "chaos", + workerQueue: "main", + }); + expect(dequeued.length).toBe(1); + + // Crash between the Postgres commit and the Redis append of ONE transition. + faults.arm("afterPgBeforeRedis", { times: 1, runId: run.id }); + const attempt = await engine.startRunAttempt({ + runId: dequeued[0]!.run.id, + snapshotId: dequeued[0]!.snapshot.id, + }); + faults.disarm("afterPgBeforeRedis"); + + // The boundary was actually reached. Without this the test could pass having proved nothing. + expect(faults.fired("afterPgBeforeRedis")).toBe(1); + + // Postgres committed the attempt bump; the run is not stuck and not lost. + expect(attempt.run.attemptNumber).toBe(1); + const pgRun = await prisma.taskRun.findFirstOrThrow({ where: { id: run.id } }); + expect(pgRun.attemptNumber).toBe(1); + + // The gap handed the run to the repair job rather than failing the caller. + expect(harness.repairs).toHaveLength(1); + expect(harness.repairs[0]!.runId).toBe(run.id); + + // Reads still resolve: the run's state machine is readable, so nothing hangs. + const data = await engine.getRunExecutionData({ runId: run.id }); + assertNonNullable(data); + + // The bound: one crash costs at most one attempt number. + expect(pgRun.attemptNumber! - 1).toBeLessThanOrEqual(1); + } finally { + await engine.quit(); + await harness.quit(); + } + } + ); + + containerTest( + "afterRedisBirthBeforePg: no run is created, and the next trigger succeeds", + async ({ prisma, redisOptions }) => { + const faults = createFaultInjector({ error: (b) => new InjectedSnapshotFault(b) }); + const harness = buildDecoratedStore({ + prisma, + redisOptions, + mode: "redis-read", + readPercent: 100, + faults: faults.hook, + }); + const engine = new RunEngine(engineOptions(prisma, redisOptions, harness, 200) as never); + + try { + const environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + await setupBackgroundWorker(engine, environment, "chaos-task"); + + faults.arm("afterRedisBirthBeforePg", { times: 1 }); + await expect( + engine.trigger(triggerArgs("chaos-task", environment), prisma) + ).rejects.toBeInstanceOf(InjectedSnapshotFault); + expect(faults.fired("afterRedisBirthBeforePg")).toBe(1); + + // The harmless state: no run row, so nothing can ever read a run that has no snapshot. + const runsAfterCrash = await prisma.taskRun.count(); + expect(runsAfterCrash).toBe(0); + + // A crashed birth must not poison the path: the next trigger runs to completion. + const run = await engine.trigger(triggerArgs("chaos-task", environment), prisma); + await setTimeout(500); + const dequeued = await engine.dequeueFromWorkerQueue({ + consumerId: "chaos", + workerQueue: "main", + }); + const attempt = await engine.startRunAttempt({ + runId: dequeued[0]!.run.id, + snapshotId: dequeued[0]!.snapshot.id, + }); + await engine.completeRunAttempt({ + runId: run.id, + snapshotId: attempt.snapshot.id, + completion: { + ok: true, + id: run.id, + output: `{"done":true}`, + outputType: "application/json", + }, + }); + + const finished = await prisma.taskRun.findFirstOrThrow({ where: { id: run.id } }); + expect(finished.status).toBe("COMPLETED_SUCCESSFULLY"); + expect(finished.attemptNumber).toBe(1); + } finally { + await engine.quit(); + await harness.quit(); + } + } + ); + + containerTest( + "midFlushRetry: a crash during a retry still converges through the repair job", + async ({ prisma, redisOptions }) => { + const faults = createFaultInjector({ error: (b) => new InjectedSnapshotFault(b) }); + // A dead port makes attempt 0 fail FOR REAL, which is the only way the retry boundary is + // reachable: an injected fault at attempt 0 is treated as a dead process and skips the + // retries entirely. Arming midFlushRetry alone would fire nothing and pass for the wrong + // reason, which is what the fired() assertion below catches. + const harness = buildDecoratedStore({ + prisma, + redisOptions: { ...(redisOptions as object), port: 1, retryStrategy: () => null } as never, + mode: "dual-write", + faults: faults.hook, + }); + const engine = new RunEngine(engineOptions(prisma, redisOptions, harness, 200) as never); + + try { + const environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + await setupBackgroundWorker(engine, environment, "chaos-task"); + + faults.arm("midFlushRetry", { times: 1 }); + const run = await engine.trigger(triggerArgs("chaos-task", environment), prisma); + await setTimeout(500); + + // The birth append failed for real and, before redis-only, that is survivable: Postgres is + // authoritative and the run exists. + const pgRun = await prisma.taskRun.findFirstOrThrow({ where: { id: run.id } }); + expect(pgRun.id).toBe(run.id); + + const dequeued = await engine.dequeueFromWorkerQueue({ + consumerId: "chaos", + workerQueue: "main", + }); + expect(dequeued.length).toBe(1); + + const attempt = await engine.startRunAttempt({ + runId: dequeued[0]!.run.id, + snapshotId: dequeued[0]!.snapshot.id, + }); + + // The retry boundary was genuinely reached, not merely armed. + expect(faults.fired("midFlushRetry")).toBeGreaterThanOrEqual(1); + + // The run converges regardless: Postgres holds every snapshot at this dial position. + expect(attempt.run.attemptNumber).toBe(1); + const data = await engine.getRunExecutionData({ runId: run.id }); + assertNonNullable(data); + expect(data.snapshot.executionStatus).toBe("EXECUTING"); + } finally { + await engine.quit(); + await harness.quit(); + } + } + ); + + containerTest( + "a crash-stalled run rejects a stale snapshot rather than hanging", + async ({ prisma, redisOptions }) => { + const faults = createFaultInjector({ error: (b) => new InjectedSnapshotFault(b) }); + const harness = buildDecoratedStore({ + prisma, + redisOptions, + mode: "redis-read", + readPercent: 100, + faults: faults.hook, + }); + const engine = new RunEngine(engineOptions(prisma, redisOptions, harness, 200) as never); + + try { + const environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + await setupBackgroundWorker(engine, environment, "chaos-task"); + const run = await engine.trigger(triggerArgs("chaos-task", environment), prisma); + await setTimeout(500); + + faults.arm("afterPgBeforeRedis", { times: 1, runId: run.id }); + const dequeued = await engine.dequeueFromWorkerQueue({ + consumerId: "chaos", + workerQueue: "main", + }); + faults.disarm("afterPgBeforeRedis"); + expect(faults.fired("afterPgBeforeRedis")).toBe(1); + + // Postgres advanced; the Redis head did not. Reads come from Redis, so the caller now holds + // a snapshot id that no longer matches what the read store reports as latest. + // + // The contract is that this SURFACES rather than corrupts: the next operation to validate + // against latest rejects with a stale-snapshot error, which is the same answer a caller gets + // from an ordinary lost race. It does not hang, and it does not silently execute against the + // wrong state. + await expect( + engine.startRunAttempt({ + runId: dequeued[0]!.run.id, + snapshotId: dequeued[0]!.snapshot.id, + }) + ).rejects.toThrow(/Snapshot changed/); + + // The run is still readable and still has a coherent state machine. + const data = await engine.getRunExecutionData({ runId: run.id }); + assertNonNullable(data); + + // The gap was handed to the repair job, which is the compensator the protocol names. + expect(harness.repairs.length).toBeGreaterThanOrEqual(1); + expect(harness.repairs.some((r) => r.runId === run.id)).toBe(true); + + // And no attempt was burned by the rejection itself: the bound is per crash, and the + // rejected call never reached the attempt bump. + const pgRun = await prisma.taskRun.findFirstOrThrow({ where: { id: run.id } }); + expect(pgRun.attemptNumber ?? 0).toBeLessThanOrEqual(faults.fired("afterPgBeforeRedis")); + } finally { + await engine.quit(); + await harness.quit(); + } + } + ); + + containerTest( + "two crashes cost at most two attempts, and the divergence does not amplify", + async ({ prisma, redisOptions }) => { + const faults = createFaultInjector({ error: (b) => new InjectedSnapshotFault(b) }); + // dual-write, so reads still come from Postgres and the run can be driven forward through the + // normal API. That isolates the property under test — how many attempts two crashes cost — + // from the stale-read rejection the previous test covers. + const harness = buildDecoratedStore({ + prisma, + redisOptions, + mode: "dual-write", + faults: faults.hook, + }); + const engine = new RunEngine(engineOptions(prisma, redisOptions, harness, 200) as never); + + try { + const environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + await setupBackgroundWorker(engine, environment, "chaos-task"); + const run = await engine.trigger(triggerArgs("chaos-task", environment), prisma); + await setTimeout(500); + + faults.arm("afterPgBeforeRedis", { times: 2, runId: run.id }); + + const dequeued = await engine.dequeueFromWorkerQueue({ + consumerId: "chaos", + workerQueue: "main", + }); + const attempt = await engine.startRunAttempt({ + runId: dequeued[0]!.run.id, + snapshotId: dequeued[0]!.snapshot.id, + }); + await engine.completeRunAttempt({ + runId: run.id, + snapshotId: attempt.snapshot.id, + completion: { + ok: true, + id: run.id, + output: `{"done":true}`, + outputType: "application/json", + }, + }); + + const crashes = faults.fired("afterPgBeforeRedis"); + expect(crashes).toBeGreaterThanOrEqual(1); + + const pgRun = await prisma.taskRun.findFirstOrThrow({ where: { id: run.id } }); + + // pgAttempt - maxLoggedAttempt <= crashCount. Each crash costs at most one attempt, and the + // divergence does not amplify: two crashes never cost three. A flat bound of one was + // refuted by the model check, so the assertion is against the crash count, not a constant. + expect((pgRun.attemptNumber ?? 0) - 1).toBeLessThanOrEqual(crashes); + + // Postgres holds every snapshot at this dial position, so the run still converges. + expect(pgRun.status).toBe("COMPLETED_SUCCESSFULLY"); + expect(harness.repairs.length).toBe(crashes); + } finally { + await engine.quit(); + await harness.quit(); + } + } + ); +}); diff --git a/internal-packages/run-engine/src/engine/tests/snapshotStoreReadGate.test.ts b/internal-packages/run-engine/src/engine/tests/snapshotStoreReadGate.test.ts index 097f92a5cd0..17c186d074d 100644 --- a/internal-packages/run-engine/src/engine/tests/snapshotStoreReadGate.test.ts +++ b/internal-packages/run-engine/src/engine/tests/snapshotStoreReadGate.test.ts @@ -87,7 +87,12 @@ describe("snapshot store read gate", () => { await engine.completeRunAttempt({ runId: run.id, snapshotId: attempt.snapshot.id, - completion: { ok: true, id: run.id, output: `{"done":true}`, outputType: "application/json" }, + completion: { + ok: true, + id: run.id, + output: `{"done":true}`, + outputType: "application/json", + }, }); const finished = await prisma.taskRun.findFirstOrThrow({ where: { id: run.id } }); @@ -186,9 +191,7 @@ describe("snapshot store read gate", () => { .catch((error: unknown) => ({ threw: (error as Error).constructor.name })); const postgresOnly = buildDecoratedStore({ prisma, redisOptions, mode: "off" }); - const engineOff = new RunEngine( - engineOptions(prisma, redisOptions, postgresOnly) as never - ); + const engineOff = new RunEngine(engineOptions(prisma, redisOptions, postgresOnly) as never); let viaPostgres: unknown; try { viaPostgres = await engineOff @@ -260,39 +263,42 @@ describe("snapshot store read gate", () => { } ); - containerTest("falls back to Postgres for a pre-cutover run", async ({ prisma, redisOptions }) => { - // A run created while the dial was off has no keyspace. Turning reads on must not lose it. - const off = buildDecoratedStore({ prisma, redisOptions, mode: "off" }); - const engineOff = new RunEngine(engineOptions(prisma, redisOptions, off) as never); - - let runId: string; - let environment: any; - try { - environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); - await setupBackgroundWorker(engineOff, environment, "gate-task"); - const run = await engineOff.trigger(triggerArgs("gate-task", environment, 5), prisma); - runId = run.id; - await setTimeout(500); - } finally { - await engineOff.quit(); - await off.quit(); - } + containerTest( + "falls back to Postgres for a pre-cutover run", + async ({ prisma, redisOptions }) => { + // A run created while the dial was off has no keyspace. Turning reads on must not lose it. + const off = buildDecoratedStore({ prisma, redisOptions, mode: "off" }); + const engineOff = new RunEngine(engineOptions(prisma, redisOptions, off) as never); - const on = buildDecoratedStore({ - prisma, - redisOptions, - mode: "redis-read", - readPercent: 100, - }); - const engineOn = new RunEngine(engineOptions(prisma, redisOptions, on) as never); - try { - const data = await engineOn.getRunExecutionData({ runId }); - assertNonNullable(data); - expect(data.snapshot.executionStatus).toBe("QUEUED"); - expect(on.reads.some((r) => r.source === "postgres")).toBe(true); - } finally { - await engineOn.quit(); - await on.quit(); + let runId: string; + let environment: any; + try { + environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + await setupBackgroundWorker(engineOff, environment, "gate-task"); + const run = await engineOff.trigger(triggerArgs("gate-task", environment, 5), prisma); + runId = run.id; + await setTimeout(500); + } finally { + await engineOff.quit(); + await off.quit(); + } + + const on = buildDecoratedStore({ + prisma, + redisOptions, + mode: "redis-read", + readPercent: 100, + }); + const engineOn = new RunEngine(engineOptions(prisma, redisOptions, on) as never); + try { + const data = await engineOn.getRunExecutionData({ runId }); + assertNonNullable(data); + expect(data.snapshot.executionStatus).toBe("QUEUED"); + expect(on.reads.some((r) => r.source === "postgres")).toBe(true); + } finally { + await engineOn.quit(); + await on.quit(); + } } - }); + ); }); diff --git a/internal-packages/run-store/scripts/generateDelegatingRunStore.ts b/internal-packages/run-store/scripts/generateDelegatingRunStore.ts index df901f3a2f7..c827e10c753 100644 --- a/internal-packages/run-store/scripts/generateDelegatingRunStore.ts +++ b/internal-packages/run-store/scripts/generateDelegatingRunStore.ts @@ -176,10 +176,14 @@ export class DelegatingRunStore implements RunStore { ${readonlyProperties // Indexed access rather than the written type, so the getter needs no import of its own and // follows the interface if that type is ever changed. - .map((p) => ` get ${p.name}(): RunStore["${p.name}"] {\n return this.delegate.${p.name};\n }`) + .map( + (p) => ` get ${p.name}(): RunStore["${p.name}"] {\n return this.delegate.${p.name};\n }` + ) .join("\n\n")}${readonlyProperties.length > 0 ? "\n\n" : ""}${unique - .map((n) => ` ${n}(...args: any[]): any {\n return (this.delegate as any).${n}(...args);\n }`) - .join("\n\n")} + .map( + (n) => ` ${n}(...args: any[]): any {\n return (this.delegate as any).${n}(...args);\n }` + ) + .join("\n\n")} } ` ); diff --git a/internal-packages/run-store/src/PostgresRunStore.snapshotWrites.test.ts b/internal-packages/run-store/src/PostgresRunStore.snapshotWrites.test.ts index 81d3d3e658e..dda0f483bf5 100644 --- a/internal-packages/run-store/src/PostgresRunStore.snapshotWrites.test.ts +++ b/internal-packages/run-store/src/PostgresRunStore.snapshotWrites.test.ts @@ -150,7 +150,9 @@ describe("PostgresRunStore snapshotWrites flag", () => { { select: { id: true } } ); - expect((await prisma.taskRun.findFirstOrThrow({ where: { id: run.id } })).status).toBe("EXPIRED"); + expect((await prisma.taskRun.findFirstOrThrow({ where: { id: run.id } })).status).toBe( + "EXPIRED" + ); expect(await prisma.taskRunExecutionSnapshot.count({ where: { runId: run.id } })).toBe(0); }); @@ -226,7 +228,9 @@ describe("PostgresRunStore snapshotWrites flag", () => { }, }); - expect((await prisma.taskRun.findFirstOrThrow({ where: { id: run.id } })).status).toBe("DEQUEUED"); + expect((await prisma.taskRun.findFirstOrThrow({ where: { id: run.id } })).status).toBe( + "DEQUEUED" + ); expect(await prisma.taskRunExecutionSnapshot.count({ where: { runId: run.id } })).toBe(0); }); diff --git a/internal-packages/run-store/src/PostgresRunStore.ts b/internal-packages/run-store/src/PostgresRunStore.ts index c88c7464508..0f00f5848fe 100644 --- a/internal-packages/run-store/src/PostgresRunStore.ts +++ b/internal-packages/run-store/src/PostgresRunStore.ts @@ -744,6 +744,8 @@ export class PostgresRunStore implements RunStore { const snapshotCreate = { id: params.snapshot.id, + createdAt: params.snapshot.createdAt, + updatedAt: params.snapshot.createdAt, engine: params.snapshot.engine, executionStatus: params.snapshot.executionStatus, description: params.snapshot.description, @@ -831,6 +833,8 @@ export class PostgresRunStore implements RunStore { const snapshotCreate = { id: params.snapshot.id, + createdAt: params.snapshot.createdAt, + updatedAt: params.snapshot.createdAt, engine: params.snapshot.engine, executionStatus: params.snapshot.executionStatus, description: params.snapshot.description, @@ -942,6 +946,8 @@ export class PostgresRunStore implements RunStore { costInCents: data.costInCents, ...this.#nestedSnapshot({ id: data.snapshot.id, + createdAt: data.snapshot.createdAt, + updatedAt: data.snapshot.createdAt, executionStatus: data.snapshot.executionStatus, description: data.snapshot.description, runStatus: data.snapshot.runStatus, @@ -1147,6 +1153,8 @@ export class PostgresRunStore implements RunStore { error: data.error as Prisma.InputJsonValue, ...this.#nestedSnapshot({ id: data.snapshot.id, + createdAt: data.snapshot.createdAt, + updatedAt: data.snapshot.createdAt, engine: data.snapshot.engine, executionStatus: data.snapshot.executionStatus, description: data.snapshot.description, @@ -1277,6 +1285,8 @@ export class PostgresRunStore implements RunStore { maxAttempts: data.maxAttempts ?? undefined, ...this.#nestedSnapshot({ id: data.snapshot.id, + createdAt: data.snapshot.createdAt, + updatedAt: data.snapshot.createdAt, engine: "V2", executionStatus: "PENDING_EXECUTING", description: "Run was dequeued for execution", @@ -1382,6 +1392,8 @@ export class PostgresRunStore implements RunStore { error: data.error as Prisma.InputJsonValue, ...this.#nestedSnapshot({ id: data.snapshot.id, + createdAt: data.snapshot.createdAt, + updatedAt: data.snapshot.createdAt, engine: data.snapshot.engine, executionStatus: data.snapshot.executionStatus, description: data.snapshot.description, @@ -1454,6 +1466,8 @@ export class PostgresRunStore implements RunStore { ...(data.snapshot && this.#nestedSnapshot({ id: data.snapshot.id, + createdAt: data.snapshot.createdAt, + updatedAt: data.snapshot.createdAt, engine: "V2", executionStatus: data.snapshot.executionStatus ?? "DELAYED", description: @@ -1984,6 +1998,7 @@ export class PostgresRunStore implements RunStore { ): Promise> { const { id, + createdAt, run, snapshot, previousSnapshotId, @@ -2014,7 +2029,7 @@ export class PostgresRunStore implements RunStore { ); } - const now = new Date(); + const now = createdAt ?? new Date(); return { id, engine: "V2", @@ -2049,6 +2064,8 @@ export class PostgresRunStore implements RunStore { const newSnapshot = await prisma.taskRunExecutionSnapshot.create({ data: { id, + createdAt, + updatedAt: createdAt, engine: "V2", executionStatus: snapshot.executionStatus, description: snapshot.description, diff --git a/internal-packages/run-store/src/redisSnapshotStore.sinceCreatedAt.test.ts b/internal-packages/run-store/src/redisSnapshotStore.sinceCreatedAt.test.ts index 7e61e619b78..f1394be597e 100644 --- a/internal-packages/run-store/src/redisSnapshotStore.sinceCreatedAt.test.ts +++ b/internal-packages/run-store/src/redisSnapshotStore.sinceCreatedAt.test.ts @@ -24,8 +24,7 @@ function entry(runId: string, id: string, createdAt: string): SnapshotEntryInput }; } -const at = (seconds: number) => - new Date(Date.UTC(2026, 0, 1, 0, 0, seconds)).toISOString(); +const at = (seconds: number) => new Date(Date.UTC(2026, 0, 1, 0, 0, seconds)).toISOString(); async function seed( store: RedisSnapshotStore, @@ -42,26 +41,29 @@ async function seed( } describe("getSinceCreatedAt", () => { - redisTest("returns only entries newer than the cursor, oldest first", async ({ redisOptions }) => { - const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); - try { - const runId = "run_window"; - await seed( - store, - runId, - [0, 1, 2, 3, 4].map((n) => ({ id: `snap_${n}`, createdAt: at(n) })) - ); - - const result = await store.getSinceCreatedAt(runId, at(1)); - - expect(result.kind).toBe("hit"); - if (result.kind !== "hit") return; - // Ascending, matching what the engine hands its caller after its own reverse(). - expect(result.entries.map((e) => e.id)).toEqual(["snap_2", "snap_3", "snap_4"]); - } finally { - await store.quit(); + redisTest( + "returns only entries newer than the cursor, oldest first", + async ({ redisOptions }) => { + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + try { + const runId = "run_window"; + await seed( + store, + runId, + [0, 1, 2, 3, 4].map((n) => ({ id: `snap_${n}`, createdAt: at(n) })) + ); + + const result = await store.getSinceCreatedAt(runId, at(1)); + + expect(result.kind).toBe("hit"); + if (result.kind !== "hit") return; + // Ascending, matching what the engine hands its caller after its own reverse(). + expect(result.entries.map((e) => e.id)).toEqual(["snap_2", "snap_3", "snap_4"]); + } finally { + await store.quit(); + } } - }); + ); redisTest("misses when the run has no keyspace", async ({ redisOptions }) => { const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); diff --git a/internal-packages/run-store/src/runStoreMethodNames.ts b/internal-packages/run-store/src/runStoreMethodNames.ts index 989c7dc37bb..d3dd0868676 100644 --- a/internal-packages/run-store/src/runStoreMethodNames.ts +++ b/internal-packages/run-store/src/runStoreMethodNames.ts @@ -78,6 +78,4 @@ export const RUN_STORE_METHOD_NAMES = [ ] as const; // Data properties the base exposes as getters over the delegate, not as forwarders. -export const RUN_STORE_PROPERTY_NAMES = [ - "primaryReadClient", -] as const; +export const RUN_STORE_PROPERTY_NAMES = ["primaryReadClient"] as const; diff --git a/internal-packages/run-store/src/snapshotEntry.parity.test.ts b/internal-packages/run-store/src/snapshotEntry.parity.test.ts index 8c176178e0a..8e0a11fafbe 100644 --- a/internal-packages/run-store/src/snapshotEntry.parity.test.ts +++ b/internal-packages/run-store/src/snapshotEntry.parity.test.ts @@ -23,6 +23,13 @@ import { } from "./testFixtures/snapshotIdFixture.js"; /** + * NOTE ON createdAt. An earlier version of this suite built the expected entry with + * `createdAt: row.createdAt` and then asserted the two matched, which is tautological and hid a + * real divergence: seven of the eight write sites stamped the entry from the app clock while + * Postgres stamped its own column default, so the stores held different instants. The builders are + * now given an INDEPENDENT instant, and the row must carry that same value because the decorator + * passes it through to Postgres. + * * Compares only what the entry claims. The Redis model carries no `updatedAt` and no join rows, and * it holds `createdAt` as an ISO string, so those are checked separately or not at all. */ @@ -45,6 +52,8 @@ function assertParity(entry: SnapshotEntryInput, row: Record) { expect(row.runnerId ?? undefined).toBe(entry.runnerId ?? undefined); expect(row.isValid).toBe(entry.error === undefined); expect((row.createdAt as Date).toISOString()).toBe(entry.createdAt); + // Write-once rows: both columns hold the one instant, so a Prisma-stamped updatedAt would drift. + expect((row.updatedAt as Date).toISOString()).toBe(entry.createdAt); } function birthSnapshot(id: string, env: SnapshotFixtureEnv) { @@ -172,7 +181,10 @@ describe("entry to Postgres row parity", () => { ); const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); - assertParity(entryFromCompletion({ id, runId: run.id, createdAt: row.createdAt }, snapshot), row); + assertParity( + entryFromCompletion({ id, runId: run.id, createdAt: row.createdAt }, snapshot), + row + ); }); postgresTest("expireRun", async ({ prisma }) => { @@ -253,7 +265,10 @@ describe("entry to Postgres row parity", () => { }); const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); - assertParity(entryFromReschedule({ id, runId: run.id, createdAt: row.createdAt }, snapshot), row); + assertParity( + entryFromReschedule({ id, runId: run.id, createdAt: row.createdAt }, snapshot), + row + ); }); postgresTest("rescheduleRun with every value supplied", async ({ prisma }) => { @@ -277,7 +292,10 @@ describe("entry to Postgres row parity", () => { }); const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); - assertParity(entryFromReschedule({ id, runId: run.id, createdAt: row.createdAt }, snapshot), row); + assertParity( + entryFromReschedule({ id, runId: run.id, createdAt: row.createdAt }, snapshot), + row + ); }); postgresTest("lockRunToWorker", async ({ prisma }) => { @@ -395,3 +413,88 @@ describe("entry to Postgres row parity", () => { ); }); }); + +// The clock-provenance guard. Independent of the builders above: it asserts that what Postgres +// stores is the instant the CALLER supplied, not one the database chose. Without this, a snapshot +// has two different creation times depending on which store answers, the compared field can never +// reach zero divergence, and the since-window cursor resolved from one store misfilters the window +// walked in the other. +describe("createdAt provenance", () => { + postgresTest("Postgres stores the caller's instant, not its own", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const { run, env } = await setupSnapshotIdFixture(prisma); + const id = generateInternalId(); + // Far enough from now that a database default could never coincide with it. + const stamp = new Date(Date.now() - 5 * 60 * 1000); + + await store.createExecutionSnapshot({ + id, + createdAt: stamp, + run: { id: run.id, status: "EXECUTING", attemptNumber: 1 }, + snapshot: { executionStatus: "EXECUTING", description: "Run started" }, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }); + + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); + expect(row.createdAt.toISOString()).toBe(stamp.toISOString()); + expect(row.updatedAt.toISOString()).toBe(stamp.toISOString()); + }); + + postgresTest("an absent instant still takes the database default", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const { run, env } = await setupSnapshotIdFixture(prisma); + const id = generateInternalId(); + const before = new Date(Date.now() - 1000); + + // Mode off supplies nothing, so Postgres must behave exactly as it always has. This is what + // keeps the merge test true. + await store.createExecutionSnapshot({ + id, + run: { id: run.id, status: "EXECUTING", attemptNumber: 1 }, + snapshot: { executionStatus: "EXECUTING", description: "Run started" }, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }); + + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); + expect(row.createdAt.getTime()).toBeGreaterThan(before.getTime()); + }); + + postgresTest("a nested write site stores the caller's instant too", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const { run, env } = await setupSnapshotIdFixture(prisma); + const id = generateInternalId(); + const stamp = new Date(Date.now() - 5 * 60 * 1000); + + await store.expireRun( + run.id, + { + error: { type: "STRING_ERROR", raw: "expired" }, + completedAt: new Date(), + expiredAt: new Date(), + snapshot: { + id, + createdAt: stamp, + engine: "V2", + executionStatus: "FINISHED", + description: "Run expired", + runStatus: "EXPIRED", + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }, + }, + { select: { id: true } } + ); + + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); + expect(row.createdAt.toISOString()).toBe(stamp.toISOString()); + expect(row.updatedAt.toISOString()).toBe(stamp.toISOString()); + }); +}); diff --git a/internal-packages/run-store/src/snapshotOrphanSweeper.test.ts b/internal-packages/run-store/src/snapshotOrphanSweeper.test.ts index 68a9d99cb6d..81e4ea729ef 100644 --- a/internal-packages/run-store/src/snapshotOrphanSweeper.test.ts +++ b/internal-packages/run-store/src/snapshotOrphanSweeper.test.ts @@ -50,7 +50,11 @@ describe("SnapshotOrphanSweeper", () => { const env = await seedSnapshotEnvironment(prisma); const runId = generateInternalId(); // Non-terminal append, so no expiry is ever set — the lost-TTL-set case. - await store.append({ entry: birthEntry(runId, env, new Date()), kind: "birth", isTerminal: false }); + await store.append({ + entry: birthEntry(runId, env, new Date()), + kind: "birth", + isTerminal: false, + }); await prisma.taskRun.create({ data: { ...buildCreateRunData(runId, env), status: "COMPLETED_SUCCESSFULLY" }, }); @@ -130,7 +134,11 @@ describe("SnapshotOrphanSweeper", () => { const old = new Date(Date.now() - 2 * ORPHAN_AGE_MS); // The crashed birth: an entry, no Postgres run, non-terminal so no expiry. - await store.append({ entry: birthEntry(runId, env, old), kind: "birth", isTerminal: false }); + await store.append({ + entry: birthEntry(runId, env, old), + kind: "birth", + isTerminal: false, + }); await store.append({ entry: birthEntry(runId, env, old), kind: "transition", @@ -169,7 +177,11 @@ describe("SnapshotOrphanSweeper", () => { const env = await seedSnapshotEnvironment(prisma); const runId = generateInternalId(); // Written just now: the Postgres insert of a healthy birth may still be in flight. - await store.append({ entry: birthEntry(runId, env, new Date()), kind: "birth", isTerminal: false }); + await store.append({ + entry: birthEntry(runId, env, new Date()), + kind: "birth", + isTerminal: false, + }); const result = await sweeper.sweep(); @@ -279,7 +291,11 @@ describe("SnapshotOrphanSweeper", () => { const env = await seedSnapshotEnvironment(prisma); const runId = generateInternalId(); const old = new Date(Date.now() - 2 * ORPHAN_AGE_MS); - await store.append({ entry: birthEntry(runId, env, old), kind: "birth", isTerminal: false }); + await store.append({ + entry: birthEntry(runId, env, old), + kind: "birth", + isTerminal: false, + }); // A failed lookup says nothing about whether the run exists, and rule 2 deletes a whole // keyspace. The sweep must resolve rather than throw, and must reap nothing. @@ -309,7 +325,11 @@ describe("SnapshotOrphanSweeper", () => { const old = new Date(Date.now() - 2 * ORPHAN_AGE_MS); const orphans = Array.from({ length: 5 }, () => generateInternalId()); for (const runId of orphans) { - await store.append({ entry: birthEntry(runId, env, old), kind: "birth", isTerminal: false }); + await store.append({ + entry: birthEntry(runId, env, old), + kind: "birth", + isTerminal: false, + }); } const result = await sweeper.sweep({ batchSize: 2 }); diff --git a/internal-packages/run-store/src/snapshotReadShapes.test.ts b/internal-packages/run-store/src/snapshotReadShapes.test.ts index afa33dd3da2..258731a60b5 100644 --- a/internal-packages/run-store/src/snapshotReadShapes.test.ts +++ b/internal-packages/run-store/src/snapshotReadShapes.test.ts @@ -67,7 +67,9 @@ describe("matchSinceCursorLookup", () => { }); it("refuses an unknown top-level key", () => { - expect(matchSinceCursorLookup({ ...cursorArgs, orderBy: { createdAt: "desc" } })).toBeUndefined(); + expect( + matchSinceCursorLookup({ ...cursorArgs, orderBy: { createdAt: "desc" } }) + ).toBeUndefined(); }); it("refuses anything that is not an argument object", () => { @@ -108,9 +110,7 @@ describe("matchSinceWindow", () => { }); it("refuses ascending order", () => { - expect( - matchSinceWindow({ ...windowArgs, orderBy: { createdAt: "asc" } }) - ).toBeUndefined(); + expect(matchSinceWindow({ ...windowArgs, orderBy: { createdAt: "asc" } })).toBeUndefined(); }); it("refuses a window that does not filter to valid entries", () => { diff --git a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.reads.test.ts b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.reads.test.ts index 1510773e1ce..42924e34c34 100644 --- a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.reads.test.ts +++ b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.reads.test.ts @@ -101,32 +101,29 @@ describe("snapshot reads", () => { } }); - containerTest( - "returns the same payload Postgres would", - async ({ prisma, redisOptions }) => { - const { decorated, redis } = build(prisma as never, redisOptions as never); - const postgresOnly = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); - try { - const env = await seedSnapshotEnvironment(prisma); - const runId = await seedRun(decorated, env); - await decorated.createExecutionSnapshot(snapshotInput(runId, env, "Run started")); - - const fromRedis = await decorated.findLatestExecutionSnapshot(runId); - const fromPostgres = await postgresOnly.findLatestExecutionSnapshot(runId); - - expect(fromRedis!.id).toBe(fromPostgres!.id); - expect(fromRedis!.executionStatus).toBe(fromPostgres!.executionStatus); - expect(fromRedis!.description).toBe(fromPostgres!.description); - expect(fromRedis!.runStatus).toBe(fromPostgres!.runStatus); - expect(fromRedis!.attemptNumber).toBe(fromPostgres!.attemptNumber); - expect(fromRedis!.isValid).toBe(fromPostgres!.isValid); - expect(fromRedis!.environmentId).toBe(fromPostgres!.environmentId); - expect(fromRedis!.createdAt.toISOString()).toBe(fromPostgres!.createdAt.toISOString()); - } finally { - await redis.quit(); - } + containerTest("returns the same payload Postgres would", async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + const postgresOnly = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, env); + await decorated.createExecutionSnapshot(snapshotInput(runId, env, "Run started")); + + const fromRedis = await decorated.findLatestExecutionSnapshot(runId); + const fromPostgres = await postgresOnly.findLatestExecutionSnapshot(runId); + + expect(fromRedis!.id).toBe(fromPostgres!.id); + expect(fromRedis!.executionStatus).toBe(fromPostgres!.executionStatus); + expect(fromRedis!.description).toBe(fromPostgres!.description); + expect(fromRedis!.runStatus).toBe(fromPostgres!.runStatus); + expect(fromRedis!.attemptNumber).toBe(fromPostgres!.attemptNumber); + expect(fromRedis!.isValid).toBe(fromPostgres!.isValid); + expect(fromRedis!.environmentId).toBe(fromPostgres!.environmentId); + expect(fromRedis!.createdAt.toISOString()).toBe(fromPostgres!.createdAt.toISOString()); + } finally { + await redis.quit(); } - ); + }); containerTest( "reads a foreign environment as not found, so the caller's 404 still fires", diff --git a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.transitions.test.ts b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.transitions.test.ts index 311b46cc780..e6468c4f09f 100644 --- a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.transitions.test.ts +++ b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.transitions.test.ts @@ -181,9 +181,7 @@ describe("transition write ordering", () => { }); expect(await redis.getById(runId, row.id)).toBeNull(); - expect(repairs).toEqual([ - { runId, snapshotId: row.id, executionStatus: "FINISHED" }, - ]); + expect(repairs).toEqual([{ runId, snapshotId: row.id, executionStatus: "FINISHED" }]); } finally { await redis.quit(); } @@ -409,31 +407,34 @@ describe("transition write ordering", () => { } ); - containerTest("appends for the standalone createExecutionSnapshot", async ({ prisma, redisOptions }) => { - const { decorated, redis } = harness(prisma as never, redisOptions as never); - try { - const env = await seedSnapshotEnvironment(prisma); - const runId = generateInternalId(); - await seedBirth(decorated, redis, runId, env); + containerTest( + "appends for the standalone createExecutionSnapshot", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = harness(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + await seedBirth(decorated, redis, runId, env); - const created = await decorated.createExecutionSnapshot({ - run: { id: runId, status: "EXECUTING", attemptNumber: 1 }, - snapshot: { executionStatus: "EXECUTING", description: "Run started" }, - environmentId: env.id, - environmentType: env.type, - projectId: env.projectId, - organizationId: env.organizationId, - }); + const created = await decorated.createExecutionSnapshot({ + run: { id: runId, status: "EXECUTING", attemptNumber: 1 }, + snapshot: { executionStatus: "EXECUTING", description: "Run started" }, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }); - const read = await redis.getById(runId, created.id); - expect(read).not.toBeNull(); - expect(read!.entry.executionStatus).toBe("EXECUTING"); - // The standalone path is the one whose delegate returns the row, so both stores agree exactly. - expect(read!.entry.createdAt).toBe(created.createdAt.toISOString()); - } finally { - await redis.quit(); + const read = await redis.getById(runId, created.id); + expect(read).not.toBeNull(); + expect(read!.entry.executionStatus).toBe("EXECUTING"); + // The standalone path is the one whose delegate returns the row, so both stores agree exactly. + expect(read!.entry.createdAt).toBe(created.createdAt.toISOString()); + } finally { + await redis.quit(); + } } - }); + ); containerTest("writes nothing to Redis at mode off", async ({ prisma, redisOptions }) => { const { decorated, redis } = harness(prisma as never, redisOptions as never, { mode: "off" }); diff --git a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.ts b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.ts index 30655d727db..46a57c6b3bf 100644 --- a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.ts +++ b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.ts @@ -14,7 +14,13 @@ import { Logger } from "@trigger.dev/core/logger"; import { generateInternalId } from "@trigger.dev/core/v3/isomorphic"; import { DelegatingRunStore } from "./delegatingRunStore.js"; -import type { RedisSnapshotStore, SnapshotEntryInput, SnapshotRead } from "./redisSnapshotStore.js"; +import type { + CompletedWaitpointRef, + RedisSnapshotStore, + SnapshotEntryInput, + SnapshotRead, +} from "./redisSnapshotStore.js"; +import { deriveOrder } from "./redisSnapshotStore.js"; import { entryFromCompletion, entryFromCreateExecutionSnapshot, @@ -37,15 +43,19 @@ import type { RunStore, TaskRunWithWaitpoint, } from "./types.js"; -import { - matchSinceCursorLookup, - matchSinceWindow, -} from "./snapshotReadShapes.js"; +import { matchSinceCursorLookup, matchSinceWindow } from "./snapshotReadShapes.js"; +import { boundedIn } from "@trigger.dev/database"; import type { Prisma, PrismaClientOrTransaction, TaskRun } from "@trigger.dev/database"; /** One initial attempt plus three retries, per the write protocol. */ const APPEND_ATTEMPTS = 4; +/** + * Matches the engine's own chunked waitpoint fetch. A batch can complete a thousand waitpoints at + * once, and an unbounded `in:` makes each distinct list length its own prepared statement. + */ +const WAITPOINT_CHUNK_SIZE = 100; + /** * The rollout dial. Postgres stays fully written and authoritative in every position before * `redis-only`, so every earlier position rolls back losslessly by turning the dial down. @@ -88,7 +98,13 @@ export type TaskRunExecutionSnapshotStoreOptions = { * an intercepted write does its Postgres half and pushes its entry here instead of appending, and * the outer instance flushes the buffer after the transaction commits. */ - staging?: SnapshotEntryInput[]; + staging?: StagedAppend[]; +}; + +/** One deferred append: the entry, plus the wait cycle it carries, if any. */ +export type StagedAppend = { + entry: SnapshotEntryInput; + completedWaitpoints?: CompletedWaitpointRef[]; }; export class TaskRunExecutionSnapshotStore extends DelegatingRunStore { @@ -99,7 +115,7 @@ export class TaskRunExecutionSnapshotStore extends DelegatingRunStore { protected readonly faults?: SnapshotFaultInjector; protected readonly metrics?: DecoratorMetrics; protected readonly logger: Logger; - protected readonly staging?: SnapshotEntryInput[]; + protected readonly staging?: StagedAppend[]; constructor(delegate: RunStore, options: TaskRunExecutionSnapshotStoreOptions) { super(delegate); @@ -140,15 +156,20 @@ export class TaskRunExecutionSnapshotStore extends DelegatingRunStore { return this.delegate.runInTransaction(runId, fn); } - const staged: SnapshotEntryInput[] = []; + const staged: StagedAppend[] = []; const result = await this.delegate.runInTransaction(runId, (store, tx) => fn(this.#wrap(store, staged), tx) ); // The transaction committed. Only now can a snapshot claim its partner is durable. - for (const entry of staged) { - await this.#appendTransition("runInTransaction", entry); + for (const item of staged) { + await this.#appendTransition( + "runInTransaction", + item.entry, + undefined, + item.completedWaitpoints + ); } return result; @@ -177,7 +198,7 @@ export class TaskRunExecutionSnapshotStore extends DelegatingRunStore { * the write-ordering logic in exactly one place. Passing no buffer gives a plain decorator that * appends immediately; passing one makes it stage instead. */ - #wrap(store: RunStore, staging?: SnapshotEntryInput[]): TaskRunExecutionSnapshotStore { + #wrap(store: RunStore, staging?: StagedAppend[]): TaskRunExecutionSnapshotStore { return new TaskRunExecutionSnapshotStore(store, { store: this.redis, mode: this.mode, @@ -203,7 +224,7 @@ export class TaskRunExecutionSnapshotStore extends DelegatingRunStore { } const ctx = this.#context(params.data.id, params.snapshot.id); - const snapshot = { ...params.snapshot, id: ctx.id }; + const snapshot = { ...params.snapshot, id: ctx.id, createdAt: ctx.createdAt }; await this.#appendBirth("createRun", entryFromCreateRun(ctx, snapshot)); @@ -219,7 +240,7 @@ export class TaskRunExecutionSnapshotStore extends DelegatingRunStore { } const ctx = this.#context(params.data.id, params.snapshot.id); - const snapshot = { ...params.snapshot, id: ctx.id }; + const snapshot = { ...params.snapshot, id: ctx.id, createdAt: ctx.createdAt }; await this.#appendBirth("createCancelledRun", entryFromCreateRun(ctx, snapshot)); @@ -248,7 +269,10 @@ export class TaskRunExecutionSnapshotStore extends DelegatingRunStore { } const ctx = this.#context(runId, data.snapshot.id); - const withId = { ...data, snapshot: { ...data.snapshot, id: ctx.id } }; + const withId = { + ...data, + snapshot: { ...data.snapshot, id: ctx.id, createdAt: ctx.createdAt }, + }; const result = await this.delegate.completeAttemptSuccess(runId, withId, args, tx); @@ -270,7 +294,10 @@ export class TaskRunExecutionSnapshotStore extends DelegatingRunStore { } const ctx = this.#context(runId, data.snapshot.id); - const withId = { ...data, snapshot: { ...data.snapshot, id: ctx.id } }; + const withId = { + ...data, + snapshot: { ...data.snapshot, id: ctx.id, createdAt: ctx.createdAt }, + }; const result = await this.delegate.expireRun(runId, withId as never, args, tx); @@ -294,7 +321,10 @@ export class TaskRunExecutionSnapshotStore extends DelegatingRunStore { } const ctx = this.#context(runId, data.snapshot.id); - const withId = { ...data, snapshot: { ...data.snapshot, id: ctx.id } }; + const withId = { + ...data, + snapshot: { ...data.snapshot, id: ctx.id, createdAt: ctx.createdAt }, + }; const result = await this.delegate.expireParkedRun(runId, withId as never, tx); @@ -317,7 +347,10 @@ export class TaskRunExecutionSnapshotStore extends DelegatingRunStore { } const ctx = this.#context(runId, data.snapshot.id); - const withId = { ...data, snapshot: { ...data.snapshot, id: ctx.id } }; + const withId = { + ...data, + snapshot: { ...data.snapshot, id: ctx.id, createdAt: ctx.createdAt }, + }; const result = await this.delegate.rescheduleRun(runId, withId, tx); @@ -337,13 +370,17 @@ export class TaskRunExecutionSnapshotStore extends DelegatingRunStore { // This is the one transition whose input already carries both an id and the previous snapshot // id, so it is also the one that can append under a compare-and-set on the current head. const ctx = { id: data.snapshot.id, runId, createdAt: new Date() }; + const withStamp = { ...data, snapshot: { ...data.snapshot, createdAt: ctx.createdAt } }; - const result = await this.delegate.lockRunToWorker(runId, data, tx); + const result = await this.delegate.lockRunToWorker(runId, withStamp, tx); await this.#appendTransition( "lockRunToWorker", - entryFromLock(ctx, data.snapshot), - data.snapshot.previousSnapshotId + entryFromLock(ctx, withStamp.snapshot), + withStamp.snapshot.previousSnapshotId, + // The lock site already carries the resolved order, so the refs are rebuilt from it rather + // than re-derived: its index IS the position in that list. + withStamp.snapshot.completedWaitpointOrder.map((id, index) => ({ id, index })) ); return result; } @@ -357,14 +394,18 @@ export class TaskRunExecutionSnapshotStore extends DelegatingRunStore { } const ctx = this.#context(input.run.id, input.id); - const created = await this.delegate.createExecutionSnapshot({ ...input, id: ctx.id }, tx); + const created = await this.delegate.createExecutionSnapshot( + { ...input, id: ctx.id, createdAt: ctx.createdAt }, + tx + ); // The standalone path is the only one whose delegate returns the row, so its entry can take the // exact createdAt Postgres recorded rather than the decorator's own clock. await this.#appendTransition( "createExecutionSnapshot", - entryFromCreateExecutionSnapshot({ ...ctx, createdAt: created.createdAt }, input), - input.previousSnapshotId + entryFromCreateExecutionSnapshot(ctx, input), + input.previousSnapshotId, + input.completedWaitpoints ); return created; } @@ -446,12 +487,13 @@ export class TaskRunExecutionSnapshotStore extends DelegatingRunStore { async #appendTransition( site: string, entry: SnapshotEntryInput, - expectedCur?: string + expectedCur?: string, + completedWaitpoints?: CompletedWaitpointRef[] ): Promise { if (this.staging) { // Inside a transaction the append cannot run until the Postgres side commits, or a rollback // leaves Redis holding a transition that never happened. - this.staging.push(entry); + this.staging.push({ entry, ...(completedWaitpoints && { completedWaitpoints }) }); return; } @@ -462,11 +504,14 @@ export class TaskRunExecutionSnapshotStore extends DelegatingRunStore { snapshotId: entry.id, }); + const cycle = await this.#resolveCycle(entry.runId, completedWaitpoints); + const result = await this.redis.append({ entry, kind: "transition", isTerminal: isTerminalEntry(entry), ...(expectedCur !== undefined && { expectedCur }), + ...(cycle && { cycle }), }); this.#recordOutcome(site, entry, result); @@ -496,6 +541,52 @@ export class TaskRunExecutionSnapshotStore extends DelegatingRunStore { } } + /** + * Decides whether this append mints a new wait cycle or points at the one already there. + * + * A resume append carries a newly-differing id set, so it mints a cycle and the record set is + * written once. Every copy-forward append that follows re-passes the SAME list, and re-minting on + * each would rewrite the record set once per entry in the resume chain — the write amplification + * the pointer model exists to remove. So an unchanged id set carries the previous cycleSeq + * forward and writes no key. + * + * The extra read only happens for an append that actually carries waitpoints, which is the resume + * path rather than the hot path. + * + * `records` is deliberately left unset. The record envelope belongs to the waitpoint lane and + * ships empty in this build, so dual-write never re-versions the entry when it arrives. + */ + async #resolveCycle( + runId: string, + completedWaitpoints?: CompletedWaitpointRef[] + ): Promise< + | { kind: "new"; completedWaitpoints: CompletedWaitpointRef[] } + | { kind: "carryForward"; cycleSeq: number } + | undefined + > { + if (!completedWaitpoints || completedWaitpoints.length === 0) { + return undefined; + } + + const order = deriveOrder(completedWaitpoints); + + try { + const head = await this.redis.getLatest(runId); + const previous = head?.completedWaitpointIds?.order; + + if (head?.cycle && previous && sameOrder(previous, order)) { + return { kind: "carryForward", cycleSeq: head.cycle.cycleSeq }; + } + } catch (error) { + // A failed probe must not lose the waitpoints. Minting a fresh cycle is the safe direction: + // it costs one duplicated record set, where a wrong carryForward would point at another + // cycle's ids. + this.logger.warn("snapshot cycle probe failed, minting a new cycle", { runId, error }); + } + + return { kind: "new", completedWaitpoints }; + } + /** * None of the four append outcomes is a failure, and none of them enqueues a repair. * @@ -564,7 +655,7 @@ export class TaskRunExecutionSnapshotStore extends DelegatingRunStore { } this.metrics?.recordRead("findLatestExecutionSnapshot", "redis"); - return this.#hydrate(read, runId, client); + return this.#hydrate(read, runId, client, { hydrateWaitpointRows: true }); } override async findExecutionSnapshot( @@ -592,9 +683,7 @@ export class TaskRunExecutionSnapshotStore extends DelegatingRunStore { } as unknown as Prisma.TaskRunExecutionSnapshotGetPayload; } - override async findManyExecutionSnapshots< - T extends Prisma.TaskRunExecutionSnapshotFindManyArgs, - >( + override async findManyExecutionSnapshots( args: Prisma.SelectSubset, client?: ReadClient ): Promise[]> { @@ -617,8 +706,10 @@ export class TaskRunExecutionSnapshotStore extends DelegatingRunStore { // The engine asks for createdAt DESC and reverses app-side; the store returns ascending. const descending = [...result.entries].reverse(); + // Rows are hydrated for no entry here: the engine fetches the head's waitpoints itself, from + // the ids this call's head row reports. Each row still carries its own order. const hydrated = await Promise.all( - descending.map((entry) => this.#hydrate(entry, shape.runId, client, { waitpoints: false })) + descending.map((entry) => this.#hydrate(entry, shape.runId, client)) ); return hydrated as unknown as Prisma.TaskRunExecutionSnapshotGetPayload[]; } @@ -675,7 +766,7 @@ export class TaskRunExecutionSnapshotStore extends DelegatingRunStore { read: SnapshotRead, runId: string, client?: ReadClient, - opts?: { waitpoints?: boolean } + opts?: { hydrateWaitpointRows?: boolean } ): Promise< Prisma.TaskRunExecutionSnapshotGetPayload<{ include: { completedWaitpoints: true; checkpoint: true }; @@ -687,22 +778,18 @@ export class TaskRunExecutionSnapshotStore extends DelegatingRunStore { ? await this.#hydrateCheckpoint(runId, read.id, client) : null; - let completedWaitpoints: unknown[] = []; - let completedWaitpointOrder: string[] = []; + // `completedWaitpointOrder` is a scalar column, NOT the join. The engine reads it off the head + // row as the index oracle that gives each completed waitpoint its position in a batch, so it + // must be populated even when the waitpoint ROWS are not fetched. Returning an empty order here + // resumes every batched triggerAndWait with `index: undefined`. + const ids = + read.completedWaitpointIds ?? (await this.redis.getSnapshotWaitpointIds(runId, read.id)); + const completedWaitpointOrder = ids.order; - if (opts?.waitpoints !== false) { - const ids = - read.completedWaitpointIds ?? (await this.redis.getSnapshotWaitpointIds(runId, read.id)); - completedWaitpointOrder = ids.order; - - if (ids.distinctIds.length > 0) { - completedWaitpoints = await this.delegate.findManyWaitpoints( - { where: { id: { in: ids.distinctIds } } }, - client, - runId - ); - } - } + // The rows themselves are head-only, mirroring the engine's own N x M avoidance. + const completedWaitpoints = opts?.hydrateWaitpointRows + ? await this.#fetchWaitpointsInChunks(ids.distinctIds, runId, client) + : []; return { id: read.id, @@ -726,6 +813,7 @@ export class TaskRunExecutionSnapshotStore extends DelegatingRunStore { isValid: read.isValid, error: entry.error ?? null, createdAt: new Date(entry.createdAt as string), + // A snapshot row is write-once, so both columns hold the one instant the decorator minted. updatedAt: new Date(entry.createdAt as string), checkpoint, completedWaitpoints, @@ -734,6 +822,30 @@ export class TaskRunExecutionSnapshotStore extends DelegatingRunStore { }>; } + /** + * Chunked, and bounded within each chunk, mirroring the engine's own waitpoint fetch. The run id + * routes each chunk to the owning store rather than fanning every one across both databases. + */ + async #fetchWaitpointsInChunks( + waitpointIds: string[], + runId: string, + client?: ReadClient + ): Promise { + if (waitpointIds.length === 0) return []; + + const all: unknown[] = []; + for (let i = 0; i < waitpointIds.length; i += WAITPOINT_CHUNK_SIZE) { + const chunk = waitpointIds.slice(i, i + WAITPOINT_CHUNK_SIZE); + const rows = await this.delegate.findManyWaitpoints( + { where: { id: { in: boundedIn(chunk) } } }, + client, + runId + ); + all.push(...rows); + } + return all; + } + /** * Reads the checkpoint row through the snapshot the delegate still holds, so the read stays * residency-aware: the run id in the where is what routes it to the owning database, and the @@ -772,3 +884,8 @@ export class TaskRunExecutionSnapshotStore extends DelegatingRunStore { } } } + +/** Position-sensitive: the same ids in a different order are a different wait cycle. */ +function sameOrder(a: string[], b: string[]): boolean { + return a.length === b.length && a.every((id, index) => id === b[index]); +} diff --git a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.waitpointCycles.test.ts b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.waitpointCycles.test.ts new file mode 100644 index 00000000000..1879fe1246b --- /dev/null +++ b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.waitpointCycles.test.ts @@ -0,0 +1,363 @@ +// The completed-waitpoint path, which every other suite here was blind to. +// +// Two defects hid behind that blindness. The decorator passed no cycle to `append`, so no +// wp: key was ever written and the Redis waitpoint side was permanently empty. And the +// since-window hydration returned an empty `completedWaitpointOrder`, which is the index oracle the +// engine uses to give each completed waitpoint its position in a batch — an empty order resumes +// every batched triggerAndWait with `index: undefined`. +// +// So these tests all use a snapshot that ACTUALLY carries waitpoints. A test that does not cannot +// tell a working cycle from a missing one. +import { describe, expect } from "vitest"; +import { containerTest } from "@internal/testcontainers"; +import { createRedisClient } from "@internal/redis"; +import { generateInternalId } from "@trigger.dev/core/v3/isomorphic"; +import { PostgresRunStore } from "./PostgresRunStore.js"; +import { RedisSnapshotStore } from "./redisSnapshotStore.js"; +import { entryFromCreateRun } from "./snapshotEntry.js"; +import { TaskRunExecutionSnapshotStore } from "./taskRunExecutionSnapshotStore.js"; +import type { RunStore } from "./types.js"; +import { + buildCreateRunData, + seedSnapshotEnvironment, + seedSnapshotWaitpoints, + type SnapshotFixtureEnv, +} from "./testFixtures/snapshotIdFixture.js"; + +const COMPLETED_TTL_MS = 72 * 60 * 60 * 1000; + +function build( + prisma: never, + redisOptions: never, + mode: "dual-write" | "redis-read" = "redis-read" +) { + const redis = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + const writes: { site: string; outcome: string }[] = []; + const decorated = new TaskRunExecutionSnapshotStore( + new PostgresRunStore({ prisma, readOnlyPrisma: prisma }) as unknown as RunStore, + { + store: redis, + mode, + readPercent: 100, + metrics: { + recordWrite: (site, outcome) => writes.push({ site, outcome }), + recordAppendFailed: () => {}, + recordRead: () => {}, + }, + } + ); + return { decorated, redis, writes }; +} + +async function seedRun( + decorated: TaskRunExecutionSnapshotStore, + redis: RedisSnapshotStore, + env: SnapshotFixtureEnv +): Promise { + const runId = generateInternalId(); + const snapshot = { + id: generateInternalId(), + engine: "V2" as const, + executionStatus: "RUN_CREATED" as const, + description: "Run was created", + runStatus: "PENDING" as const, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }; + await redis.append({ + entry: entryFromCreateRun({ id: snapshot.id, runId, createdAt: new Date() }, snapshot), + kind: "birth", + isTerminal: false, + }); + await decorated.createRun({ data: buildCreateRunData(runId, env), snapshot }); + return runId; +} + +function resumeInput( + runId: string, + env: SnapshotFixtureEnv, + completedWaitpoints: { id: string; index?: number }[], + description = "Run resumed" +) { + return { + id: generateInternalId(), + run: { id: runId, status: "EXECUTING" as const, attemptNumber: 1 }, + snapshot: { executionStatus: "EXECUTING" as const, description }, + completedWaitpoints, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }; +} + +describe("completed-waitpoint cycles", () => { + containerTest("a resume append mints a cycle key", async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, redis, env); + const [wpA, wpB] = await seedSnapshotWaitpoints(prisma, env, 2); + + const created = await decorated.createExecutionSnapshot( + resumeInput(runId, env, [ + { id: wpA, index: 0 }, + { id: wpB, index: 1 }, + ]) + ); + + // The key exists at all — before the fix, none was ever written. + const cycleKeys = await probe.keys(`snap:{${runId}}:wp:*`); + expect(cycleKeys.length).toBe(1); + + const ids = await redis.getSnapshotWaitpointIds(runId, created.id); + expect(ids.present).toBe(true); + expect(ids.order).toEqual([wpA, wpB]); + expect(ids.distinctIds).toEqual([wpA, wpB]); + } finally { + await Promise.all([redis.quit(), probe.quit().catch(() => {})]); + } + }); + + containerTest( + "a copy-forward reuses the cycle and writes no second key", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, redis, env); + const [wpA, wpB] = await seedSnapshotWaitpoints(prisma, env, 2); + const waitpoints = [ + { id: wpA, index: 0 }, + { id: wpB, index: 1 }, + ]; + + await decorated.createExecutionSnapshot(resumeInput(runId, env, waitpoints, "resume")); + // The same id set again: this is the copy-forward every dequeue and checkpoint site does. + const second = await decorated.createExecutionSnapshot( + resumeInput(runId, env, waitpoints, "carry one") + ); + const third = await decorated.createExecutionSnapshot( + resumeInput(runId, env, waitpoints, "carry two") + ); + + // Still ONE key. Re-minting per entry is the write amplification the pointer model removes. + expect((await probe.keys(`snap:{${runId}}:wp:*`)).length).toBe(1); + + // And every entry still resolves the same order. + for (const id of [second.id, third.id]) { + expect((await redis.getSnapshotWaitpointIds(runId, id)).order).toEqual([wpA, wpB]); + } + } finally { + await Promise.all([redis.quit(), probe.quit().catch(() => {})]); + } + } + ); + + containerTest( + "a newly-differing id set mints a second cycle", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, redis, env); + const [wpA, wpB] = await seedSnapshotWaitpoints(prisma, env, 2); + + await decorated.createExecutionSnapshot( + resumeInput(runId, env, [{ id: wpA, index: 0 }], "first wait") + ); + const second = await decorated.createExecutionSnapshot( + resumeInput(runId, env, [{ id: wpB, index: 0 }], "second wait") + ); + + expect((await probe.keys(`snap:{${runId}}:wp:*`)).length).toBe(2); + expect((await redis.getSnapshotWaitpointIds(runId, second.id)).order).toEqual([wpB]); + } finally { + await Promise.all([redis.quit(), probe.quit().catch(() => {})]); + } + } + ); + + containerTest( + "the same ids in a different order are a new cycle", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, redis, env); + const [wpA, wpB] = await seedSnapshotWaitpoints(prisma, env, 2); + + await decorated.createExecutionSnapshot( + resumeInput(runId, env, [ + { id: wpA, index: 0 }, + { id: wpB, index: 1 }, + ]) + ); + // Order IS the index oracle, so a reordering is a different cycle, not a carry-forward. + const reordered = await decorated.createExecutionSnapshot( + resumeInput(runId, env, [ + { id: wpB, index: 0 }, + { id: wpA, index: 1 }, + ]) + ); + + expect((await probe.keys(`snap:{${runId}}:wp:*`)).length).toBe(2); + expect((await redis.getSnapshotWaitpointIds(runId, reordered.id)).order).toEqual([ + wpB, + wpA, + ]); + } finally { + await Promise.all([redis.quit(), probe.quit().catch(() => {})]); + } + } + ); + + containerTest("a repeated id keeps both of its positions", async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, redis, env); + const [wpX] = await seedSnapshotWaitpoints(prisma, env, 1); + + // One run batched twice under a single idempotency key: the id repeats, and each position + // must survive, because the runner matches results to positions. + const created = await decorated.createExecutionSnapshot( + resumeInput(runId, env, [ + { id: wpX, index: 0 }, + { id: wpX, index: 1 }, + ]) + ); + + const ids = await redis.getSnapshotWaitpointIds(runId, created.id); + expect(ids.order).toEqual([wpX, wpX]); + expect(ids.distinctIds).toEqual([wpX]); + } finally { + await redis.quit(); + } + }); + + containerTest( + "findLatestExecutionSnapshot returns the index oracle", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, redis, env); + const [wpA, wpB] = await seedSnapshotWaitpoints(prisma, env, 2); + await decorated.createExecutionSnapshot( + resumeInput(runId, env, [ + { id: wpA, index: 0 }, + { id: wpB, index: 1 }, + ]) + ); + + const latest = await decorated.findLatestExecutionSnapshot(runId); + + // completedWaitpointOrder is a scalar column, not the join. Empty here means every batched + // waitpoint resumes with index undefined. + expect(latest!.completedWaitpointOrder).toEqual([wpA, wpB]); + } finally { + await redis.quit(); + } + } + ); + + containerTest( + "the since-window head carries the index oracle", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, redis, env); + const [wpA, wpB] = await seedSnapshotWaitpoints(prisma, env, 2); + const first = await decorated.createExecutionSnapshot( + resumeInput(runId, env, [], "before the wait") + ); + await new Promise((resolve) => setTimeout(resolve, 5)); + await decorated.createExecutionSnapshot( + resumeInput(runId, env, [ + { id: wpA, index: 0 }, + { id: wpB, index: 1 }, + ]) + ); + + const window = await decorated.findManyExecutionSnapshots({ + where: { runId, isValid: true, createdAt: { gt: first.createdAt } }, + include: { checkpoint: true }, + orderBy: { createdAt: "desc" }, + take: 50, + }); + + // The head is first in a descending window. This is the row the engine reads the oracle off. + expect(window[0]!.completedWaitpointOrder).toEqual([wpA, wpB]); + } finally { + await redis.quit(); + } + } + ); + + containerTest( + "lockRunToWorker carries its resolved order into the cycle", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, redis, env); + const [wpA, wpB] = await seedSnapshotWaitpoints(prisma, env, 2); + const head = await redis.getLatest(runId); + const snapshotId = generateInternalId(); + + await decorated.lockRunToWorker(runId, { + lockedAt: new Date(), + lockedById: undefined, + lockedToVersionId: undefined, + lockedQueueId: undefined, + startedAt: new Date(), + baseCostInCents: 0, + machinePreset: "small-1x", + taskVersion: "1.0.0", + snapshot: { + id: snapshotId, + previousSnapshotId: head!.id, + attemptNumber: 1, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + completedWaitpointIds: [wpA, wpB], + completedWaitpointOrder: [wpA, wpB], + }, + } as never); + + const ids = await redis.getSnapshotWaitpointIds(runId, snapshotId); + expect(ids.order).toEqual([wpA, wpB]); + } finally { + await redis.quit(); + } + } + ); + + containerTest( + "an append with no waitpoints writes no cycle key", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, redis, env); + + await decorated.createExecutionSnapshot(resumeInput(runId, env, [], "no waitpoints")); + + expect(await probe.keys(`snap:{${runId}}:wp:*`)).toEqual([]); + } finally { + await Promise.all([redis.quit(), probe.quit().catch(() => {})]); + } + } + ); +}); diff --git a/internal-packages/run-store/src/testFixtures/snapshotIdFixture.ts b/internal-packages/run-store/src/testFixtures/snapshotIdFixture.ts index 17c0248312b..e0b8a1a2107 100644 --- a/internal-packages/run-store/src/testFixtures/snapshotIdFixture.ts +++ b/internal-packages/run-store/src/testFixtures/snapshotIdFixture.ts @@ -116,6 +116,38 @@ export async function seedSnapshotWorker( return { workerId: worker.id, taskId: task.id }; } +/** + * Seeds real Waitpoint rows and returns their ids. The legacy completed-waitpoint join carries a + * real foreign key, so an invented id fails the constraint rather than the assertion — the test + * then reports a fixture fault as if it were a defect in the code under test. + */ +export async function seedSnapshotWaitpoints( + prisma: PrismaClient, + env: SnapshotFixtureEnv, + count: number +): Promise { + const ids: string[] = []; + + for (let i = 0; i < count; i++) { + const suffix = generateInternalId().slice(-12); + const waitpoint = await prisma.waitpoint.create({ + data: { + friendlyId: `waitpoint_${suffix}`, + type: "MANUAL", + status: "COMPLETED", + completedAt: new Date(), + idempotencyKey: `idem_${suffix}`, + userProvidedIdempotencyKey: false, + projectId: env.projectId, + environmentId: env.id, + }, + }); + ids.push(waitpoint.id); + } + + return ids; +} + /** * Seeds an environment plus one run in `status`, with no execution snapshot. The suites that use it * assert on the snapshot rows a store method writes, so the run must start with none. diff --git a/internal-packages/run-store/src/types.ts b/internal-packages/run-store/src/types.ts index 5bde5950459..93fffe0ee88 100644 --- a/internal-packages/run-store/src/types.ts +++ b/internal-packages/run-store/src/types.ts @@ -29,6 +29,11 @@ export type IdempotencyKeyRunMatch = { }; export type CreateRunSnapshotInput = { + /** Caller-minted creation instant. Absent, Postgres applies its own default. The decorator sets + * it so a snapshot carries the SAME instant in Postgres and in the Redis store: the field is + * compared directly under dual-write, and the since-window cursor is resolved from one store and + * applied in the other, so two different instants misfilter that window. */ + createdAt?: Date; id?: string; engine: "V2"; executionStatus: TaskRunExecutionStatus; @@ -43,6 +48,11 @@ export type CreateRunSnapshotInput = { }; export type CompletionSnapshotInput = { + /** Caller-minted creation instant. Absent, Postgres applies its own default. The decorator sets + * it so a snapshot carries the SAME instant in Postgres and in the Redis store: the field is + * compared directly under dual-write, and the since-window cursor is resolved from one store and + * applied in the other, so two different instants misfilter that window. */ + createdAt?: Date; /** Caller-minted snapshot id. Absent, Prisma's `@default(cuid())` supplies one. The decorator * sets it so a snapshot carries the same id in Postgres and in the Redis store. */ id?: string; @@ -67,6 +77,11 @@ export type PromotePendingVersionArgs = { }; export type ExpireSnapshotInput = { + /** Caller-minted creation instant. Absent, Postgres applies its own default. The decorator sets + * it so a snapshot carries the SAME instant in Postgres and in the Redis store: the field is + * compared directly under dual-write, and the since-window cursor is resolved from one store and + * applied in the other, so two different instants misfilter that window. */ + createdAt?: Date; /** Caller-minted snapshot id. Absent, Prisma's `@default(cuid())` supplies one. The decorator * sets it so a snapshot carries the same id in Postgres and in the Redis store. */ id?: string; @@ -81,6 +96,11 @@ export type ExpireSnapshotInput = { }; export type RescheduleSnapshotInput = { + /** Caller-minted creation instant. Absent, Postgres applies its own default. The decorator sets + * it so a snapshot carries the SAME instant in Postgres and in the Redis store: the field is + * compared directly under dual-write, and the since-window cursor is resolved from one store and + * applied in the other, so two different instants misfilter that window. */ + createdAt?: Date; /** Caller-minted snapshot id. Absent, Prisma's `@default(cuid())` supplies one. The decorator * sets it so a snapshot carries the same id in Postgres and in the Redis store. */ id?: string; @@ -94,6 +114,11 @@ export type RescheduleSnapshotInput = { }; export type LockSnapshotInput = { + /** Caller-minted creation instant. Absent, Postgres applies its own default. The decorator sets + * it so a snapshot carries the SAME instant in Postgres and in the Redis store: the field is + * compared directly under dual-write, and the since-window cursor is resolved from one store and + * applied in the other, so two different instants misfilter that window. */ + createdAt?: Date; id: string; previousSnapshotId: string; attemptNumber?: number; @@ -301,6 +326,11 @@ export type TaskRunWithWaitpoint = TaskRun & { associatedWaitpoint: Waitpoint | * input — callers pass the high-level shape, not a raw Prisma `data`/`include`. */ export type CreateExecutionSnapshotInput = { + /** Caller-minted creation instant. Absent, Postgres applies its own default. The decorator sets + * it so a snapshot carries the SAME instant in Postgres and in the Redis store: the field is + * compared directly under dual-write, and the since-window cursor is resolved from one store and + * applied in the other, so two different instants misfilter that window. */ + createdAt?: Date; /** Caller-minted snapshot id. Absent, Prisma's `@default(cuid())` supplies one. The decorator * sets it so a snapshot carries the same id in Postgres and in the Redis store. */ id?: string; From ea0e17bdc3697271760ad6180009394900d37a3e Mon Sep 17 00:00:00 2001 From: Daniel Sutton Date: Mon, 24 Aug 2026 17:48:20 +0100 Subject: [PATCH 12/56] chore(run-store): treat the run-store scripts directory as an entry point The generator that emits the pass-through store base is a runnable script, not dead code, and the same glob covers any script added there later. --- knip.json | 3 +++ 1 file changed, 3 insertions(+) diff --git a/knip.json b/knip.json index 84456756ca1..8c32f5e72c3 100644 --- a/knip.json +++ b/knip.json @@ -37,6 +37,9 @@ "internal-packages/observability-map": { "entry": ["src/index.ts", "fixtures/**/*.{js,mjs,cjs,ts,mts,cts,tsx}"] }, + "internal-packages/run-store": { + "entry": ["scripts/**/*.{js,mjs,cjs,ts,mts,cts}"] + }, "internal-packages/otlp-importer": { "ignoreDependencies": ["ts-proto"] }, From 1d4eb7ba467e9dacad3b35e6bf42d1ea956c93f4 Mon Sep 17 00:00:00 2001 From: Daniel Sutton Date: Mon, 24 Aug 2026 18:11:44 +0100 Subject: [PATCH 13/56] fix(run-store): address review on the sweep, and make the timestamp parity real The sweep discovered keyspaces by their cur key, which the append script writes only when an entry is valid. A keyspace whose entries all carry an error has no cur and no index, so neither sweep rule could ever see it and it leaked with no expiry, which is the same unbounded leak the second rule exists to close. It now scans on the entry hash, which every append writes, and the age probe falls back to the newest instant in that hash when the index is empty. Enumerating a run's cycle keys used KEYS. That command iterates the whole database and blocks while it does, and a hash tag routes a key without scoping the scan, so a sweep pass would have issued one full keyspace scan per run. It now reads the dense cycle high-water counter the append script maintains, which is the same source the store's own terminal-expiry loop uses, and pipelines the existence checks into one round trip. The timestamp parity assertion was still tautological. The previous commit added a note saying the builders receive an independent instant and did not change the builder calls, which kept reading the value off the row under test. Every case now mints one instant, passes it to the store, and gives the builder the same value, so a write site that stops forwarding the caller's instant fails here. Also documents what an injected fault actually does at each write path, since only the birth path rethrows, and scopes a run count in the chaos suite to the environment under test. --- .../engine/tests/snapshotStoreChaos.test.ts | 4 +- .../src/snapshotEntry.parity.test.ts | 58 +++++++++----- .../run-store/src/snapshotFaultInjection.ts | 12 ++- .../src/snapshotOrphanSweeper.test.ts | 41 ++++++++++ .../run-store/src/snapshotOrphanSweeper.ts | 79 +++++++++++++++++-- 5 files changed, 164 insertions(+), 30 deletions(-) diff --git a/internal-packages/run-engine/src/engine/tests/snapshotStoreChaos.test.ts b/internal-packages/run-engine/src/engine/tests/snapshotStoreChaos.test.ts index a727c9bcc1f..ea3447acc56 100644 --- a/internal-packages/run-engine/src/engine/tests/snapshotStoreChaos.test.ts +++ b/internal-packages/run-engine/src/engine/tests/snapshotStoreChaos.test.ts @@ -186,7 +186,9 @@ describe("snapshot store crash boundaries", () => { expect(faults.fired("afterRedisBirthBeforePg")).toBe(1); // The harmless state: no run row, so nothing can ever read a run that has no snapshot. - const runsAfterCrash = await prisma.taskRun.count(); + const runsAfterCrash = await prisma.taskRun.count({ + where: { runtimeEnvironmentId: environment.id }, + }); expect(runsAfterCrash).toBe(0); // A crashed birth must not poison the path: the next trigger runs to completion. diff --git a/internal-packages/run-store/src/snapshotEntry.parity.test.ts b/internal-packages/run-store/src/snapshotEntry.parity.test.ts index 8e0a11fafbe..635160fd831 100644 --- a/internal-packages/run-store/src/snapshotEntry.parity.test.ts +++ b/internal-packages/run-store/src/snapshotEntry.parity.test.ts @@ -24,11 +24,14 @@ import { /** * NOTE ON createdAt. An earlier version of this suite built the expected entry with - * `createdAt: row.createdAt` and then asserted the two matched, which is tautological and hid a - * real divergence: seven of the eight write sites stamped the entry from the app clock while - * Postgres stamped its own column default, so the stores held different instants. The builders are - * now given an INDEPENDENT instant, and the row must carry that same value because the decorator - * passes it through to Postgres. + * `createdAt: row.createdAt`, reading the value off the row it was checking and then asserting the + * two matched. That can never fail, and it hid a real divergence: seven of the eight write sites + * stamped the entry from the app clock while Postgres stamped its own column default, so the two + * stores held different instants for one snapshot. + * + * Every case now mints ONE instant, passes it to the store call, and gives the builder the same + * value. The row must carry it because the write site forwards it. A write site that stops + * forwarding the caller's instant fails here. * * Compares only what the entry claims. The Redis model carries no `updatedAt` and no join rows, and * it holds `createdAt` as an ISO string, so those are checked separately or not at all. @@ -56,9 +59,13 @@ function assertParity(entry: SnapshotEntryInput, row: Record) { expect((row.updatedAt as Date).toISOString()).toBe(entry.createdAt); } +/** Five minutes in the past, so a database default could never coincide with it. */ +const independentStamp = new Date(Date.now() - 5 * 60 * 1000); + function birthSnapshot(id: string, env: SnapshotFixtureEnv) { return { id, + createdAt: independentStamp, engine: "V2" as const, executionStatus: "RUN_CREATED" as const, description: "Run was created", @@ -81,7 +88,7 @@ describe("entry to Postgres row parity", () => { await store.createRun({ data: buildCreateRunData(runId, env), snapshot }); const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); - assertParity(entryFromCreateRun({ id, runId, createdAt: row.createdAt }, snapshot), row); + assertParity(entryFromCreateRun({ id, runId, createdAt: independentStamp }, snapshot), row); }); postgresTest("createRun with an associated waitpoint, legacy schema", async ({ prisma }) => { @@ -107,7 +114,7 @@ describe("entry to Postgres row parity", () => { }); const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); - assertParity(entryFromCreateRun({ id, runId, createdAt: row.createdAt }, snapshot), row); + assertParity(entryFromCreateRun({ id, runId, createdAt: independentStamp }, snapshot), row); }); postgresTest("createRun carries the worker and runner ids", async ({ prisma }) => { @@ -121,7 +128,7 @@ describe("entry to Postgres row parity", () => { await store.createRun({ data: buildCreateRunData(runId, env), snapshot }); const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); - assertParity(entryFromCreateRun({ id, runId, createdAt: row.createdAt }, snapshot), row); + assertParity(entryFromCreateRun({ id, runId, createdAt: independentStamp }, snapshot), row); }); postgresTest("createCancelledRun", async ({ prisma }) => { @@ -149,7 +156,7 @@ describe("entry to Postgres row parity", () => { }); const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); - assertParity(entryFromCreateRun({ id, runId, createdAt: row.createdAt }, snapshot), row); + assertParity(entryFromCreateRun({ id, runId, createdAt: independentStamp }, snapshot), row); }); postgresTest("completeAttemptSuccess", async ({ prisma }) => { @@ -158,6 +165,7 @@ describe("entry to Postgres row parity", () => { const id = generateInternalId(); const snapshot = { id, + createdAt: independentStamp, executionStatus: "FINISHED" as const, description: "Run completed", runStatus: "COMPLETED_SUCCESSFULLY" as const, @@ -182,7 +190,7 @@ describe("entry to Postgres row parity", () => { const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); assertParity( - entryFromCompletion({ id, runId: run.id, createdAt: row.createdAt }, snapshot), + entryFromCompletion({ id, runId: run.id, createdAt: independentStamp }, snapshot), row ); }); @@ -193,6 +201,7 @@ describe("entry to Postgres row parity", () => { const id = generateInternalId(); const snapshot = { id, + createdAt: independentStamp, engine: "V2" as const, executionStatus: "FINISHED" as const, description: "Run expired", @@ -215,7 +224,10 @@ describe("entry to Postgres row parity", () => { ); const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); - assertParity(entryFromExpire({ id, runId: run.id, createdAt: row.createdAt }, snapshot), row); + assertParity( + entryFromExpire({ id, runId: run.id, createdAt: independentStamp }, snapshot), + row + ); }); postgresTest("expireParkedRun", async ({ prisma }) => { @@ -224,6 +236,7 @@ describe("entry to Postgres row parity", () => { const id = generateInternalId(); const snapshot = { id, + createdAt: independentStamp, engine: "V2" as const, executionStatus: "FINISHED" as const, description: "Parked run expired", @@ -244,7 +257,10 @@ describe("entry to Postgres row parity", () => { expect(result.count).toBe(1); const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); - assertParity(entryFromExpire({ id, runId: run.id, createdAt: row.createdAt }, snapshot), row); + assertParity( + entryFromExpire({ id, runId: run.id, createdAt: independentStamp }, snapshot), + row + ); }); postgresTest("rescheduleRun with every default applied", async ({ prisma }) => { @@ -253,6 +269,7 @@ describe("entry to Postgres row parity", () => { const id = generateInternalId(); const snapshot = { id, + createdAt: independentStamp, environmentId: env.id, environmentType: env.type, projectId: env.projectId, @@ -266,7 +283,7 @@ describe("entry to Postgres row parity", () => { const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); assertParity( - entryFromReschedule({ id, runId: run.id, createdAt: row.createdAt }, snapshot), + entryFromReschedule({ id, runId: run.id, createdAt: independentStamp }, snapshot), row ); }); @@ -277,6 +294,7 @@ describe("entry to Postgres row parity", () => { const id = generateInternalId(); const snapshot = { id, + createdAt: independentStamp, environmentId: env.id, environmentType: env.type, projectId: env.projectId, @@ -293,7 +311,7 @@ describe("entry to Postgres row parity", () => { const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); assertParity( - entryFromReschedule({ id, runId: run.id, createdAt: row.createdAt }, snapshot), + entryFromReschedule({ id, runId: run.id, createdAt: independentStamp }, snapshot), row ); }); @@ -314,6 +332,7 @@ describe("entry to Postgres row parity", () => { const id = generateInternalId(); const snapshot = { id, + createdAt: independentStamp, previousSnapshotId: previous.id, attemptNumber: 1, environmentId: env.id, @@ -337,7 +356,7 @@ describe("entry to Postgres row parity", () => { }); const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); - assertParity(entryFromLock({ id, runId: run.id, createdAt: row.createdAt }, snapshot), row); + assertParity(entryFromLock({ id, runId: run.id, createdAt: independentStamp }, snapshot), row); }); postgresTest("createExecutionSnapshot, the standalone site", async ({ prisma }) => { @@ -346,6 +365,7 @@ describe("entry to Postgres row parity", () => { const id = generateInternalId(); const input = { id, + createdAt: independentStamp, run: { id: run.id, status: "EXECUTING" as const, attemptNumber: 2 }, snapshot: { executionStatus: "EXECUTING" as const, description: "Run started" }, environmentId: env.id, @@ -359,7 +379,7 @@ describe("entry to Postgres row parity", () => { const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); expect(created.id).toBe(id); assertParity( - entryFromCreateExecutionSnapshot({ id, runId: run.id, createdAt: row.createdAt }, input), + entryFromCreateExecutionSnapshot({ id, runId: run.id, createdAt: independentStamp }, input), row ); }); @@ -370,6 +390,7 @@ describe("entry to Postgres row parity", () => { const id = generateInternalId(); const input = { id, + createdAt: independentStamp, run: { id: run.id, status: "DEQUEUED" as const, attemptNumber: 1 }, snapshot: { executionStatus: "PENDING_EXECUTING" as const, description: "Run was dequeued" }, environmentId: env.id, @@ -383,7 +404,7 @@ describe("entry to Postgres row parity", () => { const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); expect(row.runStatus).toBe("PENDING"); assertParity( - entryFromCreateExecutionSnapshot({ id, runId: run.id, createdAt: row.createdAt }, input), + entryFromCreateExecutionSnapshot({ id, runId: run.id, createdAt: independentStamp }, input), row ); }); @@ -394,6 +415,7 @@ describe("entry to Postgres row parity", () => { const id = generateInternalId(); const input = { id, + createdAt: independentStamp, run: { id: run.id, status: "EXECUTING" as const, attemptNumber: 1 }, snapshot: { executionStatus: "EXECUTING" as const, description: "Stale write" }, error: "snapshot is not the latest", @@ -408,7 +430,7 @@ describe("entry to Postgres row parity", () => { const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); expect(row.isValid).toBe(false); assertParity( - entryFromCreateExecutionSnapshot({ id, runId: run.id, createdAt: row.createdAt }, input), + entryFromCreateExecutionSnapshot({ id, runId: run.id, createdAt: independentStamp }, input), row ); }); diff --git a/internal-packages/run-store/src/snapshotFaultInjection.ts b/internal-packages/run-store/src/snapshotFaultInjection.ts index 50029b9518d..83abf43b179 100644 --- a/internal-packages/run-store/src/snapshotFaultInjection.ts +++ b/internal-packages/run-store/src/snapshotFaultInjection.ts @@ -20,9 +20,15 @@ export type SnapshotFaultInjector = ( ) => void; /** - * Thrown by a test injector. The write path tells this apart from a real append failure: an injected - * fault models a process that died, so it is rethrown rather than retried, while a real failure is - * retried and then handed to the repair job. + * Thrown by a test injector. The write path tells this apart from a real append failure, because an + * injected fault models a process that died rather than a call that failed. The two write paths then + * do different things with it, and both differ from a real failure: + * + * - A transition skips its remaining retries, hands the run to the repair job, and does NOT rethrow. + * Postgres has already committed, so the caller must not see an error. + * - A birth rethrows, so the Postgres insert never runs and the crash leaves an orphaned keyspace + * with no run row, which is the harmless state that ordering exists to produce. + * - A real append failure is retried, and only then handed to the repair job. */ export class InjectedSnapshotFault extends Error { readonly boundary: SnapshotFaultBoundary; diff --git a/internal-packages/run-store/src/snapshotOrphanSweeper.test.ts b/internal-packages/run-store/src/snapshotOrphanSweeper.test.ts index 81e4ea729ef..fc3bdcca39f 100644 --- a/internal-packages/run-store/src/snapshotOrphanSweeper.test.ts +++ b/internal-packages/run-store/src/snapshotOrphanSweeper.test.ts @@ -310,6 +310,47 @@ describe("SnapshotOrphanSweeper", () => { } ); + containerTest( + "discovers and reaps a keyspace whose entries are all invalid", + async ({ prisma, redisOptions }) => { + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + const runStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const sweeper = new SnapshotOrphanSweeper({ + redisOptions, + runStore: runStore as unknown as RunStore, + completedTtlMs: COMPLETED_TTL_MS, + orphanAgeMs: ORPHAN_AGE_MS, + }); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + const old = new Date(Date.now() - 2 * ORPHAN_AGE_MS); + + // The append script writes `cur` and indexes the entry only when it is valid, so a keyspace + // whose entries all carry an error has neither. A sweep that discovers keyspaces by their + // `cur` key would never see this one, and neither rule would ever apply to it. + await store.append({ + entry: { ...birthEntry(runId, env, old), error: "stale write" }, + kind: "birth", + isTerminal: false, + }); + + const keys = snapshotKeys(runId); + expect(await probe.exists(keys.e)).toBe(1); + expect(await probe.exists(keys.cur)).toBe(0); + + const result = await sweeper.sweep(); + + expect(result.deleted).toBe(1); + expect(await probe.exists(keys.e)).toBe(0); + expect(await probe.exists(keys.seq)).toBe(0); + } finally { + await Promise.all([store.quit(), sweeper.quit(), probe.quit().catch(() => {})]); + } + } + ); + containerTest("processes every keyspace across batches", async ({ prisma, redisOptions }) => { const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); const runStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); diff --git a/internal-packages/run-store/src/snapshotOrphanSweeper.ts b/internal-packages/run-store/src/snapshotOrphanSweeper.ts index f94876da37c..6ce5c574c2f 100644 --- a/internal-packages/run-store/src/snapshotOrphanSweeper.ts +++ b/internal-packages/run-store/src/snapshotOrphanSweeper.ts @@ -108,10 +108,14 @@ export class SnapshotOrphanSweeper { let cursor = "0"; do { + // Match on the entry hash, not on `cur`. The append script writes `cur` only when the entry + // is valid, so a keyspace whose entries are all invalid would never be discovered and would + // leak with no expiry, which is the same unbounded leak rule 2 exists to close. `e` is + // written by every append. const [next, keys] = await this.#redis.scan( cursor, "MATCH", - `${this.#prefix}{*}:cur`, + `${this.#prefix}{*}:e`, "COUNT", batchSize ); @@ -212,15 +216,41 @@ export class SnapshotOrphanSweeper { result.deleted += 1; } - /** Every key for one run: the four core keys plus each wait-cycle key. */ + /** + * Every key for one run: the four core keys plus each wait-cycle key. + * + * The cycle keys are enumerated from the `c` high-water field on the seq hash, which the append + * script mints densely with HINCRBY, so 1..high covers every wp key that was ever written. This + * is the same source the store's own terminal-expiry loop uses. + * + * It deliberately does NOT use `KEYS`. That command iterates the whole database and blocks while + * it does, and a hash tag routes a key without scoping the scan, so one sweep pass over a batch + * would issue a full keyspace scan per run. + * + * The trade-off: if the seq hash is evicted while a wp key survives, `high` reads 0 and that + * orphaned cycle key is left behind. That is the right way to be wrong here. Leaving one small + * key costs bytes, where scanning the keyspace to find it costs every hot-path client latency on + * every pass. + */ async #allKeys(runId: string): Promise { const core = snapshotKeys(runId); - // Scoped to one hash tag, so this is a lookup inside a single slot rather than a keyspace scan. - const cycles = await this.#redis.keys(`${this.#prefix}{${runId}}:wp:*`); + + const high = Number((await this.#redis.hget(core.seq, "c")) ?? "0"); + const cycles: string[] = []; + for (let n = 1; n <= high; n++) { + cycles.push(`${this.#prefix}{${runId}}:wp:${n}`); + } + const candidates = [core.e, core.idx, core.cur, core.seq, ...cycles]; - const exists = await Promise.all(candidates.map((key) => this.#redis.exists(key))); - return candidates.filter((_key, index) => exists[index] === 1); + // One round trip for the whole set, rather than one per candidate. + const pipeline = this.#redis.pipeline(); + for (const key of candidates) { + pipeline.exists(key); + } + const replies = await pipeline.exec(); + + return candidates.filter((_key, index) => replies?.[index]?.[1] === 1); } /** @@ -229,11 +259,17 @@ export class SnapshotOrphanSweeper { */ async #newestEntryAgeMs(runId: string): Promise { const core = snapshotKeys(runId); + const newest = await this.#redis.zrevrange(core.idx, 0, 0); const id = newest[0]; - if (!id) return undefined; - const raw = await this.#redis.hget(core.e, id); + const raw = id + ? await this.#redis.hget(core.e, id) + : // The index holds valid entries only, so an all-invalid keyspace has an empty index. Fall + // back to the newest instant in the entry hash, or that keyspace is never old enough to + // reap and the leak survives the scan fix above. + await this.#newestRawFromEntries(core.e); + if (!raw) return undefined; try { @@ -246,6 +282,33 @@ export class SnapshotOrphanSweeper { } } + /** + * The newest entry document in the hash, by its own createdAt. Only reached for a keyspace with + * no index, which is rare, so the whole-hash read is acceptable where it would not be on the + * indexed path. + */ + async #newestRawFromEntries(eKey: string): Promise { + const all = await this.#redis.hgetall(eKey); + let newestRaw: string | undefined; + let newestAt = -Infinity; + + for (const [field, raw] of Object.entries(all)) { + // Sidecar fields hang off the entry ids as `#s` and `#c`; skip them. + if (field.includes("#")) continue; + try { + const at = Date.parse((JSON.parse(raw) as { createdAt?: string }).createdAt ?? ""); + if (!Number.isNaN(at) && at > newestAt) { + newestAt = at; + newestRaw = raw; + } + } catch { + continue; + } + } + + return newestRaw; + } + #runIdFrom(key: string): string | undefined { const open = key.indexOf("{"); const close = key.indexOf("}", open + 1); From 94b9c68325d63c0e72807e658e92b60dc26c61a3 Mon Sep 17 00:00:00 2001 From: Daniel Sutton Date: Tue, 25 Aug 2026 09:26:17 +0100 Subject: [PATCH 14/56] test(run-store): prove interface parity in the compiler, not in a fake Review asked why the pass-through base is tested against a hand-built delegate rather than a real store. Checking what the compiler already guarantees showed the test's own stated reason was wrong, and that one of its cases could not fail. implements RunStore already rejects a missing member with TS2420, so the claim that a method added later would become a silent hole was not true. The case comparing the class against the generated name list could not detect a parse miss either, because both the class and the list come from one parse of the interface, so a miss drops the member from both sides. The generator's comment asserting otherwise was false. Parity now lives where it can actually fail: assertions tying the name lists to keyof RunStore in both directions, and one rejecting a public member the class declares and the interface does not. They sit in src rather than in a test, because the build config excludes test files, so a type assertion written in a test is never checked. Each was verified by making it fail. What the compiler cannot see is inside the forwarder bodies, since every one is typed (...args: any[]): any. A forwarder wired to the wrong member, or dropping an argument, typechecks cleanly. The remaining probe covers exactly that, using a per-member sentinel so a misrouted body returns the wrong value rather than merely returning something. Verified by rewiring a forwarder: typecheck passes, the probe fails and names the member. Renames the double to forwardingProbe across both suites and says at the top why a container cannot replace it: no database is involved in whether a pass-through passes through. --- .../scripts/generateDelegatingRunStore.ts | 63 +++++++++++- .../run-store/src/delegatingRunStore.test.ts | 95 +++++++++++-------- .../run-store/src/delegatingRunStore.ts | 14 +++ .../run-store/src/runStoreMethodNames.ts | 37 +++++++- .../taskRunExecutionSnapshotStore.off.test.ts | 30 +++--- 5 files changed, 182 insertions(+), 57 deletions(-) diff --git a/internal-packages/run-store/scripts/generateDelegatingRunStore.ts b/internal-packages/run-store/scripts/generateDelegatingRunStore.ts index c827e10c753..84a6ade0ce7 100644 --- a/internal-packages/run-store/scripts/generateDelegatingRunStore.ts +++ b/internal-packages/run-store/scripts/generateDelegatingRunStore.ts @@ -7,9 +7,18 @@ // pnpm exec tsx scripts/generateDelegatingRunStore.ts // // The interface is scanned directly rather than through the TypeScript compiler API, because -// `require("typescript")` resolves to a stub in this workspace. A parsing miss cannot pass silently: -// delegatingRunStore.test.ts asserts the class and the interface hold exactly the same member set, -// and `implements RunStore` fails typecheck if a method is absent. +// `require("typescript")` resolves to a stub in this workspace. +// +// A parsing miss cannot pass silently, but NOT because of the runtime suite: that compares the class +// against the name list, and both come from this one parse, so a miss drops the member from both +// sides and the comparison still holds. Two compile-time checks catch it instead: +// +// - `implements RunStore` on the generated class fails with TS2420 when a member is absent. +// - The assertions emitted into runStoreMethodNames.ts tie the name lists to `keyof RunStore`, +// in both directions, so a dropped or invented name fails typecheck. +// +// Those live in src rather than in a test because tsconfig.build.json excludes test files, so a +// type-level assertion written in a test is never checked by CI. import { readFileSync, writeFileSync } from "node:fs"; import { dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; @@ -145,8 +154,11 @@ const header = `// GENERATED by scripts/generateDelegatingRunStore.ts. Do not ed writeFileSync( join(root, "src/runStoreMethodNames.ts"), `${header} -// Every method the RunStore interface declares. The decorator suites enumerate this, so a method -// added to the interface and not to the base fails a test instead of becoming a silent hole. +import type { RunStore } from "./types.js"; + +// Every method the RunStore interface declares. The forwarding probe enumerates this to drive one +// call per member; member PRESENCE is proved by the compiler, in the assertions at the foot of this +// file and by \`implements RunStore\` on the generated class. export const RUN_STORE_METHOD_NAMES = [ ${unique.map((n) => ` "${n}",`).join("\n")} ] as const; @@ -155,6 +167,33 @@ ${unique.map((n) => ` "${n}",`).join("\n")} export const RUN_STORE_PROPERTY_NAMES = [ ${readonlyProperties.map((p) => ` "${p.name}",`).join("\n")} ] as const; + +// --------------------------------------------------------------------------- +// Parity with the interface, checked by the compiler. +// +// The lists above are produced by parsing types.ts. These assertions compare them +// against \`keyof RunStore\`, which the compiler derives from the interface itself, +// so a name this generator failed to parse, or invented, is a build failure rather +// than a silent gap. Both directions are checked: a missing name and an extra one. +// --------------------------------------------------------------------------- + +type RunStoreMemberName = + | (typeof RUN_STORE_METHOD_NAMES)[number] + | (typeof RUN_STORE_PROPERTY_NAMES)[number]; + +/** Fails when the interface declares a member the generator did not emit. */ +type _EveryInterfaceMemberIsListed = [Exclude] extends [never] + ? true + : never; +const _everyInterfaceMemberIsListed: _EveryInterfaceMemberIsListed = true; +void _everyInterfaceMemberIsListed; + +/** Fails when the generator emitted a name the interface does not declare. */ +type _EveryListedNameIsOnTheInterface = [Exclude] extends [never] + ? true + : never; +const _everyListedNameIsOnTheInterface: _EveryListedNameIsOnTheInterface = true; +void _everyListedNameIsOnTheInterface; ` ); @@ -185,6 +224,20 @@ ${readonlyProperties ) .join("\n\n")} } + +// \`implements\` above fails when a member of the interface is MISSING here. It says nothing about a +// member that should not exist, so the reverse direction is asserted too: a public member this class +// declares and the interface does not is a build failure. +// +// \`protected delegate\` is correctly absent from \`keyof\`, so the constructor parameter does not +// trip this. +type _ClassDeclaresNoExtraMembers = [ + Exclude, +] extends [never] + ? true + : never; +const _classParity: _ClassDeclaresNoExtraMembers = true; +void _classParity; ` ); diff --git a/internal-packages/run-store/src/delegatingRunStore.test.ts b/internal-packages/run-store/src/delegatingRunStore.test.ts index 7065f801d71..74d6a9e9afb 100644 --- a/internal-packages/run-store/src/delegatingRunStore.test.ts +++ b/internal-packages/run-store/src/delegatingRunStore.test.ts @@ -1,57 +1,88 @@ -// The base must forward EVERY RunStore member. A method added to the interface and not to the base -// is a silent hole in the decorator built on top of it, so this suite enumerates the generated name -// lists rather than restating them by hand. Regenerate the base and both lists together: -// pnpm exec tsx scripts/generateDelegatingRunStore.ts +// What this suite covers, and why it does not use a container. +// +// `DelegatingRunStore` is generated and holds no logic: every member forwards to a delegate. Three +// properties of it are already proved by the compiler and are NOT retested here: +// +// - a member of the interface missing from the class -> `implements RunStore`, TS2420 +// - a public member the interface does not declare -> the parity assertion in the base +// - a name list out of step with the interface -> the assertions in runStoreMethodNames +// +// What the compiler cannot see is inside the forwarder bodies, because each one is typed +// `(...args: any[]): any`. A forwarder wired to the wrong delegate method, dropping an argument, or +// reading a property once at construction instead of on each access, all typecheck cleanly. Those +// are template-correctness properties of the generator's output, and they are what is tested below. +// +// A Testcontainers-backed store cannot demonstrate them. It would mean calling all 70 methods with +// valid arguments and valid foreign-key state, and a real return value cannot show that arguments +// arrived untouched the way a per-member sentinel can. The probe below is not a stand-in for a +// database: no database is involved in whether a pass-through passes through. Behaviour against a +// real store is covered by the container suites for the decorator built on this base. import { describe, expect, it } from "vitest"; import { DelegatingRunStore } from "./delegatingRunStore.js"; import { RUN_STORE_METHOD_NAMES, RUN_STORE_PROPERTY_NAMES } from "./runStoreMethodNames.js"; import type { RunStore } from "./types.js"; -function recordingDelegate(): { store: RunStore; calls: { name: string; args: unknown[] }[] } { - const calls: { name: string; args: unknown[] }[] = []; - const store: Record = {}; +type ProbedCall = { name: string; args: unknown[] }; - for (const name of RUN_STORE_METHOD_NAMES) { - store[name] = (...args: unknown[]) => { - calls.push({ name, args }); - return `result:${name}`; - }; - } - for (const name of RUN_STORE_PROPERTY_NAMES) { - store[name] = `property:${name}`; - } +/** + * A delegate that records what was called on it and answers with a per-member sentinel, so a + * forwarder wired to the wrong member returns the wrong sentinel and fails loudly. + */ +function forwardingProbe(): { store: RunStore; calls: ProbedCall[] } { + const calls: ProbedCall[] = []; + + const store = new Proxy({} as Record, { + get(_target, prop: string) { + if ((RUN_STORE_PROPERTY_NAMES as readonly string[]).includes(prop)) { + return `property:${prop}`; + } + return (...args: unknown[]) => { + calls.push({ name: prop, args }); + return `result:${prop}`; + }; + }, + }); return { store: store as unknown as RunStore, calls }; } describe("DelegatingRunStore", () => { - it("forwards every RunStore method to the delegate, arguments untouched", () => { - const { store, calls } = recordingDelegate(); + it("forwards every method to the member of the same name", () => { + const { store, calls } = forwardingProbe(); const base = new DelegatingRunStore(store) as unknown as Record< string, (...args: unknown[]) => unknown >; for (const name of RUN_STORE_METHOD_NAMES) { - expect(base[name]("arg-one", "arg-two")).toBe(`result:${name}`); + // The sentinel is per member, so a body forwarding to a different method fails here rather + // than passing because both happened to return something. + expect(base[name]()).toBe(`result:${name}`); } expect(calls.map((c) => c.name)).toEqual([...RUN_STORE_METHOD_NAMES]); - for (const call of calls) { - expect(call.args).toEqual(["arg-one", "arg-two"]); - } }); - it("reads every RunStore data property from the delegate", () => { - const { store } = recordingDelegate(); - const base = new DelegatingRunStore(store) as unknown as Record; + it("forwards arguments untouched", () => { + const { store, calls } = forwardingProbe(); + const base = new DelegatingRunStore(store) as unknown as Record< + string, + (...args: unknown[]) => unknown + >; + const args = ["first", { second: true }, undefined, 4]; + + for (const name of RUN_STORE_METHOD_NAMES) { + base[name](...args); + } - for (const name of RUN_STORE_PROPERTY_NAMES) { - expect(base[name]).toBe(`property:${name}`); + for (const call of calls) { + expect(call.args).toEqual(args); } }); it("reads a data property live, so a delegate that changes is not cached", () => { + // A getter is the only correct shape here. Capturing the value in the constructor would + // typecheck and would then serve a stale client for the life of the decorator. const store = { primaryReadClient: "first" } as unknown as RunStore; const base = new DelegatingRunStore(store); @@ -59,14 +90,4 @@ describe("DelegatingRunStore", () => { (store as unknown as Record).primaryReadClient = "second"; expect(base.primaryReadClient).toBe("second" as unknown); }); - - it("declares exactly the members the interface declares, and no others", () => { - const own = Object.getOwnPropertyNames(DelegatingRunStore.prototype) - .filter((name) => name !== "constructor") - .sort(); - - const expected = [...RUN_STORE_METHOD_NAMES, ...RUN_STORE_PROPERTY_NAMES].sort(); - - expect(own).toEqual(expected); - }); }); diff --git a/internal-packages/run-store/src/delegatingRunStore.ts b/internal-packages/run-store/src/delegatingRunStore.ts index 6266ac5abd1..621242f75cf 100644 --- a/internal-packages/run-store/src/delegatingRunStore.ts +++ b/internal-packages/run-store/src/delegatingRunStore.ts @@ -298,3 +298,17 @@ export class DelegatingRunStore implements RunStore { return (this.delegate as any).findManyWaitpointTags(...args); } } + +// `implements` above fails when a member of the interface is MISSING here. It says nothing about a +// member that should not exist, so the reverse direction is asserted too: a public member this class +// declares and the interface does not is a build failure. +// +// `protected delegate` is correctly absent from `keyof`, so the constructor parameter does not +// trip this. +type _ClassDeclaresNoExtraMembers = [Exclude] extends [ + never, +] + ? true + : never; +const _classParity: _ClassDeclaresNoExtraMembers = true; +void _classParity; diff --git a/internal-packages/run-store/src/runStoreMethodNames.ts b/internal-packages/run-store/src/runStoreMethodNames.ts index d3dd0868676..eff7589fcfe 100644 --- a/internal-packages/run-store/src/runStoreMethodNames.ts +++ b/internal-packages/run-store/src/runStoreMethodNames.ts @@ -2,8 +2,11 @@ // Regenerate after any change to the RunStore interface: // pnpm exec tsx scripts/generateDelegatingRunStore.ts -// Every method the RunStore interface declares. The decorator suites enumerate this, so a method -// added to the interface and not to the base fails a test instead of becoming a silent hole. +import type { RunStore } from "./types.js"; + +// Every method the RunStore interface declares. The forwarding probe enumerates this to drive one +// call per member; member PRESENCE is proved by the compiler, in the assertions at the foot of this +// file and by `implements RunStore` on the generated class. export const RUN_STORE_METHOD_NAMES = [ "runInTransaction", "createRun", @@ -22,7 +25,6 @@ export const RUN_STORE_METHOD_NAMES = [ "lockRunToWorker", "parkPendingVersion", "promotePendingVersionRuns", - "expireParkedRun", "suspendForCheckpoint", "resumeFromCheckpoint", "rescheduleRun", @@ -79,3 +81,32 @@ export const RUN_STORE_METHOD_NAMES = [ // Data properties the base exposes as getters over the delegate, not as forwarders. export const RUN_STORE_PROPERTY_NAMES = ["primaryReadClient"] as const; + +// --------------------------------------------------------------------------- +// Parity with the interface, checked by the compiler. +// +// The lists above are produced by parsing types.ts. These assertions compare them +// against `keyof RunStore`, which the compiler derives from the interface itself, +// so a name this generator failed to parse, or invented, is a build failure rather +// than a silent gap. Both directions are checked: a missing name and an extra one. +// --------------------------------------------------------------------------- + +type RunStoreMemberName = + | (typeof RUN_STORE_METHOD_NAMES)[number] + | (typeof RUN_STORE_PROPERTY_NAMES)[number]; + +/** Fails when the interface declares a member the generator did not emit. */ +type _EveryInterfaceMemberIsListed = [Exclude] extends [never] + ? true + : never; +const _everyInterfaceMemberIsListed: _EveryInterfaceMemberIsListed = true; +void _everyInterfaceMemberIsListed; + +/** Fails when the generator emitted a name the interface does not declare. */ +type _EveryListedNameIsOnTheInterface = [Exclude] extends [ + never, +] + ? true + : never; +const _everyListedNameIsOnTheInterface: _EveryListedNameIsOnTheInterface = true; +void _everyListedNameIsOnTheInterface; diff --git a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.off.test.ts b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.off.test.ts index 975c68c371a..4f788b5ec27 100644 --- a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.off.test.ts +++ b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.off.test.ts @@ -17,23 +17,29 @@ function explodingRedisStore(): RedisSnapshotStore { }); } -function recordingDelegate(): { store: RunStore; calls: string[] } { +/** + * Records what the decorator forwarded, and answers with a per-member sentinel. No database is + * involved in whether mode off is a pass-through, so none is started; the behavioural suites for + * every other mode run against a real Postgres and a real Redis. + */ +function forwardingProbe(): { store: RunStore; calls: string[] } { const calls: string[] = []; - const store: Record = {}; - for (const name of RUN_STORE_METHOD_NAMES) { - store[name] = (...args: unknown[]) => { - calls.push(name); - return `result:${name}`; - }; - } + const store = new Proxy({} as Record, { + get(_target, prop: string) { + return (...args: unknown[]) => { + calls.push(prop); + return `result:${prop}`; + }; + }, + }); return { store: store as unknown as RunStore, calls }; } describe("TaskRunExecutionSnapshotStore at mode off", () => { it("defaults to mode off", () => { - const { store } = recordingDelegate(); + const { store } = forwardingProbe(); const decorated = new TaskRunExecutionSnapshotStore(store, { store: explodingRedisStore() }); @@ -41,7 +47,7 @@ describe("TaskRunExecutionSnapshotStore at mode off", () => { }); it("forwards every method to the delegate and never calls Redis", async () => { - const { store, calls } = recordingDelegate(); + const { store, calls } = forwardingProbe(); const decorated = new TaskRunExecutionSnapshotStore(store, { store: explodingRedisStore(), mode: "off", @@ -56,7 +62,7 @@ describe("TaskRunExecutionSnapshotStore at mode off", () => { }); it("hands the delegate's own store to a transaction callback", async () => { - const inner = recordingDelegate().store; + const inner = forwardingProbe().store; let seen: unknown; const delegate = { runInTransaction: async ( @@ -80,7 +86,7 @@ describe("TaskRunExecutionSnapshotStore at mode off", () => { }); it("reports every other dial position as one that writes Redis", () => { - const { store } = recordingDelegate(); + const { store } = forwardingProbe(); const modes = ["dual-write", "compare", "redis-read", "redis-only"] as const; for (const mode of modes) { From f01d299636fe3d81483a6e314bbeaf4ad72a44f6 Mon Sep 17 00:00:00 2001 From: Daniel Sutton Date: Tue, 25 Aug 2026 09:27:20 +0100 Subject: [PATCH 15/56] fix(run-store): restore a name dropped from the generated list A member was removed from the generated list while verifying that the new parity assertion fails when one goes missing, and the restore did not run, so the verification state was committed. Regenerated from the interface. The assertion did its job: typecheck rejects the list, naming the missing member. --- internal-packages/run-store/src/runStoreMethodNames.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/internal-packages/run-store/src/runStoreMethodNames.ts b/internal-packages/run-store/src/runStoreMethodNames.ts index eff7589fcfe..ffb07a7de07 100644 --- a/internal-packages/run-store/src/runStoreMethodNames.ts +++ b/internal-packages/run-store/src/runStoreMethodNames.ts @@ -25,6 +25,7 @@ export const RUN_STORE_METHOD_NAMES = [ "lockRunToWorker", "parkPendingVersion", "promotePendingVersionRuns", + "expireParkedRun", "suspendForCheckpoint", "resumeFromCheckpoint", "rescheduleRun", From b02b42683f2815f5c309e774f93642a46ea9b3c6 Mon Sep 17 00:00:00 2001 From: Daniel Sutton Date: Tue, 25 Aug 2026 11:34:31 +0100 Subject: [PATCH 16/56] fix(run-store): keep the fork guard on a staged append, and survive a client key prefix Two defects from review, both silent. An append staged inside a transaction dropped its expected-head argument, and the post-commit flush passed undefined in its place. That disabled the compare-and-set for every snapshot written inside a transaction, which is the path both engine transaction writers use, so a stale append that the store would have refused as forked was written instead and became the head. The expectation now travels with the staged entry. The sweep built its scan pattern without the client key prefix. ioredis prepends that prefix to keys for ordinary commands but not to a SCAN MATCH pattern, and returns matched keys with it still attached, so a prefixed client made the sweep match nothing and report a clean pass. The engine sets a prefix on every other Redis client it builds, so this would have surfaced at wiring time as a reaper that silently protected nothing. Also removes a keyPrefix option on the sweep that could never work: the keyspace prefix belongs to snapshotKeys in the store, which writes snap: keys unconditionally, so there was no other keyspace to point it at. Both fixes have a test verified by reintroducing the defect: the staged stale append is written without the guard, and the prefixed sweep scans nothing. --- .../src/snapshotOrphanSweeper.test.ts | 44 +++++++++++++++ .../run-store/src/snapshotOrphanSweeper.ts | 26 +++++++-- ...kRunExecutionSnapshotStore.staging.test.ts | 55 +++++++++++++++++++ .../src/taskRunExecutionSnapshotStore.ts | 14 ++++- 4 files changed, 132 insertions(+), 7 deletions(-) diff --git a/internal-packages/run-store/src/snapshotOrphanSweeper.test.ts b/internal-packages/run-store/src/snapshotOrphanSweeper.test.ts index fc3bdcca39f..b0d39d27e56 100644 --- a/internal-packages/run-store/src/snapshotOrphanSweeper.test.ts +++ b/internal-packages/run-store/src/snapshotOrphanSweeper.test.ts @@ -351,6 +351,50 @@ describe("SnapshotOrphanSweeper", () => { } ); + containerTest( + "still finds keyspaces when the client carries a key prefix", + async ({ prisma, redisOptions }) => { + // ioredis prepends its keyPrefix to keys for ordinary commands, but NOT to a SCAN MATCH + // pattern, and it returns matched keys with the prefix still on them. The engine sets a + // prefix on every other Redis client it builds, so a sweep that ignored this would match + // nothing and report a clean pass: a safety net that silently protects nothing. + const prefixed = { ...(redisOptions as object), keyPrefix: "engine:" } as never; + const store = new RedisSnapshotStore({ + redisOptions: prefixed, + completedTtlMs: COMPLETED_TTL_MS, + }); + const runStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const sweeper = new SnapshotOrphanSweeper({ + redisOptions: prefixed, + runStore: runStore as unknown as RunStore, + completedTtlMs: COMPLETED_TTL_MS, + orphanAgeMs: ORPHAN_AGE_MS, + }); + const probe = createRedisClient(prefixed, { onError: () => {} }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + const old = new Date(Date.now() - 2 * ORPHAN_AGE_MS); + + await store.append({ + entry: birthEntry(runId, env, old), + kind: "birth", + isTerminal: false, + cycle: { kind: "new", completedWaitpoints: [{ id: "w_1", index: 0 }] }, + }); + + const result = await sweeper.sweep(); + + expect(result.scanned).toBe(1); + expect(result.deleted).toBe(1); + expect(await probe.exists(snapshotKeys(runId).e)).toBe(0); + expect(await probe.exists(`snap:{${runId}}:wp:1`)).toBe(0); + } finally { + await Promise.all([store.quit(), sweeper.quit(), probe.quit().catch(() => {})]); + } + } + ); + containerTest("processes every keyspace across batches", async ({ prisma, redisOptions }) => { const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); const runStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); diff --git a/internal-packages/run-store/src/snapshotOrphanSweeper.ts b/internal-packages/run-store/src/snapshotOrphanSweeper.ts index 6ce5c574c2f..4d3eed674b4 100644 --- a/internal-packages/run-store/src/snapshotOrphanSweeper.ts +++ b/internal-packages/run-store/src/snapshotOrphanSweeper.ts @@ -37,6 +37,13 @@ const FINAL = new Set(FINAL_RUN_STATUSES); /** Comfortably above run-creation latency, so a birth in flight is never mistaken for an orphan. */ const DEFAULT_ORPHAN_AGE_MS = 24 * 60 * 60 * 1000; + +/** + * The keyspace prefix, owned by `snapshotKeys` in the store rather than configurable here. A sweep + * that could be pointed at a different prefix would be a fiction: the store writes `snap:` keys + * unconditionally, so there is no other keyspace to point it at. + */ +const SNAPSHOT_KEYSPACE_PREFIX = "snap:"; const DEFAULT_BATCH_SIZE = 1000; export type SweepResult = { @@ -64,7 +71,6 @@ export type SnapshotOrphanSweeperOptions = { runStore: RunStore; completedTtlMs: number; orphanAgeMs?: number; - keyPrefix?: string; logger?: Logger; }; @@ -73,7 +79,13 @@ export class SnapshotOrphanSweeper { readonly #runStore: RunStore; readonly #completedTtlMs: number; readonly #orphanAgeMs: number; - readonly #prefix: string; + /** + * The ioredis client-level prefix, which is NOT the keyspace prefix. ioredis prepends it to keys + * for ordinary commands, but it does not prepend it to a SCAN MATCH pattern, and it does return + * matched keys with it still attached. Unhandled, a prefixed client makes the sweep match nothing + * and report a clean pass, which is the worst outcome for a safety net. + */ + readonly #clientPrefix: string; readonly #logger: Logger; #quit?: Promise; @@ -82,7 +94,7 @@ export class SnapshotOrphanSweeper { this.#runStore = options.runStore; this.#completedTtlMs = options.completedTtlMs; this.#orphanAgeMs = options.orphanAgeMs ?? DEFAULT_ORPHAN_AGE_MS; - this.#prefix = options.keyPrefix ?? "snap:"; + this.#clientPrefix = (options.redisOptions.keyPrefix as string | undefined) ?? ""; this.#redis = createRedisClient(options.redisOptions, { onError: (error) => this.#logger.error("SnapshotOrphanSweeper redis client error", { error }), }); @@ -115,7 +127,7 @@ export class SnapshotOrphanSweeper { const [next, keys] = await this.#redis.scan( cursor, "MATCH", - `${this.#prefix}{*}:e`, + `${this.#clientPrefix}${SNAPSHOT_KEYSPACE_PREFIX}{*}:e`, "COUNT", batchSize ); @@ -238,7 +250,7 @@ export class SnapshotOrphanSweeper { const high = Number((await this.#redis.hget(core.seq, "c")) ?? "0"); const cycles: string[] = []; for (let n = 1; n <= high; n++) { - cycles.push(`${this.#prefix}{${runId}}:wp:${n}`); + cycles.push(`${SNAPSHOT_KEYSPACE_PREFIX}{${runId}}:wp:${n}`); } const candidates = [core.e, core.idx, core.cur, core.seq, ...cycles]; @@ -309,6 +321,10 @@ export class SnapshotOrphanSweeper { return newestRaw; } + /** + * The run id is whatever sits inside the hash tag, so a client prefix on the returned key does not + * need stripping: `engine:snap:{run_x}:e` and `snap:{run_x}:e` both yield `run_x`. + */ #runIdFrom(key: string): string | undefined { const open = key.indexOf("{"); const close = key.indexOf("}", open + 1); diff --git a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.staging.test.ts b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.staging.test.ts index 3b75ca4ca05..9952f569e2b 100644 --- a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.staging.test.ts +++ b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.staging.test.ts @@ -138,6 +138,61 @@ describe("the staging facade", () => { } }); + containerTest( + "keeps the fork guard on an append staged inside a transaction", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + await seedBirth(decorated, redis, runId, env); + const id = generateInternalId(); + + // A stale expectation: this names a head that was never current. Outside a transaction the + // append is rejected as forked and never written. Staging must not weaken that, or a write + // the store would have refused becomes the head purely because it ran inside a transaction. + await decorated.runInTransaction(runId, async (store, tx) => { + await store.createExecutionSnapshot( + { ...snapshotInput(runId, env, id, "stale"), previousSnapshotId: generateInternalId() }, + tx + ); + }); + + expect(await redis.getById(runId, id)).toBeNull(); + + const head = await redis.getLatest(runId); + expect(head?.id).not.toBe(id); + } finally { + await redis.quit(); + } + } + ); + + containerTest( + "honours a correct expectation on an append staged inside a transaction", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + await seedBirth(decorated, redis, runId, env); + const head = await redis.getLatest(runId); + const id = generateInternalId(); + + await decorated.runInTransaction(runId, async (store, tx) => { + await store.createExecutionSnapshot( + { ...snapshotInput(runId, env, id, "expected"), previousSnapshotId: head!.id }, + tx + ); + }); + + expect((await redis.getById(runId, id))?.entry.description).toBe("expected"); + } finally { + await redis.quit(); + } + } + ); + containerTest( "hands the transaction callback a decorated store", async ({ prisma, redisOptions }) => { diff --git a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.ts b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.ts index 46a57c6b3bf..4b466eca268 100644 --- a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.ts +++ b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.ts @@ -104,6 +104,12 @@ export type TaskRunExecutionSnapshotStoreOptions = { /** One deferred append: the entry, plus the wait cycle it carries, if any. */ export type StagedAppend = { entry: SnapshotEntryInput; + /** + * The head this append expects, carried through staging so the compare-and-set survives the + * deferral. Dropping it would silently disable the fork guard for every snapshot written inside a + * transaction, and a stale append that should be rejected would instead become the head. + */ + expectedCur?: string; completedWaitpoints?: CompletedWaitpointRef[]; }; @@ -167,7 +173,7 @@ export class TaskRunExecutionSnapshotStore extends DelegatingRunStore { await this.#appendTransition( "runInTransaction", item.entry, - undefined, + item.expectedCur, item.completedWaitpoints ); } @@ -493,7 +499,11 @@ export class TaskRunExecutionSnapshotStore extends DelegatingRunStore { if (this.staging) { // Inside a transaction the append cannot run until the Postgres side commits, or a rollback // leaves Redis holding a transition that never happened. - this.staging.push({ entry, ...(completedWaitpoints && { completedWaitpoints }) }); + this.staging.push({ + entry, + ...(expectedCur !== undefined && { expectedCur }), + ...(completedWaitpoints && { completedWaitpoints }), + }); return; } From 7bba6a895766cc4c49fa013c412312810fab3149 Mon Sep 17 00:00:00 2001 From: Daniel Sutton Date: Tue, 25 Aug 2026 12:16:48 +0100 Subject: [PATCH 17/56] chore(run-store): drop the pass-through generator, keep its output as source The generator was scaffolding for a one-off job: writing 70 near-identical forwarders. Keeping it meant carrying a hand-rolled scanner over the interface body, because the TypeScript compiler API is not resolvable in this workspace, which is more machinery than a file that changes only when the interface does. The two files it produced are now maintained by hand, and their headers say so. Nothing is lost, because the generator was never what guaranteed they were right. That is the compiler: implements RunStore rejects a missing member, and the parity assertions tie both name lists to keyof RunStore in each direction and reject a public member the interface does not declare. Each was re-verified by making it fail after the generator was removed. Also drops the knip entry that existed only to treat that script as an entry point. --- .../scripts/generateDelegatingRunStore.ts | 247 ------------------ .../run-store/src/delegatingRunStore.ts | 8 +- .../run-store/src/runStoreMethodNames.ts | 9 +- knip.json | 3 - 4 files changed, 11 insertions(+), 256 deletions(-) delete mode 100644 internal-packages/run-store/scripts/generateDelegatingRunStore.ts diff --git a/internal-packages/run-store/scripts/generateDelegatingRunStore.ts b/internal-packages/run-store/scripts/generateDelegatingRunStore.ts deleted file mode 100644 index 84a6ade0ce7..00000000000 --- a/internal-packages/run-store/scripts/generateDelegatingRunStore.ts +++ /dev/null @@ -1,247 +0,0 @@ -// One-off generator for the RunStore pass-through base. -// -// The base class is mechanical: 80-odd near-identical forwarders. Generating it removes the chance -// of a hand-typo that no test would catch, and turns "did we miss a method" into a diff rather than -// a review. Re-run after any change to the RunStore interface: -// -// pnpm exec tsx scripts/generateDelegatingRunStore.ts -// -// The interface is scanned directly rather than through the TypeScript compiler API, because -// `require("typescript")` resolves to a stub in this workspace. -// -// A parsing miss cannot pass silently, but NOT because of the runtime suite: that compares the class -// against the name list, and both come from this one parse, so a miss drops the member from both -// sides and the comparison still holds. Two compile-time checks catch it instead: -// -// - `implements RunStore` on the generated class fails with TS2420 when a member is absent. -// - The assertions emitted into runStoreMethodNames.ts tie the name lists to `keyof RunStore`, -// in both directions, so a dropped or invented name fails typecheck. -// -// Those live in src rather than in a test because tsconfig.build.json excludes test files, so a -// type-level assertion written in a test is never checked by CI. -import { readFileSync, writeFileSync } from "node:fs"; -import { dirname, join } from "node:path"; -import { fileURLToPath } from "node:url"; - -const root = join(dirname(fileURLToPath(import.meta.url)), ".."); -const source = readFileSync(join(root, "src/types.ts"), "utf8"); - -/** Replaces every comment and string body with spaces, so a brace inside one cannot move the depth. */ -function blankCommentsAndStrings(text: string): string { - let out = ""; - let i = 0; - while (i < text.length) { - const two = text.slice(i, i + 2); - if (two === "//") { - const end = text.indexOf("\n", i); - const stop = end === -1 ? text.length : end; - out += " ".repeat(stop - i); - i = stop; - } else if (two === "/*") { - const end = text.indexOf("*/", i + 2); - const stop = end === -1 ? text.length : end + 2; - out += text.slice(i, stop).replace(/[^\n]/g, " "); - i = stop; - } else if (text[i] === '"' || text[i] === "'" || text[i] === "`") { - const quote = text[i]; - let j = i + 1; - while (j < text.length && text[j] !== quote) { - j += text[j] === "\\" ? 2 : 1; - } - out += quote + " ".repeat(Math.max(0, j - i - 1)) + (text[j] ?? ""); - i = j + 1; - } else { - out += text[i]; - i += 1; - } - } - return out; -} - -const blanked = blankCommentsAndStrings(source); - -const declaration = "export interface RunStore {"; -const start = blanked.indexOf(declaration); -if (start === -1) { - throw new Error("export interface RunStore not found in src/types.ts"); -} - -const bodyStart = start + declaration.length; -let depth = 1; -let bodyEnd = bodyStart; -while (bodyEnd < blanked.length && depth > 0) { - const ch = blanked[bodyEnd]; - if (ch === "{") depth += 1; - else if (ch === "}") depth -= 1; - if (depth > 0) bodyEnd += 1; -} -if (depth !== 0) { - throw new Error("unbalanced braces while reading the RunStore interface body"); -} - -const body = blanked.slice(bodyStart, bodyEnd); - -// Members are separated by `;` at nesting depth 0. Only `{}`, `()` and `[]` count towards depth: -// angle brackets cannot, because `=>` in a callback parameter type carries an unmatched `>`. -function splitMembers(text: string): { offset: number; length: number }[] { - const spans: { offset: number; length: number }[] = []; - let level = 0; - let from = 0; - for (let i = 0; i < text.length; i++) { - const ch = text[i]; - if (ch === "{" || ch === "(" || ch === "[") level += 1; - else if (ch === "}" || ch === ")" || ch === "]") level -= 1; - else if (ch === ";" && level === 0) { - spans.push({ offset: from, length: i - from }); - from = i + 1; - } - } - if (text.slice(from).trim().length > 0) { - spans.push({ offset: from, length: text.length - from }); - } - return spans; -} - -const methods: string[] = []; -const readonlyProperties: { name: string; type: string }[] = []; -const mutableProperties: string[] = []; - -for (const span of splitMembers(body)) { - const blankedMember = body.slice(span.offset, span.offset + span.length).trim(); - if (blankedMember.length === 0) continue; - - // A method: an optional name then `(` or `<`. A property: a name then `:`. - const asMethod = /^([A-Za-z_$][\w$]*)\s*\??\s*[(<]/.exec(blankedMember); - if (asMethod) { - methods.push(asMethod[1]); - continue; - } - - const asProperty = /^(readonly\s+)?([A-Za-z_$][\w$]*)\s*(\??)\s*:([\s\S]*)$/.exec(blankedMember); - if (asProperty) { - const [, isReadonly, name, , type] = asProperty; - if (isReadonly) { - readonlyProperties.push({ name, type: type.trim() }); - } else { - mutableProperties.push(name); - } - continue; - } - - throw new Error(`could not classify a RunStore member: ${blankedMember.slice(0, 80)}`); -} - -if (mutableProperties.length > 0) { - // A writable data property cannot be forwarded by a getter alone, so the base would silently hold - // its own copy instead of the delegate's. Handle it by hand before regenerating. - throw new Error( - `RunStore declares writable data properties the generator cannot forward: ${mutableProperties.join(", ")}` - ); -} - -const unique = [...new Set(methods)]; -if (unique.length === 0) { - throw new Error("RunStore declares no methods, which cannot be right"); -} - -const memberNames = [...unique, ...readonlyProperties.map((p) => p.name)]; - -const header = `// GENERATED by scripts/generateDelegatingRunStore.ts. Do not edit by hand. -// Regenerate after any change to the RunStore interface: -// pnpm exec tsx scripts/generateDelegatingRunStore.ts -`; - -writeFileSync( - join(root, "src/runStoreMethodNames.ts"), - `${header} -import type { RunStore } from "./types.js"; - -// Every method the RunStore interface declares. The forwarding probe enumerates this to drive one -// call per member; member PRESENCE is proved by the compiler, in the assertions at the foot of this -// file and by \`implements RunStore\` on the generated class. -export const RUN_STORE_METHOD_NAMES = [ -${unique.map((n) => ` "${n}",`).join("\n")} -] as const; - -// Data properties the base exposes as getters over the delegate, not as forwarders. -export const RUN_STORE_PROPERTY_NAMES = [ -${readonlyProperties.map((p) => ` "${p.name}",`).join("\n")} -] as const; - -// --------------------------------------------------------------------------- -// Parity with the interface, checked by the compiler. -// -// The lists above are produced by parsing types.ts. These assertions compare them -// against \`keyof RunStore\`, which the compiler derives from the interface itself, -// so a name this generator failed to parse, or invented, is a build failure rather -// than a silent gap. Both directions are checked: a missing name and an extra one. -// --------------------------------------------------------------------------- - -type RunStoreMemberName = - | (typeof RUN_STORE_METHOD_NAMES)[number] - | (typeof RUN_STORE_PROPERTY_NAMES)[number]; - -/** Fails when the interface declares a member the generator did not emit. */ -type _EveryInterfaceMemberIsListed = [Exclude] extends [never] - ? true - : never; -const _everyInterfaceMemberIsListed: _EveryInterfaceMemberIsListed = true; -void _everyInterfaceMemberIsListed; - -/** Fails when the generator emitted a name the interface does not declare. */ -type _EveryListedNameIsOnTheInterface = [Exclude] extends [never] - ? true - : never; -const _everyListedNameIsOnTheInterface: _EveryListedNameIsOnTheInterface = true; -void _everyListedNameIsOnTheInterface; -` -); - -writeFileSync( - join(root, "src/delegatingRunStore.ts"), - `${header} -// A pass-through over another RunStore. It exists so a decorator can override the handful of methods -// it cares about and inherit the rest, instead of restating 80-odd forwarders alongside real logic. -// -// Arguments and return values are forwarded untouched. The \`any\` signatures carry each method's -// whole overload set through one forwarder, which is the single thing a generated base cannot -// preserve; a subclass that overrides a method restates the real signature there. -/* eslint-disable @typescript-eslint/no-explicit-any */ -import type { RunStore } from "./types.js"; - -export class DelegatingRunStore implements RunStore { - constructor(protected readonly delegate: RunStore) {} - -${readonlyProperties - // Indexed access rather than the written type, so the getter needs no import of its own and - // follows the interface if that type is ever changed. - .map( - (p) => ` get ${p.name}(): RunStore["${p.name}"] {\n return this.delegate.${p.name};\n }` - ) - .join("\n\n")}${readonlyProperties.length > 0 ? "\n\n" : ""}${unique - .map( - (n) => ` ${n}(...args: any[]): any {\n return (this.delegate as any).${n}(...args);\n }` - ) - .join("\n\n")} -} - -// \`implements\` above fails when a member of the interface is MISSING here. It says nothing about a -// member that should not exist, so the reverse direction is asserted too: a public member this class -// declares and the interface does not is a build failure. -// -// \`protected delegate\` is correctly absent from \`keyof\`, so the constructor parameter does not -// trip this. -type _ClassDeclaresNoExtraMembers = [ - Exclude, -] extends [never] - ? true - : never; -const _classParity: _ClassDeclaresNoExtraMembers = true; -void _classParity; -` -); - -console.log( - `generated ${unique.length} forwarders and ${readonlyProperties.length} getters ` + - `(${memberNames.length} members total)` -); diff --git a/internal-packages/run-store/src/delegatingRunStore.ts b/internal-packages/run-store/src/delegatingRunStore.ts index 621242f75cf..e158ada0037 100644 --- a/internal-packages/run-store/src/delegatingRunStore.ts +++ b/internal-packages/run-store/src/delegatingRunStore.ts @@ -1,6 +1,8 @@ -// GENERATED by scripts/generateDelegatingRunStore.ts. Do not edit by hand. -// Regenerate after any change to the RunStore interface: -// pnpm exec tsx scripts/generateDelegatingRunStore.ts +// Maintained by hand. It was scaffolded once, and the scaffolding is gone. +// +// When RunStore gains or loses a member, add or remove the forwarder here. You do not have to +// remember: `implements RunStore` fails with TS2420 on a member that is missing, and the assertion +// at the foot of this file fails on a public member the interface does not declare. // A pass-through over another RunStore. It exists so a decorator can override the handful of methods // it cares about and inherit the rest, instead of restating 80-odd forwarders alongside real logic. diff --git a/internal-packages/run-store/src/runStoreMethodNames.ts b/internal-packages/run-store/src/runStoreMethodNames.ts index ffb07a7de07..3d8fbc63c77 100644 --- a/internal-packages/run-store/src/runStoreMethodNames.ts +++ b/internal-packages/run-store/src/runStoreMethodNames.ts @@ -1,6 +1,9 @@ -// GENERATED by scripts/generateDelegatingRunStore.ts. Do not edit by hand. -// Regenerate after any change to the RunStore interface: -// pnpm exec tsx scripts/generateDelegatingRunStore.ts +// Maintained by hand, alongside delegatingRunStore.ts. It was scaffolded once, and the scaffolding +// is gone. +// +// When RunStore gains or loses a member, update the matching list here. You do not have to remember: +// the assertions at the foot of this file tie both lists to `keyof RunStore` in both directions, so +// a missing or invented name is a build failure that names the member. import type { RunStore } from "./types.js"; diff --git a/knip.json b/knip.json index 92760110a87..c6e8aee8977 100644 --- a/knip.json +++ b/knip.json @@ -38,9 +38,6 @@ "internal-packages/observability-map": { "entry": ["src/index.ts", "fixtures/**/*.{js,mjs,cjs,ts,mts,cts,tsx}"] }, - "internal-packages/run-store": { - "entry": ["scripts/**/*.{js,mjs,cjs,ts,mts,cts}"] - }, "internal-packages/otlp-importer": { "ignoreDependencies": ["ts-proto"] }, From a553562ed40833ccd2a828a73a0bdd814824a842 Mon Sep 17 00:00:00 2001 From: Daniel Sutton Date: Tue, 25 Aug 2026 12:30:06 +0100 Subject: [PATCH 18/56] test(run-engine): guard the sweeper's copy of the terminal-status list The sweeper needs to know which run statuses are terminal and cannot import the list, because run-engine depends on run-store rather than the other way round. The copy's comment claimed a parity test kept the two equal. No such test existed, so the claim was false and the copy could drift silently. Drift is not symmetric. A status added to the engine and not the copy makes the sweep treat a finished run as live and never apply its completion expiry. A status removed from the engine and not the copy makes it treat a live run as finished, and that reaps state a run is still using. Verified by removing a status and rebuilding: the test fails and reports seven members against eight. --- .../engine/tests/finalRunStatusParity.test.ts | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 internal-packages/run-engine/src/engine/tests/finalRunStatusParity.test.ts diff --git a/internal-packages/run-engine/src/engine/tests/finalRunStatusParity.test.ts b/internal-packages/run-engine/src/engine/tests/finalRunStatusParity.test.ts new file mode 100644 index 00000000000..50eba9d3a04 --- /dev/null +++ b/internal-packages/run-engine/src/engine/tests/finalRunStatusParity.test.ts @@ -0,0 +1,20 @@ +// The snapshot sweeper needs to know which run statuses are terminal, and it cannot import that +// list: run-engine depends on run-store, not the other way round. So the list is duplicated, and +// this is the only thing that keeps the copy honest. +// +// Without it, a status added here and not there makes the sweeper treat a finished run as live and +// never apply its completion expiry. A status removed here and not there makes it treat a live run +// as finished. The second one reaps state a run is still using. +import { describe, expect, it } from "vitest"; +import { FINAL_RUN_STATUSES } from "@internal/run-store"; +import { getFinalRunStatuses } from "../statuses.js"; + +describe("terminal run statuses", () => { + it("match between the engine and the snapshot sweeper", () => { + expect([...FINAL_RUN_STATUSES].sort()).toEqual([...getFinalRunStatuses()].sort()); + }); + + it("are not empty, so the comparison cannot pass vacuously", () => { + expect(FINAL_RUN_STATUSES.length).toBeGreaterThan(0); + }); +}); From 0f6c6d1dbdcc2de679b5961db579cf4fe1e15eb6 Mon Sep 17 00:00:00 2001 From: Daniel Sutton Date: Tue, 25 Aug 2026 12:54:39 +0100 Subject: [PATCH 19/56] refactor(run-store): type every forwarder on the pass-through base The forwarders were (...args: any[]): any, so the compiler could not see inside them. A body that called the wrong delegate member, or reordered its arguments, typechecked cleanly. That is not a theoretical gap: it is why a runtime probe existed to catch it, and it is the same shape of hole that let three other defects on this branch pass a green suite. Every member now restates its interface signature and forwards its arguments by name, so both mistakes are compile errors. Verified by making them: a forward to the wrong member produces two type errors, and swapping two arguments produces one. Seven members are overloaded. TypeScript cannot express a single body that satisfies an overload set, so their overloads are declared for callers and their one implementation forwards through a cast. That cast is now the only place the compiler is not checking the forward. The probe shrinks to what is left: those seven casts, a dropped OPTIONAL argument (omitting a trailing tx compiles and silently stops forwarding the transaction), and whether the data property is read live or captured once. Its header states which of those the compiler already covers. Headers on both files now describe what they are rather than that they were once scaffolded. --- .../run-store/src/delegatingRunStore.test.ts | 71 +- .../run-store/src/delegatingRunStore.ts | 966 +++++++++++++----- .../run-store/src/runStoreMethodNames.ts | 11 +- 3 files changed, 753 insertions(+), 295 deletions(-) diff --git a/internal-packages/run-store/src/delegatingRunStore.test.ts b/internal-packages/run-store/src/delegatingRunStore.test.ts index 74d6a9e9afb..7fdcd9ac07b 100644 --- a/internal-packages/run-store/src/delegatingRunStore.test.ts +++ b/internal-packages/run-store/src/delegatingRunStore.test.ts @@ -1,32 +1,51 @@ -// What this suite covers, and why it does not use a container. +// What this suite covers, and why it is now small. // -// `DelegatingRunStore` is generated and holds no logic: every member forwards to a delegate. Three -// properties of it are already proved by the compiler and are NOT retested here: +// `DelegatingRunStore` restates every interface signature and forwards its arguments by name, so +// most of what a pass-through can get wrong is a compile error rather than a test failure: // -// - a member of the interface missing from the class -> `implements RunStore`, TS2420 -// - a public member the interface does not declare -> the parity assertion in the base -// - a name list out of step with the interface -> the assertions in runStoreMethodNames +// member of the interface missing -> `implements RunStore`, TS2420 +// public member the interface lacks -> the parity assertion in the base +// forwarded to the wrong delegate member -> argument types do not match, TS2345/TS2322 +// arguments reordered -> same // -// What the compiler cannot see is inside the forwarder bodies, because each one is typed -// `(...args: any[]): any`. A forwarder wired to the wrong delegate method, dropping an argument, or -// reading a property once at construction instead of on each access, all typecheck cleanly. Those -// are template-correctness properties of the generator's output, and they are what is tested below. +// Three things remain invisible to the compiler, and they are what is left here. // -// A Testcontainers-backed store cannot demonstrate them. It would mean calling all 70 methods with -// valid arguments and valid foreign-key state, and a real return value cannot show that arguments -// arrived untouched the way a per-member sentinel can. The probe below is not a stand-in for a -// database: no database is involved in whether a pass-through passes through. Behaviour against a -// real store is covered by the container suites for the decorator built on this base. +// First, the seven overloaded members. TypeScript cannot express one body that satisfies an overload +// set, so their single implementation forwards through a cast, and the cast is exactly where a +// wrong-member forward would stop being a type error. +// +// Second, a dropped OPTIONAL argument. Omitting a trailing `tx` compiles cleanly and silently stops +// forwarding the caller's transaction. +// +// Third, whether a data property is read live or captured once at construction. Both typecheck; only +// one is correct. +// +// No database is involved in whether a pass-through passes through, so none is started. Behaviour +// against a real store is covered by the container suites for the decorator built on this base. import { describe, expect, it } from "vitest"; import { DelegatingRunStore } from "./delegatingRunStore.js"; import { RUN_STORE_METHOD_NAMES, RUN_STORE_PROPERTY_NAMES } from "./runStoreMethodNames.js"; import type { RunStore } from "./types.js"; +/** + * The members whose implementation forwards through a cast, because they are overloaded. These are + * the only methods where the compiler is not already checking the forward. + */ +const OVERLOADED_MEMBERS = [ + "finalizeRun", + "findRun", + "findRunOrThrow", + "findRunOnPrimary", + "findRunOrThrowOnPrimary", + "findRuns", + "findRunsByIds", +] as const; + type ProbedCall = { name: string; args: unknown[] }; /** - * A delegate that records what was called on it and answers with a per-member sentinel, so a - * forwarder wired to the wrong member returns the wrong sentinel and fails loudly. + * Records what was called and answers with a per-member sentinel, so a forward to the wrong member + * returns the wrong value rather than merely returning something. */ function forwardingProbe(): { store: RunStore; calls: ProbedCall[] } { const calls: ProbedCall[] = []; @@ -55,15 +74,20 @@ describe("DelegatingRunStore", () => { >; for (const name of RUN_STORE_METHOD_NAMES) { - // The sentinel is per member, so a body forwarding to a different method fails here rather - // than passing because both happened to return something. expect(base[name]()).toBe(`result:${name}`); } expect(calls.map((c) => c.name)).toEqual([...RUN_STORE_METHOD_NAMES]); }); - it("forwards arguments untouched", () => { + it("covers every overloaded member, so the list cannot rot", () => { + // If a member gains or loses overloads, the cast set changes and this suite should follow. + for (const name of OVERLOADED_MEMBERS) { + expect(RUN_STORE_METHOD_NAMES).toContain(name); + } + }); + + it("forwards arguments untouched through an overloaded member's cast", () => { const { store, calls } = forwardingProbe(); const base = new DelegatingRunStore(store) as unknown as Record< string, @@ -71,18 +95,19 @@ describe("DelegatingRunStore", () => { >; const args = ["first", { second: true }, undefined, 4]; - for (const name of RUN_STORE_METHOD_NAMES) { + for (const name of OVERLOADED_MEMBERS) { base[name](...args); } + // The overloaded implementations apply the whole argument list, so every argument survives, + // including a trailing optional the typed members would legitimately drop. for (const call of calls) { expect(call.args).toEqual(args); } + expect(calls.map((c) => c.name)).toEqual([...OVERLOADED_MEMBERS]); }); it("reads a data property live, so a delegate that changes is not cached", () => { - // A getter is the only correct shape here. Capturing the value in the constructor would - // typecheck and would then serve a stale client for the life of the decorator. const store = { primaryReadClient: "first" } as unknown as RunStore; const base = new DelegatingRunStore(store); diff --git a/internal-packages/run-store/src/delegatingRunStore.ts b/internal-packages/run-store/src/delegatingRunStore.ts index e158ada0037..008ccfce1c5 100644 --- a/internal-packages/run-store/src/delegatingRunStore.ts +++ b/internal-packages/run-store/src/delegatingRunStore.ts @@ -1,312 +1,746 @@ -// Maintained by hand. It was scaffolded once, and the scaffolding is gone. +// A pass-through over another RunStore. // -// When RunStore gains or loses a member, add or remove the forwarder here. You do not have to -// remember: `implements RunStore` fails with TS2420 on a member that is missing, and the assertion -// at the foot of this file fails on a public member the interface does not declare. - -// A pass-through over another RunStore. It exists so a decorator can override the handful of methods -// it cares about and inherit the rest, instead of restating 80-odd forwarders alongside real logic. +// It exists so a decorator can override the handful of methods it cares about and inherit the rest. +// +// Every member restates the interface signature and forwards its arguments BY NAME, so the +// forwarding is itself type-checked: a body that called the wrong delegate method, or dropped an +// argument, does not compile. That is the whole point of the shape. An untyped forwarder would let +// both mistakes through, because a pass-through has no other behaviour to catch them. // -// Arguments and return values are forwarded untouched. The `any` signatures carry each method's -// whole overload set through one forwarder, which is the single thing a generated base cannot -// preserve; a subclass that overrides a method restates the real signature there. -/* eslint-disable @typescript-eslint/no-explicit-any */ -import type { RunStore } from "./types.js"; +// Seven members are overloaded. Their overloads are declared so callers keep the full contract, and +// their single implementation signature is the one place a cast appears: TypeScript cannot express +// one body that satisfies an overload set without it. +// +// Keeping this in step with the interface is not a matter of memory. `implements RunStore` rejects a +// member that is missing, and the assertion at the foot of the file rejects one the interface never +// declared. + +import type { + BatchTaskRun, + BatchTaskRunItemStatus, + Prisma, + PrismaClientOrTransaction, + TaskRun, + TaskRunStatus, + WaitpointTag, +} from "@trigger.dev/database"; +import type { TaskRunError } from "@trigger.dev/core/v3/schemas"; +import type { Residency } from "@trigger.dev/core/v3/isomorphic"; +import type { + ClearIdempotencyKeyInput, + CompletionSnapshotInput, + CreateBatchTaskRunData, + CreateCancelledRunInput, + CreateExecutionSnapshotInput, + CreateFailedRunInput, + CreateRunInput, + ExpireSnapshotInput, + FinalizeRunData, + ForWaitpointCompletionContext, + IdempotencyKeyRunMatch, + LockRunData, + PromotePendingVersionArgs, + ReadClient, + RescheduleSnapshotInput, + RewriteDebouncedRunData, + RunStore, + TaskRunWithWaitpoint, + WaitpointColocationOptions, +} from "./types.js"; export class DelegatingRunStore implements RunStore { constructor(protected readonly delegate: RunStore) {} - get primaryReadClient(): RunStore["primaryReadClient"] { + runInTransaction( + runId: string | undefined, + fn: (store: RunStore, tx: PrismaClientOrTransaction) => Promise + ): Promise { + return this.delegate.runInTransaction(runId, fn); + } + + createRun(params: CreateRunInput, tx?: PrismaClientOrTransaction): Promise { + return this.delegate.createRun(params, tx); + } + + createCancelledRun( + params: CreateCancelledRunInput, + tx?: PrismaClientOrTransaction + ): Promise { + return this.delegate.createCancelledRun(params, tx); + } + + createFailedRun( + params: CreateFailedRunInput, + tx?: PrismaClientOrTransaction + ): Promise { + return this.delegate.createFailedRun(params, tx); + } + + startAttempt( + runId: string, + data: { attemptNumber: number; executedAt?: Date; isWarmStart: boolean }, + args: { select: S }, + tx?: PrismaClientOrTransaction + ): Promise> { + return this.delegate.startAttempt(runId, data, args, tx); + } + + completeAttemptSuccess( + runId: string, + data: { + completedAt: Date; + output?: string; + outputType: string; + usageDurationMs: number; + costInCents: number; + snapshot: CompletionSnapshotInput; + }, + args: { select: S }, + tx?: PrismaClientOrTransaction + ): Promise> { + return this.delegate.completeAttemptSuccess(runId, data, args, tx); + } + + recordRetryOutcome( + runId: string, + data: { machinePreset?: string; usageDurationMs: number; costInCents: number }, + args: { select: S }, + tx?: PrismaClientOrTransaction + ): Promise> { + return this.delegate.recordRetryOutcome(runId, data, args, tx); + } + + requeueRun( + runId: string, + args: { select: S }, + tx?: PrismaClientOrTransaction + ): Promise> { + return this.delegate.requeueRun(runId, args, tx); + } + + recordBulkActionMembership( + runId: string, + bulkActionId: string, + tx?: PrismaClientOrTransaction + ): Promise { + return this.delegate.recordBulkActionMembership(runId, bulkActionId, tx); + } + + cancelRun( + runId: string, + data: { + completedAt?: Date; + error: TaskRunError; + bulkActionId?: string; + usageDurationMs?: number; + costInCents?: number; + }, + args: { select: S }, + tx?: PrismaClientOrTransaction + ): Promise> { + return this.delegate.cancelRun(runId, data, args, tx); + } + + failRunPermanently( + runId: string, + data: { + status: TaskRunStatus; + completedAt: Date; + error: TaskRunError; + usageDurationMs: number; + costInCents: number; + }, + args: { select: S }, + tx?: PrismaClientOrTransaction + ): Promise> { + return this.delegate.failRunPermanently(runId, data, args, tx); + } + + finalizeRun( + runId: string, + data: FinalizeRunData, + args: { select: S }, + tx?: PrismaClientOrTransaction + ): Promise>; + finalizeRun( + runId: string, + data: FinalizeRunData, + args: { include: I }, + tx?: PrismaClientOrTransaction + ): Promise>; + finalizeRun( + runId: string, + data: FinalizeRunData, + tx?: PrismaClientOrTransaction + ): Promise; + finalizeRun(...args: unknown[]): unknown { + return (this.delegate.finalizeRun as (...a: unknown[]) => unknown).apply(this.delegate, args); + } + + expireRun( + runId: string, + data: { + error: TaskRunError; + completedAt: Date; + expiredAt: Date; + snapshot: ExpireSnapshotInput; + }, + args: { select: S }, + tx?: PrismaClientOrTransaction + ): Promise> { + return this.delegate.expireRun(runId, data, args, tx); + } + + expireRunsBatch( + runIds: string[], + data: { error: TaskRunError; now: Date }, + tx?: PrismaClientOrTransaction + ): Promise { + return this.delegate.expireRunsBatch(runIds, data, tx); + } + + lockRunToWorker( + runId: string, + data: LockRunData, + tx?: PrismaClientOrTransaction + ): Promise> { + return this.delegate.lockRunToWorker(runId, data, tx); + } + + parkPendingVersion( + runId: string, + data: { statusReason: string }, + args: { select: S }, + tx?: PrismaClientOrTransaction + ): Promise> { + return this.delegate.parkPendingVersion(runId, data, args, tx); + } + + promotePendingVersionRuns( + runId: string, + args?: PromotePendingVersionArgs, + tx?: PrismaClientOrTransaction + ): Promise<{ count: number }> { + return this.delegate.promotePendingVersionRuns(runId, args, tx); + } + + expireParkedRun( + runId: string, + data: { + error: TaskRunError; + completedAt: Date; + expiredAt: Date; + statusReason: string; + snapshot: ExpireSnapshotInput; + }, + tx?: PrismaClientOrTransaction + ): Promise<{ count: number }> { + return this.delegate.expireParkedRun(runId, data, tx); + } + + suspendForCheckpoint( + runId: string, + args: { include: I }, + tx?: PrismaClientOrTransaction + ): Promise> { + return this.delegate.suspendForCheckpoint(runId, args, tx); + } + + resumeFromCheckpoint( + runId: string, + args: { select: S }, + tx?: PrismaClientOrTransaction + ): Promise> { + return this.delegate.resumeFromCheckpoint(runId, args, tx); + } + + rescheduleRun( + runId: string, + data: { delayUntil: Date; queueTimestamp?: Date; snapshot?: RescheduleSnapshotInput }, + tx?: PrismaClientOrTransaction + ): Promise { + return this.delegate.rescheduleRun(runId, data, tx); + } + + enqueueDelayedRun( + runId: string, + data: { queuedAt: Date }, + tx?: PrismaClientOrTransaction + ): Promise { + return this.delegate.enqueueDelayedRun(runId, data, tx); + } + + rewriteDebouncedRun( + runId: string, + data: RewriteDebouncedRunData, + tx?: PrismaClientOrTransaction + ): Promise { + return this.delegate.rewriteDebouncedRun(runId, data, tx); + } + + updateMetadata( + runId: string, + data: { + metadata: string | null; + metadataType?: string; + metadataVersion: { increment: number }; + updatedAt: Date; + }, + options: { expectedMetadataVersion?: number }, + tx?: PrismaClientOrTransaction + ): Promise<{ count: number }> { + return this.delegate.updateMetadata(runId, data, options, tx); + } + + clearIdempotencyKey( + params: ClearIdempotencyKeyInput, + tx?: PrismaClientOrTransaction + ): Promise<{ count: number }> { + return this.delegate.clearIdempotencyKey(params, tx); + } + + pushTags( + runId: string, + tags: string[], + where: { runtimeEnvironmentId: string }, + tx?: PrismaClientOrTransaction + ): Promise<{ updatedAt: Date }> { + return this.delegate.pushTags(runId, tags, where, tx); + } + + pushRealtimeStream( + runId: string, + streamId: string, + tx?: PrismaClientOrTransaction + ): Promise { + return this.delegate.pushRealtimeStream(runId, streamId, tx); + } + + get primaryReadClient(): ReadClient { return this.delegate.primaryReadClient; } - runInTransaction(...args: any[]): any { - return (this.delegate as any).runInTransaction(...args); - } - - createRun(...args: any[]): any { - return (this.delegate as any).createRun(...args); - } - - createCancelledRun(...args: any[]): any { - return (this.delegate as any).createCancelledRun(...args); - } - - createFailedRun(...args: any[]): any { - return (this.delegate as any).createFailedRun(...args); - } - - startAttempt(...args: any[]): any { - return (this.delegate as any).startAttempt(...args); - } - - completeAttemptSuccess(...args: any[]): any { - return (this.delegate as any).completeAttemptSuccess(...args); - } - - recordRetryOutcome(...args: any[]): any { - return (this.delegate as any).recordRetryOutcome(...args); - } - - requeueRun(...args: any[]): any { - return (this.delegate as any).requeueRun(...args); - } - - recordBulkActionMembership(...args: any[]): any { - return (this.delegate as any).recordBulkActionMembership(...args); - } - - cancelRun(...args: any[]): any { - return (this.delegate as any).cancelRun(...args); - } - - failRunPermanently(...args: any[]): any { - return (this.delegate as any).failRunPermanently(...args); - } - - finalizeRun(...args: any[]): any { - return (this.delegate as any).finalizeRun(...args); - } - - expireRun(...args: any[]): any { - return (this.delegate as any).expireRun(...args); - } - - expireRunsBatch(...args: any[]): any { - return (this.delegate as any).expireRunsBatch(...args); - } - - lockRunToWorker(...args: any[]): any { - return (this.delegate as any).lockRunToWorker(...args); - } - - parkPendingVersion(...args: any[]): any { - return (this.delegate as any).parkPendingVersion(...args); - } - - promotePendingVersionRuns(...args: any[]): any { - return (this.delegate as any).promotePendingVersionRuns(...args); - } - - expireParkedRun(...args: any[]): any { - return (this.delegate as any).expireParkedRun(...args); - } - - suspendForCheckpoint(...args: any[]): any { - return (this.delegate as any).suspendForCheckpoint(...args); - } - - resumeFromCheckpoint(...args: any[]): any { - return (this.delegate as any).resumeFromCheckpoint(...args); - } - - rescheduleRun(...args: any[]): any { - return (this.delegate as any).rescheduleRun(...args); - } - - enqueueDelayedRun(...args: any[]): any { - return (this.delegate as any).enqueueDelayedRun(...args); - } - - rewriteDebouncedRun(...args: any[]): any { - return (this.delegate as any).rewriteDebouncedRun(...args); - } - - updateMetadata(...args: any[]): any { - return (this.delegate as any).updateMetadata(...args); - } - - clearIdempotencyKey(...args: any[]): any { - return (this.delegate as any).clearIdempotencyKey(...args); - } - - pushTags(...args: any[]): any { - return (this.delegate as any).pushTags(...args); - } - - pushRealtimeStream(...args: any[]): any { - return (this.delegate as any).pushRealtimeStream(...args); - } - - findRun(...args: any[]): any { - return (this.delegate as any).findRun(...args); - } - - findRunOrThrow(...args: any[]): any { - return (this.delegate as any).findRunOrThrow(...args); - } - - findRunOnPrimary(...args: any[]): any { - return (this.delegate as any).findRunOnPrimary(...args); - } - - findRunOrThrowOnPrimary(...args: any[]): any { - return (this.delegate as any).findRunOrThrowOnPrimary(...args); - } - - findRuns(...args: any[]): any { - return (this.delegate as any).findRuns(...args); - } - - findRunsByIds(...args: any[]): any { - return (this.delegate as any).findRunsByIds(...args); - } - - findRunsByIdempotencyKeys(...args: any[]): any { - return (this.delegate as any).findRunsByIdempotencyKeys(...args); - } - - createBatchTaskRunItem(...args: any[]): any { - return (this.delegate as any).createBatchTaskRunItem(...args); - } - - findLatestExecutionSnapshot(...args: any[]): any { - return (this.delegate as any).findLatestExecutionSnapshot(...args); - } - - findExecutionSnapshot(...args: any[]): any { - return (this.delegate as any).findExecutionSnapshot(...args); - } - - findManyExecutionSnapshots(...args: any[]): any { - return (this.delegate as any).findManyExecutionSnapshots(...args); - } - - createExecutionSnapshot(...args: any[]): any { - return (this.delegate as any).createExecutionSnapshot(...args); - } - - findSnapshotCompletedWaitpointIds(...args: any[]): any { - return (this.delegate as any).findSnapshotCompletedWaitpointIds(...args); - } - - findSnapshotCompletedWaitpointIdsWithPresence(...args: any[]): any { - return (this.delegate as any).findSnapshotCompletedWaitpointIdsWithPresence(...args); - } - - findWaitpointConnectedRunIds(...args: any[]): any { - return (this.delegate as any).findWaitpointConnectedRunIds(...args); - } - - findWaitpointCompletedSnapshotIds(...args: any[]): any { - return (this.delegate as any).findWaitpointCompletedSnapshotIds(...args); - } - - blockRunWithWaitpointEdges(...args: any[]): any { - return (this.delegate as any).blockRunWithWaitpointEdges(...args); - } - - countPendingWaitpoints(...args: any[]): any { - return (this.delegate as any).countPendingWaitpoints(...args); - } - - countPendingWaitpointsWithPresence(...args: any[]): any { - return (this.delegate as any).countPendingWaitpointsWithPresence(...args); - } - - createWaitpoint(...args: any[]): any { - return (this.delegate as any).createWaitpoint(...args); - } - - upsertWaitpoint(...args: any[]): any { - return (this.delegate as any).upsertWaitpoint(...args); - } - - findWaitpoint(...args: any[]): any { - return (this.delegate as any).findWaitpoint(...args); - } - - findWaitpointOnPrimary(...args: any[]): any { - return (this.delegate as any).findWaitpointOnPrimary(...args); - } - - findManyWaitpoints(...args: any[]): any { - return (this.delegate as any).findManyWaitpoints(...args); - } - - updateWaitpoint(...args: any[]): any { - return (this.delegate as any).updateWaitpoint(...args); - } - - updateManyWaitpoints(...args: any[]): any { - return (this.delegate as any).updateManyWaitpoints(...args); - } - - forWaitpointCompletion(...args: any[]): any { - return (this.delegate as any).forWaitpointCompletion(...args); - } - - findManyTaskRunWaitpoints(...args: any[]): any { - return (this.delegate as any).findManyTaskRunWaitpoints(...args); - } - - deleteManyTaskRunWaitpoints(...args: any[]): any { - return (this.delegate as any).deleteManyTaskRunWaitpoints(...args); + findRun( + where: Prisma.TaskRunWhereInput, + args: { select: S }, + client?: ReadClient + ): Promise | null>; + findRun( + where: Prisma.TaskRunWhereInput, + args: { include: I }, + client?: ReadClient + ): Promise | null>; + findRun(where: Prisma.TaskRunWhereInput, client?: ReadClient): Promise; + findRun(...args: unknown[]): unknown { + return (this.delegate.findRun as (...a: unknown[]) => unknown).apply(this.delegate, args); + } + + findRunOrThrow( + where: Prisma.TaskRunWhereInput, + args: { select: S }, + client?: ReadClient + ): Promise>; + findRunOrThrow( + where: Prisma.TaskRunWhereInput, + args: { include: I }, + client?: ReadClient + ): Promise>; + findRunOrThrow(where: Prisma.TaskRunWhereInput, client?: ReadClient): Promise; + findRunOrThrow(...args: unknown[]): unknown { + return (this.delegate.findRunOrThrow as (...a: unknown[]) => unknown).apply( + this.delegate, + args + ); + } + + findRunOnPrimary( + where: Prisma.TaskRunWhereInput, + args: { select: S } + ): Promise | null>; + findRunOnPrimary( + where: Prisma.TaskRunWhereInput, + args: { include: I } + ): Promise | null>; + findRunOnPrimary(where: Prisma.TaskRunWhereInput): Promise; + findRunOnPrimary(...args: unknown[]): unknown { + return (this.delegate.findRunOnPrimary as (...a: unknown[]) => unknown).apply( + this.delegate, + args + ); + } + + findRunOrThrowOnPrimary( + where: Prisma.TaskRunWhereInput, + args: { select: S } + ): Promise>; + findRunOrThrowOnPrimary( + where: Prisma.TaskRunWhereInput, + args: { include: I } + ): Promise>; + findRunOrThrowOnPrimary(where: Prisma.TaskRunWhereInput): Promise; + findRunOrThrowOnPrimary(...args: unknown[]): unknown { + return (this.delegate.findRunOrThrowOnPrimary as (...a: unknown[]) => unknown).apply( + this.delegate, + args + ); + } + + findRuns( + args: { + where: Prisma.TaskRunWhereInput; + select: S; + orderBy?: Prisma.TaskRunOrderByWithRelationInput | Prisma.TaskRunOrderByWithRelationInput[]; + take?: number; + skip?: number; + cursor?: Prisma.TaskRunWhereUniqueInput; + }, + client?: ReadClient + ): Promise[]>; + findRuns( + args: { + where: Prisma.TaskRunWhereInput; + include: I; + orderBy?: Prisma.TaskRunOrderByWithRelationInput | Prisma.TaskRunOrderByWithRelationInput[]; + take?: number; + skip?: number; + cursor?: Prisma.TaskRunWhereUniqueInput; + }, + client?: ReadClient + ): Promise[]>; + findRuns( + args: { + where: Prisma.TaskRunWhereInput; + orderBy?: Prisma.TaskRunOrderByWithRelationInput | Prisma.TaskRunOrderByWithRelationInput[]; + take?: number; + skip?: number; + cursor?: Prisma.TaskRunWhereUniqueInput; + }, + client?: ReadClient + ): Promise; + findRuns(...args: unknown[]): unknown { + return (this.delegate.findRuns as (...a: unknown[]) => unknown).apply(this.delegate, args); + } + + findRunsByIds( + ids: string[], + args: { select: S }, + client?: ReadClient + ): Promise>>; + findRunsByIds( + ids: string[], + args: { include: I }, + client?: ReadClient + ): Promise>>; + findRunsByIds(ids: string[], client?: ReadClient): Promise>; + findRunsByIds(...args: unknown[]): unknown { + return (this.delegate.findRunsByIds as (...a: unknown[]) => unknown).apply(this.delegate, args); + } + + findRunsByIdempotencyKeys( + args: { runtimeEnvironmentId: string; taskIdentifier: string; idempotencyKeys: string[] }, + client?: ReadClient + ): Promise { + return this.delegate.findRunsByIdempotencyKeys(args, client); + } + + createBatchTaskRunItem( + data: { batchTaskRunId: string; taskRunId: string; status: BatchTaskRunItemStatus }, + tx?: PrismaClientOrTransaction + ): Promise { + return this.delegate.createBatchTaskRunItem(data, tx); + } + + findLatestExecutionSnapshot( + runId: string, + client?: ReadClient, + // When set, scopes the read to this environment (tenant boundary); a run in another env reads as + // not-found. Omit to read regardless of environment (internal callers). + environmentId?: string + ): Promise | null> { + return this.delegate.findLatestExecutionSnapshot(runId, client); + } + + findExecutionSnapshot( + args: Prisma.SelectSubset, + client?: ReadClient + ): Promise | null> { + return this.delegate.findExecutionSnapshot(args, client); + } + + findManyExecutionSnapshots( + args: Prisma.SelectSubset, + client?: ReadClient + ): Promise[]> { + return this.delegate.findManyExecutionSnapshots(args, client); + } + + createExecutionSnapshot( + input: CreateExecutionSnapshotInput, + tx?: PrismaClientOrTransaction + ): Promise> { + return this.delegate.createExecutionSnapshot(input, tx); + } + + findSnapshotCompletedWaitpointIds( + snapshotId: string, + client?: ReadClient, + runId?: string + ): Promise { + return this.delegate.findSnapshotCompletedWaitpointIds(snapshotId, client, runId); + } + + findSnapshotCompletedWaitpointIdsWithPresence( + snapshotId: string, + client?: ReadClient, + runId?: string + ): Promise<{ present: boolean; ids: string[] }> { + return this.delegate.findSnapshotCompletedWaitpointIdsWithPresence(snapshotId, client, runId); + } + + findWaitpointConnectedRunIds(waitpointId: string, client?: ReadClient): Promise { + return this.delegate.findWaitpointConnectedRunIds(waitpointId, client); + } + + findWaitpointCompletedSnapshotIds(waitpointId: string, client?: ReadClient): Promise { + return this.delegate.findWaitpointCompletedSnapshotIds(waitpointId, client); + } + + blockRunWithWaitpointEdges(params: { + runId: string; + waitpointIds: string[]; + projectId: string; + spanIdToComplete?: string; + batchId?: string; + batchIndex?: number; + tx?: PrismaClientOrTransaction; + }): Promise { + return this.delegate.blockRunWithWaitpointEdges(params); } - findTaskRunAttempt(...args: any[]): any { - return (this.delegate as any).findTaskRunAttempt(...args); + countPendingWaitpoints( + waitpointIds: string[], + client?: ReadClient, + runId?: string + ): Promise { + return this.delegate.countPendingWaitpoints(waitpointIds, client, runId); } - createTaskRunCheckpoint(...args: any[]): any { - return (this.delegate as any).createTaskRunCheckpoint(...args); + countPendingWaitpointsWithPresence( + waitpointIds: string[], + client?: ReadClient + ): Promise<{ pendingIds: string[]; presentIds: string[] }> { + return this.delegate.countPendingWaitpointsWithPresence(waitpointIds, client); } - createBatchTaskRun(...args: any[]): any { - return (this.delegate as any).createBatchTaskRun(...args); + createWaitpoint( + args: Prisma.SelectSubset, + tx?: PrismaClientOrTransaction, + opts?: WaitpointColocationOptions + ): Promise> { + return this.delegate.createWaitpoint(args, tx, opts); } - updateBatchTaskRun(...args: any[]): any { - return (this.delegate as any).updateBatchTaskRun(...args); + upsertWaitpoint( + args: Prisma.SelectSubset, + tx?: PrismaClientOrTransaction, + opts?: WaitpointColocationOptions + ): Promise> { + return this.delegate.upsertWaitpoint(args, tx, opts); } - findBatchTaskRunById(...args: any[]): any { - return (this.delegate as any).findBatchTaskRunById(...args); + findWaitpoint( + args: Prisma.SelectSubset, + client?: ReadClient, + opts?: WaitpointColocationOptions + ): Promise | null> { + return this.delegate.findWaitpoint(args, client, opts); } - findBatchTaskRunByFriendlyId(...args: any[]): any { - return (this.delegate as any).findBatchTaskRunByFriendlyId(...args); + findWaitpointOnPrimary( + args: Prisma.SelectSubset + ): Promise | null> { + return this.delegate.findWaitpointOnPrimary(args); } - findBatchTaskRunByIdempotencyKey(...args: any[]): any { - return (this.delegate as any).findBatchTaskRunByIdempotencyKey(...args); + findManyWaitpoints( + args: Prisma.SelectSubset, + client?: ReadClient, + runId?: string + ): Promise[]> { + return this.delegate.findManyWaitpoints(args, client, runId); } - updateManyBatchTaskRun(...args: any[]): any { - return (this.delegate as any).updateManyBatchTaskRun(...args); + updateWaitpoint( + args: Prisma.SelectSubset, + tx?: PrismaClientOrTransaction, + opts?: WaitpointColocationOptions + ): Promise> { + return this.delegate.updateWaitpoint(args, tx, opts); } - countBatchTaskRunItems(...args: any[]): any { - return (this.delegate as any).countBatchTaskRunItems(...args); + updateManyWaitpoints( + args: Prisma.WaitpointUpdateManyArgs, + tx?: PrismaClientOrTransaction + ): Promise { + return this.delegate.updateManyWaitpoints(args, tx); } - updateManyBatchTaskRunItems(...args: any[]): any { - return (this.delegate as any).updateManyBatchTaskRunItems(...args); + forWaitpointCompletion( + waitpointId: string, + context: ForWaitpointCompletionContext + ): Promise { + return this.delegate.forWaitpointCompletion(waitpointId, context); } - findManyBatchTaskRunItems(...args: any[]): any { - return (this.delegate as any).findManyBatchTaskRunItems(...args); + findManyTaskRunWaitpoints( + args: Prisma.SelectSubset, + client?: ReadClient + ): Promise[]> { + return this.delegate.findManyTaskRunWaitpoints(args, client); } - findBatchTaskRunItem(...args: any[]): any { - return (this.delegate as any).findBatchTaskRunItem(...args); + deleteManyTaskRunWaitpoints( + args: Prisma.TaskRunWaitpointDeleteManyArgs, + tx?: PrismaClientOrTransaction + ): Promise { + return this.delegate.deleteManyTaskRunWaitpoints(args, tx); + } + + findTaskRunAttempt( + args: Prisma.SelectSubset, + client?: ReadClient + ): Promise | null> { + return this.delegate.findTaskRunAttempt(args, client); } - upsertWaitpointTag(...args: any[]): any { - return (this.delegate as any).upsertWaitpointTag(...args); + createTaskRunCheckpoint( + args: Prisma.SelectSubset, + ownerRunId?: string, + tx?: PrismaClientOrTransaction + ): Promise> { + return this.delegate.createTaskRunCheckpoint(args, ownerRunId, tx); } - findManyWaitpointTags(...args: any[]): any { - return (this.delegate as any).findManyWaitpointTags(...args); + createBatchTaskRun( + data: CreateBatchTaskRunData, + tx?: PrismaClientOrTransaction + ): Promise { + return this.delegate.createBatchTaskRun(data, tx); + } + + updateBatchTaskRun( + args: { + where: Prisma.BatchTaskRunWhereUniqueInput; + data: Prisma.BatchTaskRunUpdateInput; + select: S; + }, + tx?: PrismaClientOrTransaction + ): Promise> { + return this.delegate.updateBatchTaskRun(args, tx); + } + + findBatchTaskRunById( + id: string, + args?: { include?: T }, + client?: ReadClient + ): Promise | null> { + return this.delegate.findBatchTaskRunById(id, args, client); + } + + findBatchTaskRunByFriendlyId( + friendlyId: string, + environmentId: string, + args?: { include?: T }, + client?: ReadClient + ): Promise | null> { + return this.delegate.findBatchTaskRunByFriendlyId(friendlyId, environmentId, args, client); + } + + findBatchTaskRunByIdempotencyKey( + environmentId: string, + idempotencyKey: string, + args?: { include?: T }, + client?: ReadClient + ): Promise | null> { + return this.delegate.findBatchTaskRunByIdempotencyKey( + environmentId, + idempotencyKey, + args, + client + ); + } + + updateManyBatchTaskRun( + args: Prisma.BatchTaskRunUpdateManyArgs, + tx?: PrismaClientOrTransaction + ): Promise { + return this.delegate.updateManyBatchTaskRun(args, tx); + } + + countBatchTaskRunItems( + where: { batchTaskRunId: string; status?: BatchTaskRunItemStatus }, + client?: ReadClient + ): Promise { + return this.delegate.countBatchTaskRunItems(where, client); + } + + updateManyBatchTaskRunItems( + args: Prisma.BatchTaskRunItemUpdateManyArgs, + tx?: PrismaClientOrTransaction + ): Promise { + return this.delegate.updateManyBatchTaskRunItems(args, tx); + } + + findManyBatchTaskRunItems( + where: { taskRunId?: string; batchTaskRunId?: string }, + args?: { include?: I }, + client?: ReadClient + ): Promise[]> { + return this.delegate.findManyBatchTaskRunItems(where, args, client); + } + + findBatchTaskRunItem( + where: { batchTaskRunId: string; taskRunId?: string }, + args?: { include?: I }, + client?: ReadClient + ): Promise | null> { + return this.delegate.findBatchTaskRunItem(where, args, client); + } + + upsertWaitpointTag( + data: { environmentId: string; name: string; projectId: string; id?: string }, + tx?: PrismaClientOrTransaction, + // A tag has no owning run to co-locate with; when no minted `id` pins it by id-shape, a + // minted-new env's tags read this residency (NEW) so they land with the env's tokens/runs + // instead of defaulting to LEGACY. Single-store impls ignore it. + residency?: Residency + ): Promise { + return this.delegate.upsertWaitpointTag(data, tx); + } + + findManyWaitpointTags( + args: { + where: Prisma.WaitpointTagWhereInput; + orderBy?: + | Prisma.WaitpointTagOrderByWithRelationInput + | Prisma.WaitpointTagOrderByWithRelationInput[]; + take?: number; + skip?: number; + }, + client?: ReadClient + ): Promise { + return this.delegate.findManyWaitpointTags(args, client); } } -// `implements` above fails when a member of the interface is MISSING here. It says nothing about a +// `implements` above rejects a member of the interface that is missing here. It says nothing about a // member that should not exist, so the reverse direction is asserted too: a public member this class // declares and the interface does not is a build failure. // -// `protected delegate` is correctly absent from `keyof`, so the constructor parameter does not -// trip this. +// `protected delegate` is correctly absent from `keyof`, so the constructor parameter does not trip +// this. type _ClassDeclaresNoExtraMembers = [Exclude] extends [ never, ] diff --git a/internal-packages/run-store/src/runStoreMethodNames.ts b/internal-packages/run-store/src/runStoreMethodNames.ts index 3d8fbc63c77..2b10bd766d0 100644 --- a/internal-packages/run-store/src/runStoreMethodNames.ts +++ b/internal-packages/run-store/src/runStoreMethodNames.ts @@ -1,10 +1,9 @@ -// Maintained by hand, alongside delegatingRunStore.ts. It was scaffolded once, and the scaffolding -// is gone. +// The member names of RunStore, as data. +// +// The decorator suites enumerate this to drive one call per member. Member PRESENCE is not proved +// here: that is the compiler's job, through `implements RunStore` on the pass-through base and the +// assertions at the foot of this file. // -// When RunStore gains or loses a member, update the matching list here. You do not have to remember: -// the assertions at the foot of this file tie both lists to `keyof RunStore` in both directions, so -// a missing or invented name is a build failure that names the member. - import type { RunStore } from "./types.js"; // Every method the RunStore interface declares. The forwarding probe enumerates this to drive one From e03a18566a8897a4e9682cdba0095c6cab306992 Mon Sep 17 00:00:00 2001 From: Daniel Sutton Date: Tue, 25 Aug 2026 13:08:09 +0100 Subject: [PATCH 20/56] fix(run-store): forward the parameters two members were dropping Typing the forwarders closed the wrong-member and reordered-argument holes but not this one: omitting a trailing OPTIONAL argument still compiles. Two forwarders did exactly that, because the retyping pass read parameter names with a pattern that a preceding inline comment defeated, and both affected parameters happened to be optional and commented. The effects were silent and not small. findLatestExecutionSnapshot stopped applying its tenant scope, so a direct use of the base could read across the environment boundary. upsertWaitpointTag stopped applying its residency hint, so a tag write for a new-database environment would land on legacy. A source-level guard now asserts that every single-signature member forwards exactly the parameters it declares, in order. It reads the interface and the base and compares them, because that property is invisible to the compiler by definition. It carries a vacuity check, so a parse failure fails the suite instead of quietly matching nothing, and that check earned itself immediately by catching a parser that skipped every generic member. Verified: with a parameter dropped again, typecheck reports zero errors and the guard names the member and the missing argument. --- .../src/delegatingRunStore.forwarding.test.ts | 188 ++++++++++++++++++ .../run-store/src/delegatingRunStore.ts | 4 +- 2 files changed, 190 insertions(+), 2 deletions(-) create mode 100644 internal-packages/run-store/src/delegatingRunStore.forwarding.test.ts diff --git a/internal-packages/run-store/src/delegatingRunStore.forwarding.test.ts b/internal-packages/run-store/src/delegatingRunStore.forwarding.test.ts new file mode 100644 index 00000000000..081e9efe1d4 --- /dev/null +++ b/internal-packages/run-store/src/delegatingRunStore.forwarding.test.ts @@ -0,0 +1,188 @@ +// Every declared parameter must actually reach the delegate. +// +// The compiler cannot check this. A forwarder that omits a trailing OPTIONAL argument compiles +// cleanly, and the effect is silent: `findLatestExecutionSnapshot` would stop applying its tenant +// scope, and `upsertWaitpointTag` would stop applying its residency hint, so a write would land on +// the wrong database. Both of those shipped in this file before this test existed. +// +// So this reads the source of the base against the source of the interface and asserts that each +// forward passes exactly the parameters its signature declares, in order. Source-level, because +// that is the only place the property is visible. +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; + +const dir = join(import.meta.dirname); +const interfaceSource = readFileSync(join(dir, "types.ts"), "utf8"); +const baseSource = readFileSync(join(dir, "delegatingRunStore.ts"), "utf8"); + +/** Replaces comments and string bodies with spaces, so neither can shift a brace depth. */ +function blank(text: string): string { + let out = ""; + let i = 0; + while (i < text.length) { + const two = text.slice(i, i + 2); + if (two === "//") { + const end = text.indexOf("\n", i); + const stop = end === -1 ? text.length : end; + out += " ".repeat(stop - i); + i = stop; + } else if (two === "/*") { + const end = text.indexOf("*/", i + 2); + const stop = end === -1 ? text.length : end + 2; + out += text.slice(i, stop).replace(/[^\n]/g, " "); + i = stop; + } else if (text[i] === '"' || text[i] === "'" || text[i] === "`") { + const quote = text[i]; + let j = i + 1; + while (j < text.length && text[j] !== quote) j += text[j] === "\\" ? 2 : 1; + out += quote + " ".repeat(Math.max(0, j - i - 1)) + (text[j] ?? ""); + i = j + 1; + } else { + out += text[i]; + i += 1; + } + } + return out; +} + +function interfaceBody(source: string): string { + const blanked = blank(source); + const decl = "export interface RunStore {"; + const start = blanked.indexOf(decl) + decl.length; + let depth = 1; + let end = start; + while (depth > 0 && end < blanked.length) { + if (blanked[end] === "{") depth += 1; + else if (blanked[end] === "}") depth -= 1; + if (depth > 0) end += 1; + } + return blanked.slice(start, end); +} + +/** Splits a balanced parameter list on top-level commas. */ +function splitParams(signature: string): string[] { + // A generic member reads `name(...)`, so the parameter list starts after the + // balanced angle block, not at the first parenthesis. + let searchFrom = 0; + const angle = signature.indexOf("<"); + const paren = signature.indexOf("("); + if (angle !== -1 && angle < paren) { + let angleDepth = 0; + for (let i = angle; i < signature.length; i++) { + if (signature[i] === "<") angleDepth += 1; + else if (signature[i] === ">") { + angleDepth -= 1; + if (angleDepth === 0) { + searchFrom = i; + break; + } + } + } + } + + const open = signature.indexOf("(", searchFrom); + let depth = 0; + let close = open; + for (let i = open; i < signature.length; i++) { + if ("([{<".includes(signature[i]!)) depth += 1; + else if (")]}>".includes(signature[i]!)) { + depth -= 1; + if (depth === 0) { + close = i; + break; + } + } + } + const inner = signature.slice(open + 1, close); + const parts: string[] = []; + let level = 0; + let current = ""; + for (const ch of inner) { + if ("([{<".includes(ch)) level += 1; + else if (")]}>".includes(ch)) level -= 1; + if (ch === "," && level === 0) { + parts.push(current); + current = ""; + } else { + current += ch; + } + } + if (current.trim()) parts.push(current); + return parts; +} + +function paramNames(signature: string): string[] { + return splitParams(signature) + .map((p) => /^\s*([A-Za-z_$][\w$]*)\s*\??\s*:/.exec(p)?.[1]) + .filter((n): n is string => Boolean(n)); +} + +/** Member name to its declared parameter names, for members with a single signature. */ +function declaredParams(): Map { + const body = interfaceBody(interfaceSource); + const spans: string[] = []; + let level = 0; + let from = 0; + for (let i = 0; i < body.length; i++) { + const ch = body[i]!; + if ("{([".includes(ch)) level += 1; + else if ("})]".includes(ch)) level -= 1; + else if (ch === ";" && level === 0) { + spans.push(body.slice(from, i)); + from = i + 1; + } + } + + const seen = new Map(); + for (const span of spans) { + const match = /^\s*(?:readonly\s+)?([A-Za-z_$][\w$]*)\s*\??\s*[(<]/.exec(span); + if (!match) continue; + const name = match[1]!; + seen.set(name, [...(seen.get(name) ?? []), paramNames(span)]); + } + + // Overloaded members forward through a cast and apply the whole argument list, so they are not + // subject to this check. + return new Map( + [...seen].filter(([, sigs]) => sigs.length === 1).map(([n, sigs]) => [n, sigs[0]!]) + ); +} + +describe("the pass-through forwards every declared parameter", () => { + const declared = declaredParams(); + + it("parsed the interface, so a parse failure cannot pass this suite", () => { + expect(declared.size).toBeGreaterThan(50); + expect(declared.get("expireParkedRun")).toEqual(["runId", "data", "tx"]); + expect(declared.get("findLatestExecutionSnapshot")).toEqual([ + "runId", + "client", + "environmentId", + ]); + }); + + it("passes exactly the declared parameters, in order, for every single-signature member", () => { + const wrong: string[] = []; + + for (const [name, params] of declared) { + const forward = new RegExp(`return this\\.delegate\\.${name}\\(([^;]*)\\);`).exec(baseSource); + + if (!forward) { + wrong.push(`${name}: no forward found`); + continue; + } + + const passed = forward[1]! + .split(",") + .map((a) => a.trim()) + .filter(Boolean); + + if (passed.join(",") !== params.join(",")) { + wrong.push(`${name}: declares (${params.join(", ")}) but forwards (${passed.join(", ")})`); + } + } + + expect(wrong).toEqual([]); + }); +}); diff --git a/internal-packages/run-store/src/delegatingRunStore.ts b/internal-packages/run-store/src/delegatingRunStore.ts index 008ccfce1c5..c7fe3225c76 100644 --- a/internal-packages/run-store/src/delegatingRunStore.ts +++ b/internal-packages/run-store/src/delegatingRunStore.ts @@ -459,7 +459,7 @@ export class DelegatingRunStore implements RunStore { ): Promise | null> { - return this.delegate.findLatestExecutionSnapshot(runId, client); + return this.delegate.findLatestExecutionSnapshot(runId, client, environmentId); } findExecutionSnapshot( @@ -717,7 +717,7 @@ export class DelegatingRunStore implements RunStore { // instead of defaulting to LEGACY. Single-store impls ignore it. residency?: Residency ): Promise { - return this.delegate.upsertWaitpointTag(data, tx); + return this.delegate.upsertWaitpointTag(data, tx, residency); } findManyWaitpointTags( From e9ce90989945d09fb5ac95a2a76a737529eaed5b Mon Sep 17 00:00:00 2001 From: Daniel Sutton Date: Tue, 25 Aug 2026 13:27:57 +0100 Subject: [PATCH 21/56] fix(run-store): keep completed waitpoints that have no batch index A completed waitpoint with no batch index was invisible to every Redis read, so a run resumed from the store lost that wait's result while Postgres still returned it. That is every wait.for, every single triggerAndWait and every token: the engine passes index as batchIndex ?? undefined, so only waits inside a batch carry one. The cause was reading the id set out of the ordered list. That list is the index oracle and its positions ARE the indexes, so it can only ever hold indexed ids, and deduping it yields a set missing exactly the index-less ones. Postgres has no such restriction: its completed-waitpoint join records every id. The cycle key now carries the complete distinct set in its own field, written when the cycle is minted and read back beside the order. The order keeps its meaning and stays index-only. Two tests: one asserting an index-less wait survives a round trip with an empty order, and one asserting the set matches the Postgres join for a mix of indexed and index-less waits. Verified by deriving the set from the order again, which makes the waitpoint vanish. The suites missed this because every earlier case gave each waitpoint an index. --- .../run-store/src/redisSnapshotStore.ts | 46 ++++++++++++-- ...utionSnapshotStore.waitpointCycles.test.ts | 63 +++++++++++++++++++ 2 files changed, 103 insertions(+), 6 deletions(-) diff --git a/internal-packages/run-store/src/redisSnapshotStore.ts b/internal-packages/run-store/src/redisSnapshotStore.ts index e36179c6041..88c18d00595 100644 --- a/internal-packages/run-store/src/redisSnapshotStore.ts +++ b/internal-packages/run-store/src/redisSnapshotStore.ts @@ -29,6 +29,19 @@ export function deriveOrder(completedWaitpoints: CompletedWaitpointRef[]): strin .map((w) => w.id); } +/** + * The COMPLETE distinct set of completed-waitpoint ids, including those with no batch index. + * + * This is deliberately not `deriveOrder` deduped. `order` is the index oracle and carries only + * batch-indexed ids, because its positions ARE the indexes. A wait with no batch index (every + * `wait.for`, every single `triggerAndWait`, every token) has no position and is absent from it, + * while Postgres records it in the completed-waitpoint join like any other. Reading the id set back + * from `order` therefore loses exactly those waits, and a run resumed from Redis loses their results. + */ +export function deriveDistinctIds(completedWaitpoints: CompletedWaitpointRef[]): string[] { + return [...new Set(completedWaitpoints.map((w) => w.id))]; +} + // isValid is derived, never stored, so the entry JSON stays byte-identical to the caller's document. export function isValidFor(entry: { error?: unknown }): boolean { return !entry.error; @@ -270,12 +283,14 @@ export class RedisSnapshotStore { let cycleMode = "none"; let cycleSeqIn = "0"; let orderJson = ""; + let distinctJson = ""; let records = ""; let orderCount = "0"; if (args.cycle?.kind === "new") { const order = deriveOrder(args.cycle.completedWaitpoints); cycleMode = "new"; orderJson = JSON.stringify(order); + distinctJson = JSON.stringify(deriveDistinctIds(args.cycle.completedWaitpoints)); records = args.cycle.records ? JSON.stringify(args.cycle.records) : ""; orderCount = String(order.length); } else if (args.cycle?.kind === "carryForward") { @@ -300,7 +315,8 @@ export class RedisSnapshotStore { records, orderCount, args.expectedCur ?? "", - args.expectedCur !== undefined ? "1" : "0" + args.expectedCur !== undefined ? "1" : "0", + distinctJson )) as string[]; return this.#interpretAppend(reply, raw, orderJson, records, args.entry.runId); @@ -407,7 +423,7 @@ export class RedisSnapshotStore { return this.#timed("getSnapshotWaitpointIds", async () => { const k = snapshotKeys(runId); const reply = await this.redis.readSnapshotWaitpointIds(k.e, k.idx, k.cur, k.seq, snapshotId); - return decodeWaitpointIds(reply[0] === "1", reply[1] ?? ""); + return decodeWaitpointIds(reply[0] === "1", reply[1] ?? "", reply[2] ?? ""); }); } @@ -588,6 +604,13 @@ export class RedisSnapshotStore { if not cs then return '' end return redis.call('HGET', wpKey(cs), 'order') or '' end + -- The complete id set, which is NOT the order deduped: order holds only batch-indexed ids. + local function distinctFor(pointer) + if not pointer then return '' end + local cs = string.match(pointer, '^(%d+):') + if not cs then return '' end + return redis.call('HGET', wpKey(cs), 'distinct') or '' + end `; this.redis.defineCommand("appendSnapshotEntry", { @@ -607,6 +630,9 @@ export class RedisSnapshotStore { local orderCount = ARGV[11] local expectedCur = ARGV[12] local casEnabled = ARGV[13] == '1' + -- The COMPLETE distinct id set. Not the order deduped: order omits every id with no batch + -- index, and those ids still have to come back on a read. + local distinctJson = ARGV[14] -- Liveness is TWO anchors: e and seq. All keys get the same PEXPIRE but expire independently -- (or seq can vanish under maxmemory eviction while e survives), so checking e alone lets a @@ -641,7 +667,7 @@ export class RedisSnapshotStore { -- The STORE mints cycleSeq, so the sequence is dense by construction and the terminal -- PEXPIRE loop from 1..c is correct. cycleSeq = redis.call('HINCRBY', seqKey, 'c', 1) - redis.call('HSET', wpKey(cycleSeq), 'order', orderJson, 'count', orderCount) + redis.call('HSET', wpKey(cycleSeq), 'order', orderJson, 'count', orderCount, 'distinct', distinctJson) if records ~= '' then redis.call('HSET', wpKey(cycleSeq), 'records', records) else @@ -736,7 +762,7 @@ export class RedisSnapshotStore { return { '0', '' } end local pointer = redis.call('HGET', eKey, id .. '#c') - return { '1', orderFor(pointer) } + return { '1', orderFor(pointer), distinctFor(pointer) } `, }); @@ -848,9 +874,16 @@ export class RedisSnapshotStore { } } -export function decodeWaitpointIds(present: boolean, orderJson: string): WaitpointIds { +export function decodeWaitpointIds( + present: boolean, + orderJson: string, + distinctJson = "" +): WaitpointIds { const order: string[] = orderJson === "" ? [] : (JSON.parse(orderJson) as string[]); - return { present, distinctIds: [...new Set(order)], order }; + // The complete set is stored separately, because `order` omits every id with no batch index. + const distinctIds: string[] = + distinctJson === "" ? [...new Set(order)] : (JSON.parse(distinctJson) as string[]); + return { present, distinctIds, order }; } declare module "@internal/redis" { @@ -873,6 +906,7 @@ declare module "@internal/redis" { orderCount: string, expectedCur: string, casEnabled: string, + distinctJson: string, callback?: Callback ): Result; readSnapshotById( diff --git a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.waitpointCycles.test.ts b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.waitpointCycles.test.ts index 1879fe1246b..9f7832a16c6 100644 --- a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.waitpointCycles.test.ts +++ b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.waitpointCycles.test.ts @@ -242,6 +242,69 @@ describe("completed-waitpoint cycles", () => { } }); + containerTest( + "keeps a completed waitpoint that has no batch index", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, redis, env); + const [wpA] = await seedSnapshotWaitpoints(prisma, env, 1); + + // Every wait.for, every single triggerAndWait and every token resumes with no batch index: + // the engine passes `index: b.batchIndex ?? undefined`. Postgres records the id in the + // completed-waitpoint join regardless. The ordered list cannot hold it, because its + // positions ARE the indexes, so the complete set has to be stored separately or the wait's + // result vanishes on a Redis read. + const created = await decorated.createExecutionSnapshot( + resumeInput(runId, env, [{ id: wpA }], "single wait") + ); + + const ids = await redis.getSnapshotWaitpointIds(runId, created.id); + expect(ids.present).toBe(true); + expect(ids.distinctIds).toEqual([wpA]); + // No position, so it is absent from the oracle. That part is correct. + expect(ids.order).toEqual([]); + } finally { + await redis.quit(); + } + } + ); + + containerTest( + "matches the Postgres join for a mix of indexed and index-less waits", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, redis, env); + const [wpA, wpB, wpC] = await seedSnapshotWaitpoints(prisma, env, 3); + + const created = await decorated.createExecutionSnapshot( + resumeInput( + runId, + env, + [{ id: wpA, index: 0 }, { id: wpB }, { id: wpC, index: 1 }], + "mixed wait" + ) + ); + + // Parity with what Postgres holds is the actual requirement: the engine iterates the rows + // this set fetches, and uses the order only to assign each one its index. + const fromRedis = await redis.getSnapshotWaitpointIds(runId, created.id); + const fromPostgres = await new PostgresRunStore({ + prisma, + readOnlyPrisma: prisma, + }).findSnapshotCompletedWaitpointIds(created.id, undefined, runId); + + expect([...fromRedis.distinctIds].sort()).toEqual([...fromPostgres].sort()); + expect(fromRedis.order).toEqual([wpA, wpC]); + } finally { + await redis.quit(); + } + } + ); + containerTest( "findLatestExecutionSnapshot returns the index oracle", async ({ prisma, redisOptions }) => { From 57018fcbf66ec041c26f2180fb3f94dd39b31e58 Mon Sep 17 00:00:00 2001 From: Daniel Sutton Date: Tue, 25 Aug 2026 13:43:38 +0100 Subject: [PATCH 22/56] fix(run-store): three more index-less waitpoint losses on the read path The previous fix stored the complete id set but left three places still deriving it from the ordered list, and the ordered list holds only batch-indexed ids. A carry-forward decided on the order alone. Two DIFFERENT single waits both present an empty order, so they compared equal, the second inherited the first's cycle, and a read returned the wrong waitpoint entirely. The comparison now requires the id set to match as well. The dequeue site built its Redis refs from the ordered list while the delegate connects the complete set in Postgres, so an index-less waitpoint reached Postgres and never reached Redis. Refs are now built from the complete set, with the index taken from the ordered list where the id appears in it. The entry decode derived the set from the order too, which meant getLatest and getById returned an incomplete set. That is the hot read: findLatestExecutionSnapshot hydrates the waitpoint rows from it, so a resume would have fetched no row at all for a single wait. The read scripts now return the stored set alongside the order. Four tests, each verified against its own defect: two consecutive single waits keep separate cycles, a repeated one still carries forward, the dequeue snapshot keeps an index-less id, and the hot read hydrates its row. --- .../run-store/src/redisSnapshotStore.ts | 42 ++++--- .../src/taskRunExecutionSnapshotStore.ts | 55 ++++++-- ...utionSnapshotStore.waitpointCycles.test.ts | 118 ++++++++++++++++++ 3 files changed, 194 insertions(+), 21 deletions(-) diff --git a/internal-packages/run-store/src/redisSnapshotStore.ts b/internal-packages/run-store/src/redisSnapshotStore.ts index 88c18d00595..78efdc24706 100644 --- a/internal-packages/run-store/src/redisSnapshotStore.ts +++ b/internal-packages/run-store/src/redisSnapshotStore.ts @@ -457,11 +457,12 @@ export class RedisSnapshotStore { } const headOrder = reply[1] ?? ""; + const headDistinct = reply[2] ?? ""; const rows: SnapshotRead[] = []; // Tracks whether the Lua-chosen head row (always the first, i === 2) itself survives the // env filter below -- headOrder must never be attributed to a different, surviving row. let headSurvived = false; - for (let i = 2; i + 3 < reply.length; i += 4) { + for (let i = 3; i + 3 < reply.length; i += 4) { // orderKnown is false here: headOrder covers only the head row, resolved separately below. const decoded = this.#decode( [reply[i], reply[i + 1], reply[i + 2], reply[i + 3], ""], @@ -471,13 +472,17 @@ export class RedisSnapshotStore { ); if (decoded) { rows.push(decoded); - if (i === 2) headSurvived = true; + if (i === 3) headSurvived = true; } } rows.reverse(); const head = headSurvived ? rows[rows.length - 1] : undefined; - const headWaitpointIds = decodeWaitpointIds(head !== undefined, head ? headOrder : ""); + const headWaitpointIds = decodeWaitpointIds( + head !== undefined, + head ? headOrder : "", + head ? headDistinct : "" + ); if (head) { head.completedWaitpointIds = headWaitpointIds; if (head.cycle) { @@ -516,11 +521,12 @@ export class RedisSnapshotStore { if (reply === null) return { kind: "miss" }; const headOrder = reply[1] ?? ""; + const headDistinct = reply[2] ?? ""; const rows: SnapshotRead[] = []; // Tracks whether the Lua-chosen head row (always the first, i === 2) survives the env filter, // so headOrder is never attributed to a different, surviving row. let headSurvived = false; - for (let i = 2; i + 3 < reply.length; i += 4) { + for (let i = 3; i + 3 < reply.length; i += 4) { const decoded = this.#decode( [reply[i], reply[i + 1], reply[i + 2], reply[i + 3], ""], opts?.environmentId, @@ -529,13 +535,17 @@ export class RedisSnapshotStore { ); if (decoded) { rows.push(decoded); - if (i === 2) headSurvived = true; + if (i === 3) headSurvived = true; } } rows.reverse(); const head = headSurvived ? rows[rows.length - 1] : undefined; - const headWaitpointIds = decodeWaitpointIds(head !== undefined, head ? headOrder : ""); + const headWaitpointIds = decodeWaitpointIds( + head !== undefined, + head ? headOrder : "", + head ? headDistinct : "" + ); if (head) { head.completedWaitpointIds = headWaitpointIds; if (head.cycle) { @@ -568,7 +578,7 @@ export class RedisSnapshotStore { orderKnown: boolean ): SnapshotRead | null { if (!reply || reply.length === 0) return null; - const [id, raw, seqStr, pointer, orderJson] = reply; + const [id, raw, seqStr, pointer, orderJson, distinctJson] = reply; const entry = JSON.parse(raw) as Record; if (environmentId !== undefined && entry.environmentId !== environmentId) return null; const read: SnapshotRead = { @@ -582,7 +592,7 @@ export class RedisSnapshotStore { const [cs, count] = pointer.split(":"); read.cycle = { cycleSeq: Number(cs), count: Number(count) }; if (orderKnown) { - const ids = decodeWaitpointIds(true, orderJson); + const ids = decodeWaitpointIds(true, orderJson, distinctJson ?? ""); read.completedWaitpointIds = ids; this.#checkCycleMismatch(runId, Number(count), ids.order.length); } @@ -737,7 +747,7 @@ export class RedisSnapshotStore { local vals = redis.call('HMGET', eKey, id, id .. '#s', id .. '#c') if not vals[1] then return nil end -- Coerce every element: a Lua false TRUNCATES the returned array at that position. - return { id, vals[1], vals[2] or '', vals[3] or '', orderFor(vals[3]) } + return { id, vals[1], vals[2] or '', vals[3] or '', orderFor(vals[3]), distinctFor(vals[3]) } `, }); @@ -749,7 +759,7 @@ export class RedisSnapshotStore { if not cur then return nil end local vals = redis.call('HMGET', eKey, cur, cur .. '#s', cur .. '#c') if not vals[1] then return nil end - return { cur, vals[1], vals[2] or '', vals[3] or '', orderFor(vals[3]) } + return { cur, vals[1], vals[2] or '', vals[3] or '', orderFor(vals[3]), distinctFor(vals[3]) } `, }); @@ -785,7 +795,7 @@ export class RedisSnapshotStore { -- compare is a chronological compare. Walking newest-first lets the scan stop at the first -- entry at or before the cursor, which makes its length the length of the ANSWER rather -- than the length of the run's history. - local out = { '', '' } + local out = { '', '', '' } local headId = nil local offset = 0 local page = limit @@ -820,7 +830,9 @@ export class RedisSnapshotStore { end if headId then - out[2] = orderFor(redis.call('HGET', eKey, headId .. '#c')) + local headPointer = redis.call('HGET', eKey, headId .. '#c') + out[2] = orderFor(headPointer) + out[3] = distinctFor(headPointer) end return out `, @@ -852,7 +864,7 @@ export class RedisSnapshotStore { -- The head is the newest SURVIVING entry, and it is the only one whose cycle key is read. -- Deriving the order after the loop keeps it paired with the row it is attached to: a row -- dropped for a missing body must not donate its cycle data to the next one. - local out = { sinceRaw, '' } + local out = { sinceRaw, '', '' } local headId = nil for i = 1, #ids do local id = ids[i] @@ -866,7 +878,9 @@ export class RedisSnapshotStore { end end if headId then - out[2] = orderFor(redis.call('HGET', eKey, headId .. '#c')) + local headPointer = redis.call('HGET', eKey, headId .. '#c') + out[2] = orderFor(headPointer) + out[3] = distinctFor(headPointer) end return out `, diff --git a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.ts b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.ts index 4b466eca268..82d57e04a2f 100644 --- a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.ts +++ b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.ts @@ -20,7 +20,7 @@ import type { SnapshotEntryInput, SnapshotRead, } from "./redisSnapshotStore.js"; -import { deriveOrder } from "./redisSnapshotStore.js"; +import { deriveDistinctIds, deriveOrder } from "./redisSnapshotStore.js"; import { entryFromCompletion, entryFromCreateExecutionSnapshot, @@ -384,9 +384,13 @@ export class TaskRunExecutionSnapshotStore extends DelegatingRunStore { "lockRunToWorker", entryFromLock(ctx, withStamp.snapshot), withStamp.snapshot.previousSnapshotId, - // The lock site already carries the resolved order, so the refs are rebuilt from it rather - // than re-derived: its index IS the position in that list. - withStamp.snapshot.completedWaitpointOrder.map((id, index) => ({ id, index })) + // Built from the COMPLETE id set, which is what the delegate connects in Postgres, with the + // index taken from the ordered list where the id appears in it. Building from the ordered list + // instead would drop every id with no batch index, exactly the ids Postgres still records. + lockCycleRefs( + withStamp.snapshot.completedWaitpointIds, + withStamp.snapshot.completedWaitpointOrder + ) ); return result; } @@ -579,12 +583,21 @@ export class TaskRunExecutionSnapshotStore extends DelegatingRunStore { } const order = deriveOrder(completedWaitpoints); + const distinct = deriveDistinctIds(completedWaitpoints); try { const head = await this.redis.getLatest(runId); - const previous = head?.completedWaitpointIds?.order; - - if (head?.cycle && previous && sameOrder(previous, order)) { + const previousIds = head?.completedWaitpointIds; + + // Both halves must match. Comparing the order alone is not enough: it holds only indexed ids, + // so two DIFFERENT single waits both present an empty order and would compare equal, and the + // second would inherit the first's waitpoint set instead of minting its own. + if ( + head?.cycle && + previousIds && + sameOrder(previousIds.order, order) && + sameSet(previousIds.distinctIds, distinct) + ) { return { kind: "carryForward", cycleSeq: head.cycle.cycleSeq }; } } catch (error) { @@ -899,3 +912,31 @@ export class TaskRunExecutionSnapshotStore extends DelegatingRunStore { function sameOrder(a: string[], b: string[]): boolean { return a.length === b.length && a.every((id, index) => id === b[index]); } + +/** + * Turns the lock site's two lists into cycle refs. `completedWaitpointIds` is the complete set the + * delegate connects; `completedWaitpointOrder` gives a position only to the ids that have one, and a + * repeated id keeps each of its positions. + */ +function lockCycleRefs(ids: string[], order: string[]): { id: string; index?: number }[] { + const refs: { id: string; index?: number }[] = []; + const indexed = new Set(); + + order.forEach((id, index) => { + refs.push({ id, index }); + indexed.add(id); + }); + + for (const id of ids) { + if (!indexed.has(id)) refs.push({ id }); + } + + return refs; +} + +/** Membership only, for the id set, which has no meaningful order. */ +function sameSet(a: string[], b: string[]): boolean { + if (a.length !== b.length) return false; + const seen = new Set(a); + return b.every((id) => seen.has(id)); +} diff --git a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.waitpointCycles.test.ts b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.waitpointCycles.test.ts index 9f7832a16c6..80d98aeef0e 100644 --- a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.waitpointCycles.test.ts +++ b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.waitpointCycles.test.ts @@ -305,6 +305,124 @@ describe("completed-waitpoint cycles", () => { } ); + containerTest( + "two consecutive index-less waits do not share a cycle", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, redis, env); + const [wpA, wpB] = await seedSnapshotWaitpoints(prisma, env, 2); + + // Neither wait has a batch index, so both present an EMPTY order. Deciding carry-forward on + // the order alone makes them compare equal, and the second silently inherits the first's + // waitpoint set: its own result is never stored and a read returns the wrong id. + const first = await decorated.createExecutionSnapshot( + resumeInput(runId, env, [{ id: wpA }], "first single wait") + ); + const second = await decorated.createExecutionSnapshot( + resumeInput(runId, env, [{ id: wpB }], "second single wait") + ); + + expect((await redis.getSnapshotWaitpointIds(runId, first.id)).distinctIds).toEqual([wpA]); + expect((await redis.getSnapshotWaitpointIds(runId, second.id)).distinctIds).toEqual([wpB]); + expect((await probe.keys(`snap:{${runId}}:wp:*`)).length).toBe(2); + } finally { + await Promise.all([redis.quit(), probe.quit().catch(() => {})]); + } + } + ); + + containerTest( + "the same index-less wait repeated does still carry forward", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, redis, env); + const [wpA] = await seedSnapshotWaitpoints(prisma, env, 1); + + // The copy-forward case must survive the stricter comparison: the same id set, still one key. + await decorated.createExecutionSnapshot(resumeInput(runId, env, [{ id: wpA }], "wait")); + const carried = await decorated.createExecutionSnapshot( + resumeInput(runId, env, [{ id: wpA }], "carry") + ); + + expect((await probe.keys(`snap:{${runId}}:wp:*`)).length).toBe(1); + expect((await redis.getSnapshotWaitpointIds(runId, carried.id)).distinctIds).toEqual([wpA]); + } finally { + await Promise.all([redis.quit(), probe.quit().catch(() => {})]); + } + } + ); + + containerTest( + "the dequeue snapshot keeps an index-less waitpoint", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, redis, env); + const [wpA, wpB] = await seedSnapshotWaitpoints(prisma, env, 2); + const head = await redis.getLatest(runId); + const snapshotId = generateInternalId(); + + // Postgres connects completedWaitpointIds, the COMPLETE set. Building the Redis refs from + // completedWaitpointOrder instead drops every id that has no position in it. + await decorated.lockRunToWorker(runId, { + lockedAt: new Date(), + lockedById: undefined, + lockedToVersionId: undefined, + lockedQueueId: undefined, + startedAt: new Date(), + baseCostInCents: 0, + machinePreset: "small-1x", + taskVersion: "1.0.0", + snapshot: { + id: snapshotId, + previousSnapshotId: head!.id, + attemptNumber: 1, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + completedWaitpointIds: [wpA, wpB], + completedWaitpointOrder: [wpA], + }, + } as never); + + const ids = await redis.getSnapshotWaitpointIds(runId, snapshotId); + expect([...ids.distinctIds].sort()).toEqual([wpA, wpB].sort()); + expect(ids.order).toEqual([wpA]); + } finally { + await redis.quit(); + } + } + ); + + containerTest( + "findLatestExecutionSnapshot hydrates an index-less waitpoint row", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, redis, env); + const [wpA] = await seedSnapshotWaitpoints(prisma, env, 1); + + await decorated.createExecutionSnapshot(resumeInput(runId, env, [{ id: wpA }], "single")); + + // The hot read hydrates the rows from the id set, so an incomplete set means the resume + // gets no waitpoint at all. + const latest = await decorated.findLatestExecutionSnapshot(runId); + expect(latest!.completedWaitpoints.map((w) => w.id)).toEqual([wpA]); + } finally { + await redis.quit(); + } + } + ); + containerTest( "findLatestExecutionSnapshot returns the index oracle", async ({ prisma, redisOptions }) => { From cff7fb9535cfb15c271d6031e056627e6dbe2713 Mon Sep 17 00:00:00 2001 From: Daniel Sutton Date: Tue, 25 Aug 2026 13:57:04 +0100 Subject: [PATCH 23/56] fix(run-store): hydrate the column the read path was omitting A sweep for values derived where they should be read found one more. The hydrated payload left out lastHeartbeatAt entirely, so a Redis-served read returned undefined for it where Postgres returns null. No code writes that column, so null is not a guess: it is the only value Postgres ever holds. The effect was small but constant, on every read served from Redis, and it is the kind of difference a comparator has to either explain or chase. Guarded by comparing the KEY SET of the two payloads rather than their values, so a column omitted by the hydrator fails as a missing key rather than passing as an absent value. Verified by removing the line again: the test names the column. Also covers the timestamp write on both schema variants. updatedAt is declared @updatedAt, which Prisma manages, so whether an explicit value survives a create is a property of the client rather than of the schema, and the two variants are separately generated clients. Agreeing declarations were not evidence. Both honour the caller's instant. --- ...ostgresRunStore.snapshotTimestamps.test.ts | 132 ++++++++++++++++++ ...askRunExecutionSnapshotStore.reads.test.ts | 23 +++ .../src/taskRunExecutionSnapshotStore.ts | 4 + 3 files changed, 159 insertions(+) create mode 100644 internal-packages/run-store/src/PostgresRunStore.snapshotTimestamps.test.ts diff --git a/internal-packages/run-store/src/PostgresRunStore.snapshotTimestamps.test.ts b/internal-packages/run-store/src/PostgresRunStore.snapshotTimestamps.test.ts new file mode 100644 index 00000000000..8026f2f99ec --- /dev/null +++ b/internal-packages/run-store/src/PostgresRunStore.snapshotTimestamps.test.ts @@ -0,0 +1,132 @@ +// The caller's instant must land in BOTH timestamp columns, on BOTH schema variants. +// +// `updatedAt` is declared `@updatedAt`, which Prisma manages itself, so whether an explicit value +// survives a create is a property of the client rather than of the schema. The two variants are +// separately generated clients over separately declared schemas, so agreeing declarations are not +// evidence that they agree in behaviour. This asserts it on each. +// +// It matters because the decorator writes one instant to both stores. If Prisma overrode it here, +// Postgres and Redis would hold different values for a column the comparator checks for equality, +// on every snapshot. +import { heteroPostgresTest, heteroRunOpsPostgresTest } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import type { RunOpsPrismaClient } from "@internal/run-ops-database"; +import { generateInternalId } from "@trigger.dev/core/v3/isomorphic"; +import { describe, expect } from "vitest"; +import { PostgresRunStore } from "./PostgresRunStore.js"; + +type AnyClient = PrismaClient | RunOpsPrismaClient; + +/** Five minutes in the past, so a column default could never coincide with it. */ +const STAMP = new Date(Date.now() - 5 * 60 * 1000); + +async function writeSnapshot( + prisma: AnyClient, + schemaVariant: "legacy" | "dedicated", + suffix: string +) { + const scope = + schemaVariant === "dedicated" + ? { + environmentId: `env_${suffix}`, + projectId: `proj_${suffix}`, + organizationId: `org_${suffix}`, + } + : await seedLegacyScope(prisma as PrismaClient, suffix); + + const store = new PostgresRunStore({ + prisma: prisma as never, + readOnlyPrisma: prisma as never, + schemaVariant, + }); + + const runId = generateInternalId(); + const id = generateInternalId(); + + await (prisma as PrismaClient).taskRun.create({ + data: { + id: runId, + engine: "V2", + status: "PENDING", + friendlyId: `run_${suffix}`, + runtimeEnvironmentId: scope.environmentId, + environmentType: "DEVELOPMENT", + organizationId: scope.organizationId, + projectId: scope.projectId, + taskIdentifier: "my-task", + payload: "{}", + payloadType: "application/json", + traceContext: {}, + traceId: `trace_${suffix}`, + spanId: `span_${suffix}`, + queue: "task/my-task", + isTest: false, + taskEventStore: "taskEvent", + depth: 0, + } as never, + }); + + await store.createExecutionSnapshot({ + id, + createdAt: STAMP, + run: { id: runId, status: "EXECUTING", attemptNumber: 1 }, + snapshot: { executionStatus: "EXECUTING", description: "Run started" }, + environmentId: scope.environmentId, + environmentType: "DEVELOPMENT", + projectId: scope.projectId, + organizationId: scope.organizationId, + }); + + return (prisma as PrismaClient).taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); +} + +async function seedLegacyScope(prisma: PrismaClient, suffix: string) { + const organization = await prisma.organization.create({ + data: { title: `Org ${suffix}`, slug: `org-${suffix}` }, + }); + const project = await prisma.project.create({ + data: { + name: `Project ${suffix}`, + slug: `project-${suffix}`, + externalRef: `proj_${suffix}`, + organizationId: organization.id, + }, + }); + const environment = await prisma.runtimeEnvironment.create({ + data: { + type: "DEVELOPMENT", + slug: `dev-${suffix}`, + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_dev_${suffix}`, + pkApiKey: `pk_dev_${suffix}`, + shortcode: `short_${suffix}`, + }, + }); + return { + environmentId: environment.id, + projectId: project.id, + organizationId: organization.id, + }; +} + +describe("snapshot timestamps are the caller's, on both schema variants", () => { + heteroPostgresTest("legacy client honours the supplied instant", async ({ prisma14 }) => { + const row = await writeSnapshot(prisma14, "legacy", "tsleg"); + + expect(row.createdAt.toISOString()).toBe(STAMP.toISOString()); + // The one Prisma manages. If it overrode the value, the two stores would disagree here on + // every snapshot. + expect(row.updatedAt.toISOString()).toBe(STAMP.toISOString()); + }); + + heteroRunOpsPostgresTest( + "dedicated client honours the supplied instant", + async ({ prisma17 }) => { + const row = await writeSnapshot(prisma17, "dedicated", "tsded"); + + expect(row.createdAt.toISOString()).toBe(STAMP.toISOString()); + expect(row.updatedAt.toISOString()).toBe(STAMP.toISOString()); + } + ); +}); diff --git a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.reads.test.ts b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.reads.test.ts index 42924e34c34..a458e9070ed 100644 --- a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.reads.test.ts +++ b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.reads.test.ts @@ -125,6 +125,29 @@ describe("snapshot reads", () => { } }); + containerTest( + "returns the same field set Postgres does, key for key", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + const postgresOnly = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, env); + await decorated.createExecutionSnapshot(snapshotInput(runId, env, "Run started")); + + const fromRedis = await decorated.findLatestExecutionSnapshot(runId); + const fromPostgres = await postgresOnly.findLatestExecutionSnapshot(runId); + + // Not a value comparison: a column the hydrator forgets is absent rather than wrong, so it + // shows up as a missing KEY. lastHeartbeatAt was omitted this way and read back undefined + // where Postgres returns null, on every Redis-served read. + expect(Object.keys(fromRedis!).sort()).toEqual(Object.keys(fromPostgres!).sort()); + } finally { + await redis.quit(); + } + } + ); + containerTest( "reads a foreign environment as not found, so the caller's 404 still fires", async ({ prisma, redisOptions }) => { diff --git a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.ts b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.ts index 82d57e04a2f..11771132998 100644 --- a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.ts +++ b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.ts @@ -832,6 +832,10 @@ export class TaskRunExecutionSnapshotStore extends DelegatingRunStore { workerId: entry.workerId ?? null, runnerId: entry.runnerId ?? null, metadata: entry.metadata ?? null, + // A column no code writes, so Postgres returns null for it on every row. The entry does not + // carry it, and omitting it here would hand back undefined where Postgres hands back null, + // on every single read served from Redis. + lastHeartbeatAt: null, completedWaitpointOrder, isValid: read.isValid, error: entry.error ?? null, From bde40b3499c8b046200579117e96465434d2df3a Mon Sep 17 00:00:00 2001 From: Daniel Sutton Date: Tue, 25 Aug 2026 14:12:51 +0100 Subject: [PATCH 24/56] fix(run-store): four defects from an adversarial sweep of the read path An independent pass hunting one shape, a value derived where it should be read, found these. None was reachable from a test that existed. The hot read paid a second Redis call in its most common case. An entry with no wait cycle has no waitpoints by construction, and the hydrator asked the store to confirm that rather than concluding it, on every read of a run that is not resuming from a wait. It now distinguishes the three cases and only asks when it genuinely does not know. decodeWaitpointIds still reconstructed the id set from the ordered list when the stored set was absent. That is the sixth instance of the bug fixed five times, surviving as a fallback. It is unreachable today, because both fields are written by one command, but the reconstruction is lossy by nature and the loss is silent. A missing set beside a non-empty order now reports the entry as not present, which sends the caller to Postgres. The window read checked one liveness anchor where the append script deliberately checks two and explains why. An index lost to eviction while the entry hash survived would have reported an empty hit rather than a miss, so the poll would have returned nothing new for the rest of the run's life while Postgres held the transitions. The wrapped store handle dropped the staging buffer, so a handle taken inside a transaction would have appended before the commit. No caller writes a snapshot through it today. Also restores excess-property checking on the nested snapshot writes. Routing them through a generic helper let a typo'd field name compile and fail at runtime; a concrete parameter type brings the check back at the five sites that pass a fresh literal. Verified: a bogus field is now TS2353. --- .../run-store/src/PostgresRunStore.ts | 10 ++++++++- .../run-store/src/redisSnapshotStore.ts | 22 +++++++++++++++---- .../src/taskRunExecutionSnapshotStore.ts | 12 ++++++++-- 3 files changed, 37 insertions(+), 7 deletions(-) diff --git a/internal-packages/run-store/src/PostgresRunStore.ts b/internal-packages/run-store/src/PostgresRunStore.ts index 0f00f5848fe..980bc6b4569 100644 --- a/internal-packages/run-store/src/PostgresRunStore.ts +++ b/internal-packages/run-store/src/PostgresRunStore.ts @@ -666,7 +666,15 @@ export class PostgresRunStore implements RunStore { * key and `undefined` alike, so spreading an empty object drops the nested write entirely rather * than sending an empty one. */ - #nestedSnapshot(create: T): { executionSnapshots: { create: T } } | Record { + #nestedSnapshot( + create: Prisma.TaskRunExecutionSnapshotUncheckedCreateWithoutRunInput + ): + | { + executionSnapshots: { + create: Prisma.TaskRunExecutionSnapshotUncheckedCreateWithoutRunInput; + }; + } + | Record { return this.snapshotWrites ? { executionSnapshots: { create } } : {}; } diff --git a/internal-packages/run-store/src/redisSnapshotStore.ts b/internal-packages/run-store/src/redisSnapshotStore.ts index 78efdc24706..bc447b37d25 100644 --- a/internal-packages/run-store/src/redisSnapshotStore.ts +++ b/internal-packages/run-store/src/redisSnapshotStore.ts @@ -785,7 +785,11 @@ export class RedisSnapshotStore { -- A run with no keyspace is a MISS, so the caller falls back to Postgres. A run that has one -- and nothing newer is an empty HIT, so it does not fall back for a window it owns. - if redis.call('EXISTS', eKey) == 0 then return nil end + -- + -- Both anchors, for the reason the append script gives: keys expire independently, and an + -- index lost to eviction while the entry hash survives would otherwise report an empty HIT + -- on every poll for the rest of the run's life, with Postgres holding the transitions. + if redis.call('EXISTS', eKey) == 0 or redis.call('EXISTS', idxKey) == 0 then return nil end -- STRICTLY greater than the cursor, and same-millisecond entries are dropped. Postgres -- serves this window with createdAt > cursor and drops them too; a Redis read that is more @@ -894,9 +898,19 @@ export function decodeWaitpointIds( distinctJson = "" ): WaitpointIds { const order: string[] = orderJson === "" ? [] : (JSON.parse(orderJson) as string[]); - // The complete set is stored separately, because `order` omits every id with no batch index. - const distinctIds: string[] = - distinctJson === "" ? [...new Set(order)] : (JSON.parse(distinctJson) as string[]); + + // The complete set is stored separately, because `order` omits every id with no batch index, so + // deduping the order to recover it silently drops every wait that has none. + // + // A cycle key always holds both fields, written by one command, so a missing `distinct` beside a + // NON-EMPTY `order` means the invariant is broken. Reconstructing from the order there would be + // the same lossy shortcut this field exists to remove, and the loss would be silent. Report the + // entry as not present instead, which sends the caller to Postgres. + if (distinctJson === "" && order.length > 0) { + return { present: false, distinctIds: [], order: [] }; + } + + const distinctIds: string[] = distinctJson === "" ? [] : (JSON.parse(distinctJson) as string[]); return { present, distinctIds, order }; } diff --git a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.ts b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.ts index 11771132998..282817b6df5 100644 --- a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.ts +++ b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.ts @@ -196,7 +196,9 @@ export class TaskRunExecutionSnapshotStore extends DelegatingRunStore { return store; } - return this.#wrap(store); + // Carry the staging buffer through. Without it, a handle taken inside a transaction appends + // immediately, which is the exact ordering the facade exists to prevent. + return this.#wrap(store, this.staging); } /** @@ -805,8 +807,14 @@ export class TaskRunExecutionSnapshotStore extends DelegatingRunStore { // row as the index oracle that gives each completed waitpoint its position in a batch, so it // must be populated even when the waitpoint ROWS are not fetched. Returning an empty order here // resumes every batched triggerAndWait with `index: undefined`. + // Three cases, and only the last needs a second Redis call. The read already carries the ids + // when the store decoded them. An entry with no wait cycle has no waitpoints by construction, + // which is the common case and used to cost a round trip to rediscover. Anything else asks. const ids = - read.completedWaitpointIds ?? (await this.redis.getSnapshotWaitpointIds(runId, read.id)); + read.completedWaitpointIds ?? + (read.cycle === undefined + ? { present: true, distinctIds: [], order: [] } + : await this.redis.getSnapshotWaitpointIds(runId, read.id)); const completedWaitpointOrder = ids.order; // The rows themselves are head-only, mirroring the engine's own N x M avoidance. From c4469f70d29f600fbf116009bc79eda9ee76911b Mon Sep 17 00:00:00 2001 From: Daniel Sutton Date: Tue, 25 Aug 2026 14:32:13 +0100 Subject: [PATCH 25/56] fix(run-store): a refused wait cycle no longer leaves an unreadable head Two paths reached the same silent hang, and neither had a test. When the store refuses a carried pointer it was still writing the entry, which then became the run's head with no pointer at all. A read of that answers present-with-nothing, and present-with-nothing is precisely the signal that tells the engine's read-repair it does not need to look, so the runner got a waitpoint-less continue and dropped it. Refusing the pointer stays right; the append now mints a fresh cycle from the refs the caller carried, in the same atomic call, so the entry always has a pointer that can be trusted. Refs are optional and only the fallback needs them, so callers that supply none keep the previous behaviour. The second path needs no refusal at all. An entry whose cycle key has gone still carries its pointer, and the read answered empty for it too. That is reachable by eviction and also by the completion expiry, which is applied to every key for a run at one moment but lets them expire independently. Reads now report such an entry as not present, which sends the caller to Postgres, where the join rows still are. The hot read and the window both fall back rather than serve it. Three tests. The refusal is driven at the store, because the decorator cannot reach it on purpose: its probe sees the id set no longer matches and mints a new cycle, so the refusal only happens when the key vanishes between probe and append. Each verified against its own defect. --- .../run-store/src/PostgresRunStore.ts | 4 +- .../run-store/src/redisSnapshotStore.ts | 101 ++++++++++++--- .../src/taskRunExecutionSnapshotStore.ts | 21 +++- ...utionSnapshotStore.waitpointCycles.test.ts | 119 ++++++++++++++++++ 4 files changed, 226 insertions(+), 19 deletions(-) diff --git a/internal-packages/run-store/src/PostgresRunStore.ts b/internal-packages/run-store/src/PostgresRunStore.ts index 980bc6b4569..007dc08092c 100644 --- a/internal-packages/run-store/src/PostgresRunStore.ts +++ b/internal-packages/run-store/src/PostgresRunStore.ts @@ -666,9 +666,7 @@ export class PostgresRunStore implements RunStore { * key and `undefined` alike, so spreading an empty object drops the nested write entirely rather * than sending an empty one. */ - #nestedSnapshot( - create: Prisma.TaskRunExecutionSnapshotUncheckedCreateWithoutRunInput - ): + #nestedSnapshot(create: Prisma.TaskRunExecutionSnapshotUncheckedCreateWithoutRunInput): | { executionSnapshots: { create: Prisma.TaskRunExecutionSnapshotUncheckedCreateWithoutRunInput; diff --git a/internal-packages/run-store/src/redisSnapshotStore.ts b/internal-packages/run-store/src/redisSnapshotStore.ts index bc447b37d25..1160bd3ab4f 100644 --- a/internal-packages/run-store/src/redisSnapshotStore.ts +++ b/internal-packages/run-store/src/redisSnapshotStore.ts @@ -175,6 +175,12 @@ export type SnapshotRead = { raw: string; cycle?: CompletedWaitpointsPointer; completedWaitpointIds?: WaitpointIds; + /** + * The entry points at a cycle key that no longer exists, so its waitpoints are unreachable rather + * than absent. A caller must not treat this as an empty set: it has to fall back to Postgres, + * which still holds the join rows. + */ + danglingCycle?: boolean; }; export type AppendResult = @@ -266,7 +272,19 @@ export class RedisSnapshotStore { completedWaitpoints: CompletedWaitpointRef[]; records?: CompletedWaitpointRecord[]; } - | { kind: "carryForward"; cycleSeq: number }; + | { + kind: "carryForward"; + cycleSeq: number; + /** + * The same refs a `new` cycle would carry. A carry the store refuses falls back to + * minting inside the same call, and it cannot do that without them: with no refs there is + * nothing to mint from, so the entry is written with no pointer, as before. + * + * Every production caller supplies them. Omitting them gives up the fallback. + */ + completedWaitpoints?: CompletedWaitpointRef[]; + records?: CompletedWaitpointRecord[]; + }; }): Promise { if (args.entry.completedWaitpoints !== undefined) { throw new Error( @@ -296,6 +314,16 @@ export class RedisSnapshotStore { } else if (args.cycle?.kind === "carryForward") { cycleMode = "carry"; cycleSeqIn = String(args.cycle.cycleSeq); + + // Carried for the refusal path only. The script uses these solely when it declines the + // pointer and mints a replacement, and can only do that when the caller supplied them. + if (args.cycle.completedWaitpoints) { + const order = deriveOrder(args.cycle.completedWaitpoints); + orderJson = JSON.stringify(order); + records = args.cycle.records ? JSON.stringify(args.cycle.records) : ""; + orderCount = String(order.length); + distinctJson = JSON.stringify(deriveDistinctIds(args.cycle.completedWaitpoints)); + } } const reply = (await this.redis.appendSnapshotEntry( @@ -423,6 +451,16 @@ export class RedisSnapshotStore { return this.#timed("getSnapshotWaitpointIds", async () => { const k = snapshotKeys(runId); const reply = await this.redis.readSnapshotWaitpointIds(k.e, k.idx, k.cur, k.seq, snapshotId); + // A dangling pointer means this entry's waitpoints are unreachable, not absent. Reporting + // `present: false` is what sends the caller to Postgres, which still holds the join rows. + if (reply[3] === "1") { + this.metrics?.recordCycleMismatch(); + this.logger.warn("RedisSnapshotStore snapshot points at a cycle key that is gone", { + runId, + snapshotId, + }); + return { present: false, distinctIds: [], order: [] }; + } return decodeWaitpointIds(reply[0] === "1", reply[1] ?? "", reply[2] ?? ""); }); } @@ -578,7 +616,7 @@ export class RedisSnapshotStore { orderKnown: boolean ): SnapshotRead | null { if (!reply || reply.length === 0) return null; - const [id, raw, seqStr, pointer, orderJson, distinctJson] = reply; + const [id, raw, seqStr, pointer, orderJson, distinctJson, dangling] = reply; const entry = JSON.parse(raw) as Record; if (environmentId !== undefined && entry.environmentId !== environmentId) return null; const read: SnapshotRead = { @@ -591,6 +629,14 @@ export class RedisSnapshotStore { if (pointer) { const [cs, count] = pointer.split(":"); read.cycle = { cycleSeq: Number(cs), count: Number(count) }; + if (dangling === "1") { + read.danglingCycle = true; + this.metrics?.recordCycleMismatch(); + this.logger.warn("RedisSnapshotStore entry points at a cycle key that is gone", { + runId, + snapshotId: id, + }); + } if (orderKnown) { const ids = decodeWaitpointIds(true, orderJson, distinctJson ?? ""); read.completedWaitpointIds = ids; @@ -614,6 +660,17 @@ export class RedisSnapshotStore { if not cs then return '' end return redis.call('HGET', wpKey(cs), 'order') or '' end + -- A pointer whose cycle key is GONE. Not the same as having no pointer: this entry should + -- have waitpoints and cannot produce them, so a read must refuse rather than answer empty. + -- Reachable by eviction, and by the completion TTL, which is applied to every key for a run + -- at the same moment but lets them expire independently. + local function danglingFor(pointer) + if not pointer then return '0' end + local cs = string.match(pointer, '^(%d+):') + if not cs then return '0' end + if redis.call('EXISTS', wpKey(cs)) == 0 then return '1' end + return '0' + end -- The complete id set, which is NOT the order deduped: order holds only batch-indexed ids. local function distinctFor(pointer) if not pointer then return '' end @@ -673,27 +730,43 @@ export class RedisSnapshotStore { local cycleSeq = 0 local mismatch = 0 - if cycleMode == 'new' then - -- The STORE mints cycleSeq, so the sequence is dense by construction and the terminal - -- PEXPIRE loop from 1..c is correct. - cycleSeq = redis.call('HINCRBY', seqKey, 'c', 1) - redis.call('HSET', wpKey(cycleSeq), 'order', orderJson, 'count', orderCount, 'distinct', distinctJson) + + -- The STORE mints cycleSeq, so the sequence is dense by construction and the terminal + -- PEXPIRE loop from 1..c is correct. + local function mintCycle() + local minted = redis.call('HINCRBY', seqKey, 'c', 1) + redis.call('HSET', wpKey(minted), 'order', orderJson, 'count', orderCount, 'distinct', distinctJson) if records ~= '' then - redis.call('HSET', wpKey(cycleSeq), 'records', records) + redis.call('HSET', wpKey(minted), 'records', records) else -- A new cycle owns the whole key: a lost seq counter can re-mint a cycleSeq whose key -- still holds another cycle's records, and order/count stay mutually consistent so the -- mismatch check cannot see it. No-op on a fresh key. - redis.call('HDEL', wpKey(cycleSeq), 'records') + redis.call('HDEL', wpKey(minted), 'records') end + return minted + end + + if cycleMode == 'new' then + cycleSeq = mintCycle() elseif cycleMode == 'carry' then - -- Attach a pointer only if this incarnation actually minted the cycle. seq can be - -- evicted while a wp: key survives, so a bare key-exists check would adopt a dead + -- Attach the CARRIED pointer only if this incarnation actually minted that cycle. seq can + -- be evicted while a wp: key survives, so a bare key-exists check would adopt a dead -- incarnation's order and records under a consistent count, invisibly. local minted = tonumber(redis.call('HGET', seqKey, 'c') or '0') local c = redis.call('HGET', wpKey(cycleSeqIn), 'count') if not c or minted < cycleSeqIn then + -- Refusing the pointer is right. Writing the entry WITHOUT one is not: it becomes the + -- head with no waitpoints, and a read of it answers present-with-nothing, which is the + -- one answer that tells the engine's repair it need not look. Mint a fresh cycle from + -- the refs the caller carried, in this same atomic call, so the entry always has a + -- pointer that can be trusted. The mismatch is still reported, for the metric. mismatch = 1 + -- Only possible when the caller carried the refs. With none there is nothing to mint + -- from, and the entry is written with no pointer, which is the older behaviour. + if distinctJson ~= '' then + cycleSeq = mintCycle() + end else cycleSeq = cycleSeqIn orderCount = c @@ -747,7 +820,7 @@ export class RedisSnapshotStore { local vals = redis.call('HMGET', eKey, id, id .. '#s', id .. '#c') if not vals[1] then return nil end -- Coerce every element: a Lua false TRUNCATES the returned array at that position. - return { id, vals[1], vals[2] or '', vals[3] or '', orderFor(vals[3]), distinctFor(vals[3]) } + return { id, vals[1], vals[2] or '', vals[3] or '', orderFor(vals[3]), distinctFor(vals[3]), danglingFor(vals[3]) } `, }); @@ -759,7 +832,7 @@ export class RedisSnapshotStore { if not cur then return nil end local vals = redis.call('HMGET', eKey, cur, cur .. '#s', cur .. '#c') if not vals[1] then return nil end - return { cur, vals[1], vals[2] or '', vals[3] or '', orderFor(vals[3]), distinctFor(vals[3]) } + return { cur, vals[1], vals[2] or '', vals[3] or '', orderFor(vals[3]), distinctFor(vals[3]), danglingFor(vals[3]) } `, }); @@ -772,7 +845,7 @@ export class RedisSnapshotStore { return { '0', '' } end local pointer = redis.call('HGET', eKey, id .. '#c') - return { '1', orderFor(pointer), distinctFor(pointer) } + return { '1', orderFor(pointer), distinctFor(pointer), danglingFor(pointer) } `, }); diff --git a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.ts b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.ts index 282817b6df5..e96aef76172 100644 --- a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.ts +++ b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.ts @@ -577,7 +577,7 @@ export class TaskRunExecutionSnapshotStore extends DelegatingRunStore { completedWaitpoints?: CompletedWaitpointRef[] ): Promise< | { kind: "new"; completedWaitpoints: CompletedWaitpointRef[] } - | { kind: "carryForward"; cycleSeq: number } + | { kind: "carryForward"; cycleSeq: number; completedWaitpoints: CompletedWaitpointRef[] } | undefined > { if (!completedWaitpoints || completedWaitpoints.length === 0) { @@ -600,7 +600,11 @@ export class TaskRunExecutionSnapshotStore extends DelegatingRunStore { sameOrder(previousIds.order, order) && sameSet(previousIds.distinctIds, distinct) ) { - return { kind: "carryForward", cycleSeq: head.cycle.cycleSeq }; + return { + kind: "carryForward", + cycleSeq: head.cycle.cycleSeq, + completedWaitpoints, + }; } } catch (error) { // A failed probe must not lose the waitpoints. Minting a fresh cycle is the safe direction: @@ -679,6 +683,14 @@ export class TaskRunExecutionSnapshotStore extends DelegatingRunStore { return this.delegate.findLatestExecutionSnapshot(runId, client, environmentId); } + if (read.danglingCycle) { + // The entry says it has waitpoints and the cycle key holding them is gone. Serving it would + // hand back an empty set that looks authoritative, and the run would resume with no waits. + // Postgres still has the join rows. + this.metrics?.recordRead("findLatestExecutionSnapshot", "postgres"); + return this.delegate.findLatestExecutionSnapshot(runId, client, environmentId); + } + this.metrics?.recordRead("findLatestExecutionSnapshot", "redis"); return this.#hydrate(read, runId, client, { hydrateWaitpointRows: true }); } @@ -727,6 +739,11 @@ export class TaskRunExecutionSnapshotStore extends DelegatingRunStore { return this.delegate.findManyExecutionSnapshots(args, client); } + if (result.entries.some((entry) => entry.danglingCycle)) { + this.metrics?.recordRead("findManyExecutionSnapshots", "postgres"); + return this.delegate.findManyExecutionSnapshots(args, client); + } + this.metrics?.recordRead("findManyExecutionSnapshots", "redis"); // The engine asks for createdAt DESC and reverses app-side; the store returns ascending. diff --git a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.waitpointCycles.test.ts b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.waitpointCycles.test.ts index 80d98aeef0e..9f40e54c599 100644 --- a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.waitpointCycles.test.ts +++ b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.waitpointCycles.test.ts @@ -423,6 +423,125 @@ describe("completed-waitpoint cycles", () => { } ); + containerTest( + "a refused carry mints a fresh cycle rather than writing a pointerless head", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, redis, env); + const [wpA] = await seedSnapshotWaitpoints(prisma, env, 1); + const snapshotId = generateInternalId(); + + // Driven at the store, because the decorator cannot reach this state deliberately: its + // probe reads the head first, sees the id set no longer matches, and mints a new cycle. + // The refusal is only reachable when the key vanishes BETWEEN that probe and the append, + // which is a race. Naming a cycle this incarnation never minted reproduces the same + // refusal deterministically. + const result = await redis.append({ + entry: { + id: snapshotId, + engine: "V2", + executionStatus: "EXECUTING", + description: "carry a cycle that was never minted", + runId, + runStatus: "EXECUTING", + createdAt: new Date().toISOString(), + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }, + kind: "transition", + isTerminal: false, + cycle: { + kind: "carryForward", + cycleSeq: 9999, + completedWaitpoints: [{ id: wpA, index: 0 }], + }, + }); + + expect(result.outcome).toBe("written"); + if (result.outcome !== "written") return; + + // Refusing the pointer is right. Writing the entry with NO pointer is not: it becomes the + // head, and a read of it answers present-with-nothing, which is the one answer that stops + // the engine's read-repair from looking. + expect(result.cycleMismatch).toBe(true); + expect(result.cycleSeq).toBeGreaterThan(0); + + const ids = await redis.getSnapshotWaitpointIds(runId, snapshotId); + expect(ids.present).toBe(true); + expect(ids.distinctIds).toEqual([wpA]); + } finally { + await redis.quit(); + } + } + ); + + containerTest( + "a dangling pointer reads as not present, not as an empty set", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, redis, env); + const [wpA] = await seedSnapshotWaitpoints(prisma, env, 1); + + const created = await decorated.createExecutionSnapshot( + resumeInput(runId, env, [{ id: wpA, index: 0 }], "resume") + ); + + // The entry keeps its pointer and the cycle key goes. Reachable by eviction, and by the + // completion TTL, which is set on every key for a run at once but expires them separately. + for (const key of await probe.keys(`snap:{${runId}}:wp:*`)) await probe.del(key); + + const ids = await redis.getSnapshotWaitpointIds(runId, created.id); + // present:false is what sends the caller to Postgres, which still holds the join rows. + // present:true with an empty set would suppress the engine's read-repair. + expect(ids.present).toBe(false); + expect(ids.distinctIds).toEqual([]); + + // And the projections the engine actually calls fall back rather than answering empty. + const withPresence = await decorated.findSnapshotCompletedWaitpointIdsWithPresence( + created.id, + undefined, + runId + ); + expect(withPresence.ids).toEqual([wpA]); + } finally { + await Promise.all([redis.quit(), probe.quit().catch(() => {})]); + } + } + ); + + containerTest( + "the hot read falls back to Postgres when the cycle key is gone", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, redis, env); + const [wpA] = await seedSnapshotWaitpoints(prisma, env, 1); + + await decorated.createExecutionSnapshot( + resumeInput(runId, env, [{ id: wpA, index: 0 }], "resume") + ); + for (const key of await probe.keys(`snap:{${runId}}:wp:*`)) await probe.del(key); + + const latest = await decorated.findLatestExecutionSnapshot(runId); + + // Served from Postgres, so the waitpoint is still there and the resume is not silently + // stripped of it. + expect(latest!.completedWaitpoints.map((w) => w.id)).toEqual([wpA]); + } finally { + await Promise.all([redis.quit(), probe.quit().catch(() => {})]); + } + } + ); + containerTest( "findLatestExecutionSnapshot returns the index oracle", async ({ prisma, redisOptions }) => { From b964f867d902e634f61efcd45285913b8bcb32aa Mon Sep 17 00:00:00 2001 From: Daniel Sutton Date: Tue, 25 Aug 2026 14:36:11 +0100 Subject: [PATCH 26/56] fix(run-store): ignore the read cohort at the last dial position At that position Postgres holds no snapshot rows, so a run routed away from Redis by the cohort percentage reads nothing at all. The percentage is only meaningful while both stores hold the data. Fixing it in the dial rather than documenting the constraint makes the combination unreachable, instead of leaving three settings that have to agree by convention. --- .../src/taskRunExecutionSnapshotStore.readCohort.test.ts | 9 +++++++++ .../run-store/src/taskRunExecutionSnapshotStore.ts | 6 ++++++ 2 files changed, 15 insertions(+) diff --git a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.readCohort.test.ts b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.readCohort.test.ts index 12ad2e5fc54..e33f7a9b253 100644 --- a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.readCohort.test.ts +++ b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.readCohort.test.ts @@ -46,6 +46,15 @@ describe("the read cohort", () => { expect(ids.every((id) => !store.readsFromRedis(id))).toBe(true); }); + it("ignores the dial at redis-only, whatever it is set to", () => { + // Postgres holds no snapshot rows at that position, so a run routed away from Redis reads + // nothing at all. The percentage is meaningful only while both stores hold the data. + for (const percent of [0, 1, 50, 99]) { + const store = probe("redis-only", percent); + expect(ids.every((id) => store.readsFromRedis(id))).toBe(true); + } + }); + it("gives one run the same answer every time", () => { // A run that changed store between two reads of one poll could show the log going backwards. const store = probe("redis-read", 50); diff --git a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.ts b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.ts index e96aef76172..f4a46cb0414 100644 --- a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.ts +++ b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.ts @@ -655,6 +655,12 @@ export class TaskRunExecutionSnapshotStore extends DelegatingRunStore { */ protected readsFromRedis(runId: string): boolean { if (this.mode !== "redis-read" && this.mode !== "redis-only") return false; + + // At `redis-only` the cohort dial has no meaning. Postgres holds no snapshot rows at that + // position, so a run routed away from Redis reads nothing at all. Ignoring the percentage here + // makes that misconfiguration unreachable rather than merely documented. + if (this.mode === "redis-only") return true; + if (this.readPercent >= 100) return true; if (this.readPercent <= 0) return false; From ba91f75e62bf03af37c49746e9b74329920de0dc Mon Sep 17 00:00:00 2001 From: Daniel Sutton Date: Tue, 25 Aug 2026 17:45:44 +0100 Subject: [PATCH 27/56] fix(run-store): make the snapshot sweep and store correct against a Redis cluster SCAN carries no key, so a cluster cannot route it: one connection iterates one node's keyspace and then reports a completed cursor. The sweep now fans out over every master, resolved per pass so a failover cannot leave it scanning a stale node list, and reports how many it covered. Rule 2 deletes a whole keyspace when the run lookup returns no row. That lookup partitions ids by residency and reads each store's replica, so an absent row is not proof of absence. Deletion now needs the keyspace to be seen absent in two separate passes, and any run found to exist clears its mark. Both window reads returned the head's waitpoint order without its dangling flag, so a head whose cycle key had expired came back with an empty order rather than falling back to Postgres, losing every position on a batched resume. Also lets both classes take a caller-built client so they can reach a cluster at all, and gives the sweep a deadline and an abort signal so a pass can stop inside its budget instead of being killed mid-cursor. --- internal-packages/redis/src/index.ts | 21 +- .../run-store/src/redisSnapshotStore.ts | 87 ++++- .../src/snapshotOrphanSweeper.cluster.test.ts | 157 ++++++++ .../src/snapshotOrphanSweeper.confirm.test.ts | 357 ++++++++++++++++++ .../src/snapshotOrphanSweeper.test.ts | 41 +- .../run-store/src/snapshotOrphanSweeper.ts | 299 +++++++++++++-- ...utionSnapshotStore.waitpointCycles.test.ts | 43 +++ 7 files changed, 952 insertions(+), 53 deletions(-) create mode 100644 internal-packages/run-store/src/snapshotOrphanSweeper.cluster.test.ts create mode 100644 internal-packages/run-store/src/snapshotOrphanSweeper.confirm.test.ts diff --git a/internal-packages/redis/src/index.ts b/internal-packages/redis/src/index.ts index 6fb30b9a4bf..622efe613fc 100644 --- a/internal-packages/redis/src/index.ts +++ b/internal-packages/redis/src/index.ts @@ -1,7 +1,24 @@ -import { Redis, type RedisOptions } from "ioredis"; +import { type Cluster, Redis, type RedisOptions } from "ioredis"; import { Logger } from "@trigger.dev/core/logger"; -export { Redis, type Callback, type RedisOptions, type Result, type RedisCommander } from "ioredis"; +export { + Redis, + Cluster, + type Callback, + type RedisOptions, + type ClusterNode, + type ClusterOptions, + type Result, + type RedisCommander, +} from "ioredis"; + +/** + * Either endpoint shape. A component that only issues key-addressed commands works against both, so + * it should accept this rather than pin itself to a standalone connection. Commands with no key — + * SCAN above all — do NOT fan out across a cluster, so anything that issues one must iterate + * `cluster.nodes("master")` itself. + */ +export type RedisClient = Redis | Cluster; /** * Reply-error -> reconnect mapping. Without this hook, an ElastiCache diff --git a/internal-packages/run-store/src/redisSnapshotStore.ts b/internal-packages/run-store/src/redisSnapshotStore.ts index 1160bd3ab4f..2339ab0dd5e 100644 --- a/internal-packages/run-store/src/redisSnapshotStore.ts +++ b/internal-packages/run-store/src/redisSnapshotStore.ts @@ -1,7 +1,7 @@ import { createRedisClient, type Callback, - type Redis, + type RedisClient, type RedisOptions, type Result, } from "@internal/redis"; @@ -205,8 +205,20 @@ export type SnapshotStoreMetrics = { recordLatency(op: string, ms: number): void; }; -export type RedisSnapshotStoreOptions = { - redisOptions: RedisOptions; +/** + * How the store reaches Redis. Exactly one of the two, enforced by the type rather than a runtime + * check: `never` on the opposite member makes both "neither" and "both" a compile error. + * + * `client` exists because production points at a Valkey/Redis CLUSTER, and cluster topology is not + * this package's business. Every command the store issues is key-addressed and every key carries a + * `{runId}` hashtag, so one slot serves a whole run and both endpoint shapes behave identically. + * A caller-supplied client is owned by the caller: `quit()` leaves it open. + */ +export type RedisSnapshotStoreConnection = + | { client: RedisClient; redisOptions?: never } + | { client?: never; redisOptions: RedisOptions }; + +export type RedisSnapshotStoreOptions = RedisSnapshotStoreConnection & { completedTtlMs: number; sinceLimit?: number; highWater?: { entryBytes?: number; cycleKeyBytes?: number; cycleCount?: number }; @@ -214,13 +226,25 @@ export type RedisSnapshotStoreOptions = { logger?: Logger; }; +/** + * Both window scripts return four leading slots before the first row: the id-cursor variant's + * `sinceRaw`, the head's order, the head's distinct set, and the head's dangling flag. Rows follow + * in four-element groups, so the head row is the group at this offset. + * + * Named because the offset drifted out of the comments describing it twice, and the second drift + * arrived in the change that fixed the first. + */ +const WINDOW_HEAD_ROW_INDEX = 4; + const SKIPPED = "skipped"; const FORKED = "forked"; const WRITTEN = "written"; const DUPLICATE = "duplicate"; export class RedisSnapshotStore { - private readonly redis: Redis; + private readonly redis: RedisClient; + /** Only a client this class opened may be closed by it. */ + private readonly ownsClient: boolean; private readonly logger: Logger; private readonly completedTtlMs: number; private readonly sinceLimit: number; @@ -234,15 +258,22 @@ export class RedisSnapshotStore { this.sinceLimit = options.sinceLimit ?? 50; this.metrics = options.metrics; this.highWater = options.highWater ?? {}; - this.redis = createRedisClient(options.redisOptions, { - onError: (error) => this.logger.error("RedisSnapshotStore redis client error", { error }), - }); + this.ownsClient = options.client === undefined; + this.redis = + options.client ?? + createRedisClient(options.redisOptions, { + onError: (error) => this.logger.error("RedisSnapshotStore redis client error", { error }), + }); this.#registerCommands(); } async quit(): Promise { // Idempotent and error-swallowing: every test calls this in a `finally`, and a double quit() // (or one after a failed connect) must never mask the real assertion failure. + // + // An injected client is the caller's. Closing it here would take down a connection shared with + // the sweeper or with another component, so a borrowed client is left open. + if (!this.ownsClient) return; if (!this.#quit) { this.#quit = this.redis.quit().then( () => undefined, @@ -496,11 +527,12 @@ export class RedisSnapshotStore { const headOrder = reply[1] ?? ""; const headDistinct = reply[2] ?? ""; + const headDangling = reply[3] ?? ""; const rows: SnapshotRead[] = []; - // Tracks whether the Lua-chosen head row (always the first, i === 2) itself survives the + // Tracks whether the Lua-chosen head row (always the first, WINDOW_HEAD_ROW_INDEX) survives the // env filter below -- headOrder must never be attributed to a different, surviving row. let headSurvived = false; - for (let i = 3; i + 3 < reply.length; i += 4) { + for (let i = WINDOW_HEAD_ROW_INDEX; i + 3 < reply.length; i += 4) { // orderKnown is false here: headOrder covers only the head row, resolved separately below. const decoded = this.#decode( [reply[i], reply[i + 1], reply[i + 2], reply[i + 3], ""], @@ -510,7 +542,7 @@ export class RedisSnapshotStore { ); if (decoded) { rows.push(decoded); - if (i === 3) headSurvived = true; + if (i === WINDOW_HEAD_ROW_INDEX) headSurvived = true; } } @@ -523,6 +555,13 @@ export class RedisSnapshotStore { ); if (head) { head.completedWaitpointIds = headWaitpointIds; + // A head whose cycle key has expired carries an empty order that means "unknown", not + // "none". The caller cannot distinguish those, so it has to be told, or it resumes a batch + // with every position lost. This is what makes the decorator's Postgres fallback reachable + // on the since-window path as well as the hot read. + if (headDangling === "1") { + head.danglingCycle = true; + } if (head.cycle) { this.#checkCycleMismatch(runId, head.cycle.count, headWaitpointIds.order.length); } @@ -560,11 +599,12 @@ export class RedisSnapshotStore { const headOrder = reply[1] ?? ""; const headDistinct = reply[2] ?? ""; + const headDangling = reply[3] ?? ""; const rows: SnapshotRead[] = []; - // Tracks whether the Lua-chosen head row (always the first, i === 2) survives the env filter, + // Tracks whether the Lua-chosen head row (always the first, WINDOW_HEAD_ROW_INDEX) survives the env filter, // so headOrder is never attributed to a different, surviving row. let headSurvived = false; - for (let i = 3; i + 3 < reply.length; i += 4) { + for (let i = WINDOW_HEAD_ROW_INDEX; i + 3 < reply.length; i += 4) { const decoded = this.#decode( [reply[i], reply[i + 1], reply[i + 2], reply[i + 3], ""], opts?.environmentId, @@ -573,7 +613,7 @@ export class RedisSnapshotStore { ); if (decoded) { rows.push(decoded); - if (i === 3) headSurvived = true; + if (i === WINDOW_HEAD_ROW_INDEX) headSurvived = true; } } @@ -586,6 +626,13 @@ export class RedisSnapshotStore { ); if (head) { head.completedWaitpointIds = headWaitpointIds; + // A head whose cycle key has expired carries an empty order that means "unknown", not + // "none". The caller cannot distinguish those, so it has to be told, or it resumes a batch + // with every position lost. This is what makes the decorator's Postgres fallback reachable + // on the since-window path as well as the hot read. + if (headDangling === "1") { + head.danglingCycle = true; + } if (head.cycle) { this.#checkCycleMismatch(runId, head.cycle.count, headWaitpointIds.order.length); } @@ -872,7 +919,7 @@ export class RedisSnapshotStore { -- compare is a chronological compare. Walking newest-first lets the scan stop at the first -- entry at or before the cursor, which makes its length the length of the ANSWER rather -- than the length of the run's history. - local out = { '', '', '' } + local out = { '', '', '', '' } local headId = nil local offset = 0 local page = limit @@ -910,6 +957,11 @@ export class RedisSnapshotStore { local headPointer = redis.call('HGET', eKey, headId .. '#c') out[2] = orderFor(headPointer) out[3] = distinctFor(headPointer) + -- The head's cycle key can expire while its entry survives: the completion TTL is applied + -- per key. Without this flag the head returns an EMPTY order and the caller cannot tell + -- that from a head that genuinely had no indexed waitpoints, so a batched resume loses + -- every position instead of falling back to Postgres. + out[4] = danglingFor(headPointer) end return out `, @@ -941,7 +993,7 @@ export class RedisSnapshotStore { -- The head is the newest SURVIVING entry, and it is the only one whose cycle key is read. -- Deriving the order after the loop keeps it paired with the row it is attached to: a row -- dropped for a missing body must not donate its cycle data to the next one. - local out = { sinceRaw, '', '' } + local out = { sinceRaw, '', '', '' } local headId = nil for i = 1, #ids do local id = ids[i] @@ -958,6 +1010,11 @@ export class RedisSnapshotStore { local headPointer = redis.call('HGET', eKey, headId .. '#c') out[2] = orderFor(headPointer) out[3] = distinctFor(headPointer) + -- The head's cycle key can expire while its entry survives: the completion TTL is applied + -- per key. Without this flag the head returns an EMPTY order and the caller cannot tell + -- that from a head that genuinely had no indexed waitpoints, so a batched resume loses + -- every position instead of falling back to Postgres. + out[4] = danglingFor(headPointer) end return out `, diff --git a/internal-packages/run-store/src/snapshotOrphanSweeper.cluster.test.ts b/internal-packages/run-store/src/snapshotOrphanSweeper.cluster.test.ts new file mode 100644 index 00000000000..d8e70d5b3ec --- /dev/null +++ b/internal-packages/run-store/src/snapshotOrphanSweeper.cluster.test.ts @@ -0,0 +1,157 @@ +// Production points at a Valkey/Redis CLUSTER. SCAN carries no key, so a cluster cannot route it: +// one connection iterates ONE node's keyspace and then returns a completed cursor. A single-client +// sweep would therefore report {scanned, expired, deleted, skipped} looking exactly like a clean +// pass, having examined roughly 1/N of the keyspace, and the rest would leak with nothing left to +// revisit it. Both sweep rules close unbounded leaks, and TRI-13453 gates the rollout dial on an +// OBSERVED sweep pass, so a false green here is the worst failure this component has. +// +// Everything the sweep does after the scan is key-addressed and a cluster client routes it without +// help, so the node list is the whole of the exposure. These tests pin that decision directly. +// +// There is no Redis-cluster container fixture in the repo (@internal/testcontainers ships slot +// arithmetic, not a cluster), so the cluster cases drive a real ioredis `Cluster` object that has +// never connected and assert which method the code reaches for. That is a test of our branch, not +// a simulation of Redis. A true multi-node integration test wants a cluster fixture, and the ticket +// building the cluster client is the one placed to add it. +import { describe, expect, it } from "vitest"; +import { Cluster, Redis } from "@internal/redis"; +import { containerTest } from "@internal/testcontainers"; +import { generateInternalId } from "@trigger.dev/core/v3/isomorphic"; +import { PostgresRunStore } from "./PostgresRunStore.js"; +import { clientPrefixOf, scanTargetsOf, SnapshotOrphanSweeper } from "./snapshotOrphanSweeper.js"; + +/** An ioredis Cluster that is never connected. `lazyConnect` keeps the constructor from dialling. */ +function offlineCluster(options?: { keyPrefix?: string }): Cluster { + return new Cluster([{ host: "127.0.0.1", port: 7000 }], { + lazyConnect: true, + redisOptions: options?.keyPrefix ? { keyPrefix: options.keyPrefix } : undefined, + }); +} + +describe("scanTargetsOf", () => { + it("returns the one connection for a standalone client", () => { + const client = new Redis({ lazyConnect: true, port: 65000 }); + try { + expect(scanTargetsOf(client)).toEqual([client]); + } finally { + client.disconnect(); + } + }); + + it("returns every master for a cluster, and never a replica", async () => { + const cluster = offlineCluster(); + const masters = [ + new Redis({ lazyConnect: true, port: 65001 }), + new Redis({ lazyConnect: true, port: 65002 }), + new Redis({ lazyConnect: true, port: 65003 }), + ]; + const replica = new Redis({ lazyConnect: true, port: 65004 }); + + const asked: string[] = []; + // The assertion that matters: the sweep asks for "master" specifically. Asking for "all" would + // scan replicas too and act twice on one keyspace. + (cluster as unknown as { nodes: (role: string) => Redis[] }).nodes = (role: string) => { + asked.push(role); + return role === "master" ? masters : [...masters, replica]; + }; + + expect(scanTargetsOf(cluster)).toEqual(masters); + expect(asked).toEqual(["master"]); + expect(scanTargetsOf(cluster)).not.toContain(replica); + + for (const client of [...masters, replica]) client.disconnect(); + cluster.disconnect(); + }); + + it("resolves the node list per call, so a failover is picked up", () => { + const cluster = offlineCluster(); + let generation = 0; + (cluster as unknown as { nodes: () => Redis[] }).nodes = () => { + generation += 1; + return Array.from({ length: generation }, (_v, i) => new Redis({ lazyConnect: true, port: 65100 + i })); + }; + + expect(scanTargetsOf(cluster)).toHaveLength(1); + expect(scanTargetsOf(cluster)).toHaveLength(2); + cluster.disconnect(); + }); +}); + +describe("clientPrefixOf", () => { + it("reads the top-level keyPrefix on a standalone client", () => { + const client = new Redis({ lazyConnect: true, port: 65000, keyPrefix: "engine:" }); + try { + expect(clientPrefixOf(client)).toBe("engine:"); + } finally { + client.disconnect(); + } + }); + + it("reads the nested redisOptions.keyPrefix on a cluster", () => { + // On a Cluster the prefix lives under redisOptions. Reading the top level yields "", every + // SCAN MATCH then misses, and the pass reports a clean sweep of nothing. + const cluster = offlineCluster({ keyPrefix: "engine:" }); + try { + expect(clientPrefixOf(cluster)).toBe("engine:"); + } finally { + cluster.disconnect(); + } + }); + + it("is empty when no prefix is configured, for either shape", () => { + const client = new Redis({ lazyConnect: true, port: 65000 }); + const cluster = offlineCluster(); + try { + expect(clientPrefixOf(client)).toBe(""); + expect(clientPrefixOf(cluster)).toBe(""); + } finally { + client.disconnect(); + cluster.disconnect(); + } + }); +}); + +describe("SweepResult.nodes", () => { + containerTest( + "a standalone pass reports the one connection it covered", + async ({ prisma, redisOptions }) => { + const runStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const sweeper = new SnapshotOrphanSweeper({ + redisOptions, + runStore, + completedTtlMs: 72 * 60 * 60 * 1000, + }); + + try { + const result = await sweeper.sweep({ dryRun: true }); + // Without this field a pass that covered one node of six is indistinguishable from a + // complete one, which is exactly the false green the fan-out exists to prevent. + expect(result.nodes).toBe(1); + } finally { + await sweeper.quit(); + } + } + ); +}); + +describe("client ownership", () => { + containerTest("quit() leaves a borrowed client open", async ({ prisma, redisOptions }) => { + const runStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const client = new Redis(redisOptions); + const sweeper = new SnapshotOrphanSweeper({ + client, + runStore, + completedTtlMs: 72 * 60 * 60 * 1000, + }); + + try { + await sweeper.quit(); + // The store and the sweep can share one cluster client. If quit() closed a client it did not + // open, the first component to shut down would take the other one's connection with it. + await client.set(`ownership:${generateInternalId()}`, "1"); + expect(await client.ping()).toBe("PONG"); + } finally { + client.disconnect(); + } + }); +}); diff --git a/internal-packages/run-store/src/snapshotOrphanSweeper.confirm.test.ts b/internal-packages/run-store/src/snapshotOrphanSweeper.confirm.test.ts new file mode 100644 index 00000000000..d617ca27834 --- /dev/null +++ b/internal-packages/run-store/src/snapshotOrphanSweeper.confirm.test.ts @@ -0,0 +1,357 @@ +// Rule 2 deletes a whole keyspace on the strength of "findRunsByIds returned no row for it". The +// catch in #sweepBatch covers a lookup that THROWS; it cannot see a lookup that succeeds and is +// incomplete, and a row that exists but did not come back reads exactly like a run that never +// existed. `findRunsByIds` partitions ids by residency and asks each store only for its own, and +// with no client passed it reads each store's replica — both sound today, but neither is something +// this delete path can verify. +// +// A false negative leaks keys, which is bounded and recoverable. A false positive destroys a live +// run's execution state. So deletion requires two sightings across the confirm window, and these +// tests pin that: a single pass never deletes, however old the keyspace. +import { describe, expect } from "vitest"; +import { containerTest } from "@internal/testcontainers"; +import { createRedisClient } from "@internal/redis"; +import { generateInternalId } from "@trigger.dev/core/v3/isomorphic"; +import { PostgresRunStore } from "./PostgresRunStore.js"; +import { RedisSnapshotStore, snapshotKeys } from "./redisSnapshotStore.js"; +import { entryFromCreateRun } from "./snapshotEntry.js"; +import { SnapshotOrphanSweeper } from "./snapshotOrphanSweeper.js"; +import type { RunStore } from "./types.js"; +import { + buildCreateRunData, + seedSnapshotEnvironment, + type SnapshotFixtureEnv, +} from "./testFixtures/snapshotIdFixture.js"; + +const COMPLETED_TTL_MS = 72 * 60 * 60 * 1000; +const ORPHAN_AGE_MS = 60 * 60 * 1000; + +function birthEntry(runId: string, env: SnapshotFixtureEnv, createdAt: Date) { + const snapshot = { + id: generateInternalId(), + engine: "V2" as const, + executionStatus: "RUN_CREATED" as const, + description: "Run was created", + runStatus: "PENDING" as const, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }; + return entryFromCreateRun({ id: snapshot.id, runId, createdAt }, snapshot); +} + +describe("rule 2 requires a second sighting", () => { + containerTest( + "one pass marks an orphan and deletes nothing, however old the keyspace", + async ({ prisma, redisOptions }) => { + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + const runStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const sweeper = new SnapshotOrphanSweeper({ + redisOptions, + runStore: runStore as unknown as RunStore, + completedTtlMs: COMPLETED_TTL_MS, + orphanAgeMs: ORPHAN_AGE_MS, + confirmOrphanAfterMs: 0, + }); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + // A thousand times the age gate. Age is not what holds the deletion back. + const ancient = new Date(Date.now() - 1000 * ORPHAN_AGE_MS); + await store.append({ + entry: birthEntry(runId, env, ancient), + kind: "birth", + isTerminal: false, + }); + + const first = await sweeper.sweep(); + + expect(first.deleted).toBe(0); + expect(first.pendingDeletion).toBe(1); + expect(await probe.exists(snapshotKeys(runId).e)).toBe(1); + + const second = await sweeper.sweep(); + + expect(second.deleted).toBe(1); + expect(await probe.exists(snapshotKeys(runId).e)).toBe(0); + } finally { + await Promise.all([store.quit(), sweeper.quit(), probe.quit().catch(() => {})]); + } + } + ); + + containerTest( + "a marked keyspace is not deleted until the confirm window has passed", + async ({ prisma, redisOptions }) => { + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + const runStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const sweeper = new SnapshotOrphanSweeper({ + redisOptions, + runStore: runStore as unknown as RunStore, + completedTtlMs: COMPLETED_TTL_MS, + orphanAgeMs: ORPHAN_AGE_MS, + // An hour. Passes minutes apart must not convert a candidate. + confirmOrphanAfterMs: 60 * 60 * 1000, + }); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + await store.append({ + entry: birthEntry(runId, env, new Date(Date.now() - 2 * ORPHAN_AGE_MS)), + kind: "birth", + isTerminal: false, + }); + + await sweeper.sweep(); + const second = await sweeper.sweep(); + const third = await sweeper.sweep(); + + expect(second.deleted).toBe(0); + expect(second.pendingDeletion).toBe(1); + expect(third.deleted).toBe(0); + expect(await probe.exists(snapshotKeys(runId).e)).toBe(1); + } finally { + await Promise.all([store.quit(), sweeper.quit(), probe.quit().catch(() => {})]); + } + } + ); + + containerTest( + "a transient miss followed by a found run does not delete, and does not leave the keyspace pre-authorised", + async ({ prisma, redisOptions }) => { + // The case the guard exists for. Pass 1 gets an incomplete answer and marks the keyspace. + // Pass 2 sees the run, so it must clear the mark: were the mark to survive, a LATER genuine + // absence would delete on its own first sighting and the two-sighting rule would be gone. + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + const real = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + + let lie = true; + const flaky = { + ...real, + findRunsByIds: (...args: unknown[]) => + lie + ? // Succeeds and is incomplete: exactly what the catch cannot see. + Promise.resolve(new Map()) + : (real.findRunsByIds as (...rest: unknown[]) => Promise>).apply( + real, + args + ), + } as unknown as RunStore; + + const sweeper = new SnapshotOrphanSweeper({ + redisOptions, + runStore: flaky, + completedTtlMs: COMPLETED_TTL_MS, + orphanAgeMs: ORPHAN_AGE_MS, + confirmOrphanAfterMs: 0, + }); + + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + await store.append({ + entry: birthEntry(runId, env, new Date(Date.now() - 2 * ORPHAN_AGE_MS)), + kind: "birth", + isTerminal: false, + }); + // The run is alive and terminal in Postgres the whole time. Only the lookup lies. + await prisma.taskRun.create({ + data: { ...buildCreateRunData(runId, env), status: "COMPLETED_SUCCESSFULLY" }, + }); + + const first = await sweeper.sweep(); + expect(first.deleted).toBe(0); + expect(first.pendingDeletion).toBe(1); + + lie = false; + const second = await sweeper.sweep(); + expect(second.deleted).toBe(0); + expect(await probe.exists(snapshotKeys(runId).e)).toBe(1); + + // The run vanishes for real. With the mark cleared this is a first sighting again. + lie = true; + const third = await sweeper.sweep(); + expect(third.deleted).toBe(0); + expect(third.pendingDeletion).toBe(1); + expect(await probe.exists(snapshotKeys(runId).e)).toBe(1); + + const fourth = await sweeper.sweep(); + expect(fourth.deleted).toBe(1); + } finally { + await Promise.all([store.quit(), sweeper.quit(), probe.quit().catch(() => {})]); + } + } + ); +}); + +describe("the marker cannot expire out from under a candidate", () => { + containerTest( + "a marked keyspace still carries its mark after the whole keyspace is re-read", + async ({ prisma, redisOptions }) => { + // The marker used to be a key with its own TTL derived from the confirm window, which could + // be shorter than the interval between passes: the marker written at T was gone by + // T+interval, every pass wrote a fresh one, and rule 2 deleted nothing while reporting clean. + // It is now a field on the run's `seq` hash, so it lives exactly as long as the keyspace and + // there is no lifetime left to misconfigure. + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + const runStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const sweeper = new SnapshotOrphanSweeper({ + redisOptions, + runStore: runStore as unknown as RunStore, + completedTtlMs: COMPLETED_TTL_MS, + orphanAgeMs: ORPHAN_AGE_MS, + confirmOrphanAfterMs: 0, + }); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + await store.append({ + entry: birthEntry(runId, env, new Date(Date.now() - 2 * ORPHAN_AGE_MS)), + kind: "birth", + isTerminal: false, + }); + + expect((await sweeper.sweep()).pendingDeletion).toBe(1); + + // The mark is a field on seq, and it carries no expiry of its own. + expect(await probe.hget(snapshotKeys(runId).seq, "orph")).not.toBeNull(); + expect(await probe.pttl(snapshotKeys(runId).seq)).toBe(-1); + + expect((await sweeper.sweep()).deleted).toBe(1); + // And it went with the keyspace rather than outliving it. + expect(await probe.exists(snapshotKeys(runId).seq)).toBe(0); + } finally { + await Promise.all([store.quit(), sweeper.quit(), probe.quit().catch(() => {})]); + } + } + ); +}); + +describe("a run seen alive clears its marker", () => { + containerTest( + "a lie, then a LIVE run, then a lie again does not delete", + async ({ prisma, redisOptions }) => { + // The hole a long marker lifetime opens. Only terminal runs used to clear the marker, so a + // keyspace marked by an incomplete lookup and then seen ALIVE kept its mark. A later genuine + // absence would then find a mature marker and delete on what is really a first sighting. + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + const real = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + + let lie = true; + const flaky = { + ...real, + findRunsByIds: (...args: unknown[]) => + lie + ? Promise.resolve(new Map()) + : (real.findRunsByIds as (...rest: unknown[]) => Promise>).apply( + real, + args + ), + } as unknown as RunStore; + + const sweeper = new SnapshotOrphanSweeper({ + redisOptions, + runStore: flaky, + completedTtlMs: COMPLETED_TTL_MS, + orphanAgeMs: ORPHAN_AGE_MS, + confirmOrphanAfterMs: 0, + }); + + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + await store.append({ + entry: birthEntry(runId, env, new Date(Date.now() - 2 * ORPHAN_AGE_MS)), + kind: "birth", + isTerminal: false, + }); + // EXECUTING, not terminal. This run never reaches rule 1, so rule 1 cannot be what clears + // the mark; a SUSPENDED run can legitimately sit here for weeks. + await prisma.taskRun.create({ + data: { ...buildCreateRunData(runId, env), status: "EXECUTING" }, + }); + + expect((await sweeper.sweep()).pendingDeletion).toBe(1); + + lie = false; + const seenAlive = await sweeper.sweep(); + expect(seenAlive.skipped).toBeGreaterThan(0); + expect(seenAlive.deleted).toBe(0); + + lie = true; + const afterAlive = await sweeper.sweep(); + // A first sighting again, because being seen alive cleared the mark. + expect(afterAlive.deleted).toBe(0); + expect(afterAlive.pendingDeletion).toBe(1); + expect(await probe.exists(snapshotKeys(runId).e)).toBe(1); + } finally { + await Promise.all([store.quit(), sweeper.quit(), probe.quit().catch(() => {})]); + } + } + ); +}); + +describe("a pass can stop inside a budget", () => { + containerTest("an already-passed deadline yields a partial pass", async ({ prisma, redisOptions }) => { + const runStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const sweeper = new SnapshotOrphanSweeper({ + redisOptions, + runStore: runStore as unknown as RunStore, + completedTtlMs: COMPLETED_TTL_MS, + }); + + try { + const result = await sweeper.sweep({ deadline: Date.now() - 1 }); + + // redis-worker redelivers a job that outlives its visibility timeout, and nothing extends it, + // so a pass that cannot stop on its own runs concurrently with itself. + expect(result.partial).toBe(true); + expect(result.scanned).toBe(0); + } finally { + await sweeper.quit(); + } + }); + + containerTest("an aborted signal yields a partial pass", async ({ prisma, redisOptions }) => { + const runStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const sweeper = new SnapshotOrphanSweeper({ + redisOptions, + runStore: runStore as unknown as RunStore, + completedTtlMs: COMPLETED_TTL_MS, + }); + const controller = new AbortController(); + controller.abort(); + + try { + const result = await sweeper.sweep({ signal: controller.signal }); + expect(result.partial).toBe(true); + } finally { + await sweeper.quit(); + } + }); + + containerTest("a pass with budget to spare is not partial", async ({ prisma, redisOptions }) => { + const runStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const sweeper = new SnapshotOrphanSweeper({ + redisOptions, + runStore: runStore as unknown as RunStore, + completedTtlMs: COMPLETED_TTL_MS, + }); + + try { + const result = await sweeper.sweep({ deadline: Date.now() + 60_000 }); + expect(result.partial).toBe(false); + } finally { + await sweeper.quit(); + } + }); +}); diff --git a/internal-packages/run-store/src/snapshotOrphanSweeper.test.ts b/internal-packages/run-store/src/snapshotOrphanSweeper.test.ts index b0d39d27e56..7d7eb2aca0d 100644 --- a/internal-packages/run-store/src/snapshotOrphanSweeper.test.ts +++ b/internal-packages/run-store/src/snapshotOrphanSweeper.test.ts @@ -44,6 +44,10 @@ describe("SnapshotOrphanSweeper", () => { runStore: runStore as unknown as RunStore, completedTtlMs: COMPLETED_TTL_MS, orphanAgeMs: ORPHAN_AGE_MS, + // Rule 2 needs two sightings. These cases are about WHICH keyspaces it picks, not about + // the confirm window, so the window is zero and they sweep twice. The window itself has + // its own tests below. + confirmOrphanAfterMs: 0, }); const probe = createRedisClient(redisOptions, { onError: () => {} }); try { @@ -87,6 +91,10 @@ describe("SnapshotOrphanSweeper", () => { runStore: runStore as unknown as RunStore, completedTtlMs: COMPLETED_TTL_MS, orphanAgeMs: ORPHAN_AGE_MS, + // Rule 2 needs two sightings. These cases are about WHICH keyspaces it picks, not about + // the confirm window, so the window is zero and they sweep twice. The window itself has + // its own tests below. + confirmOrphanAfterMs: 0, }); const probe = createRedisClient(redisOptions, { onError: () => {} }); try { @@ -126,6 +134,10 @@ describe("SnapshotOrphanSweeper", () => { runStore: runStore as unknown as RunStore, completedTtlMs: COMPLETED_TTL_MS, orphanAgeMs: ORPHAN_AGE_MS, + // Rule 2 needs two sightings. These cases are about WHICH keyspaces it picks, not about + // the confirm window, so the window is zero and they sweep twice. The window itself has + // its own tests below. + confirmOrphanAfterMs: 0, }); const probe = createRedisClient(redisOptions, { onError: () => {} }); try { @@ -149,6 +161,7 @@ describe("SnapshotOrphanSweeper", () => { const cyclesBefore = await probe.keys(`snap:{${runId}}:wp:*`); expect(cyclesBefore.length).toBeGreaterThan(0); + await sweeper.sweep(); const result = await sweeper.sweep(); expect(result.deleted).toBe(1); @@ -171,6 +184,7 @@ describe("SnapshotOrphanSweeper", () => { runStore: runStore as unknown as RunStore, completedTtlMs: COMPLETED_TTL_MS, orphanAgeMs: ORPHAN_AGE_MS, + confirmOrphanAfterMs: 0, }); const probe = createRedisClient(redisOptions, { onError: () => {} }); try { @@ -203,6 +217,10 @@ describe("SnapshotOrphanSweeper", () => { runStore: runStore as unknown as RunStore, completedTtlMs: COMPLETED_TTL_MS, orphanAgeMs: ORPHAN_AGE_MS, + // Rule 2 needs two sightings. These cases are about WHICH keyspaces it picks, not about + // the confirm window, so the window is zero and they sweep twice. The window itself has + // its own tests below. + confirmOrphanAfterMs: 0, }); const probe = createRedisClient(redisOptions, { onError: () => {} }); try { @@ -242,6 +260,7 @@ describe("SnapshotOrphanSweeper", () => { runStore: runStore as unknown as RunStore, completedTtlMs: COMPLETED_TTL_MS, orphanAgeMs: ORPHAN_AGE_MS, + confirmOrphanAfterMs: 0, }); const probe = createRedisClient(redisOptions, { onError: () => {} }); try { @@ -262,7 +281,11 @@ describe("SnapshotOrphanSweeper", () => { const result = await sweeper.sweep({ dryRun: true }); - expect(result.deleted).toBe(1); + // A dry pass writes no marker, so an unconfirmed rule 2 candidate reports as pending rather + // than as a deletion. That is what a real pass would do at this instant, which is the honest + // answer for a preview: nothing is confirmed yet, so nothing would be deleted yet. + expect(result.deleted).toBe(0); + expect(result.pendingDeletion).toBe(1); expect(result.expired).toBe(1); expect(await probe.exists(snapshotKeys(orphan).e)).toBe(1); expect(await probe.pttl(snapshotKeys(terminal).e)).toBe(-1); @@ -285,6 +308,10 @@ describe("SnapshotOrphanSweeper", () => { runStore: failing, completedTtlMs: COMPLETED_TTL_MS, orphanAgeMs: ORPHAN_AGE_MS, + // Rule 2 needs two sightings. These cases are about WHICH keyspaces it picks, not about + // the confirm window, so the window is zero and they sweep twice. The window itself has + // its own tests below. + confirmOrphanAfterMs: 0, }); const probe = createRedisClient(redisOptions, { onError: () => {} }); try { @@ -320,6 +347,10 @@ describe("SnapshotOrphanSweeper", () => { runStore: runStore as unknown as RunStore, completedTtlMs: COMPLETED_TTL_MS, orphanAgeMs: ORPHAN_AGE_MS, + // Rule 2 needs two sightings. These cases are about WHICH keyspaces it picks, not about + // the confirm window, so the window is zero and they sweep twice. The window itself has + // its own tests below. + confirmOrphanAfterMs: 0, }); const probe = createRedisClient(redisOptions, { onError: () => {} }); try { @@ -340,6 +371,7 @@ describe("SnapshotOrphanSweeper", () => { expect(await probe.exists(keys.e)).toBe(1); expect(await probe.exists(keys.cur)).toBe(0); + await sweeper.sweep(); const result = await sweeper.sweep(); expect(result.deleted).toBe(1); @@ -369,6 +401,10 @@ describe("SnapshotOrphanSweeper", () => { runStore: runStore as unknown as RunStore, completedTtlMs: COMPLETED_TTL_MS, orphanAgeMs: ORPHAN_AGE_MS, + // Rule 2 needs two sightings. These cases are about WHICH keyspaces it picks, not about + // the confirm window, so the window is zero and they sweep twice. The window itself has + // its own tests below. + confirmOrphanAfterMs: 0, }); const probe = createRedisClient(prefixed, { onError: () => {} }); try { @@ -383,6 +419,7 @@ describe("SnapshotOrphanSweeper", () => { cycle: { kind: "new", completedWaitpoints: [{ id: "w_1", index: 0 }] }, }); + await sweeper.sweep(); const result = await sweeper.sweep(); expect(result.scanned).toBe(1); @@ -403,6 +440,7 @@ describe("SnapshotOrphanSweeper", () => { runStore: runStore as unknown as RunStore, completedTtlMs: COMPLETED_TTL_MS, orphanAgeMs: ORPHAN_AGE_MS, + confirmOrphanAfterMs: 0, }); const probe = createRedisClient(redisOptions, { onError: () => {} }); try { @@ -417,6 +455,7 @@ describe("SnapshotOrphanSweeper", () => { }); } + await sweeper.sweep({ batchSize: 2 }); const result = await sweeper.sweep({ batchSize: 2 }); expect(result.deleted).toBe(5); diff --git a/internal-packages/run-store/src/snapshotOrphanSweeper.ts b/internal-packages/run-store/src/snapshotOrphanSweeper.ts index 4d3eed674b4..422217cbc84 100644 --- a/internal-packages/run-store/src/snapshotOrphanSweeper.ts +++ b/internal-packages/run-store/src/snapshotOrphanSweeper.ts @@ -11,7 +11,13 @@ // // Nothing schedules this. The engine's worker is what has to run it, and run-store cannot reach the // engine, so the wiring belongs to the ticket that owns production construction. -import { createRedisClient, type Redis, type RedisOptions } from "@internal/redis"; +import { + Cluster, + createRedisClient, + type Redis, + type RedisClient, + type RedisOptions, +} from "@internal/redis"; import { Logger } from "@trigger.dev/core/logger"; import type { TaskRunStatus } from "@trigger.dev/database"; import { snapshotKeys } from "./redisSnapshotStore.js"; @@ -46,6 +52,12 @@ const DEFAULT_ORPHAN_AGE_MS = 24 * 60 * 60 * 1000; const SNAPSHOT_KEYSPACE_PREFIX = "snap:"; const DEFAULT_BATCH_SIZE = 1000; +/** + * How long a rule 2 candidate must have been marked before it may be deleted. It has to exceed the + * interval between passes, or a candidate is never sighted twice and never converts. + */ +const DEFAULT_ORPHAN_CONFIRM_MS = 60 * 60 * 1000; + export type SweepResult = { /** Keyspaces examined. */ scanned: number; @@ -55,14 +67,39 @@ export type SweepResult = { deleted: number; /** Left alone: a live run, a young orphan, or a batch whose Postgres lookup failed. */ skipped: number; -}; - -export type SnapshotOrphanSweeperOptions = { /** - * The sweep opens its own connection rather than borrowing the store's, so a long scan can never - * stall a hot-path client. + * Rule 2 candidates that were marked but not deleted, because deletion needs a second sighting + * in a later pass. A number that never converts to `deleted` means the confirm window is longer + * than the interval between passes, or the marker TTL is shorter than it. */ - redisOptions: RedisOptions; + pendingDeletion: number; + /** + * Connections the pass iterated: every master of a cluster, or 1 standalone. Reported because the + * failure this component cannot tolerate is a false green, and a pass that covered one node of + * six is indistinguishable from a complete one by any other field here. TRI-13453 gates the dial + * on an observed sweep pass, so the observation has to carry its own coverage. + */ + nodes: number; + /** True when the pass stopped early on its deadline or abort signal, so coverage is incomplete. */ + partial: boolean; +}; + +/** + * Exactly one of `redisOptions` or `client`, enforced by the type rather than a runtime check. + * With `redisOptions` the sweep opens its OWN connection, which is the preferred shape: a long + * scan can then never stall a hot-path client. `client` exists for a caller that has already built + * a client and wants the sweep to use it; a borrowed client is left open by `quit()`. + * + * What the sweep needs is a connection of its OWN, not one it built itself. A caller pointing at a + * cluster should build a SECOND, sweep-dedicated cluster client and pass it here: that keeps a long + * scan off the hot path just as well as `redisOptions` does. Handing over the client the snapshot + * store is using is the case to avoid. + */ +export type SnapshotOrphanSweeperConnection = + | { client: RedisClient; redisOptions?: never } + | { client?: never; redisOptions: RedisOptions }; + +export type SnapshotOrphanSweeperOptions = SnapshotOrphanSweeperConnection & { /** * Resolved through the run store, not a raw client. Under the run-ops split a run row can live on * either database, and only the store knows which — a raw lookup would report a live run as an @@ -71,14 +108,32 @@ export type SnapshotOrphanSweeperOptions = { runStore: RunStore; completedTtlMs: number; orphanAgeMs?: number; + /** + * How long a rule 2 candidate must stay marked before the sweep will delete it. Defaults to one + * hour. + * + * Set it at or below the interval between passes, or the second sighting arrives too early to + * count and every candidate needs three passes instead of two. It does NOT need to exceed the + * interval; the constraint people reach for ("longer than the interval") is the wrong one and + * only costs latency. + * + * There is no marker-lifetime constraint to satisfy alongside it. The marker is a field on the + * run's `seq` hash, so it lives exactly as long as the keyspace it describes: it cannot expire + * out from under a candidate that is still waiting for its second sighting, and it cannot outlive + * a keyspace that was deleted. + */ + confirmOrphanAfterMs?: number; logger?: Logger; }; export class SnapshotOrphanSweeper { - readonly #redis: Redis; + readonly #redis: RedisClient; + /** Only a client this class opened may be closed by it. */ + readonly #ownsClient: boolean; readonly #runStore: RunStore; readonly #completedTtlMs: number; readonly #orphanAgeMs: number; + readonly #confirmOrphanAfterMs: number; /** * The ioredis client-level prefix, which is NOT the keyspace prefix. ioredis prepends it to keys * for ordinary commands, but it does not prepend it to a SCAN MATCH pattern, and it does return @@ -94,13 +149,21 @@ export class SnapshotOrphanSweeper { this.#runStore = options.runStore; this.#completedTtlMs = options.completedTtlMs; this.#orphanAgeMs = options.orphanAgeMs ?? DEFAULT_ORPHAN_AGE_MS; - this.#clientPrefix = (options.redisOptions.keyPrefix as string | undefined) ?? ""; - this.#redis = createRedisClient(options.redisOptions, { - onError: (error) => this.#logger.error("SnapshotOrphanSweeper redis client error", { error }), - }); + this.#confirmOrphanAfterMs = options.confirmOrphanAfterMs ?? DEFAULT_ORPHAN_CONFIRM_MS; + this.#ownsClient = options.client === undefined; + this.#redis = + options.client ?? + createRedisClient(options.redisOptions, { + onError: (error) => + this.#logger.error("SnapshotOrphanSweeper redis client error", { error }), + }); + this.#clientPrefix = clientPrefixOf(this.#redis); } async quit(): Promise { + // A borrowed client belongs to the caller; closing it here would take down a connection the + // snapshot store may still be using. + if (!this.#ownsClient) return; if (!this.#quit) { this.#quit = this.#redis.quit().then( () => undefined, @@ -112,37 +175,93 @@ export class SnapshotOrphanSweeper { /** * One full pass over the keyspace. `dryRun` reports what it would do and changes nothing. + * + * `deadline` and `signal` let the caller stop a pass cleanly instead of having it killed + * mid-cursor. The scheduler needs this: redis-worker moves a dequeued item's score to + * `now + visibilityTimeoutMs` and nothing extends it, so a pass that outlives its timeout is + * redelivered and runs concurrently with itself. A pass that stops inside its budget cannot. + * + * Whichever way it stops, `partial` comes back true. Reporting a truncated pass as a full one is + * the same false green as under-scanning a cluster: TRI-13453 gates the dial on an OBSERVED + * sweep pass, so the observation has to say how much of the keyspace it actually reached. */ - async sweep(opts?: { batchSize?: number; dryRun?: boolean }): Promise { + async sweep(opts?: { + batchSize?: number; + dryRun?: boolean; + /** Epoch ms. The pass stops at the next batch boundary once passed. */ + deadline?: number; + signal?: AbortSignal; + }): Promise { const batchSize = opts?.batchSize ?? DEFAULT_BATCH_SIZE; const dryRun = opts?.dryRun ?? false; - const result: SweepResult = { scanned: 0, expired: 0, deleted: 0, skipped: 0 }; - - let cursor = "0"; - do { - // Match on the entry hash, not on `cur`. The append script writes `cur` only when the entry - // is valid, so a keyspace whose entries are all invalid would never be discovered and would - // leak with no expiry, which is the same unbounded leak rule 2 exists to close. `e` is - // written by every append. - const [next, keys] = await this.#redis.scan( - cursor, - "MATCH", - `${this.#clientPrefix}${SNAPSHOT_KEYSPACE_PREFIX}{*}:e`, - "COUNT", - batchSize - ); - cursor = next; + const result: SweepResult = { + scanned: 0, + expired: 0, + deleted: 0, + skipped: 0, + pendingDeletion: 0, + nodes: 0, + partial: false, + }; + + // Checked at batch boundaries only. Stopping mid-batch would leave a run half-acted-on, and a + // batch is bounded work, so the boundary is both the safe and the timely place. + const outOfBudget = () => + opts?.signal?.aborted === true || + (opts?.deadline !== undefined && Date.now() >= opts.deadline); + + // SCAN carries no key, so a cluster cannot route it: one connection iterates ONE node's + // keyspace and then reports a completed cursor. A single-client sweep against a cluster would + // therefore return a clean-looking result having examined roughly 1/N of the keyspace, and the + // rest would leak with nothing to revisit it. Both rules are unbounded leaks when missed, so + // the pass fans out over every master and only reports done when all of them are done. + const nodes = this.#scanTargets(); + result.nodes = nodes.length; + + for (const node of nodes) { + if (outOfBudget()) { + result.partial = true; + break; + } - const runIds = [...new Set(keys.map((key) => this.#runIdFrom(key)).filter(isString))]; - if (runIds.length === 0) continue; + let cursor = "0"; + do { + // Match on the entry hash, not on `cur`. The append script writes `cur` only when the entry + // is valid, so a keyspace whose entries are all invalid would never be discovered and would + // leak with no expiry, which is the same unbounded leak rule 2 exists to close. `e` is + // written by every append. + const [next, keys] = await node.scan( + cursor, + "MATCH", + `${this.#clientPrefix}${SNAPSHOT_KEYSPACE_PREFIX}{*}:e`, + "COUNT", + batchSize + ); + cursor = next; + + const runIds = [...new Set(keys.map((key) => this.#runIdFrom(key)).filter(isString))]; + if (runIds.length === 0) continue; + + await this.#sweepBatch(runIds, dryRun, result); + + if (outOfBudget()) { + // A cursor mid-iteration means this node is not finished, so the pass is not either. + result.partial = true; + break; + } + } while (cursor !== "0"); - await this.#sweepBatch(runIds, dryRun, result); - } while (cursor !== "0"); + if (result.partial) break; + } this.#logger.log("SnapshotOrphanSweeper pass complete", { ...result, dryRun }); return result; } + #scanTargets(): Redis[] { + return scanTargetsOf(this.#redis); + } + async #sweepBatch(runIds: string[], dryRun: boolean, result: SweepResult): Promise { result.scanned += runIds.length; @@ -162,6 +281,17 @@ export class SnapshotOrphanSweeper { return; } + // Every run that EXISTS clears its rule 2 marker, live ones included. It has to be every one, + // not just the terminal ones: a keyspace marked by an incomplete lookup, then seen alive, then + // missed again would otherwise present a mature marker on what is really a first sighting, and + // the two-sighting rule would be gone exactly when it was needed. This costs one DEL per + // existing run per pass, which is the price of the guard being sound rather than nearly sound. + const present = runIds.filter((runId) => rows.has(runId)); + if (!dryRun && present.length > 0) { + // Individual commands, never one pipeline: these keys span runs, so they span cluster slots. + await Promise.all(present.map((runId) => this.#clearOrphanMarker(runId))); + } + for (const runId of runIds) { const run = rows.get(runId); @@ -206,7 +336,23 @@ export class SnapshotOrphanSweeper { result.expired += 1; } - /** Rule 2: a keyspace with no run row at all, past the age threshold. */ + /** + * Rule 2: a keyspace with no run row at all, past the age threshold. + * + * TWO SIGHTINGS ARE REQUIRED. The `catch` in #sweepBatch covers a lookup that THROWS, but it + * cannot see a lookup that succeeds and is incomplete: a row that exists but did not come back + * reads exactly like a run that never existed, and the response to that is deleting a live run's + * execution state. `findRunsByIds` routes through RoutingRunStore.#findRunsByIdSet, which + * partitions ids by residency and asks each store only for its own — and with no client passed it + * reads each store's REPLICA. Both are sound today (id classification is authoritative for runs, + * and replica lag is nowhere near the 24h age gate), but each is an assumption held somewhere + * else in the codebase, not something this delete path can check. + * + * The asymmetry decides it: a false negative leaks keys, which is bounded and recoverable, while + * a false positive destroys live state. So an absent row marks the keyspace and returns; only a + * candidate still absent in a LATER pass is deleted. Any transient incomplete answer, whatever + * its cause, has to occur twice across the confirm window to do damage. + */ async #applyRuleTwo(runId: string, dryRun: boolean, result: SweepResult): Promise { const keys = await this.#allKeys(runId); if (keys.length === 0) { @@ -221,13 +367,54 @@ export class SnapshotOrphanSweeper { return; } + const seqKey = snapshotKeys(runId).seq; + const markedAtRaw = await this.#redis.hget(seqKey, ORPHAN_MARKER_FIELD); + const markedAt = markedAtRaw === null ? undefined : Number(markedAtRaw); + + if (markedAt === undefined || Number.isNaN(markedAt)) { + if (!dryRun) { + // The TTL is a multiple of the confirm window so a candidate gets several chances to be + // sighted again, while a marker left behind by a run that turned out to be alive cannot + // linger long enough to pre-authorise a later deletion. + // The field carries no TTL of its own; it lives and dies with the seq hash, which the + // keyspace's own completion expiry already governs. That removes the marker-lifetime knob + // whose derivation was wrong in the first place. + await this.#redis.hset(seqKey, ORPHAN_MARKER_FIELD, String(Date.now())); + } + result.pendingDeletion += 1; + return; + } + + if (Date.now() - markedAt < this.#confirmOrphanAfterMs) { + result.pendingDeletion += 1; + return; + } + if (!dryRun) { + // One slot: every key here carries the same `{runId}` hash tag. The marker is a field on + // `seq`, which is in `keys`, so it goes with the keyspace rather than needing its own entry. await this.#redis.del(...keys); } result.deleted += 1; } + /** + * Clears a rule 2 marker for a keyspace whose run turned out to exist after all, so a later + * genuine absence still needs its own two sightings rather than inheriting a stale one. + * + * Only called on a path that already found a run row, and only for terminal runs — a live run + * never reaches rule 2, so it can never hold a marker, and charging every live keyspace a round + * trip to prove that would cost more than the case is worth. + */ + async #clearOrphanMarker(runId: string): Promise { + try { + await this.#redis.hdel(snapshotKeys(runId).seq, ORPHAN_MARKER_FIELD); + } catch { + // Best effort. A marker that outlives its usefulness expires on its own TTL. + } + } + /** * Every key for one run: the four core keys plus each wait-cycle key. * @@ -255,7 +442,9 @@ export class SnapshotOrphanSweeper { const candidates = [core.e, core.idx, core.cur, core.seq, ...cycles]; - // One round trip for the whole set, rather than one per candidate. + // One round trip for the whole set, rather than one per candidate. Cluster-safe: every key here + // carries the same `{runId}` hash tag, so the whole pipeline lands in one slot on one node. + // The same holds for the pexpire pipeline and the multi-key DEL above. const pipeline = this.#redis.pipeline(); for (const key of candidates) { pipeline.exists(key); @@ -333,6 +522,46 @@ export class SnapshotOrphanSweeper { } } +/** + * Every connection a pass must iterate to cover the whole keyspace: each master of a cluster, or + * the one standalone connection. Replicas are excluded — they hold the same keys as their master, + * so scanning them would double-count and act on one keyspace twice. + * + * Module-level and exported so the fan-out decision can be pinned on its own. It is the whole of + * the defect this guards against: everything the sweep does AFTER the scan is key-addressed and a + * cluster client routes it correctly without help, so the node list is the only place a cluster + * can silently cost the pass coverage. + * + * Resolved per pass, never cached: cluster topology changes under failover and resharding, and a + * stale node list is the same silent under-scan this exists to prevent. + */ + +/** + * Rule 2's "seen absent once" marker is a FIELD on the run's `seq` hash, not a key of its own. + * + * As a separate key its removal depended on the deleting call site remembering to append it to the + * DEL, which is the kind of contract a later edit breaks with no test noticing: a marker outliving + * its keyspace would let a recreated keyspace be deleted on what is really a first sighting. `seq` + * is already in `#allKeys`, so as a field the marker cannot outlive the keyspace at all. + */ +const ORPHAN_MARKER_FIELD = "orph"; + +export function scanTargetsOf(client: RedisClient): Redis[] { + return client instanceof Cluster ? client.nodes("master") : [client]; +} + +/** + * The ioredis client-level prefix for either endpoint shape. On a Cluster it lives on the nested + * `redisOptions`, not on the top-level options, and reading the wrong one yields "" — which makes + * every SCAN MATCH miss and the pass report a clean sweep of nothing. + */ +export function clientPrefixOf(client: RedisClient): string { + if (client instanceof Cluster) { + return (client.options.redisOptions?.keyPrefix as string | undefined) ?? ""; + } + return (client.options.keyPrefix as string | undefined) ?? ""; +} + function isString(value: string | undefined): value is string { return typeof value === "string"; } diff --git a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.waitpointCycles.test.ts b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.waitpointCycles.test.ts index 9f40e54c599..4bc2d0dc271 100644 --- a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.waitpointCycles.test.ts +++ b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.waitpointCycles.test.ts @@ -602,6 +602,49 @@ describe("completed-waitpoint cycles", () => { } ); + containerTest( + "the since-window falls back to Postgres when the head cycle key is gone", + async ({ prisma, redisOptions }) => { + // The hot read has always handled this. The since-window did not: its Lua returned the head's + // order and distinct set but never its dangling flag, so the decorator's fallback guard was + // dead code and an expired cycle key came back as an EMPTY order. Empty means "no indexed + // waitpoints" to the engine, which is how a batched triggerAndWait resumes with every + // position lost rather than falling back to the store that still knows them. + const { decorated, redis } = build(prisma as never, redisOptions as never); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, redis, env); + const [wpA, wpB] = await seedSnapshotWaitpoints(prisma, env, 2); + const first = await decorated.createExecutionSnapshot( + resumeInput(runId, env, [], "before the wait") + ); + await new Promise((resolve) => setTimeout(resolve, 5)); + await decorated.createExecutionSnapshot( + resumeInput(runId, env, [ + { id: wpA, index: 0 }, + { id: wpB, index: 1 }, + ]) + ); + + // The entry hash survives; only the cycle key goes. The completion TTL is applied per key, + // so this is a state the keyspace reaches on its own. + for (const key of await probe.keys(`snap:{${runId}}:wp:*`)) await probe.del(key); + + const window = await decorated.findManyExecutionSnapshots({ + where: { runId, isValid: true, createdAt: { gt: first.createdAt } }, + include: { checkpoint: true }, + orderBy: { createdAt: "desc" }, + take: 50, + }); + + expect(window[0]!.completedWaitpointOrder).toEqual([wpA, wpB]); + } finally { + await Promise.all([redis.quit(), probe.quit().catch(() => {})]); + } + } + ); + containerTest( "lockRunToWorker carries its resolved order into the cycle", async ({ prisma, redisOptions }) => { From 2ffd2753115398b4fdd9a547759bda70c719d568 Mon Sep 17 00:00:00 2001 From: Daniel Sutton Date: Tue, 25 Aug 2026 18:18:47 +0100 Subject: [PATCH 28/56] feat(redis): add a cluster-capable client to the shared package The package had no cluster support: createRedisClient builds a single-node client and there was no Cluster export. A store whose single-slot guarantee is load-bearing only against a cluster-mode endpoint therefore could not reach that endpoint at all. Cluster is exported as a value, not just a type, because callers need instanceof to fan a keyless command out over master nodes. defaultOptions (the retry strategy, maxRetriesPerRequest, and reconnectOnError) are applied to the inner per-node options, so a role swap on a cluster gets the same reconnect-and-retry treatment a single-node client already gets. The webapp's own cluster helper sets neither of the first two. The package had no test runner; adds the same minimal vitest config @internal/cache uses. --- internal-packages/redis/package.json | 4 +- internal-packages/redis/src/cluster.test.ts | 73 +++++++++++++++++++++ internal-packages/redis/src/index.ts | 57 +++++++++++++++- internal-packages/redis/vitest.config.ts | 10 +++ 4 files changed, 141 insertions(+), 3 deletions(-) create mode 100644 internal-packages/redis/src/cluster.test.ts create mode 100644 internal-packages/redis/vitest.config.ts diff --git a/internal-packages/redis/package.json b/internal-packages/redis/package.json index 14cf7f33d61..eff0a779dd7 100644 --- a/internal-packages/redis/package.json +++ b/internal-packages/redis/package.json @@ -10,6 +10,8 @@ "@trigger.dev/core": "workspace:*" }, "scripts": { - "typecheck": "tsc --noEmit" + "typecheck": "tsc --noEmit", + "test": "vitest", + "test:watch": "vitest" } } diff --git a/internal-packages/redis/src/cluster.test.ts b/internal-packages/redis/src/cluster.test.ts new file mode 100644 index 00000000000..71d2989b22f --- /dev/null +++ b/internal-packages/redis/src/cluster.test.ts @@ -0,0 +1,73 @@ +import { describe, expect, it } from "vitest"; +import { Cluster, createRedisClusterClient, defaultReconnectOnError } from "./index.js"; + +// Port 6399 is deliberately closed: these assertions are about the client the factory builds, not +// about a connection. +const NODES = [{ host: "127.0.0.1", port: 6399 }]; + +type InnerOptions = { + options: { + redisOptions?: { + reconnectOnError?: unknown; + maxRetriesPerRequest?: number; + retryStrategy?: unknown; + keyPrefix?: string; + }; + }; +}; + +function innerOptionsOf(client: Cluster) { + return (client as unknown as InnerOptions).options.redisOptions; +} + +describe("createRedisClusterClient", () => { + it("returns a Cluster instance", async () => { + const client = createRedisClusterClient({ nodes: NODES }); + try { + expect(client).toBeInstanceOf(Cluster); + } finally { + await client.quit().catch(() => undefined); + } + }); + + it("installs defaultReconnectOnError on the inner per-node options", async () => { + const client = createRedisClusterClient({ nodes: NODES }); + try { + expect(innerOptionsOf(client)?.reconnectOnError).toBe(defaultReconnectOnError); + } finally { + await client.quit().catch(() => undefined); + } + }); + + it("carries the retry defaults onto the inner options", async () => { + const client = createRedisClusterClient({ nodes: NODES }); + try { + const inner = innerOptionsOf(client); + expect(inner?.maxRetriesPerRequest).toBeTypeOf("number"); + expect(inner?.retryStrategy).toBeTypeOf("function"); + } finally { + await client.quit().catch(() => undefined); + } + }); + + it("lets caller redisOptions override the defaults", async () => { + const client = createRedisClusterClient({ + nodes: NODES, + redisOptions: { keyPrefix: "engine:", maxRetriesPerRequest: 3 }, + }); + try { + const inner = innerOptionsOf(client); + expect(inner?.keyPrefix).toBe("engine:"); + expect(inner?.maxRetriesPerRequest).toBe(3); + } finally { + await client.quit().catch(() => undefined); + } + }); + + it("keeps mapping READONLY, LOADING and UNBLOCKED to a reconnect-and-retry", () => { + expect(defaultReconnectOnError(new Error("READONLY against a read only replica"))).toBe(2); + expect(defaultReconnectOnError(new Error("LOADING Redis is loading the dataset"))).toBe(2); + expect(defaultReconnectOnError(new Error("UNBLOCKED force unblock"))).toBe(2); + expect(defaultReconnectOnError(new Error("ERR unknown command"))).toBe(false); + }); +}); diff --git a/internal-packages/redis/src/index.ts b/internal-packages/redis/src/index.ts index 6fb30b9a4bf..967f3939395 100644 --- a/internal-packages/redis/src/index.ts +++ b/internal-packages/redis/src/index.ts @@ -1,7 +1,19 @@ -import { Redis, type RedisOptions } from "ioredis"; +import { Cluster, Redis, type ClusterNode, type ClusterOptions, type RedisOptions } from "ioredis"; import { Logger } from "@trigger.dev/core/logger"; -export { Redis, type Callback, type RedisOptions, type Result, type RedisCommander } from "ioredis"; +export { + Redis, + Cluster, + type Callback, + type RedisOptions, + type ClusterNode, + type ClusterOptions, + type Result, + type RedisCommander, +} from "ioredis"; + +/** Single-node or cluster-mode. Interchangeable for callers whose keys share one hash tag. */ +export type RedisClient = Redis | Cluster; /** * Reply-error -> reconnect mapping. Without this hook, an ElastiCache @@ -68,3 +80,44 @@ export function createRedisClient( return client; } + +export type RedisClusterClientOptions = { + nodes: ClusterNode[]; + clusterOptions?: Omit; + redisOptions?: RedisOptions; +}; + +/** + * Cluster-mode client. `defaultOptions` go on the INNER per-node options, so a role swap gets the + * same reconnect-and-retry treatment a single-node client already gets. + */ +export function createRedisClusterClient( + options: RedisClusterClientOptions, + handlers?: { onError?: (err: Error) => void } +): Cluster { + const client = new Redis.Cluster(options.nodes, { + ...options.clusterOptions, + redisOptions: { + ...defaultOptions, + ...options.redisOptions, + }, + }); + + if (process.env.VITEST) { + client.on("error", () => {}); + return client; + } + + client.on("error", (error) => { + if (handlers?.onError) { + handlers.onError(error); + } else { + logger.error(`Redis cluster client error:`, { + error, + keyPrefix: options.redisOptions?.keyPrefix, + }); + } + }); + + return client; +} diff --git a/internal-packages/redis/vitest.config.ts b/internal-packages/redis/vitest.config.ts new file mode 100644 index 00000000000..e07f05e842b --- /dev/null +++ b/internal-packages/redis/vitest.config.ts @@ -0,0 +1,10 @@ +import { defineConfig } from "vitest/config"; + +export default defineConfig({ + test: { + include: ["**/*.test.ts"], + globals: true, + isolate: true, + testTimeout: 10_000, + }, +}); From 0f79a2722fa5a5b2c0816868fae9a0a10e29d277 Mon Sep 17 00:00:00 2001 From: Daniel Sutton Date: Tue, 25 Aug 2026 18:25:46 +0100 Subject: [PATCH 29/56] feat(webapp): add the snapshot-store dial flags, and strip the global key from org payloads The dial is a feature flag rather than an environment variable: a sustained append failure burns a task attempt on every state transition, so dialling down is a correctness control and cannot wait for a deploy. Two keys, because they must accept different values. The global key holds all five positions. The per-organisation key holds the write positions only: snapshot reads are global, so an organisation at a read position would read state its own writes never created. The narrow enum makes that unrepresentable rather than documented. ORG_LOCKED_FLAGS turns out to enforce nothing: it is a client-side predicate in the admin dialog and no save path consults it. The line is therefore held the same way the mint grace stamps hold it, by stripping the key in both organisation routes. --- ...i.v1.orgs.$organizationId.feature-flags.ts | 6 ++- ...i.v2.orgs.$organizationId.feature-flags.ts | 5 +- apps/webapp/app/v3/featureFlags.ts | 28 ++++++++++ apps/webapp/test/snapshotStoreFlags.test.ts | 52 +++++++++++++++++++ 4 files changed, 88 insertions(+), 3 deletions(-) create mode 100644 apps/webapp/test/snapshotStoreFlags.test.ts diff --git a/apps/webapp/app/routes/admin.api.v1.orgs.$organizationId.feature-flags.ts b/apps/webapp/app/routes/admin.api.v1.orgs.$organizationId.feature-flags.ts index db1524f8551..ae93b9e70f4 100644 --- a/apps/webapp/app/routes/admin.api.v1.orgs.$organizationId.feature-flags.ts +++ b/apps/webapp/app/routes/admin.api.v1.orgs.$organizationId.feature-flags.ts @@ -7,7 +7,7 @@ import { prisma } from "~/db.server"; import { requireAdminApiRequest } from "~/services/personalAccessToken.server"; import { controlPlaneResolver } from "~/v3/runOpsMigration/controlPlaneResolver.server"; import { selectMintBaselineSource, stampMintKindFlip } from "~/v3/runOpsMigration/mintFlipGrace"; -import { validatePartialFeatureFlags } from "~/v3/featureFlags"; +import { validatePartialFeatureFlags, withoutOrgForbiddenSnapshotKeys } from "~/v3/featureFlags"; import { flags as getGlobalFlags } from "~/v3/featureFlags.server"; const ParamsSchema = z.object({ @@ -71,9 +71,11 @@ export async function action({ request, params }: ActionFunctionArgs) { const { runOpsMintKindPrev: _ignoredPrev, runOpsMintKindFlippedAt: _ignoredFlippedAt, - ...requestedFlags + ...rawRequestedFlags } = validationResult.data; + const requestedFlags = withoutOrgForbiddenSnapshotKeys(rawRequestedFlags); + // Seed the flip baseline from the current GLOBAL mint flags so an org's FIRST per-org override // is graced from the currently-effective global kind, not the hardcoded default "cuid". const globalFlags = (await getGlobalFlags()) as Record; diff --git a/apps/webapp/app/routes/admin.api.v2.orgs.$organizationId.feature-flags.ts b/apps/webapp/app/routes/admin.api.v2.orgs.$organizationId.feature-flags.ts index 0071054bd3e..7366d84a524 100644 --- a/apps/webapp/app/routes/admin.api.v2.orgs.$organizationId.feature-flags.ts +++ b/apps/webapp/app/routes/admin.api.v2.orgs.$organizationId.feature-flags.ts @@ -11,6 +11,7 @@ import { flags as getGlobalFlags } from "~/v3/featureFlags.server"; import { FEATURE_FLAG, validatePartialFeatureFlags, + withoutOrgForbiddenSnapshotKeys, getAllFlagControlTypes, } from "~/v3/featureFlags"; import { featuresForRequest } from "~/features.server"; @@ -138,9 +139,11 @@ export async function action({ request, params }: ActionFunctionArgs) { const { runOpsMintKindPrev: _ignoredPrev, runOpsMintKindFlippedAt: _ignoredFlippedAt, - ...requestedFlags + ...rawRequestedFlags } = validationResult.data; + const requestedFlags = withoutOrgForbiddenSnapshotKeys(rawRequestedFlags); + // Seed the flip baseline from the current GLOBAL mint flags so an org's FIRST per-org override // is graced from the currently-effective global kind, not the hardcoded default "cuid". const globalFlags = (await getGlobalFlags()) as Record; diff --git a/apps/webapp/app/v3/featureFlags.ts b/apps/webapp/app/v3/featureFlags.ts index 3a88beb54bc..8406fc11a61 100644 --- a/apps/webapp/app/v3/featureFlags.ts +++ b/apps/webapp/app/v3/featureFlags.ts @@ -44,6 +44,14 @@ export const FEATURE_FLAG = { // System-wide kill switch for additional (scoped) environment API-key lookup. // Defaults off; enable during rollout once the new lookup path is trusted. additionalApiKeyLookupEnabled: "additionalApiKeyLookupEnabled", + // The execution-snapshot store rollout dial. A flag rather than an environment variable because + // a sustained append failure burns a task attempt per transition, so dial-down is a correctness + // control and cannot wait for a deploy. + snapshotStoreMode: "snapshotStoreMode", + // Per-org override, read from the org blob only. Deliberately narrower than the global key: + // snapshot reads are global, so an org at a read position would read state its own writes never + // created. Stripped from org payloads by withoutOrgForbiddenSnapshotKeys. + snapshotStoreOrgMode: "snapshotStoreOrgMode", } as const; export const FeatureFlagCatalog = { @@ -153,6 +161,14 @@ export const FeatureFlagCatalog = { [FEATURE_FLAG.additionalApiKeysEnabled]: z.boolean(), [FEATURE_FLAG.additionalApiKeyIssuanceEnabled]: z.boolean(), [FEATURE_FLAG.additionalApiKeyLookupEnabled]: z.boolean(), + [FEATURE_FLAG.snapshotStoreMode]: z.enum([ + "off", + "dual-write", + "compare", + "redis-read", + "redis-only", + ]), + [FEATURE_FLAG.snapshotStoreOrgMode]: z.enum(["off", "dual-write", "compare"]), }; export type FeatureFlagKey = keyof typeof FeatureFlagCatalog; @@ -188,8 +204,20 @@ export const ORG_LOCKED_FLAGS: FeatureFlagKey[] = [ FEATURE_FLAG.runOpsMintShardSetPrev, FEATURE_FLAG.runOpsMintShardSetFlippedAt, FEATURE_FLAG.runOpsMintShardOverride, + // The dial is deployment-wide; only snapshotStoreOrgMode is per-org. + FEATURE_FLAG.snapshotStoreMode, ]; +/** + * Drops keys an organisation must never supply. ORG_LOCKED_FLAGS is a UI predicate and no save path + * consults it, so the line is held here — the same way the mint grace stamps are stripped. + */ +export function withoutOrgForbiddenSnapshotKeys>(values: T): T { + if (!(FEATURE_FLAG.snapshotStoreMode in values)) return values; + const { [FEATURE_FLAG.snapshotStoreMode]: _dropped, ...rest } = values; + return rest as T; +} + /** * Flag groups where the operator sets a `primary` and the server computes the rest. The topology * lives here, not in the server module, because the admin page needs it too: unsetting a primary diff --git a/apps/webapp/test/snapshotStoreFlags.test.ts b/apps/webapp/test/snapshotStoreFlags.test.ts new file mode 100644 index 00000000000..9cbce38c7de --- /dev/null +++ b/apps/webapp/test/snapshotStoreFlags.test.ts @@ -0,0 +1,52 @@ +import { describe, expect, it } from "vitest"; +import { + FeatureFlagCatalog, + ORG_LOCKED_FLAGS, + withoutOrgForbiddenSnapshotKeys, +} from "~/v3/featureFlags"; + +describe("snapshot store dial catalog", () => { + it("accepts all five positions globally", () => { + for (const value of ["off", "dual-write", "compare", "redis-read", "redis-only"]) { + expect(FeatureFlagCatalog.snapshotStoreMode.safeParse(value).success).toBe(true); + } + }); + + it("rejects an unknown position", () => { + expect(FeatureFlagCatalog.snapshotStoreMode.safeParse("redis-write").success).toBe(false); + }); + + it("accepts only write positions per organisation", () => { + for (const value of ["off", "dual-write", "compare"]) { + expect(FeatureFlagCatalog.snapshotStoreOrgMode.safeParse(value).success).toBe(true); + } + // Reads are global, so an org at a read position would read state its own writes never created. + expect(FeatureFlagCatalog.snapshotStoreOrgMode.safeParse("redis-read").success).toBe(false); + expect(FeatureFlagCatalog.snapshotStoreOrgMode.safeParse("redis-only").success).toBe(false); + }); + + it("lists the global dial as org-locked", () => { + expect(ORG_LOCKED_FLAGS).toContain("snapshotStoreMode"); + }); +}); + +describe("withoutOrgForbiddenSnapshotKeys", () => { + it("removes the global dial and keeps everything else", () => { + expect( + withoutOrgForbiddenSnapshotKeys({ + snapshotStoreMode: "redis-only", + snapshotStoreOrgMode: "dual-write", + runOpsMintKind: "cuid", + }) + ).toEqual({ snapshotStoreOrgMode: "dual-write", runOpsMintKind: "cuid" }); + }); + + it("returns the same object when the dial is absent", () => { + const input = { runOpsMintKind: "cuid" }; + expect(withoutOrgForbiddenSnapshotKeys(input)).toBe(input); + }); + + it("removes the dial even when it is the only key", () => { + expect(withoutOrgForbiddenSnapshotKeys({ snapshotStoreMode: "dual-write" })).toEqual({}); + }); +}); From bbcd8c7ad64e1d21aa2e9ad0075a47cff2bf52df Mon Sep 17 00:00:00 2001 From: Daniel Sutton Date: Tue, 25 Aug 2026 18:39:58 +0100 Subject: [PATCH 30/56] feat(run-store): resolve the snapshot dial per write through an injected resolver The dial has to be movable at runtime, so `mode` stops being a constructor constant: it becomes a getter over an injected resolver, and each write guard resolves from its own entry's organizationId. Every snapshot input type already carries organizationId as a required field, so no plumbing is needed to reach it. The resolver is deliberately synchronous. Seven methods here accept a caller-supplied tx, so this class cannot see a caller's transaction boundary, and an awaited read could land inside someone else's open interactive transaction, on the same connection pool for single-DB and self-host. Two behavioural changes, both intentional: runInTransaction always installs the staging facade. It holds only a runId, and a per-organisation dial lives in that organisation's row, so it cannot know whether any write inside will reach Redis. Each entry resolves its own mode as it is staged, and an organisation at off stages nothing. forWaitpointCompletion always wraps its handle. Leaving it unwrapped was already described in-file as the one hole a future snapshot write could slip through unnoticed, and a runtime dial widens that. The guards read organizationId with optional chaining so a malformed input is still forwarded untouched at off, exactly as before. Three existing assertions covered object identity rather than behaviour and are updated to the behaviour they were standing in for: at off nothing is appended to Redis, which the exploding-Redis fixture already proves. --- ...xecutionSnapshotStore.modeResolver.test.ts | 60 +++++++++++++++++ .../taskRunExecutionSnapshotStore.off.test.ts | 28 ++++++-- ...kRunExecutionSnapshotStore.staging.test.ts | 13 ++-- .../src/taskRunExecutionSnapshotStore.ts | 64 ++++++++++++------- 4 files changed, 133 insertions(+), 32 deletions(-) create mode 100644 internal-packages/run-store/src/taskRunExecutionSnapshotStore.modeResolver.test.ts diff --git a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.modeResolver.test.ts b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.modeResolver.test.ts new file mode 100644 index 00000000000..2175de9f097 --- /dev/null +++ b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.modeResolver.test.ts @@ -0,0 +1,60 @@ +import { describe, expect, it } from "vitest"; +import { + TaskRunExecutionSnapshotStore, + type SnapshotStoreMode, + type SnapshotStoreModeResolver, +} from "./taskRunExecutionSnapshotStore.js"; + +function storeWith(options: { + mode?: SnapshotStoreMode; + modeResolver?: SnapshotStoreModeResolver; +}) { + return new TaskRunExecutionSnapshotStore({} as never, { + store: {} as never, + ...options, + }); +} + +function resolverOf(perOrg: Record, global: SnapshotStoreMode) { + return { resolve: (orgId?: string) => (orgId ? (perOrg[orgId] ?? global) : global) }; +} + +describe("TaskRunExecutionSnapshotStore mode resolution", () => { + it("prefers the resolver over the static mode", () => { + const store = storeWith({ mode: "off", modeResolver: resolverOf({}, "dual-write") }); + expect(store.mode).toBe("dual-write"); + }); + + it("falls back to the static mode when no resolver is supplied", () => { + expect(storeWith({ mode: "compare" }).mode).toBe("compare"); + }); + + it("defaults to off with neither", () => { + expect(storeWith({}).mode).toBe("off"); + }); + + it("resolves per organisation", () => { + const store = storeWith({ modeResolver: resolverOf({ org_a: "dual-write" }, "off") }); + expect(store.writesRedisForTest("org_a")).toBe(true); + expect(store.writesRedisForTest("org_b")).toBe(false); + }); + + it("lets an organisation be off while the global answer is on", () => { + const store = storeWith({ modeResolver: resolverOf({ org_a: "off" }, "dual-write") }); + expect(store.writesRedisForTest("org_a")).toBe(false); + expect(store.writesRedisForTest("org_b")).toBe(true); + }); + + it("sees a resolver answer that changes after construction", () => { + let current: SnapshotStoreMode = "off"; + const store = storeWith({ modeResolver: { resolve: () => current } }); + expect(store.mode).toBe("off"); + current = "dual-write"; + expect(store.mode).toBe("dual-write"); + }); + + it("resolves the global answer when no organisation is supplied", () => { + const store = storeWith({ modeResolver: resolverOf({ org_a: "off" }, "redis-read") }); + expect(store.writesRedisForTest()).toBe(true); + }); +}); diff --git a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.off.test.ts b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.off.test.ts index 4f788b5ec27..994d807c42a 100644 --- a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.off.test.ts +++ b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.off.test.ts @@ -53,15 +53,31 @@ describe("TaskRunExecutionSnapshotStore at mode off", () => { mode: "off", }) as unknown as Record unknown>; + // Both exceptions return a wrapped handle rather than the delegate's value verbatim, because + // the dial can move at runtime and an unwrapped handle would let a later write bypass the + // decorator with no signal. Every other method is still a pass-through, and the exploding + // Redis store is what proves none of them reaches Redis. + const WRAPPED = ["runInTransaction", "forWaitpointCompletion"]; + for (const name of RUN_STORE_METHOD_NAMES) { - if (name === "runInTransaction") continue; + if (WRAPPED.includes(name)) continue; expect(await decorated[name]("arg-one", "arg-two")).toBe(`result:${name}`); } - expect(calls).toEqual(RUN_STORE_METHOD_NAMES.filter((n) => n !== "runInTransaction")); + const handle = await decorated.forWaitpointCompletion("waitpoint", {}); + expect(handle).toBeInstanceOf(TaskRunExecutionSnapshotStore); + expect((handle as unknown as { delegate: unknown }).delegate).toBe( + "result:forWaitpointCompletion" + ); + + // forWaitpointCompletion is called after the loop, so it lands last rather than in place. + expect(calls).toEqual([ + ...RUN_STORE_METHOD_NAMES.filter((n) => !WRAPPED.includes(n)), + "forWaitpointCompletion", + ]); }); - it("hands the delegate's own store to a transaction callback", async () => { + it("wraps the transaction callback's store but never reaches Redis", async () => { const inner = forwardingProbe().store; let seen: unknown; const delegate = { @@ -82,7 +98,11 @@ describe("TaskRunExecutionSnapshotStore at mode off", () => { seen = store; }); - expect(seen).toBe(inner); + // The facade is always installed: this method holds only a runId, so it cannot know whether a + // per-organisation dial would put any write inside on Redis. The exploding Redis store is what + // proves the position still costs nothing at `off`. + expect(seen).toBeInstanceOf(TaskRunExecutionSnapshotStore); + expect(seen).not.toBe(inner); }); it("reports every other dial position as one that writes Redis", () => { diff --git a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.staging.test.ts b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.staging.test.ts index 9952f569e2b..0c86aad7d14 100644 --- a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.staging.test.ts +++ b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.staging.test.ts @@ -216,7 +216,7 @@ describe("the staging facade", () => { ); containerTest( - "hands the transaction callback the plain delegate at mode off", + "stages nothing and touches no Redis inside a transaction at mode off", async ({ prisma, redisOptions }) => { const { decorated, redis } = build(prisma as never, redisOptions as never, "off"); try { @@ -242,7 +242,10 @@ describe("the staging facade", () => { seen = store; }); - expect(seen).not.toBeInstanceOf(TaskRunExecutionSnapshotStore); + // The callback now always receives the staging facade: this method holds only a runId, and + // a per-organisation dial lives in that organisation's blob, so it cannot know whether any + // write inside will reach Redis. What must hold at `off` is that nothing is appended. + expect(seen).toBeInstanceOf(TaskRunExecutionSnapshotStore); expect(await redis.getLatest(runId)).toBeNull(); } finally { await redis.quit(); @@ -269,7 +272,7 @@ describe("the staging facade", () => { ); containerTest( - "returns the plain handle from forWaitpointCompletion at mode off", + "wraps the forWaitpointCompletion handle at mode off too", async ({ prisma, redisOptions }) => { const { decorated, redis } = build(prisma as never, redisOptions as never, "off"); try { @@ -277,7 +280,9 @@ describe("the staging facade", () => { routeKind: "MANUAL", } as never); - expect(handle).not.toBeInstanceOf(TaskRunExecutionSnapshotStore); + // Leaving it unwrapped was the one hole a future snapshot write could slip through with no + // signal, and the dial can now move at runtime, so the handle is wrapped at every position. + expect(handle).toBeInstanceOf(TaskRunExecutionSnapshotStore); } finally { await redis.quit(); } diff --git a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.ts b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.ts index f4a46cb0414..c4199550cf3 100644 --- a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.ts +++ b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.ts @@ -65,6 +65,15 @@ const WAITPOINT_CHUNK_SIZE = 100; */ export type SnapshotStoreMode = "off" | "dual-write" | "compare" | "redis-read" | "redis-only"; +/** + * Resolves the dial for one write. MUST be synchronous and MUST NOT query: seven methods here take + * a caller-supplied `tx`, so this class cannot see a caller's transaction boundary, and a read + * issued here could land inside someone else's open interactive transaction. + */ +export type SnapshotStoreModeResolver = { + resolve(organizationId?: string): SnapshotStoreMode; +}; + /** * Enqueues the existing `repairSnapshot` job for a run whose append was lost. The decorator lives in * run-store and cannot reach the engine's worker, so the binding is injected. That binding must @@ -87,6 +96,8 @@ export type TaskRunExecutionSnapshotStoreOptions = { store: RedisSnapshotStore; /** Defaults to `off`, which is a pure pass-through that never touches Redis. */ mode?: SnapshotStoreMode; + /** Takes precedence over `mode`, and is re-read on every write so the dial can move at runtime. */ + modeResolver?: SnapshotStoreModeResolver; /** Percentage of runs whose reads come from Redis at `redis-read` and `redis-only`. Defaults to 0. */ readPercent?: number; onAppendFailure?: SnapshotRepairEnqueuer; @@ -114,7 +125,8 @@ export type StagedAppend = { }; export class TaskRunExecutionSnapshotStore extends DelegatingRunStore { - readonly mode: SnapshotStoreMode; + readonly #staticMode: SnapshotStoreMode; + protected readonly modeResolver?: SnapshotStoreModeResolver; protected readonly redis: RedisSnapshotStore; protected readonly readPercent: number; protected readonly onAppendFailure?: SnapshotRepairEnqueuer; @@ -126,7 +138,8 @@ export class TaskRunExecutionSnapshotStore extends DelegatingRunStore { constructor(delegate: RunStore, options: TaskRunExecutionSnapshotStoreOptions) { super(delegate); this.redis = options.store; - this.mode = options.mode ?? "off"; + this.#staticMode = options.mode ?? "off"; + this.modeResolver = options.modeResolver; this.readPercent = options.readPercent ?? 0; this.onAppendFailure = options.onAppendFailure; this.faults = options.faults; @@ -135,9 +148,18 @@ export class TaskRunExecutionSnapshotStore extends DelegatingRunStore { this.staging = options.staging; } - /** True in every position that appends to Redis. */ - protected get writesRedis(): boolean { - return this.mode !== "off"; + get mode(): SnapshotStoreMode { + return this.modeResolver?.resolve() ?? this.#staticMode; + } + + /** True in every position that appends to Redis, for this entry's organisation. */ + protected writesRedisFor(organizationId?: string): boolean { + return (this.modeResolver?.resolve(organizationId) ?? this.#staticMode) !== "off"; + } + + /** Test seam for the per-organisation predicate. Not for production callers. */ + writesRedisForTest(organizationId?: string): boolean { + return this.writesRedisFor(organizationId); } /** @@ -156,12 +178,9 @@ export class TaskRunExecutionSnapshotStore extends DelegatingRunStore { runId: string | undefined, fn: (store: RunStore, tx: PrismaClientOrTransaction) => Promise ): Promise { - if (!this.writesRedis) { - // At `off` the callback must receive the delegate's own store, untouched, so a transaction - // behaves exactly as it does without the decorator in the chain. - return this.delegate.runInTransaction(runId, fn); - } - + // Always stage. This method holds only a runId, and a per-organisation dial lives in that + // organisation's blob, so "can any write in here reach Redis" cannot be answered here. Each + // entry resolves its own mode as it is staged, and an organisation at `off` stages nothing. const staged: StagedAppend[] = []; const result = await this.delegate.runInTransaction(runId, (store, tx) => @@ -192,10 +211,6 @@ export class TaskRunExecutionSnapshotStore extends DelegatingRunStore { ): Promise { const store = await this.delegate.forWaitpointCompletion(waitpointId, context); - if (!this.writesRedis) { - return store; - } - // Carry the staging buffer through. Without it, a handle taken inside a transaction appends // immediately, which is the exact ordering the facade exists to prevent. return this.#wrap(store, this.staging); @@ -209,8 +224,9 @@ export class TaskRunExecutionSnapshotStore extends DelegatingRunStore { #wrap(store: RunStore, staging?: StagedAppend[]): TaskRunExecutionSnapshotStore { return new TaskRunExecutionSnapshotStore(store, { store: this.redis, - mode: this.mode, + mode: this.#staticMode, readPercent: this.readPercent, + ...(this.modeResolver && { modeResolver: this.modeResolver }), logger: this.logger, ...(this.onAppendFailure && { onAppendFailure: this.onAppendFailure }), ...(this.faults && { faults: this.faults }), @@ -227,7 +243,7 @@ export class TaskRunExecutionSnapshotStore extends DelegatingRunStore { params: CreateRunInput, tx?: PrismaClientOrTransaction ): Promise { - if (!this.writesRedis) { + if (!this.writesRedisFor(params.snapshot?.organizationId)) { return this.delegate.createRun(params, tx); } @@ -243,7 +259,7 @@ export class TaskRunExecutionSnapshotStore extends DelegatingRunStore { params: CreateCancelledRunInput, tx?: PrismaClientOrTransaction ): Promise { - if (!this.writesRedis) { + if (!this.writesRedisFor(params.snapshot?.organizationId)) { return this.delegate.createCancelledRun(params, tx); } @@ -272,7 +288,7 @@ export class TaskRunExecutionSnapshotStore extends DelegatingRunStore { args: { select: S }, tx?: PrismaClientOrTransaction ): Promise> { - if (!this.writesRedis) { + if (!this.writesRedisFor(data.snapshot?.organizationId)) { return this.delegate.completeAttemptSuccess(runId, data, args, tx); } @@ -297,7 +313,7 @@ export class TaskRunExecutionSnapshotStore extends DelegatingRunStore { args: { select: S }, tx?: PrismaClientOrTransaction ): Promise> { - if (!this.writesRedis) { + if (!this.writesRedisFor(data.snapshot?.organizationId)) { return this.delegate.expireRun(runId, data as never, args, tx); } @@ -324,7 +340,7 @@ export class TaskRunExecutionSnapshotStore extends DelegatingRunStore { }, tx?: PrismaClientOrTransaction ): Promise<{ count: number }> { - if (!this.writesRedis) { + if (!this.writesRedisFor(data.snapshot?.organizationId)) { return this.delegate.expireParkedRun(runId, data as never, tx); } @@ -350,7 +366,7 @@ export class TaskRunExecutionSnapshotStore extends DelegatingRunStore { ): Promise { // The delegate writes a snapshot only when one is supplied, so an absent snapshot is a plain run // update with nothing for Redis to mirror. - if (!this.writesRedis || !data.snapshot) { + if (!data.snapshot || !this.writesRedisFor(data.snapshot?.organizationId)) { return this.delegate.rescheduleRun(runId, data, tx); } @@ -371,7 +387,7 @@ export class TaskRunExecutionSnapshotStore extends DelegatingRunStore { data: LockRunData, tx?: PrismaClientOrTransaction ): Promise>> { - if (!this.writesRedis) { + if (!this.writesRedisFor(data.snapshot?.organizationId)) { return this.delegate.lockRunToWorker(runId, data, tx); } @@ -401,7 +417,7 @@ export class TaskRunExecutionSnapshotStore extends DelegatingRunStore { input: CreateExecutionSnapshotInput, tx?: PrismaClientOrTransaction ): Promise> { - if (!this.writesRedis) { + if (!this.writesRedisFor(input?.organizationId)) { return this.delegate.createExecutionSnapshot(input, tx); } From 06b825eea57cfdd7bcc74d9dde97fdae0881cd6d Mon Sep 17 00:00:00 2001 From: Daniel Sutton Date: Tue, 25 Aug 2026 18:40:17 +0100 Subject: [PATCH 31/56] feat(webapp): add the snapshot-store env block and refuse a silent dial flip The env block carries the connection, the TTLs and the sweep tuning. It does not carry the operational dial. That is a feature flag, so it can move without a deploy. RUN_ENGINE_SNAPSHOT_STORE_MODE is only the floor used when no flag row exists, which keeps a self-host install working with no rows at all. No variable falls back to the generic REDIS_*. This endpoint is a distinct durable cluster, and a fallback would silently put execution state on the general-purpose cache. QUEUE_METRICS_REDIS_HOST is the precedent for omitting the fallback deliberately. Construction is gated on the host being set, so a dial flip in a deployment with no snapshot-store Redis would do nothing at all. The guard refuses that flip on both global save paths rather than letting an operator discover the silence later. It cannot live in validatePartialFeatureFlags, which client components import and which therefore can never read env. --- apps/webapp/app/env.server.ts | 34 ++++++++++++++ .../app/routes/admin.api.v1.feature-flags.ts | 8 ++++ .../webapp/app/routes/admin.feature-flags.tsx | 8 ++++ .../app/v3/snapshotStoreFlagGuard.server.ts | 22 +++++++++ .../test/snapshotStoreFlagGuard.test.ts | 46 +++++++++++++++++++ 5 files changed, 118 insertions(+) create mode 100644 apps/webapp/app/v3/snapshotStoreFlagGuard.server.ts create mode 100644 apps/webapp/test/snapshotStoreFlagGuard.test.ts diff --git a/apps/webapp/app/env.server.ts b/apps/webapp/app/env.server.ts index c63c79d41d6..af32e8bed8e 100644 --- a/apps/webapp/app/env.server.ts +++ b/apps/webapp/app/env.server.ts @@ -1298,6 +1298,40 @@ const EnvironmentSchema = z .string() .default(process.env.REDIS_TLS_DISABLED ?? "false"), + // Execution-snapshot store. MODE here is only the FLOOR: the operational dial is the + // snapshotStoreMode feature flag, so it can move without a deploy. + RUN_ENGINE_SNAPSHOT_STORE_MODE: z + .enum(["off", "dual-write", "compare", "redis-read", "redis-only"]) + .default("off"), + RUN_ENGINE_SNAPSHOT_STORE_COMPLETED_TTL_MS: z.coerce + .number() + .int() + .default(72 * 60 * 60 * 1000), + RUN_ENGINE_SNAPSHOT_STORE_ORPHAN_AGE_MS: z.coerce + .number() + .int() + .default(24 * 60 * 60 * 1000), + RUN_ENGINE_SNAPSHOT_STORE_CONFIRM_ORPHAN_AFTER_MS: z.coerce + .number() + .int() + .default(2 * 60 * 60 * 1000), + RUN_ENGINE_SNAPSHOT_STORE_GC_SWEEP_SCHEDULE: z.string().default("0 */6 * * *"), + RUN_ENGINE_SNAPSHOT_STORE_GC_SWEEP_JITTER_IN_MS: z.coerce.number().int().default(60_000), + // An existing run costs ~4 serial round trips and the orphan-marker clear cannot be batched + // (cross-slot pipelines are rejected), so a full pass is hours, not minutes. A budget that + // truncates every pass stops rule 2 converging, because it needs consecutive sightings. + RUN_ENGINE_SNAPSHOT_STORE_GC_SWEEP_BUDGET_MS: z.coerce.number().int().default(10_800_000), + RUN_ENGINE_SNAPSHOT_STORE_ORG_MODE_CACHE_TTL_MS: z.coerce.number().int().default(30_000), + RUN_ENGINE_SNAPSHOT_STORE_ORG_MODE_CACHE_MAX: z.coerce.number().int().default(10_000), + // No fallback to REDIS_*: this is a distinct durable endpoint and must be set explicitly, or + // execution state silently lands on the general-purpose cache. + RUN_ENGINE_SNAPSHOT_STORE_REDIS_HOST: z.string().optional(), + RUN_ENGINE_SNAPSHOT_STORE_REDIS_PORT: z.coerce.number().optional(), + RUN_ENGINE_SNAPSHOT_STORE_REDIS_USERNAME: z.string().optional(), + RUN_ENGINE_SNAPSHOT_STORE_REDIS_PASSWORD: z.string().optional(), + RUN_ENGINE_SNAPSHOT_STORE_REDIS_TLS_DISABLED: z.string().default("false"), + RUN_ENGINE_SNAPSHOT_STORE_REDIS_CLUSTER_MODE_ENABLED: z.string().default("0"), + RUN_ENGINE_DEV_PRESENCE_REDIS_HOST: z .string() .optional() diff --git a/apps/webapp/app/routes/admin.api.v1.feature-flags.ts b/apps/webapp/app/routes/admin.api.v1.feature-flags.ts index e9da02effd9..34bee6fe575 100644 --- a/apps/webapp/app/routes/admin.api.v1.feature-flags.ts +++ b/apps/webapp/app/routes/admin.api.v1.feature-flags.ts @@ -10,6 +10,7 @@ import { withoutDerivedKeys, } from "~/v3/featureFlags.server"; import { validatePartialFeatureFlags } from "~/v3/featureFlags"; +import { snapshotStoreFlagSaveError } from "~/v3/snapshotStoreFlagGuard.server"; export async function action({ request }: ActionFunctionArgs) { await requireAdminApiRequest(request); @@ -30,6 +31,13 @@ export async function action({ request }: ActionFunctionArgs) { ); } + const snapshotStoreError = snapshotStoreFlagSaveError(body as Record, { + redisHostConfigured: !!env.RUN_ENGINE_SNAPSHOT_STORE_REDIS_HOST, + }); + if (snapshotStoreError) { + return json({ error: snapshotStoreError }, { status: 400 }); + } + // Both the strip and the branch derive from the graced-group table, so adding a group needs // no edit here. Naming the keys inline is how a new group ends up writing its stamp straight // from the request body, with no lock. diff --git a/apps/webapp/app/routes/admin.feature-flags.tsx b/apps/webapp/app/routes/admin.feature-flags.tsx index e987812f520..49788195ade 100644 --- a/apps/webapp/app/routes/admin.feature-flags.tsx +++ b/apps/webapp/app/routes/admin.feature-flags.tsx @@ -17,6 +17,7 @@ import { lockedFlagsInPayload, validatePartialFeatureFlags, } from "~/v3/featureFlags"; +import { snapshotStoreFlagSaveError } from "~/v3/snapshotStoreFlagGuard.server"; import { flags as getGlobalFlags, replaceGlobalFeatureFlags } from "~/v3/featureFlags.server"; import { featuresForRequest } from "~/features.server"; import { Button } from "~/components/primitives/Buttons"; @@ -129,6 +130,13 @@ export const action = dashboardAction( ); } + const snapshotStoreError = snapshotStoreFlagSaveError(parsed.data.flags, { + redisHostConfigured: !!env.RUN_ENGINE_SNAPSHOT_STORE_REDIS_HOST, + }); + if (snapshotStoreError) { + return json({ error: snapshotStoreError }, { status: 400 }); + } + await replaceGlobalFeatureFlags(prisma, { requestedFlags: validationResult.data as Record, catalogKeys: Object.keys(getAllFlagControlTypes()) as FeatureFlagKey[], diff --git a/apps/webapp/app/v3/snapshotStoreFlagGuard.server.ts b/apps/webapp/app/v3/snapshotStoreFlagGuard.server.ts new file mode 100644 index 00000000000..8ee224cd85a --- /dev/null +++ b/apps/webapp/app/v3/snapshotStoreFlagGuard.server.ts @@ -0,0 +1,22 @@ +import { FEATURE_FLAG } from "~/v3/featureFlags"; + +/** + * Refuses a dial flip that would be silent: with no host the store is never constructed, so a flag + * past `off` has no effect. Cannot live in validatePartialFeatureFlags, which client components + * import and which therefore can never read env. + */ +export function snapshotStoreFlagSaveError( + requested: Record, + opts: { redisHostConfigured: boolean } +): string | undefined { + if (opts.redisHostConfigured) { + return undefined; + } + + const mode = requested[FEATURE_FLAG.snapshotStoreMode]; + if (typeof mode === "string" && mode !== "off") { + return `Cannot set ${FEATURE_FLAG.snapshotStoreMode} to "${mode}": RUN_ENGINE_SNAPSHOT_STORE_REDIS_HOST is not configured in this deployment, so the snapshot store is never constructed and the flag would have no effect.`; + } + + return undefined; +} diff --git a/apps/webapp/test/snapshotStoreFlagGuard.test.ts b/apps/webapp/test/snapshotStoreFlagGuard.test.ts new file mode 100644 index 00000000000..543a1c7b958 --- /dev/null +++ b/apps/webapp/test/snapshotStoreFlagGuard.test.ts @@ -0,0 +1,46 @@ +import { describe, expect, it } from "vitest"; +import { snapshotStoreFlagSaveError } from "~/v3/snapshotStoreFlagGuard.server"; + +describe("snapshotStoreFlagSaveError", () => { + it("refuses a flip past off when no host is configured", () => { + expect( + snapshotStoreFlagSaveError( + { snapshotStoreMode: "dual-write" }, + { redisHostConfigured: false } + ) + ).toMatch(/RUN_ENGINE_SNAPSHOT_STORE_REDIS_HOST/); + }); + + it("names the position it refused", () => { + expect( + snapshotStoreFlagSaveError( + { snapshotStoreMode: "redis-read" }, + { redisHostConfigured: false } + ) + ).toMatch(/redis-read/); + }); + + it("allows a flip past off once the host is configured", () => { + expect( + snapshotStoreFlagSaveError({ snapshotStoreMode: "dual-write" }, { redisHostConfigured: true }) + ).toBeUndefined(); + }); + + it("allows off with no host, because that is the default state", () => { + expect( + snapshotStoreFlagSaveError({ snapshotStoreMode: "off" }, { redisHostConfigured: false }) + ).toBeUndefined(); + }); + + it("ignores a payload that does not mention the dial", () => { + expect( + snapshotStoreFlagSaveError({ runOpsMintKind: "cuid" }, { redisHostConfigured: false }) + ).toBeUndefined(); + }); + + it("ignores a non-string dial value and leaves it to schema validation", () => { + expect( + snapshotStoreFlagSaveError({ snapshotStoreMode: 3 }, { redisHostConfigured: false }) + ).toBeUndefined(); + }); +}); From 253466c963c0f3816c55bf3cbee413ed9da279fe Mon Sep 17 00:00:00 2001 From: Daniel Sutton Date: Tue, 25 Aug 2026 18:40:17 +0100 Subject: [PATCH 32/56] feat(webapp): resolve the snapshot-store dial synchronously, never on the calling path The resolver is read on every snapshot write, which happens on each state transition while the per-run lock is held. It must therefore never query: seven decorator methods accept a caller-supplied transaction, so a read here can land inside another caller's open interactive transaction, on the same pool for single-DB and self-host. The global value comes from globalFlagsRegistry, which already holds an in-memory snapshot refreshed on an interval and is already read synchronously on the trigger hot path. The per-organisation value comes from an LRU with a bounded size and TTL, following ControlPlaneResolver. A cache miss returns the global answer and warms the cache out of band, so a cold organisation costs no round trip and a control-plane blip cannot fail a state transition. Only the narrow per-org key is read from the organisation blob; the blob is never passed as overrides for the global key, where a parsing value would win outright. Both organisation save paths now invalidate alongside the existing control-plane invalidation, so the writing process sees a dial change at once. --- ...i.v1.orgs.$organizationId.feature-flags.ts | 2 + ...i.v2.orgs.$organizationId.feature-flags.ts | 3 + .../webapp/app/v3/snapshotStoreMode.server.ts | 101 ++++++++++++++++++ apps/webapp/test/snapshotStoreMode.test.ts | 76 +++++++++++++ 4 files changed, 182 insertions(+) create mode 100644 apps/webapp/app/v3/snapshotStoreMode.server.ts create mode 100644 apps/webapp/test/snapshotStoreMode.test.ts diff --git a/apps/webapp/app/routes/admin.api.v1.orgs.$organizationId.feature-flags.ts b/apps/webapp/app/routes/admin.api.v1.orgs.$organizationId.feature-flags.ts index ae93b9e70f4..3f4327e4547 100644 --- a/apps/webapp/app/routes/admin.api.v1.orgs.$organizationId.feature-flags.ts +++ b/apps/webapp/app/routes/admin.api.v1.orgs.$organizationId.feature-flags.ts @@ -6,6 +6,7 @@ import { env } from "~/env.server"; import { prisma } from "~/db.server"; import { requireAdminApiRequest } from "~/services/personalAccessToken.server"; import { controlPlaneResolver } from "~/v3/runOpsMigration/controlPlaneResolver.server"; +import { invalidateSnapshotStoreOrgMode } from "~/v3/snapshotStoreMode.server"; import { selectMintBaselineSource, stampMintKindFlip } from "~/v3/runOpsMigration/mintFlipGrace"; import { validatePartialFeatureFlags, withoutOrgForbiddenSnapshotKeys } from "~/v3/featureFlags"; import { flags as getGlobalFlags } from "~/v3/featureFlags.server"; @@ -131,6 +132,7 @@ export async function action({ request, params }: ActionFunctionArgs) { // Org feature flags are embedded in every env of the org; drop all its cached env rows. controlPlaneResolver.invalidateOrganization(organizationId); + invalidateSnapshotStoreOrgMode(organizationId); const updatedFlagsResult = updatedOrganization.featureFlags ? validatePartialFeatureFlags(updatedOrganization.featureFlags as Record) diff --git a/apps/webapp/app/routes/admin.api.v2.orgs.$organizationId.feature-flags.ts b/apps/webapp/app/routes/admin.api.v2.orgs.$organizationId.feature-flags.ts index 7366d84a524..5a98fd4a52e 100644 --- a/apps/webapp/app/routes/admin.api.v2.orgs.$organizationId.feature-flags.ts +++ b/apps/webapp/app/routes/admin.api.v2.orgs.$organizationId.feature-flags.ts @@ -6,6 +6,7 @@ import { env } from "~/env.server"; import { prisma } from "~/db.server"; import { requireUser } from "~/services/session.server"; import { controlPlaneResolver } from "~/v3/runOpsMigration/controlPlaneResolver.server"; +import { invalidateSnapshotStoreOrgMode } from "~/v3/snapshotStoreMode.server"; import { selectMintBaselineSource, stampMintKindFlip } from "~/v3/runOpsMigration/mintFlipGrace"; import { flags as getGlobalFlags } from "~/v3/featureFlags.server"; import { @@ -124,6 +125,7 @@ export async function action({ request, params }: ActionFunctionArgs) { } controlPlaneResolver.invalidateOrganization(organizationId); + invalidateSnapshotStoreOrgMode(organizationId); return json({ success: true }); } @@ -184,6 +186,7 @@ export async function action({ request, params }: ActionFunctionArgs) { // Org feature flags are embedded in every env of the org; drop all its cached env rows. controlPlaneResolver.invalidateOrganization(organizationId); + invalidateSnapshotStoreOrgMode(organizationId); return json({ success: true }); } diff --git a/apps/webapp/app/v3/snapshotStoreMode.server.ts b/apps/webapp/app/v3/snapshotStoreMode.server.ts new file mode 100644 index 00000000000..458c655efb4 --- /dev/null +++ b/apps/webapp/app/v3/snapshotStoreMode.server.ts @@ -0,0 +1,101 @@ +import { LRUCache } from "lru-cache"; +import type { SnapshotStoreMode, SnapshotStoreModeResolver } from "@internal/run-store"; +import { $replica } from "~/db.server"; +import { env } from "~/env.server"; +import { logger } from "~/services/logger.server"; +import { singleton } from "~/utils/singleton"; +import { FEATURE_FLAG, FeatureFlagCatalog } from "~/v3/featureFlags"; +import { globalFlagsRegistry } from "~/v3/globalFlagsRegistry.server"; + +type OrgModeSource = { + /** Cache-only: returns undefined on a miss rather than querying. */ + get(organizationId: string): SnapshotStoreMode | undefined; + /** Fire-and-forget, de-duplicated per organisation, never throws. */ + refresh(organizationId: string): void; +}; + +export function buildSnapshotStoreModeResolver(deps: { + globalMode: () => SnapshotStoreMode | undefined; + orgMode: OrgModeSource; + envFloor: SnapshotStoreMode; +}): SnapshotStoreModeResolver { + return { + resolve(organizationId?: string): SnapshotStoreMode { + const global = deps.globalMode() ?? deps.envFloor; + if (!organizationId) { + return global; + } + + const override = deps.orgMode.get(organizationId); + if (override !== undefined) { + return override; + } + + // Deliberately no read here. Seven decorator methods accept a caller-supplied `tx`, so a + // query on this path can land inside another caller's open interactive transaction, on the + // same pool for single-DB and self-host. Serve the global answer, warm the cache off-path. + try { + deps.orgMode.refresh(organizationId); + } catch { + // a warm-up must never fail a state transition + } + return global; + }, + }; +} + +function createOrgModeSource(): OrgModeSource { + const cache = new LRUCache({ + max: env.RUN_ENGINE_SNAPSHOT_STORE_ORG_MODE_CACHE_MAX, + ttl: env.RUN_ENGINE_SNAPSHOT_STORE_ORG_MODE_CACHE_TTL_MS, + }); + const inFlight = new Set(); + + return { + get: (organizationId) => cache.get(organizationId), + refresh: (organizationId) => { + if (inFlight.has(organizationId)) { + return; + } + inFlight.add(organizationId); + + void $replica.organization + .findFirst({ where: { id: organizationId }, select: { featureFlags: true } }) + .then((row) => { + // Only the narrow per-org key. The blob is never passed as `overrides` for the global + // key, where a parsing override would win outright. + const raw = (row?.featureFlags as Record | null | undefined)?.[ + FEATURE_FLAG.snapshotStoreOrgMode + ]; + const parsed = FeatureFlagCatalog[FEATURE_FLAG.snapshotStoreOrgMode].safeParse(raw); + if (parsed.success) { + cache.set(organizationId, parsed.data); + } else { + cache.delete(organizationId); + } + }) + .catch((error) => { + logger.warn("snapshotStoreMode: organisation override refresh failed", { + organizationId, + error, + }); + }) + .finally(() => { + inFlight.delete(organizationId); + }); + }, + }; +} + +const orgModeSource = singleton("snapshotStoreOrgModeSource", createOrgModeSource); + +export const snapshotStoreModeResolver: SnapshotStoreModeResolver = buildSnapshotStoreModeResolver({ + globalMode: () => globalFlagsRegistry.current()?.[FEATURE_FLAG.snapshotStoreMode], + orgMode: orgModeSource, + envFloor: env.RUN_ENGINE_SNAPSHOT_STORE_MODE, +}); + +/** Called by the organisation flag save path so the writing process sees a dial change at once. */ +export function invalidateSnapshotStoreOrgMode(organizationId: string): void { + orgModeSource.refresh(organizationId); +} diff --git a/apps/webapp/test/snapshotStoreMode.test.ts b/apps/webapp/test/snapshotStoreMode.test.ts new file mode 100644 index 00000000000..3fb253ad406 --- /dev/null +++ b/apps/webapp/test/snapshotStoreMode.test.ts @@ -0,0 +1,76 @@ +import { describe, expect, it, vi } from "vitest"; +import type { SnapshotStoreMode } from "@internal/run-store"; +import { buildSnapshotStoreModeResolver } from "~/v3/snapshotStoreMode.server"; + +const noOverrides = { get: () => undefined, refresh: () => {} }; + +function build(opts: { + globalMode?: SnapshotStoreMode; + perOrg?: Record; + envFloor?: SnapshotStoreMode; + refresh?: (organizationId: string) => void; +}) { + return buildSnapshotStoreModeResolver({ + globalMode: () => opts.globalMode, + orgMode: { + get: (id: string) => opts.perOrg?.[id], + refresh: opts.refresh ?? (() => {}), + }, + envFloor: opts.envFloor ?? "off", + }); +} + +describe("snapshot store mode resolver", () => { + it("falls back to the env floor when the global snapshot is cold", () => { + expect(build({ envFloor: "off" }).resolve()).toBe("off"); + expect(build({ envFloor: "dual-write" }).resolve()).toBe("dual-write"); + }); + + it("prefers the global flag over the floor", () => { + expect(build({ globalMode: "redis-read", envFloor: "off" }).resolve()).toBe("redis-read"); + }); + + it("prefers an organisation override over the global flag", () => { + const r = build({ globalMode: "off", perOrg: { org_a: "dual-write" } }); + expect(r.resolve("org_a")).toBe("dual-write"); + expect(r.resolve("org_b")).toBe("off"); + }); + + it("lets an organisation be off while the global flag is on", () => { + const r = build({ globalMode: "dual-write", perOrg: { org_a: "off" } }); + expect(r.resolve("org_a")).toBe("off"); + expect(r.resolve("org_b")).toBe("dual-write"); + }); + + it("serves the global answer on a cold organisation and schedules a refresh", () => { + const refresh = vi.fn(); + const r = build({ globalMode: "dual-write", refresh }); + expect(r.resolve("org_cold")).toBe("dual-write"); + expect(refresh).toHaveBeenCalledWith("org_cold"); + }); + + it("never lets a refresh failure reach the caller", () => { + const refresh = vi.fn(() => { + throw new Error("control plane unreachable"); + }); + const r = build({ globalMode: "off", refresh }); + expect(() => r.resolve("org_x")).not.toThrow(); + expect(r.resolve("org_x")).toBe("off"); + }); + + it("resolves an unknown organisation to the global answer, never a throw", () => { + const r = build({ globalMode: "off", perOrg: {} }); + expect(r.resolve("org_deleted")).toBe("off"); + }); + + it("does not consult the organisation source when no organisation is supplied", () => { + const get = vi.fn(() => undefined); + const r = buildSnapshotStoreModeResolver({ + globalMode: () => "compare", + orgMode: { get, refresh: () => {} }, + envFloor: "off", + }); + expect(r.resolve()).toBe("compare"); + expect(get).not.toHaveBeenCalled(); + }); +}); From 91293152e9dd02dd3bd8ef79eee5a81143219c48 Mon Sep 17 00:00:00 2001 From: Daniel Sutton Date: Tue, 25 Aug 2026 18:40:36 +0100 Subject: [PATCH 33/56] feat(webapp): add the snapshot-store metrics factory Every instrument is created inside the factory. At module scope they would register on every boot, including deployments with no snapshot-store Redis configured, which would break this change's claim to be inert when unconfigured. The sweep gets one emitter carrying an outcome dimension. A pass that throws returns no result and would otherwise emit nothing at all, and the deploy step gates dual-write on an observed pass, so silence has to be distinguishable from success. compare_divergence and trimmed are deliberately absent. Their producers do not exist yet, and a counter pinned at zero is indistinguishable from a working one that found nothing. The guard against an instrument drifting back to module scope is a source assertion, because an instrument that is created but never incremented emits no data point and so cannot be seen through a MeterProvider. --- .../app/v3/snapshotStoreMetrics.server.ts | 63 +++++++++++++++++++ apps/webapp/test/snapshotStoreMetrics.test.ts | 36 +++++++++++ 2 files changed, 99 insertions(+) create mode 100644 apps/webapp/app/v3/snapshotStoreMetrics.server.ts create mode 100644 apps/webapp/test/snapshotStoreMetrics.test.ts diff --git a/apps/webapp/app/v3/snapshotStoreMetrics.server.ts b/apps/webapp/app/v3/snapshotStoreMetrics.server.ts new file mode 100644 index 00000000000..e30282bec36 --- /dev/null +++ b/apps/webapp/app/v3/snapshotStoreMetrics.server.ts @@ -0,0 +1,63 @@ +import type { Meter } from "@internal/tracing"; +import type { DecoratorMetrics, SnapshotStoreMetrics } from "@internal/run-store"; + +export type SnapshotSweepCounts = Record; + +/** + * Every instrument is created inside this function. At module scope they would register on every + * boot, including deployments with no snapshot-store Redis configured. + */ +export function createSnapshotStoreMetrics(meter: Meter) { + const appendTotal = meter.createCounter("run_engine.snapshot_store.append_total"); + const appendFailed = meter.createCounter("run_engine.snapshot_store.append_failed"); + const flushStaged = meter.createCounter("run_engine.snapshot_store.flush_staged"); + const readSource = meter.createCounter("run_engine.snapshot_store.read_source"); + const postExpiryWrite = meter.createCounter("run_engine.snapshot_store.post_expiry_write"); + const skippedNoKeyspace = meter.createCounter("run_engine.snapshot_store.skipped_no_keyspace"); + const cycleMismatch = meter.createCounter("run_engine.snapshot_store.cycle_mismatch"); + const entryBytes = meter.createHistogram("run_engine.snapshot_store.entry_bytes"); + const cycleKeyBytes = meter.createHistogram("run_engine.snapshot_store.cycle_key_bytes"); + const cycleCount = meter.createHistogram("run_engine.snapshot_store.cycle_count"); + const opLatency = meter.createHistogram("run_engine.snapshot_store.op_latency_ms"); + const sweepPass = meter.createCounter("run_engine.snapshot_store.sweep_pass_total"); + const sweepCounts = meter.createHistogram("run_engine.snapshot_store.sweep_counts"); + + const store: SnapshotStoreMetrics = { + recordAppend: (outcome, ttl) => appendTotal.add(1, { outcome, ttl }), + recordEntryBytes: (bytes) => entryBytes.record(bytes), + recordCycleKeyBytes: (bytes) => cycleKeyBytes.record(bytes), + recordCycleCount: (count) => cycleCount.record(count), + recordSkippedNoKeyspace: () => skippedNoKeyspace.add(1), + recordCycleMismatch: () => cycleMismatch.add(1), + recordLatency: (op, ms) => opLatency.record(ms, { op }), + }; + + const decorator: DecoratorMetrics = { + recordWrite: (site, outcome) => { + appendTotal.add(1, { site, outcome }); + if (outcome === "post_expiry") { + postExpiryWrite.add(1); + } + if (outcome === "staged") { + flushStaged.add(1); + } + }, + recordAppendFailed: (site) => appendFailed.add(1, { site }), + recordRead: (method, source) => readSource.add(1, { method, source }), + }; + + /** One emitter per pass, so a pass that throws is distinguishable from one that succeeded. */ + function recordSweepPass(outcome: string, counts?: SnapshotSweepCounts): void { + sweepPass.add(1, { outcome }); + if (!counts) { + return; + } + for (const [field, value] of Object.entries(counts)) { + if (typeof value === "number") { + sweepCounts.record(value, { field }); + } + } + } + + return { store, decorator, recordSweepPass }; +} diff --git a/apps/webapp/test/snapshotStoreMetrics.test.ts b/apps/webapp/test/snapshotStoreMetrics.test.ts new file mode 100644 index 00000000000..597d959578c --- /dev/null +++ b/apps/webapp/test/snapshotStoreMetrics.test.ts @@ -0,0 +1,36 @@ +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; + +const SOURCE_PATH = join(process.cwd(), "app/v3/snapshotStoreMetrics.server.ts"); + +describe("snapshotStoreMetrics module shape", () => { + // An instrument created but never incremented emits no data point, so a MeterProvider cannot see + // one that drifted back to module scope, where it would register on every boot. + const source = readFileSync(SOURCE_PATH, "utf8"); + const factoryAt = source.indexOf("export function createSnapshotStoreMetrics"); + + it("exports the factory", () => { + expect(factoryAt).toBeGreaterThan(-1); + }); + + it("creates every instrument inside the factory", () => { + const creations = [ + ...source.matchAll(/create(Counter|Histogram|UpDownCounter|Observable\w*)\(/g), + ]; + expect(creations.length).toBeGreaterThan(0); + for (const match of creations) { + expect(match.index).toBeGreaterThan(factoryAt); + } + }); + + it("calls getMeter nowhere at module scope", () => { + expect(source).not.toMatch(/^\s*(const|let|var)\s+\w+\s*=\s*getMeter\(/m); + }); + + it("declares no counter that has no producer in this ticket", () => { + // A counter pinned at zero looks the same as a working one that found nothing. + expect(source).not.toMatch(/compare_divergence/); + expect(source).not.toMatch(/\btrimmed\b/); + }); +}); From b9c79ca357bac5259d35acd017d297b2cd8b4255 Mon Sep 17 00:00:00 2001 From: Daniel Sutton Date: Tue, 25 Aug 2026 18:40:37 +0100 Subject: [PATCH 34/56] feat(webapp): construct the snapshot store above the router, gated on its connection The gate is the connection, not the dial. The store's options require a constructed Redis store, that store opens its socket in its constructor, and the client factory sets no lazyConnect, so building it unconditionally would open a doomed localhost connection, retrying forever, in every self-host install, developer machine and CI run. Eighty-eight webapp modules import the run store, so that is nearly every process. buildRunStore is untouched and keeps its no-env contract. A second singleton wraps the first, and the undecorated router is exported for one consumer: the orphan sweep's rule-2 lookup, which must ask Postgres whether a run row exists and must never be able to ask Redis whether Redis is an orphan. The repair enqueuer is late-bound through a module that imports nothing, because the run store cannot import the engine. An unbound enqueuer logs and drops rather than throwing an undefined callable. The construction test asserts that no client object is constructed rather than that no socket is opened. A socket assertion cannot work here: the webapp test setup mocks ioredis with a lazyConnect subclass, so nothing ever dials and such a test would pass even with the gate removed. The test file says so, to stop the weaker assertion being reintroduced as if it proved something. Cluster mode currently reaches the sweep connection but not the hot path, because the Redis snapshot store still builds its own single-node client. --- apps/webapp/app/v3/runStore.server.ts | 12 +- .../app/v3/snapshotStoreBindings.server.ts | 31 ++++ .../app/v3/snapshotStoreInstance.server.ts | 136 ++++++++++++++++++ .../test/snapshotStoreConstruction.test.ts | 69 +++++++++ 4 files changed, 247 insertions(+), 1 deletion(-) create mode 100644 apps/webapp/app/v3/snapshotStoreBindings.server.ts create mode 100644 apps/webapp/app/v3/snapshotStoreInstance.server.ts create mode 100644 apps/webapp/test/snapshotStoreConstruction.test.ts diff --git a/apps/webapp/app/v3/runStore.server.ts b/apps/webapp/app/v3/runStore.server.ts index 9ccf84b5117..5d189596d05 100644 --- a/apps/webapp/app/v3/runStore.server.ts +++ b/apps/webapp/app/v3/runStore.server.ts @@ -12,6 +12,7 @@ import { } from "~/db.server"; import { env } from "~/env.server"; import { singleton } from "~/utils/singleton"; +import { decorateWithSnapshotStore } from "./snapshotStoreInstance.server"; import { resilienceForClient, type TransactionResilienceConfig, @@ -116,7 +117,12 @@ function tryResolveRunOpsHandles() { } } -export const runStore: RunStore = singleton("RunStore", () => { +/** + * The router with no snapshot decorator. One intended consumer: the orphan sweeper's rule-2 + * lookup, which must ask Postgres whether a run row exists and must never be able to ask Redis + * whether Redis is an orphan. Every other caller wants `runStore`. + */ +export const runStoreWithoutSnapshotDecorator: RunStore = singleton("RunStore.undecorated", () => { const handles = ROUTING_ENABLED ? tryResolveRunOpsHandles() : null; // Single-store passthrough: self-host (one DB), or a context without run-ops handles. if (!handles) { @@ -137,3 +143,7 @@ export const runStore: RunStore = singleton("RunStore", () => { legacyResilience: resilienceForClient(handles.legacyWriter), }); }); + +export const runStore: RunStore = singleton("RunStore", () => + decorateWithSnapshotStore(runStoreWithoutSnapshotDecorator) +); diff --git a/apps/webapp/app/v3/snapshotStoreBindings.server.ts b/apps/webapp/app/v3/snapshotStoreBindings.server.ts new file mode 100644 index 00000000000..e10bbe796b6 --- /dev/null +++ b/apps/webapp/app/v3/snapshotStoreBindings.server.ts @@ -0,0 +1,31 @@ +import type { SnapshotRepairEnqueuer } from "@internal/run-store"; + +export type SweepPassOutcome = { + outcome: "completed" | "partial" | "skipped_locked" | "failed" | "unbound" | "aborted"; + counts?: Record; +}; + +export type SweepRunner = (opts: { + deadline: number; + signal: AbortSignal; +}) => Promise; + +/** Late-bound so the run store never has to import the engine. A third module wires both at boot. */ +let repairEnqueuer: SnapshotRepairEnqueuer | undefined; +let sweepRunner: SweepRunner | undefined; + +export function setSnapshotRepairEnqueuer(fn: SnapshotRepairEnqueuer): void { + repairEnqueuer = fn; +} + +export function getSnapshotRepairEnqueuer(): SnapshotRepairEnqueuer | undefined { + return repairEnqueuer; +} + +export function setSnapshotSweepRunner(fn: SweepRunner): void { + sweepRunner = fn; +} + +export function getSnapshotSweepRunner(): SweepRunner | undefined { + return sweepRunner; +} diff --git a/apps/webapp/app/v3/snapshotStoreInstance.server.ts b/apps/webapp/app/v3/snapshotStoreInstance.server.ts new file mode 100644 index 00000000000..420f9040b03 --- /dev/null +++ b/apps/webapp/app/v3/snapshotStoreInstance.server.ts @@ -0,0 +1,136 @@ +import { + createRedisClient, + createRedisClusterClient, + type RedisClient, + type RedisOptions, +} from "@internal/redis"; +import { + RedisSnapshotStore, + TaskRunExecutionSnapshotStore, + type RunStore, +} from "@internal/run-store"; +import { env } from "~/env.server"; +import { logger } from "~/services/logger.server"; +import { singleton } from "~/utils/singleton"; +import { getSnapshotRepairEnqueuer } from "./snapshotStoreBindings.server"; +import { snapshotStoreModeResolver } from "./snapshotStoreMode.server"; +import { createSnapshotStoreMetrics } from "./snapshotStoreMetrics.server"; +import { meter } from "./tracer.server"; + +const KEY_PREFIX = "engine:"; + +function isConfigured(): boolean { + return !!env.RUN_ENGINE_SNAPSHOT_STORE_REDIS_HOST; +} + +function redisOptions(): RedisOptions { + return { + keyPrefix: KEY_PREFIX, + host: env.RUN_ENGINE_SNAPSHOT_STORE_REDIS_HOST ?? undefined, + port: env.RUN_ENGINE_SNAPSHOT_STORE_REDIS_PORT ?? undefined, + username: env.RUN_ENGINE_SNAPSHOT_STORE_REDIS_USERNAME ?? undefined, + password: env.RUN_ENGINE_SNAPSHOT_STORE_REDIS_PASSWORD ?? undefined, + enableAutoPipelining: true, + ...(env.RUN_ENGINE_SNAPSHOT_STORE_REDIS_TLS_DISABLED === "true" ? {} : { tls: {} }), + }; +} + +function isClusterMode(): boolean { + return env.RUN_ENGINE_SNAPSHOT_STORE_REDIS_CLUSTER_MODE_ENABLED === "1"; +} + +function buildClient(name: string): RedisClient { + const options = redisOptions(); + const onError = (error: Error) => + logger.error(`snapshot store redis client error (${name})`, { error }); + + if (!isClusterMode()) { + return createRedisClient(options, { onError }); + } + + return createRedisClusterClient( + { + nodes: [ + { + host: env.RUN_ENGINE_SNAPSHOT_STORE_REDIS_HOST, + port: env.RUN_ENGINE_SNAPSHOT_STORE_REDIS_PORT, + }, + ], + redisOptions: options, + }, + { onError } + ); +} + +type Instance = { + sweepClient: RedisClient; + decorate: (store: RunStore) => RunStore; +}; + +const instance = singleton("snapshotStoreInstance", () => { + if (!isConfigured()) { + return undefined; + } + + const metrics = createSnapshotStoreMetrics(meter); + + // The sweep gets a connection of its own so a full scan of every master can never stall a + // transition append. It also backs the sweep's exclusion lock. + const sweepClient = buildClient("sweep"); + + const redisSnapshotStore = new RedisSnapshotStore({ + // The store still builds its own single-node client, so cluster mode reaches the sweep but not + // the hot path. Hand it a pre-built client once its options accept one. + redisOptions: redisOptions(), + completedTtlMs: env.RUN_ENGINE_SNAPSHOT_STORE_COMPLETED_TTL_MS, + metrics: metrics.store, + }); + + return { + sweepClient, + decorate: (store: RunStore) => + new TaskRunExecutionSnapshotStore(store, { + store: redisSnapshotStore, + modeResolver: snapshotStoreModeResolver, + // Pinned: the field defaults to 0, which would mean no read ever reaches Redis. The + // organisation is the ramp unit, so there is no percentage to ramp. + readPercent: 100, + metrics: metrics.decorator, + onAppendFailure: async (args) => { + const enqueue = getSnapshotRepairEnqueuer(); + if (!enqueue) { + logger.error("snapshot repair enqueuer is unbound; repair job dropped", args); + return; + } + await enqueue(args); + }, + }), + }; +}); + +/** Returns the store verbatim when no snapshot-store Redis is configured. */ +export function decorateWithSnapshotStore(store: RunStore): RunStore { + return instance ? instance.decorate(store) : store; +} + +export function getSnapshotSweepClient(): RedisClient | undefined { + return instance?.sweepClient; +} + +export function getSnapshotStoreConfig() { + return { + configured: isConfigured(), + mode: snapshotStoreModeResolver.resolve(), + completedTtlMs: env.RUN_ENGINE_SNAPSHOT_STORE_COMPLETED_TTL_MS, + orphanAgeMs: env.RUN_ENGINE_SNAPSHOT_STORE_ORPHAN_AGE_MS, + keyPrefix: KEY_PREFIX, + clusterMode: isClusterMode(), + }; +} + +export async function quitSnapshotStoreClients(): Promise { + if (!instance) { + return; + } + await instance.sweepClient.quit().catch(() => undefined); +} diff --git a/apps/webapp/test/snapshotStoreConstruction.test.ts b/apps/webapp/test/snapshotStoreConstruction.test.ts new file mode 100644 index 00000000000..8a76d96e586 --- /dev/null +++ b/apps/webapp/test/snapshotStoreConstruction.test.ts @@ -0,0 +1,69 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; + +// A socket-level assertion is impossible in this suite: test/setup.ts mocks ioredis with a +// LazyRedis subclass that forces lazyConnect, so no client ever dials and a "no connection was +// opened" test would pass even if the gate were broken. The property that does hold, and the one +// that matters, is that no client OBJECT is constructed — an unconstructed client cannot dial. +async function importInstanceModule() { + vi.resetModules(); + delete (globalThis as Record).__trigger_singletons; + return import("~/v3/snapshotStoreInstance.server"); +} + +afterEach(async () => { + vi.unstubAllEnvs(); + vi.resetModules(); + delete (globalThis as Record).__trigger_singletons; +}); + +describe("snapshot store construction gate", () => { + it("constructs nothing when the snapshot-store host is unset", async () => { + // The generic pair is set by test/setup.ts, so this also covers the no-fallback rule: if any + // variable in the block fell back to REDIS_HOST, the store would be constructed here. + vi.stubEnv("RUN_ENGINE_SNAPSHOT_STORE_REDIS_HOST", ""); + + const mod = await importInstanceModule(); + const sentinel = {} as never; + + expect(mod.decorateWithSnapshotStore(sentinel)).toBe(sentinel); + expect(mod.getSnapshotSweepClient()).toBeUndefined(); + expect(mod.getSnapshotStoreConfig().configured).toBe(false); + }); + + it("constructs the decorator and the sweep client once the host is set", async () => { + vi.stubEnv("RUN_ENGINE_SNAPSHOT_STORE_REDIS_HOST", "127.0.0.1"); + vi.stubEnv("RUN_ENGINE_SNAPSHOT_STORE_REDIS_PORT", "6379"); + vi.stubEnv("RUN_ENGINE_SNAPSHOT_STORE_REDIS_TLS_DISABLED", "true"); + + const mod = await importInstanceModule(); + const sentinel = {} as never; + + // Without this the negative above is satisfiable by an import that throws. + expect(mod.decorateWithSnapshotStore(sentinel)).not.toBe(sentinel); + expect(mod.getSnapshotSweepClient()).toBeDefined(); + expect(mod.getSnapshotStoreConfig().configured).toBe(true); + + await mod.quitSnapshotStoreClients(); + }); + + it("reports the resolved configuration for the boot log line", async () => { + vi.stubEnv("RUN_ENGINE_SNAPSHOT_STORE_REDIS_HOST", "127.0.0.1"); + vi.stubEnv("RUN_ENGINE_SNAPSHOT_STORE_REDIS_TLS_DISABLED", "true"); + vi.stubEnv("RUN_ENGINE_SNAPSHOT_STORE_COMPLETED_TTL_MS", "1000"); + vi.stubEnv("RUN_ENGINE_SNAPSHOT_STORE_ORPHAN_AGE_MS", "2000"); + + const mod = await importInstanceModule(); + const config = mod.getSnapshotStoreConfig(); + + expect(config).toMatchObject({ + configured: true, + mode: "off", + completedTtlMs: 1000, + orphanAgeMs: 2000, + keyPrefix: "engine:", + clusterMode: false, + }); + + await mod.quitSnapshotStoreClients(); + }); +}); From 29bbeec3019bb4571da2e3a90f5265584f54b610 Mon Sep 17 00:00:00 2001 From: Daniel Sutton Date: Tue, 25 Aug 2026 18:42:51 +0100 Subject: [PATCH 35/56] feat(run-engine): schedule the snapshot orphan sweep on the existing worker The engine owns scheduling only. The webapp owns what a pass does, because a pass needs a run store and its own Redis client, and the engine opens neither, so so the options block carries a callback, not a connection. The cron entry resolves to undefined when no runner is injected, so an unconfigured deployment registers nothing. An empty schedule falls back to the catalog default rather than passing through: an empty string is falsy, so setupCron would filter the job out with nothing in the logs naming it. That trap is live in the concurrency sweeper today, which is why the resolution is an extracted function with a test rather than an inline expression. retry maxAttempts 1 is load-bearing. A throw takes the dead-letter path, which also reschedules, so the cron chain survives a failed pass. One metric per pass, emitted in a finally with an outcome dimension. A pass that throws would otherwise emit nothing, and the deploy step gates dual-write on an observed pass, so silence had to be distinguishable from success. An occurrence delivered before the webapp sets its binding reports outcome=unbound rather than crashing on an undefined callable. --- .../run-engine/src/engine/index.ts | 73 ++++++++++++++++++- .../src/engine/snapshotSweepSchedule.test.ts | 32 ++++++++ .../src/engine/snapshotSweepSchedule.ts | 14 ++++ .../run-engine/src/engine/types.ts | 15 ++++ .../run-engine/src/engine/workerCatalog.ts | 15 ++++ 5 files changed, 148 insertions(+), 1 deletion(-) create mode 100644 internal-packages/run-engine/src/engine/snapshotSweepSchedule.test.ts create mode 100644 internal-packages/run-engine/src/engine/snapshotSweepSchedule.ts diff --git a/internal-packages/run-engine/src/engine/index.ts b/internal-packages/run-engine/src/engine/index.ts index ccfb60ca4d6..30c9ba44782 100644 --- a/internal-packages/run-engine/src/engine/index.ts +++ b/internal-packages/run-engine/src/engine/index.ts @@ -3,6 +3,7 @@ import { type Meter, type Tracer, type Counter, + type Histogram, getMeter, startSpan, trace, @@ -100,6 +101,7 @@ import type { TriggerParams, } from "./types.js"; import { createTtlWorkerCatalog } from "./ttlWorkerCatalog.js"; +import { resolveSnapshotSweepCron } from "./snapshotSweepSchedule.js"; import { workerCatalog } from "./workerCatalog.js"; import pMap from "p-map"; @@ -111,6 +113,8 @@ export class RunEngine { private logger: Logger; private tracer: Tracer; private meter: Meter; + private snapshotSweepPassCounter: Counter; + private snapshotSweepCountsHistogram: Histogram; private snapshotsSinceReplicaMissCounter: Counter; private snapshotsSinceReplicaRetryDelay: { minMs: number; maxMs: number }; private heartbeatTimeouts: HeartbeatTimeouts; @@ -251,7 +255,19 @@ export class RunEngine { ...options.worker.redis, keyPrefix: `${options.worker.redis.keyPrefix}worker:`, }, - catalog: workerCatalog, + catalog: { + ...workerCatalog, + sweepSnapshotOrphans: { + ...workerCatalog.sweepSnapshotOrphans, + cron: resolveSnapshotSweepCron({ + hasRunner: !!options.snapshotStore?.runSweep, + schedule: options.snapshotStore?.sweepSchedule, + fallback: workerCatalog.sweepSnapshotOrphans.cron, + }), + jitterInMs: + options.snapshotStore?.sweepJitterInMs ?? workerCatalog.sweepSnapshotOrphans.jitterInMs, + }, + }, concurrency: options.worker, pollIntervalMs: options.worker.pollIntervalMs, immediatePollIntervalMs: options.worker.immediatePollIntervalMs, @@ -275,6 +291,9 @@ export class RunEngine { repairSnapshot: async ({ payload }) => { await this.#handleRepairSnapshot(payload); }, + sweepSnapshotOrphans: async () => { + await this.#handleSweepSnapshotOrphans(); + }, expireRun: async ({ payload }) => { await this.ttlSystem.expireRun({ runId: payload.runId }); }, @@ -323,6 +342,19 @@ export class RunEngine { this.tracer = options.tracer; this.meter = options.meter ?? getMeter("run-engine"); + this.snapshotSweepPassCounter = this.meter.createCounter( + "run_engine.snapshot_store.sweep_pass_total", + { + description: + "Orphan-sweep passes by outcome. A pass that throws emits outcome=failed, so silence is distinguishable from success", + } + ); + + this.snapshotSweepCountsHistogram = this.meter.createHistogram( + "run_engine.snapshot_store.sweep_counts", + { description: "Per-field counts from one orphan-sweep pass" } + ); + this.snapshotsSinceReplicaMissCounter = this.meter.createCounter( "run_engine.snapshots_since.replica_miss", { @@ -2891,6 +2923,45 @@ export class RunEngine { }); } + /** + * One emitter per pass, in a finally, so a throw is reported rather than silent. The deploy step + * gates dual-write on an observed pass, so an absent metric would read as a clean sweep. + */ + async #handleSweepSnapshotOrphans() { + const runSweep = this.options.snapshotStore?.runSweep; + + if (!runSweep) { + // The cron entry is registered when the engine is constructed, which happens before the + // webapp sets the binding, so an occurrence already queued at boot can arrive unbound. + this.snapshotSweepPassCounter.add(1, { outcome: "unbound" }); + this.logger.error("sweepSnapshotOrphans ran with no sweep runner bound"); + return; + } + + const budgetMs = this.options.snapshotStore?.sweepBudgetMs ?? 10_800_000; + const controller = new AbortController(); + let outcome = "failed"; + let counts: Record | undefined; + + try { + const result = await runSweep({ + deadline: Date.now() + budgetMs, + signal: controller.signal, + }); + outcome = result.outcome; + counts = result.counts; + } catch (error) { + this.logger.error("sweepSnapshotOrphans threw", { error }); + } finally { + this.snapshotSweepPassCounter.add(1, { outcome }); + for (const [field, value] of Object.entries(counts ?? {})) { + if (typeof value === "number") { + this.snapshotSweepCountsHistogram.record(value, { field }); + } + } + } + } + async #handleRepairSnapshot({ runId, snapshotId, diff --git a/internal-packages/run-engine/src/engine/snapshotSweepSchedule.test.ts b/internal-packages/run-engine/src/engine/snapshotSweepSchedule.test.ts new file mode 100644 index 00000000000..95e9ed0738e --- /dev/null +++ b/internal-packages/run-engine/src/engine/snapshotSweepSchedule.test.ts @@ -0,0 +1,32 @@ +import { describe, expect, it } from "vitest"; +import { resolveSnapshotSweepCron } from "./snapshotSweepSchedule.js"; + +const FALLBACK = "0 */6 * * *"; + +describe("resolveSnapshotSweepCron", () => { + it("never schedules without a runner", () => { + expect(resolveSnapshotSweepCron({ hasRunner: false, fallback: FALLBACK })).toBeUndefined(); + expect( + resolveSnapshotSweepCron({ hasRunner: false, schedule: "* * * * *", fallback: FALLBACK }) + ).toBeUndefined(); + }); + + it("uses the fallback when no schedule is supplied", () => { + expect(resolveSnapshotSweepCron({ hasRunner: true, fallback: FALLBACK })).toBe(FALLBACK); + }); + + it("uses the supplied schedule", () => { + expect( + resolveSnapshotSweepCron({ hasRunner: true, schedule: "0 */12 * * *", fallback: FALLBACK }) + ).toBe("0 */12 * * *"); + }); + + it("does not let an empty schedule silently disable the job", () => { + expect(resolveSnapshotSweepCron({ hasRunner: true, schedule: "", fallback: FALLBACK })).toBe( + FALLBACK + ); + expect(resolveSnapshotSweepCron({ hasRunner: true, schedule: " ", fallback: FALLBACK })).toBe( + FALLBACK + ); + }); +}); diff --git a/internal-packages/run-engine/src/engine/snapshotSweepSchedule.ts b/internal-packages/run-engine/src/engine/snapshotSweepSchedule.ts new file mode 100644 index 00000000000..1da24e5d224 --- /dev/null +++ b/internal-packages/run-engine/src/engine/snapshotSweepSchedule.ts @@ -0,0 +1,14 @@ +/** + * `undefined` never schedules the job. An empty schedule falls back to the default rather than + * through: an empty string is falsy, so `setupCron` would filter the job out with nothing logged. + */ +export function resolveSnapshotSweepCron(opts: { + hasRunner: boolean; + schedule?: string; + fallback: string; +}): string | undefined { + if (!opts.hasRunner) { + return undefined; + } + return opts.schedule?.trim() ? opts.schedule : opts.fallback; +} diff --git a/internal-packages/run-engine/src/engine/types.ts b/internal-packages/run-engine/src/engine/types.ts index 2516776373d..47596a544a2 100644 --- a/internal-packages/run-engine/src/engine/types.ts +++ b/internal-packages/run-engine/src/engine/types.ts @@ -166,6 +166,21 @@ export type RunEngineOptions = { randomize?: boolean; }; }; + /** + * The execution-snapshot orphan sweep. The engine owns scheduling only: the webapp owns what + * runs, because a pass needs a run store and its own Redis client and the engine opens neither. + */ + snapshotStore?: { + runSweep?: (opts: { deadline: number; signal: AbortSignal }) => Promise<{ + outcome: string; + counts?: Record; + }>; + /** Cron. Absent or empty falls back to the catalog default. */ + sweepSchedule?: string; + sweepJitterInMs?: number; + /** Ceiling on one pass. Must stay below the job's visibility timeout. */ + sweepBudgetMs?: number; + }; debounce?: { redis?: RedisOptions; /** diff --git a/internal-packages/run-engine/src/engine/workerCatalog.ts b/internal-packages/run-engine/src/engine/workerCatalog.ts index 5def3071b9e..06b13ab4470 100644 --- a/internal-packages/run-engine/src/engine/workerCatalog.ts +++ b/internal-packages/run-engine/src/engine/workerCatalog.ts @@ -83,4 +83,19 @@ export const workerCatalog = { }), visibilityTimeoutMs: 30_000, }, + sweepSnapshotOrphans: { + schema: z.object({ + timestamp: z.number(), + lastTimestamp: z.number().optional(), + cron: z.string(), + }), + // The default budget plus an hour. A pass that outlives this is redelivered and runs beside + // itself; the runner's fenced lock is what actually prevents that, this is headroom. + visibilityTimeoutMs: 14_400_000, + cron: "0 */6 * * *", + jitterInMs: 60_000, + // Load-bearing. A throw takes the dead-letter path, which also reschedules, so the cron chain + // survives a failed pass. With retries it would not behave that way. + retry: { maxAttempts: 1 }, + }, }; From 3d5c9a72095c375527592d1f4edea688590ab3e0 Mon Sep 17 00:00:00 2001 From: Daniel Sutton Date: Tue, 25 Aug 2026 18:54:08 +0100 Subject: [PATCH 36/56] feat(webapp): run the snapshot orphan sweep under a fenced exclusion lock enqueueOnce provides no overlap protection. Its dedup record is the queue item itself and the ack deletes it, so it elects a winner only at start-up; steady state runs on an ack-then-reschedule chain. Nothing extends the visibility timeout either, so a long pass is redelivered and would otherwise run beside itself. An instance booting mid-pass also computes the next occurrence from now. The lock carries a unique fence and is released with a compare-and-delete. A bare DEL would let a pass that overran its own lock delete its successor's lock on release, the exact failure the lock exists to prevent, and the test for that fails if the release is weakened. Each pass reports one outcome: completed, partial, skipped_locked, failed or aborted. A pass that throws still releases its own lock and reports failed rather than emitting nothing. --- .../app/v3/snapshotSweepRunner.server.ts | 55 ++++++++ apps/webapp/test/snapshotSweepRunner.test.ts | 128 ++++++++++++++++++ 2 files changed, 183 insertions(+) create mode 100644 apps/webapp/app/v3/snapshotSweepRunner.server.ts create mode 100644 apps/webapp/test/snapshotSweepRunner.test.ts diff --git a/apps/webapp/app/v3/snapshotSweepRunner.server.ts b/apps/webapp/app/v3/snapshotSweepRunner.server.ts new file mode 100644 index 00000000000..a3da059f449 --- /dev/null +++ b/apps/webapp/app/v3/snapshotSweepRunner.server.ts @@ -0,0 +1,55 @@ +import type { RedisClient } from "@internal/redis"; +import { logger } from "~/services/logger.server"; +import type { SweepPassOutcome, SweepRunner } from "./snapshotStoreBindings.server"; + +const LOCK_KEY = "snapshot-sweep:lock"; + +// Compare-and-delete. A bare DEL would let a pass that overran its own lock delete its SUCCESSOR's +// lock on release, and two passes would then run together — the failure the lock exists to prevent. +const RELEASE_LUA = ` +if redis.call("get", KEYS[1]) == ARGV[1] then + return redis.call("del", KEYS[1]) +end +return 0 +`; + +type SweepCounts = Record; + +export function buildSnapshotSweepRunner(deps: { + client: RedisClient; + sweep: (opts: { deadline: number; signal: AbortSignal }) => Promise; + lockTtlMs: number; + fence?: () => string; +}): SweepRunner { + return async ({ deadline, signal }): Promise => { + // enqueueOnce gives no overlap protection: its dedup record IS the queue item and the ack + // deletes it, so it elects a winner only at start-up. Nothing extends the visibility timeout + // either, so a long pass is redelivered and would otherwise run beside itself. + const fence = + deps.fence?.() ?? `${process.pid}-${Date.now()}-${Math.random().toString(36).slice(2)}`; + const acquired = await deps.client.set(LOCK_KEY, fence, "PX", deps.lockTtlMs, "NX"); + + if (acquired !== "OK") { + return { outcome: "skipped_locked" }; + } + + try { + const counts = await deps.sweep({ deadline, signal }); + + if (signal.aborted) { + return { outcome: "aborted", counts }; + } + if (counts.partial === true) { + return { outcome: "partial", counts }; + } + return { outcome: "completed", counts }; + } catch (error) { + logger.error("snapshot orphan sweep pass failed", { error }); + return { outcome: "failed" }; + } finally { + await deps.client + .eval(RELEASE_LUA, 1, LOCK_KEY, fence) + .catch((error) => logger.warn("snapshot sweep lock release failed", { error })); + } + }; +} diff --git a/apps/webapp/test/snapshotSweepRunner.test.ts b/apps/webapp/test/snapshotSweepRunner.test.ts new file mode 100644 index 00000000000..6c8f0ed0a53 --- /dev/null +++ b/apps/webapp/test/snapshotSweepRunner.test.ts @@ -0,0 +1,128 @@ +import { redisTest } from "@internal/testcontainers"; +import { expect } from "vitest"; +import { createRedisClient } from "@internal/redis"; +import { buildSnapshotSweepRunner } from "~/v3/snapshotSweepRunner.server"; + +const LOCK_KEY = "snapshot-sweep:lock"; +const CLEAN = { scanned: 1, expired: 0, deleted: 0, skipped: 1, partial: false }; + +function opts() { + return { deadline: Date.now() + 10_000, signal: new AbortController().signal }; +} + +redisTest("reports completed and releases the lock", async ({ redisOptions }) => { + const client = createRedisClient(redisOptions); + try { + const runner = buildSnapshotSweepRunner({ + client, + sweep: async () => CLEAN, + lockTtlMs: 60_000, + }); + + expect((await runner(opts())).outcome).toBe("completed"); + expect(await client.get(LOCK_KEY)).toBeNull(); + } finally { + await client.quit(); + } +}); + +redisTest("reports partial when the pass truncates", async ({ redisOptions }) => { + const client = createRedisClient(redisOptions); + try { + const runner = buildSnapshotSweepRunner({ + client, + sweep: async () => ({ ...CLEAN, partial: true }), + lockTtlMs: 60_000, + }); + + const result = await runner(opts()); + expect(result.outcome).toBe("partial"); + expect(result.counts).toMatchObject({ partial: true }); + } finally { + await client.quit(); + } +}); + +redisTest("skips without running when the lock is held", async ({ redisOptions }) => { + const client = createRedisClient(redisOptions); + try { + await client.set(LOCK_KEY, "someone-else", "PX", 60_000); + let ran = false; + + const runner = buildSnapshotSweepRunner({ + client, + sweep: async () => { + ran = true; + return CLEAN; + }, + lockTtlMs: 60_000, + }); + + expect((await runner(opts())).outcome).toBe("skipped_locked"); + expect(ran).toBe(false); + expect(await client.get(LOCK_KEY)).toBe("someone-else"); + } finally { + await client.quit(); + } +}); + +redisTest("reports failed and still releases its own lock", async ({ redisOptions }) => { + const client = createRedisClient(redisOptions); + try { + const runner = buildSnapshotSweepRunner({ + client, + sweep: async () => { + throw new Error("redis down mid-pass"); + }, + lockTtlMs: 60_000, + }); + + expect((await runner(opts())).outcome).toBe("failed"); + expect(await client.get(LOCK_KEY)).toBeNull(); + } finally { + await client.quit(); + } +}); + +redisTest("an overrun pass cannot delete a successor's lock", async ({ redisOptions }) => { + const client = createRedisClient(redisOptions); + try { + const runner = buildSnapshotSweepRunner({ + client, + fence: () => "first-pass", + // Stand in for the lock expiring mid-pass and a successor claiming it. + sweep: async () => { + await client.set(LOCK_KEY, "successor", "PX", 60_000); + return CLEAN; + }, + lockTtlMs: 60_000, + }); + + await runner(opts()); + + expect(await client.get(LOCK_KEY)).toBe("successor"); + } finally { + await client.quit(); + } +}); + +redisTest("reports aborted when shutdown cancels the pass", async ({ redisOptions }) => { + const client = createRedisClient(redisOptions); + try { + const controller = new AbortController(); + const runner = buildSnapshotSweepRunner({ + client, + sweep: async () => { + controller.abort(); + return CLEAN; + }, + lockTtlMs: 60_000, + }); + + const result = await runner({ deadline: Date.now() + 10_000, signal: controller.signal }); + expect(result.outcome).toBe("aborted"); + expect(await client.get(LOCK_KEY)).toBeNull(); + } finally { + await client.quit(); + } +}); From d4bf1a4d1099b93ec75abc1667893dcb27fa08a5 Mon Sep 17 00:00:00 2001 From: Daniel Sutton Date: Tue, 25 Aug 2026 18:54:08 +0100 Subject: [PATCH 37/56] feat(webapp,run-engine): bind the append-failure hook to the existing repair job A third module imports both sides, so the run store never has to import the engine and the engine never has to know about the sweep's collaborators. It is invoked from entry.server.tsx beside the other boot registrations, and returns early when no snapshot-store Redis is configured. enqueueSnapshotRepair shares the stall watchdog's job id and its availableAt. Sharing the id is what stops the two compensators enqueueing two repairs for one run; sharing the delay is what stops the winner of a race changing when the repair runs. It now returns whether a job was actually inserted, which is what makes "exactly one repair job" directly assertable rather than inferred. The sweeper is given the undecorated router. Its rule 2 asks Postgres whether a run row exists, and it must never be able to ask Redis whether Redis is an orphan. The sweep does not yet accept a deadline, so the budget is not enforced inside a pass. The fenced lock is what keeps two passes apart, and its TTL covers the expected duration. --- apps/webapp/app/entry.server.tsx | 14 +++++ apps/webapp/app/v3/runEngine.server.ts | 13 +++++ .../app/v3/snapshotStoreWiring.server.ts | 53 +++++++++++++++++++ .../run-engine/src/engine/index.ts | 17 ++++++ .../tests/helpers/snapshotRepairEngine.ts | 24 +++++++++ .../tests/snapshotRepairBinding.test.ts | 48 +++++++++++++++++ 6 files changed, 169 insertions(+) create mode 100644 apps/webapp/app/v3/snapshotStoreWiring.server.ts create mode 100644 internal-packages/run-engine/src/engine/tests/helpers/snapshotRepairEngine.ts create mode 100644 internal-packages/run-engine/src/engine/tests/snapshotRepairBinding.test.ts diff --git a/apps/webapp/app/entry.server.tsx b/apps/webapp/app/entry.server.tsx index 074bc39d760..501f478fdf6 100644 --- a/apps/webapp/app/entry.server.tsx +++ b/apps/webapp/app/entry.server.tsx @@ -17,6 +17,8 @@ import { LocaleContextProvider } from "./components/primitives/LocaleProvider"; import type { OperatingSystemPlatform } from "./components/primitives/OperatingSystemProvider"; import { OperatingSystemContextProvider } from "./components/primitives/OperatingSystemProvider"; import { assertRunOpsSplitSentinel, Prisma } from "./db.server"; +import { assertSnapshotStoreBootFromEnv } from "./v3/snapshotStoreBoot.server"; +import { registerSnapshotStoreWiring } from "./v3/snapshotStoreWiring.server"; import { env } from "./env.server"; import { eventLoopMonitor, eventLoopUtilizationMonitor } from "./eventLoopMonitor.server"; import { logger } from "./services/logger.server"; @@ -325,6 +327,18 @@ singleton("AssertRunOpsSplitSentinel", () => { return true; }); +singleton("SnapshotStoreWiring", registerSnapshotStoreWiring); + +// Ordered after the wiring above: the boot check asserts the repair binding is set, and the +// binding is what the wiring installs. +singleton("AssertSnapshotStoreBoot", () => { + assertSnapshotStoreBootFromEnv().catch((error) => { + logger.error("Snapshot store boot check failed; refusing to start", { error }); + process.exit(1); + }); + return true; +}); + singleton("RunEngineEventBusHandlers", registerRunEngineEventBusHandlers); singleton("SetupBatchQueueCallbacks", setupBatchQueueCallbacks); // Attach the realtime run-changed publish delegations to the engine event bus. diff --git a/apps/webapp/app/v3/runEngine.server.ts b/apps/webapp/app/v3/runEngine.server.ts index 76ac35f349b..6fd490c3707 100644 --- a/apps/webapp/app/v3/runEngine.server.ts +++ b/apps/webapp/app/v3/runEngine.server.ts @@ -12,6 +12,7 @@ import { runEnginePendingVersionLookup } from "./runEnginePendingVersionLookup.s import { pickRunOpsStoreForCompletion } from "./runOpsMigration/crossSeamGuard.server"; import { runEngineControlPlaneResolver } from "./runOpsMigration/runEngineControlPlaneResolver.server"; import { runStore } from "./runStore.server"; +import { getSnapshotSweepRunner } from "./snapshotStoreBindings.server"; import { meter, tracer } from "./tracer.server"; export const engine = singleton("RunEngine", createRunEngine); @@ -241,6 +242,18 @@ function createRunEngine() { randomize: true, }, }, + snapshotStore: { + runSweep: async (opts) => { + const run = getSnapshotSweepRunner(); + if (!run) { + return { outcome: "unbound" }; + } + return run(opts); + }, + sweepSchedule: env.RUN_ENGINE_SNAPSHOT_STORE_GC_SWEEP_SCHEDULE, + sweepJitterInMs: env.RUN_ENGINE_SNAPSHOT_STORE_GC_SWEEP_JITTER_IN_MS, + sweepBudgetMs: env.RUN_ENGINE_SNAPSHOT_STORE_GC_SWEEP_BUDGET_MS, + }, // Debounce configuration debounce: { maxDebounceDurationMs: env.RUN_ENGINE_MAXIMUM_DEBOUNCE_DURATION_MS, diff --git a/apps/webapp/app/v3/snapshotStoreWiring.server.ts b/apps/webapp/app/v3/snapshotStoreWiring.server.ts new file mode 100644 index 00000000000..f6bf2806a80 --- /dev/null +++ b/apps/webapp/app/v3/snapshotStoreWiring.server.ts @@ -0,0 +1,53 @@ +import { SnapshotOrphanSweeper } from "@internal/run-store"; +import { env } from "~/env.server"; +import { logger } from "~/services/logger.server"; +import { engine } from "./runEngine.server"; +import { runStoreWithoutSnapshotDecorator } from "./runStore.server"; +import { buildSnapshotSweepRunner } from "./snapshotSweepRunner.server"; +import { setSnapshotRepairEnqueuer, setSnapshotSweepRunner } from "./snapshotStoreBindings.server"; +import { getSnapshotSweepClient } from "./snapshotStoreInstance.server"; + +/** + * The third module: it imports both sides, so neither the run store nor the engine has to import + * the other. Invoked from entry.server.tsx, beside the other boot registrations. + */ +export function registerSnapshotStoreWiring(): boolean { + const sweepClient = getSnapshotSweepClient(); + + if (!sweepClient) { + return false; + } + + setSnapshotRepairEnqueuer(async (args) => { + await engine.enqueueSnapshotRepair(args); + }); + + const sweeper = new SnapshotOrphanSweeper({ + redisOptions: { + keyPrefix: "engine:", + host: env.RUN_ENGINE_SNAPSHOT_STORE_REDIS_HOST ?? undefined, + port: env.RUN_ENGINE_SNAPSHOT_STORE_REDIS_PORT ?? undefined, + username: env.RUN_ENGINE_SNAPSHOT_STORE_REDIS_USERNAME ?? undefined, + password: env.RUN_ENGINE_SNAPSHOT_STORE_REDIS_PASSWORD ?? undefined, + ...(env.RUN_ENGINE_SNAPSHOT_STORE_REDIS_TLS_DISABLED === "true" ? {} : { tls: {} }), + }, + // The undecorated router: rule 2 asks Postgres whether a run row exists, and must never be + // able to ask Redis whether Redis is an orphan. + runStore: runStoreWithoutSnapshotDecorator, + completedTtlMs: env.RUN_ENGINE_SNAPSHOT_STORE_COMPLETED_TTL_MS, + orphanAgeMs: env.RUN_ENGINE_SNAPSHOT_STORE_ORPHAN_AGE_MS, + }); + + setSnapshotSweepRunner( + buildSnapshotSweepRunner({ + client: sweepClient, + // The sweep does not yet accept a deadline, so the budget is not enforced inside a pass. The + // fenced lock is what keeps two passes apart; its TTL covers the expected pass duration. + sweep: async () => ({ ...(await sweeper.sweep()) }), + lockTtlMs: env.RUN_ENGINE_SNAPSHOT_STORE_GC_SWEEP_BUDGET_MS + 3_600_000, + }) + ); + + logger.info("snapshot store wiring registered"); + return true; +} diff --git a/internal-packages/run-engine/src/engine/index.ts b/internal-packages/run-engine/src/engine/index.ts index 30c9ba44782..fa6773135c9 100644 --- a/internal-packages/run-engine/src/engine/index.ts +++ b/internal-packages/run-engine/src/engine/index.ts @@ -2497,6 +2497,23 @@ export class RunEngine { }; } + /** + * The append-failure compensator. Shares the stall watchdog's job id AND its availableAt, so the + * two cannot enqueue two repairs for one run and neither can win a race that changes the delay. + */ + async enqueueSnapshotRepair(payload: { + runId: string; + snapshotId: string; + executionStatus: string; + }): Promise { + return this.worker.enqueueOnce({ + id: `repair-in-progress-run:${payload.runId}`, + job: "repairSnapshot", + payload, + availableAt: new Date(Date.now() + this.repairSnapshotTimeoutMs), + }); + } + async #repairRun(runId: string, dryRun: boolean) { const snapshot = await getLatestExecutionSnapshot(this.prisma, runId, this.runStore); diff --git a/internal-packages/run-engine/src/engine/tests/helpers/snapshotRepairEngine.ts b/internal-packages/run-engine/src/engine/tests/helpers/snapshotRepairEngine.ts new file mode 100644 index 00000000000..3924a5ef4ae --- /dev/null +++ b/internal-packages/run-engine/src/engine/tests/helpers/snapshotRepairEngine.ts @@ -0,0 +1,24 @@ +// A minimal engine for asserting the append-failure repair binding. No decorator and no Redis +// snapshot store: the property under test is the job id's dedupe, which lives on the engine. +import { trace } from "@internal/tracing"; + +export function engineOptionsForSnapshotRepair(prisma: unknown, redisOptions: unknown) { + return { + prisma, + worker: { redis: redisOptions, workers: 1, tasksPerWorker: 10, pollIntervalMs: 100 }, + queue: { + redis: redisOptions, + masterQueueConsumersDisabled: true, + processWorkerQueueDebounceMs: 50, + }, + runLock: { redis: redisOptions }, + machines: { + defaultMachine: "small-1x" as const, + machines: { + "small-1x": { name: "small-1x" as const, cpu: 0.5, memory: 0.5, centsPerMs: 0.0001 }, + }, + baseCostInCents: 0.0001, + }, + tracer: trace.getTracer("test", "0.0.0"), + }; +} diff --git a/internal-packages/run-engine/src/engine/tests/snapshotRepairBinding.test.ts b/internal-packages/run-engine/src/engine/tests/snapshotRepairBinding.test.ts new file mode 100644 index 00000000000..ba14c58034e --- /dev/null +++ b/internal-packages/run-engine/src/engine/tests/snapshotRepairBinding.test.ts @@ -0,0 +1,48 @@ +import { containerTest } from "@internal/testcontainers"; +import { expect } from "vitest"; +import { RunEngine } from "../index.js"; +import { engineOptionsForSnapshotRepair } from "./helpers/snapshotRepairEngine.js"; + +containerTest( + "a second repair for the same run enqueues no second job", + async ({ prisma, redisOptions }) => { + const engine = new RunEngine(engineOptionsForSnapshotRepair(prisma, redisOptions) as never); + + try { + const payload = { + runId: "run_repair_dedupe", + snapshotId: "snap_1", + executionStatus: "EXECUTING", + }; + + // The stall watchdog uses this same job id, so the two compensators must collapse to one. + expect(await engine.enqueueSnapshotRepair(payload)).toBe(true); + expect(await engine.enqueueSnapshotRepair({ ...payload, snapshotId: "snap_2" })).toBe(false); + } finally { + await engine.quit(); + } + } +); + +containerTest("a different run gets its own repair job", async ({ prisma, redisOptions }) => { + const engine = new RunEngine(engineOptionsForSnapshotRepair(prisma, redisOptions) as never); + + try { + expect( + await engine.enqueueSnapshotRepair({ + runId: "run_a", + snapshotId: "snap_a", + executionStatus: "EXECUTING", + }) + ).toBe(true); + expect( + await engine.enqueueSnapshotRepair({ + runId: "run_b", + snapshotId: "snap_b", + executionStatus: "EXECUTING", + }) + ).toBe(true); + } finally { + await engine.quit(); + } +}); From bea262ff3d37171fa07c224d8c479e04e1d8986c Mon Sep 17 00:00:00 2001 From: Daniel Sutton Date: Tue, 25 Aug 2026 18:54:08 +0100 Subject: [PATCH 38/56] feat(webapp): add the snapshot-store boot checks Three checks, all self-skipping while the dial is off. Configuration is fail-closed: a dial past off with no host, or a non-positive TTL, refuses to start. These read static values, so an outage cannot trip them. Reachability is graded rather than blanket fail-closed. Postgres stays authoritative below redis-only, so a lost append costs nothing there and refusing to boot would bleed fleet capacity during a Redis fault to protect a write path that is currently free. At redis-only, where Redis genuinely is the store, an unreachable endpoint refuses. The test pins the graded behaviour: a blanket refusal fails it. The same check asserts the repair binding is bound. Nothing enforces that a process which appends has imported the engine module, and an unbound enqueuer loses every repair job in silence. The third check warns at redis-only, because this build always writes Postgres snapshots. Double-writing is safe, but a failed birth append rethrows at that position, so a Redis fault would fail run creation that Postgres would have covered. The boot log line stops being authoritative after boot: the dial is a flag and can move. --- .../webapp/app/v3/snapshotStoreBoot.server.ts | 112 ++++++++++++++++++ apps/webapp/test/snapshotStoreBoot.test.ts | 74 ++++++++++++ apps/webapp/test/snapshotStoreMode.test.ts | 2 - internal-packages/redis/src/index.ts | 2 +- 4 files changed, 187 insertions(+), 3 deletions(-) create mode 100644 apps/webapp/app/v3/snapshotStoreBoot.server.ts create mode 100644 apps/webapp/test/snapshotStoreBoot.test.ts diff --git a/apps/webapp/app/v3/snapshotStoreBoot.server.ts b/apps/webapp/app/v3/snapshotStoreBoot.server.ts new file mode 100644 index 00000000000..374b39bf744 --- /dev/null +++ b/apps/webapp/app/v3/snapshotStoreBoot.server.ts @@ -0,0 +1,112 @@ +import type { SnapshotStoreMode } from "@internal/run-store"; +import { logger } from "~/services/logger.server"; +import { getSnapshotRepairEnqueuer } from "./snapshotStoreBindings.server"; +import { getSnapshotStoreConfig, getSnapshotSweepClient } from "./snapshotStoreInstance.server"; + +export type SnapshotStoreBootDeps = { + mode: SnapshotStoreMode; + hostConfigured: boolean; + completedTtlMs: number; + orphanAgeMs: number; + ping: () => Promise; + repairBound: () => boolean; + log: (message: string, fields?: Record) => void; + warn: (message: string, fields?: Record) => void; +}; + +const PING_TIMEOUT_MS = 5_000; + +export async function assertSnapshotStoreBoot(deps: SnapshotStoreBootDeps): Promise { + const pastOff = deps.mode !== "off"; + + if (pastOff && !deps.hostConfigured) { + throw new Error( + `Snapshot store dial is "${deps.mode}" but RUN_ENGINE_SNAPSHOT_STORE_REDIS_HOST is unset, so nothing is constructed; refusing to start.` + ); + } + + if (pastOff && !(deps.completedTtlMs > 0)) { + throw new Error("RUN_ENGINE_SNAPSHOT_STORE_COMPLETED_TTL_MS must be a positive integer."); + } + + if (pastOff && !(deps.orphanAgeMs > 0)) { + throw new Error("RUN_ENGINE_SNAPSHOT_STORE_ORPHAN_AGE_MS must be a positive integer."); + } + + // Nothing enforces that a process which appends has imported the engine module, and an unbound + // enqueuer loses every repair job silently — which burns a task attempt per lost repair. + if (pastOff && !deps.repairBound()) { + throw new Error( + "Snapshot store dial is past off but the repair enqueuer is unbound; refusing to start." + ); + } + + if (pastOff) { + const reachable = await deps.ping(); + if (!reachable) { + if (deps.mode === "redis-only") { + throw new Error( + "Snapshot store dial is redis-only and the endpoint is unreachable; refusing to start." + ); + } + // Postgres is authoritative below redis-only, so a lost append costs nothing. Refusing here + // would bleed fleet capacity during a Redis fault to protect a write path that is free. + deps.warn("Snapshot store Redis is unreachable; booting because Postgres is authoritative", { + mode: deps.mode, + }); + } + } + + if (deps.mode === "redis-only") { + deps.warn( + "Snapshot store dial is redis-only but this build always writes Postgres snapshots. Double-writing is safe, but a Redis fault at this position fails run creation.", + { mode: deps.mode } + ); + } + + deps.log("snapshot store resolved", { + mode: deps.mode, + hostConfigured: deps.hostConfigured, + completedTtlMs: deps.completedTtlMs, + orphanAgeMs: deps.orphanAgeMs, + }); +} + +async function pingSweepClient(): Promise { + const client = getSnapshotSweepClient(); + if (!client) { + return false; + } + try { + const result = await Promise.race([ + client.ping(), + new Promise((_, reject) => + setTimeout(() => reject(new Error("ping timed out")), PING_TIMEOUT_MS) + ), + ]); + return result === "PONG"; + } catch { + return false; + } +} + +/** The env-reading adapter. The boot log line is not authoritative after boot: the dial can move. */ +export async function assertSnapshotStoreBootFromEnv(): Promise { + const config = getSnapshotStoreConfig(); + + await assertSnapshotStoreBoot({ + mode: config.mode, + hostConfigured: config.configured, + completedTtlMs: config.completedTtlMs, + orphanAgeMs: config.orphanAgeMs, + ping: pingSweepClient, + repairBound: () => !!getSnapshotRepairEnqueuer(), + log: (message, fields) => + logger.info(message, { + ...fields, + keyPrefix: config.keyPrefix, + clusterMode: config.clusterMode, + }), + warn: (message, fields) => logger.warn(message, fields), + }); +} diff --git a/apps/webapp/test/snapshotStoreBoot.test.ts b/apps/webapp/test/snapshotStoreBoot.test.ts new file mode 100644 index 00000000000..735e2d2c1d7 --- /dev/null +++ b/apps/webapp/test/snapshotStoreBoot.test.ts @@ -0,0 +1,74 @@ +import { describe, expect, it, vi } from "vitest"; +import { assertSnapshotStoreBoot } from "~/v3/snapshotStoreBoot.server"; + +function deps(overrides: Partial[0]> = {}) { + return { + mode: "off" as const, + hostConfigured: false, + completedTtlMs: 1, + orphanAgeMs: 1, + ping: async () => true, + repairBound: () => true, + log: vi.fn(), + warn: vi.fn(), + ...overrides, + }; +} + +describe("assertSnapshotStoreBoot", () => { + it("passes and logs when the dial is off and nothing is configured", async () => { + const d = deps(); + await expect(assertSnapshotStoreBoot(d)).resolves.toBeUndefined(); + expect(d.log).toHaveBeenCalled(); + }); + + it("does not probe reachability at off", async () => { + const ping = vi.fn(async () => false); + await expect(assertSnapshotStoreBoot(deps({ ping }))).resolves.toBeUndefined(); + expect(ping).not.toHaveBeenCalled(); + }); + + it("refuses a dial past off with no host", async () => { + await expect( + assertSnapshotStoreBoot(deps({ mode: "dual-write", hostConfigured: false })) + ).rejects.toThrow(/RUN_ENGINE_SNAPSHOT_STORE_REDIS_HOST/); + }); + + it("refuses a non-positive TTL once the dial is past off", async () => { + await expect( + assertSnapshotStoreBoot(deps({ mode: "dual-write", hostConfigured: true, completedTtlMs: 0 })) + ).rejects.toThrow(/COMPLETED_TTL_MS/); + await expect( + assertSnapshotStoreBoot(deps({ mode: "dual-write", hostConfigured: true, orphanAgeMs: -1 })) + ).rejects.toThrow(/ORPHAN_AGE_MS/); + }); + + it("refuses when the repair binding is unset and the dial is past off", async () => { + await expect( + assertSnapshotStoreBoot( + deps({ mode: "dual-write", hostConfigured: true, repairBound: () => false }) + ) + ).rejects.toThrow(/repair/i); + }); + + it("boots on an unreachable endpoint below redis-only, loudly", async () => { + const d = deps({ mode: "dual-write", hostConfigured: true, ping: async () => false }); + await expect(assertSnapshotStoreBoot(d)).resolves.toBeUndefined(); + expect(d.warn).toHaveBeenCalled(); + }); + + it("refuses an unreachable endpoint at redis-only", async () => { + await expect( + assertSnapshotStoreBoot( + deps({ mode: "redis-only", hostConfigured: true, ping: async () => false }) + ) + ).rejects.toThrow(/unreachable/i); + }); + + it("warns at redis-only because this build still writes Postgres snapshots", async () => { + const d = deps({ mode: "redis-only", hostConfigured: true }); + await expect(assertSnapshotStoreBoot(d)).resolves.toBeUndefined(); + const warnings = d.warn.mock.calls.flat().join(" "); + expect(warnings).toMatch(/Postgres/i); + }); +}); diff --git a/apps/webapp/test/snapshotStoreMode.test.ts b/apps/webapp/test/snapshotStoreMode.test.ts index 3fb253ad406..c21eaca914c 100644 --- a/apps/webapp/test/snapshotStoreMode.test.ts +++ b/apps/webapp/test/snapshotStoreMode.test.ts @@ -2,8 +2,6 @@ import { describe, expect, it, vi } from "vitest"; import type { SnapshotStoreMode } from "@internal/run-store"; import { buildSnapshotStoreModeResolver } from "~/v3/snapshotStoreMode.server"; -const noOverrides = { get: () => undefined, refresh: () => {} }; - function build(opts: { globalMode?: SnapshotStoreMode; perOrg?: Record; diff --git a/internal-packages/redis/src/index.ts b/internal-packages/redis/src/index.ts index 967f3939395..f0be76cd9c4 100644 --- a/internal-packages/redis/src/index.ts +++ b/internal-packages/redis/src/index.ts @@ -1,4 +1,4 @@ -import { Cluster, Redis, type ClusterNode, type ClusterOptions, type RedisOptions } from "ioredis"; +import { Redis, type Cluster, type ClusterNode, type ClusterOptions, type RedisOptions } from "ioredis"; import { Logger } from "@trigger.dev/core/logger"; export { From 6447afb5557a0c2b2a236771c2946bf2b420bf83 Mon Sep 17 00:00:00 2001 From: Daniel Sutton Date: Tue, 25 Aug 2026 18:57:29 +0100 Subject: [PATCH 39/56] fix(webapp): omit the sweep options entirely when the snapshot store is unconfigured Passing a runner that reports "unbound" still registers the cron entry, so an install with no snapshot-store Redis would schedule the job and log a failed pass every interval. The options block is now omitted, which is what makes the change genuinely inert when unconfigured. Found by working through the merge test by hand rather than assuming it held. --- apps/webapp/app/v3/runEngine.server.ts | 27 +++++++++++-------- .../src/engine/snapshotSweepSchedule.test.ts | 10 +++++++ 2 files changed, 26 insertions(+), 11 deletions(-) diff --git a/apps/webapp/app/v3/runEngine.server.ts b/apps/webapp/app/v3/runEngine.server.ts index 6fd490c3707..30a5c549433 100644 --- a/apps/webapp/app/v3/runEngine.server.ts +++ b/apps/webapp/app/v3/runEngine.server.ts @@ -13,6 +13,7 @@ import { pickRunOpsStoreForCompletion } from "./runOpsMigration/crossSeamGuard.s import { runEngineControlPlaneResolver } from "./runOpsMigration/runEngineControlPlaneResolver.server"; import { runStore } from "./runStore.server"; import { getSnapshotSweepRunner } from "./snapshotStoreBindings.server"; +import { getSnapshotStoreConfig } from "./snapshotStoreInstance.server"; import { meter, tracer } from "./tracer.server"; export const engine = singleton("RunEngine", createRunEngine); @@ -242,18 +243,22 @@ function createRunEngine() { randomize: true, }, }, - snapshotStore: { - runSweep: async (opts) => { - const run = getSnapshotSweepRunner(); - if (!run) { - return { outcome: "unbound" }; + // Omitted entirely when the snapshot store is unconfigured: passing a runner would register the + // cron job and log an unbound pass every interval on every install that does not use the store. + snapshotStore: getSnapshotStoreConfig().configured + ? { + runSweep: async (opts) => { + const run = getSnapshotSweepRunner(); + if (!run) { + return { outcome: "unbound" }; + } + return run(opts); + }, + sweepSchedule: env.RUN_ENGINE_SNAPSHOT_STORE_GC_SWEEP_SCHEDULE, + sweepJitterInMs: env.RUN_ENGINE_SNAPSHOT_STORE_GC_SWEEP_JITTER_IN_MS, + sweepBudgetMs: env.RUN_ENGINE_SNAPSHOT_STORE_GC_SWEEP_BUDGET_MS, } - return run(opts); - }, - sweepSchedule: env.RUN_ENGINE_SNAPSHOT_STORE_GC_SWEEP_SCHEDULE, - sweepJitterInMs: env.RUN_ENGINE_SNAPSHOT_STORE_GC_SWEEP_JITTER_IN_MS, - sweepBudgetMs: env.RUN_ENGINE_SNAPSHOT_STORE_GC_SWEEP_BUDGET_MS, - }, + : undefined, // Debounce configuration debounce: { maxDebounceDurationMs: env.RUN_ENGINE_MAXIMUM_DEBOUNCE_DURATION_MS, diff --git a/internal-packages/run-engine/src/engine/snapshotSweepSchedule.test.ts b/internal-packages/run-engine/src/engine/snapshotSweepSchedule.test.ts index 95e9ed0738e..65c505f92f7 100644 --- a/internal-packages/run-engine/src/engine/snapshotSweepSchedule.test.ts +++ b/internal-packages/run-engine/src/engine/snapshotSweepSchedule.test.ts @@ -30,3 +30,13 @@ describe("resolveSnapshotSweepCron", () => { ); }); }); + +describe("the unconfigured deployment", () => { + // The webapp must omit the whole options block, not pass a runner that reports unbound: a + // registered cron would log an unbound pass every interval on every install not using the store. + it("schedules nothing when the options block is absent", () => { + expect( + resolveSnapshotSweepCron({ hasRunner: false, schedule: "0 */6 * * *", fallback: FALLBACK }) + ).toBeUndefined(); + }); +}); From f901708f6e891784d1c606232e39225ff4da3006 Mon Sep 17 00:00:00 2001 From: Daniel Sutton Date: Tue, 25 Aug 2026 18:57:46 +0100 Subject: [PATCH 40/56] style(redis): reflow the ioredis import --- internal-packages/redis/src/index.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/internal-packages/redis/src/index.ts b/internal-packages/redis/src/index.ts index f0be76cd9c4..50318d54fbc 100644 --- a/internal-packages/redis/src/index.ts +++ b/internal-packages/redis/src/index.ts @@ -1,4 +1,10 @@ -import { Redis, type Cluster, type ClusterNode, type ClusterOptions, type RedisOptions } from "ioredis"; +import { + Redis, + type Cluster, + type ClusterNode, + type ClusterOptions, + type RedisOptions, +} from "ioredis"; import { Logger } from "@trigger.dev/core/logger"; export { From f7292ac29b0aaa0dd8f33fcace860121c98cb075 Mon Sep 17 00:00:00 2001 From: Daniel Sutton Date: Wed, 26 Aug 2026 09:17:11 +0100 Subject: [PATCH 41/56] fix(run-store): keep an omitted completedWaitpointOrder omitted on the live write Hoisting the value so the redis-only echo could reuse it also gave the live Prisma write a default it never had. Prisma omits an undefined key, and the column is nullable with no default, so the write stored an empty array where it used to store NULL. The echo needs a concrete array because its return type says so; the write does not, and now does not get one. --- .../run-store/src/PostgresRunStore.ts | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/internal-packages/run-store/src/PostgresRunStore.ts b/internal-packages/run-store/src/PostgresRunStore.ts index 5d9cb60185c..7af040eb99c 100644 --- a/internal-packages/run-store/src/PostgresRunStore.ts +++ b/internal-packages/run-store/src/PostgresRunStore.ts @@ -2020,11 +2020,18 @@ export class PostgresRunStore implements RunStore { error, } = input; - const completedWaitpointOrder = - completedWaitpoints - ?.filter((c) => c.index !== undefined) - .sort((a, b) => a.index! - b.index!) - .map((w) => w.id) ?? []; + // Left possibly-undefined ON PURPOSE. Prisma omits an undefined key, so the column keeps taking + // whatever it took before this method was touched: the schema declares `completedWaitpointOrder + // String[]` with no default and the column is nullable, so an omitted key stores NULL, not `{}`. + // Defaulting here would send `{}` instead and change what a live write stores. + // + // The redis-only echo below DOES need a concrete array, because it returns the row shape to the + // caller and that field is not nullable in the payload type. That default belongs to the echo, + // not to the write, so the two are kept apart. + const completedWaitpointOrder = completedWaitpoints + ?.filter((c) => c.index !== undefined) + .sort((a, b) => a.index! - b.index!) + .map((w) => w.id); // Redis-only: no row is written and the decorator owns the document. Echo the input in the shape // the caller expects, so every caller of this method keeps working while Postgres holds nothing. @@ -2054,7 +2061,7 @@ export class PostgresRunStore implements RunStore { workerId: workerId ?? null, runnerId: runnerId ?? null, metadata: snapshot.metadata ?? null, - completedWaitpointOrder, + completedWaitpointOrder: completedWaitpointOrder ?? [], isValid: !error, error: error ?? null, createdAt: now, From 243d175f71c19c59c0568dcc66ae68c419439f1d Mon Sep 17 00:00:00 2001 From: Daniel Sutton Date: Wed, 26 Aug 2026 09:42:00 +0100 Subject: [PATCH 42/56] fix(webapp,run-engine): make the unconfigured and dial-off states genuinely inert An audit of the off-by-default claim found three things active in states this change called inert. The organisation resolver deleted its cache entry when an organisation had no override, so every write for every organisation re-queried the replica, forever. That is every organisation until a ramp starts. The absence is now cached as a distinct value, so a true miss and a known-absent override are different states. The caching decision is extracted so a test can pin it: the previous test exercised its own fake and passed either way. The engine created its two sweep instruments in the constructor with no gate, so every deployment registered them, including self-hosters with nothing configured. They are now created only when a sweep runner is injected. The boot configuration check read the dial through a registry that is cold at boot, so it only ever saw the environment floor and could never see a value an operator had set. It now waits for the first flag load, with a timeout that leaves the check on the floor rather than blocking start-up. Two smaller ones. The sweep lock TTL equalled the job's visibility timeout rather than sitting below it, so redelivery and lock expiry coincided instead of being ordered. And teardown closed one of three Redis connections; the wiring module now hands back a teardown for what it built, so all three close, with the hot-path client last. --- .../webapp/app/v3/snapshotStoreBoot.server.ts | 11 +++++ .../app/v3/snapshotStoreInstance.server.ts | 14 +++++++ .../webapp/app/v3/snapshotStoreMode.server.ts | 37 +++++++++++------ .../app/v3/snapshotStoreWiring.server.ts | 4 +- apps/webapp/test/snapshotStoreMode.test.ts | 40 ++++++++++++++++++- .../run-engine/src/engine/index.ts | 36 +++++++++-------- .../run-engine/src/engine/workerCatalog.ts | 6 +-- 7 files changed, 115 insertions(+), 33 deletions(-) diff --git a/apps/webapp/app/v3/snapshotStoreBoot.server.ts b/apps/webapp/app/v3/snapshotStoreBoot.server.ts index 374b39bf744..4614a759a93 100644 --- a/apps/webapp/app/v3/snapshotStoreBoot.server.ts +++ b/apps/webapp/app/v3/snapshotStoreBoot.server.ts @@ -1,5 +1,6 @@ import type { SnapshotStoreMode } from "@internal/run-store"; import { logger } from "~/services/logger.server"; +import { globalFlagsRegistry } from "~/v3/globalFlagsRegistry.server"; import { getSnapshotRepairEnqueuer } from "./snapshotStoreBindings.server"; import { getSnapshotStoreConfig, getSnapshotSweepClient } from "./snapshotStoreInstance.server"; @@ -15,6 +16,7 @@ export type SnapshotStoreBootDeps = { }; const PING_TIMEOUT_MS = 5_000; +const FLAG_READY_TIMEOUT_MS = 10_000; export async function assertSnapshotStoreBoot(deps: SnapshotStoreBootDeps): Promise { const pastOff = deps.mode !== "off"; @@ -92,6 +94,15 @@ async function pingSweepClient(): Promise { /** The env-reading adapter. The boot log line is not authoritative after boot: the dial can move. */ export async function assertSnapshotStoreBootFromEnv(): Promise { + // Wait for the flag snapshot's first load. Without this the resolved dial is always the env + // floor, because a cold registry returns undefined, and the configuration check would only ever + // see a value no operator sets. A registry that never loads leaves the check on the floor, which + // fails toward inert. + await Promise.race([ + globalFlagsRegistry.isReady, + new Promise((resolve) => setTimeout(resolve, FLAG_READY_TIMEOUT_MS)), + ]); + const config = getSnapshotStoreConfig(); await assertSnapshotStoreBoot({ diff --git a/apps/webapp/app/v3/snapshotStoreInstance.server.ts b/apps/webapp/app/v3/snapshotStoreInstance.server.ts index 420f9040b03..16d805652d7 100644 --- a/apps/webapp/app/v3/snapshotStoreInstance.server.ts +++ b/apps/webapp/app/v3/snapshotStoreInstance.server.ts @@ -64,6 +64,7 @@ function buildClient(name: string): RedisClient { type Instance = { sweepClient: RedisClient; + redisSnapshotStore: RedisSnapshotStore; decorate: (store: RunStore) => RunStore; }; @@ -88,6 +89,7 @@ const instance = singleton("snapshotStoreInstance", () => return { sweepClient, + redisSnapshotStore, decorate: (store: RunStore) => new TaskRunExecutionSnapshotStore(store, { store: redisSnapshotStore, @@ -128,9 +130,21 @@ export function getSnapshotStoreConfig() { }; } +const extraQuits: (() => Promise)[] = []; + +/** Lets the wiring module hand back a teardown for what it built, so this owns closing everything. */ +export function registerSnapshotStoreQuit(quit: () => Promise): void { + extraQuits.push(quit); +} + export async function quitSnapshotStoreClients(): Promise { if (!instance) { return; } + for (const quit of extraQuits) { + await quit().catch(() => undefined); + } await instance.sweepClient.quit().catch(() => undefined); + // Last: an append in flight must still land. + await instance.redisSnapshotStore.quit().catch(() => undefined); } diff --git a/apps/webapp/app/v3/snapshotStoreMode.server.ts b/apps/webapp/app/v3/snapshotStoreMode.server.ts index 458c655efb4..179f700b760 100644 --- a/apps/webapp/app/v3/snapshotStoreMode.server.ts +++ b/apps/webapp/app/v3/snapshotStoreMode.server.ts @@ -7,9 +7,24 @@ import { singleton } from "~/utils/singleton"; import { FEATURE_FLAG, FeatureFlagCatalog } from "~/v3/featureFlags"; import { globalFlagsRegistry } from "~/v3/globalFlagsRegistry.server"; +/** A cached "this organisation has no override", distinct from "not cached". */ +export const NO_OVERRIDE = "__none__" as const; + +type CachedOrgMode = SnapshotStoreMode | typeof NO_OVERRIDE; + +/** + * What to cache for one organisation's blob value. An unparseable or absent value caches + * NO_OVERRIDE rather than nothing: caching nothing means every organisation without an override + * re-queries on every write, which is all of them until a ramp starts. + */ +export function cachedOrgModeFor(raw: unknown): CachedOrgMode { + const parsed = FeatureFlagCatalog[FEATURE_FLAG.snapshotStoreOrgMode].safeParse(raw); + return parsed.success ? parsed.data : NO_OVERRIDE; +} + type OrgModeSource = { - /** Cache-only: returns undefined on a miss rather than querying. */ - get(organizationId: string): SnapshotStoreMode | undefined; + /** Cache-only: returns undefined on a true miss rather than querying. */ + get(organizationId: string): CachedOrgMode | undefined; /** Fire-and-forget, de-duplicated per organisation, never throws. */ refresh(organizationId: string): void; }; @@ -26,9 +41,12 @@ export function buildSnapshotStoreModeResolver(deps: { return global; } - const override = deps.orgMode.get(organizationId); - if (override !== undefined) { - return override; + const cached = deps.orgMode.get(organizationId); + if (cached === NO_OVERRIDE) { + return global; + } + if (cached !== undefined) { + return cached; } // Deliberately no read here. Seven decorator methods accept a caller-supplied `tx`, so a @@ -45,7 +63,7 @@ export function buildSnapshotStoreModeResolver(deps: { } function createOrgModeSource(): OrgModeSource { - const cache = new LRUCache({ + const cache = new LRUCache({ max: env.RUN_ENGINE_SNAPSHOT_STORE_ORG_MODE_CACHE_MAX, ttl: env.RUN_ENGINE_SNAPSHOT_STORE_ORG_MODE_CACHE_TTL_MS, }); @@ -67,12 +85,7 @@ function createOrgModeSource(): OrgModeSource { const raw = (row?.featureFlags as Record | null | undefined)?.[ FEATURE_FLAG.snapshotStoreOrgMode ]; - const parsed = FeatureFlagCatalog[FEATURE_FLAG.snapshotStoreOrgMode].safeParse(raw); - if (parsed.success) { - cache.set(organizationId, parsed.data); - } else { - cache.delete(organizationId); - } + cache.set(organizationId, cachedOrgModeFor(raw)); }) .catch((error) => { logger.warn("snapshotStoreMode: organisation override refresh failed", { diff --git a/apps/webapp/app/v3/snapshotStoreWiring.server.ts b/apps/webapp/app/v3/snapshotStoreWiring.server.ts index f6bf2806a80..0a0148099ec 100644 --- a/apps/webapp/app/v3/snapshotStoreWiring.server.ts +++ b/apps/webapp/app/v3/snapshotStoreWiring.server.ts @@ -5,7 +5,7 @@ import { engine } from "./runEngine.server"; import { runStoreWithoutSnapshotDecorator } from "./runStore.server"; import { buildSnapshotSweepRunner } from "./snapshotSweepRunner.server"; import { setSnapshotRepairEnqueuer, setSnapshotSweepRunner } from "./snapshotStoreBindings.server"; -import { getSnapshotSweepClient } from "./snapshotStoreInstance.server"; +import { getSnapshotSweepClient, registerSnapshotStoreQuit } from "./snapshotStoreInstance.server"; /** * The third module: it imports both sides, so neither the run store nor the engine has to import @@ -48,6 +48,8 @@ export function registerSnapshotStoreWiring(): boolean { }) ); + registerSnapshotStoreQuit(() => sweeper.quit()); + logger.info("snapshot store wiring registered"); return true; } diff --git a/apps/webapp/test/snapshotStoreMode.test.ts b/apps/webapp/test/snapshotStoreMode.test.ts index c21eaca914c..0c46918bbb7 100644 --- a/apps/webapp/test/snapshotStoreMode.test.ts +++ b/apps/webapp/test/snapshotStoreMode.test.ts @@ -1,6 +1,10 @@ import { describe, expect, it, vi } from "vitest"; import type { SnapshotStoreMode } from "@internal/run-store"; -import { buildSnapshotStoreModeResolver } from "~/v3/snapshotStoreMode.server"; +import { + buildSnapshotStoreModeResolver, + cachedOrgModeFor, + NO_OVERRIDE, +} from "~/v3/snapshotStoreMode.server"; function build(opts: { globalMode?: SnapshotStoreMode; @@ -61,6 +65,40 @@ describe("snapshot store mode resolver", () => { expect(r.resolve("org_deleted")).toBe("off"); }); + it("caches an absent override rather than nothing", () => { + // Caching nothing means every organisation without an override re-queries on every write. + expect(cachedOrgModeFor(undefined)).toBe(NO_OVERRIDE); + expect(cachedOrgModeFor(null)).toBe(NO_OVERRIDE); + expect(cachedOrgModeFor("not-a-mode")).toBe(NO_OVERRIDE); + expect(cachedOrgModeFor("redis-read")).toBe(NO_OVERRIDE); + expect(cachedOrgModeFor("dual-write")).toBe("dual-write"); + }); + + it("stops querying once an absent override is cached", () => { + // Without a cached negative, every organisation with no override re-queries on every write, + // which is every organisation until a ramp starts. + const refresh = vi.fn(); + let cached: string | undefined; + const r = buildSnapshotStoreModeResolver({ + globalMode: () => "off", + orgMode: { + get: () => cached as never, + refresh: (id: string) => { + refresh(id); + cached = "__none__"; + }, + }, + envFloor: "off", + }); + + expect(r.resolve("org_a")).toBe("off"); + expect(refresh).toHaveBeenCalledTimes(1); + + expect(r.resolve("org_a")).toBe("off"); + expect(r.resolve("org_a")).toBe("off"); + expect(refresh).toHaveBeenCalledTimes(1); + }); + it("does not consult the organisation source when no organisation is supplied", () => { const get = vi.fn(() => undefined); const r = buildSnapshotStoreModeResolver({ diff --git a/internal-packages/run-engine/src/engine/index.ts b/internal-packages/run-engine/src/engine/index.ts index fa6773135c9..ee9a21b1d36 100644 --- a/internal-packages/run-engine/src/engine/index.ts +++ b/internal-packages/run-engine/src/engine/index.ts @@ -113,8 +113,8 @@ export class RunEngine { private logger: Logger; private tracer: Tracer; private meter: Meter; - private snapshotSweepPassCounter: Counter; - private snapshotSweepCountsHistogram: Histogram; + private snapshotSweepPassCounter?: Counter; + private snapshotSweepCountsHistogram?: Histogram; private snapshotsSinceReplicaMissCounter: Counter; private snapshotsSinceReplicaRetryDelay: { minMs: number; maxMs: number }; private heartbeatTimeouts: HeartbeatTimeouts; @@ -342,18 +342,22 @@ export class RunEngine { this.tracer = options.tracer; this.meter = options.meter ?? getMeter("run-engine"); - this.snapshotSweepPassCounter = this.meter.createCounter( - "run_engine.snapshot_store.sweep_pass_total", - { - description: - "Orphan-sweep passes by outcome. A pass that throws emits outcome=failed, so silence is distinguishable from success", - } - ); + // Only when the sweep is actually wired: a deployment that does not use the snapshot store + // should register no series for it at all. + if (options.snapshotStore?.runSweep) { + this.snapshotSweepPassCounter = this.meter.createCounter( + "run_engine.snapshot_store.sweep_pass_total", + { + description: + "Orphan-sweep passes by outcome. A pass that throws emits outcome=failed, so silence is distinguishable from success", + } + ); - this.snapshotSweepCountsHistogram = this.meter.createHistogram( - "run_engine.snapshot_store.sweep_counts", - { description: "Per-field counts from one orphan-sweep pass" } - ); + this.snapshotSweepCountsHistogram = this.meter.createHistogram( + "run_engine.snapshot_store.sweep_counts", + { description: "Per-field counts from one orphan-sweep pass" } + ); + } this.snapshotsSinceReplicaMissCounter = this.meter.createCounter( "run_engine.snapshots_since.replica_miss", @@ -2950,7 +2954,7 @@ export class RunEngine { if (!runSweep) { // The cron entry is registered when the engine is constructed, which happens before the // webapp sets the binding, so an occurrence already queued at boot can arrive unbound. - this.snapshotSweepPassCounter.add(1, { outcome: "unbound" }); + this.snapshotSweepPassCounter?.add(1, { outcome: "unbound" }); this.logger.error("sweepSnapshotOrphans ran with no sweep runner bound"); return; } @@ -2970,10 +2974,10 @@ export class RunEngine { } catch (error) { this.logger.error("sweepSnapshotOrphans threw", { error }); } finally { - this.snapshotSweepPassCounter.add(1, { outcome }); + this.snapshotSweepPassCounter?.add(1, { outcome }); for (const [field, value] of Object.entries(counts ?? {})) { if (typeof value === "number") { - this.snapshotSweepCountsHistogram.record(value, { field }); + this.snapshotSweepCountsHistogram?.record(value, { field }); } } } diff --git a/internal-packages/run-engine/src/engine/workerCatalog.ts b/internal-packages/run-engine/src/engine/workerCatalog.ts index 06b13ab4470..d7742255636 100644 --- a/internal-packages/run-engine/src/engine/workerCatalog.ts +++ b/internal-packages/run-engine/src/engine/workerCatalog.ts @@ -89,9 +89,9 @@ export const workerCatalog = { lastTimestamp: z.number().optional(), cron: z.string(), }), - // The default budget plus an hour. A pass that outlives this is redelivered and runs beside - // itself; the runner's fenced lock is what actually prevents that, this is headroom. - visibilityTimeoutMs: 14_400_000, + // The default budget plus two hours, so it stays strictly above the runner's lock TTL + // (budget plus one hour). Ordered that way, a lock outlives the delivery it belongs to. + visibilityTimeoutMs: 18_000_000, cron: "0 */6 * * *", jitterInMs: 60_000, // Load-bearing. A throw takes the dead-letter path, which also reschedules, so the cron chain From 07c3398ab40dc8593d475ef764271414a04ddad8 Mon Sep 17 00:00:00 2001 From: Daniel Sutton Date: Wed, 26 Aug 2026 11:00:29 +0100 Subject: [PATCH 43/56] chore(run-store): apply oxfmt to the two new sweeper test files --- .../src/snapshotOrphanSweeper.cluster.test.ts | 5 ++- .../src/snapshotOrphanSweeper.confirm.test.ts | 35 ++++++++++--------- 2 files changed, 23 insertions(+), 17 deletions(-) diff --git a/internal-packages/run-store/src/snapshotOrphanSweeper.cluster.test.ts b/internal-packages/run-store/src/snapshotOrphanSweeper.cluster.test.ts index d8e70d5b3ec..90df7bfcc1d 100644 --- a/internal-packages/run-store/src/snapshotOrphanSweeper.cluster.test.ts +++ b/internal-packages/run-store/src/snapshotOrphanSweeper.cluster.test.ts @@ -68,7 +68,10 @@ describe("scanTargetsOf", () => { let generation = 0; (cluster as unknown as { nodes: () => Redis[] }).nodes = () => { generation += 1; - return Array.from({ length: generation }, (_v, i) => new Redis({ lazyConnect: true, port: 65100 + i })); + return Array.from( + { length: generation }, + (_v, i) => new Redis({ lazyConnect: true, port: 65100 + i }) + ); }; expect(scanTargetsOf(cluster)).toHaveLength(1); diff --git a/internal-packages/run-store/src/snapshotOrphanSweeper.confirm.test.ts b/internal-packages/run-store/src/snapshotOrphanSweeper.confirm.test.ts index d617ca27834..40c90bc4774 100644 --- a/internal-packages/run-store/src/snapshotOrphanSweeper.confirm.test.ts +++ b/internal-packages/run-store/src/snapshotOrphanSweeper.confirm.test.ts @@ -301,25 +301,28 @@ describe("a run seen alive clears its marker", () => { }); describe("a pass can stop inside a budget", () => { - containerTest("an already-passed deadline yields a partial pass", async ({ prisma, redisOptions }) => { - const runStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); - const sweeper = new SnapshotOrphanSweeper({ - redisOptions, - runStore: runStore as unknown as RunStore, - completedTtlMs: COMPLETED_TTL_MS, - }); + containerTest( + "an already-passed deadline yields a partial pass", + async ({ prisma, redisOptions }) => { + const runStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const sweeper = new SnapshotOrphanSweeper({ + redisOptions, + runStore: runStore as unknown as RunStore, + completedTtlMs: COMPLETED_TTL_MS, + }); - try { - const result = await sweeper.sweep({ deadline: Date.now() - 1 }); + try { + const result = await sweeper.sweep({ deadline: Date.now() - 1 }); - // redis-worker redelivers a job that outlives its visibility timeout, and nothing extends it, - // so a pass that cannot stop on its own runs concurrently with itself. - expect(result.partial).toBe(true); - expect(result.scanned).toBe(0); - } finally { - await sweeper.quit(); + // redis-worker redelivers a job that outlives its visibility timeout, and nothing extends it, + // so a pass that cannot stop on its own runs concurrently with itself. + expect(result.partial).toBe(true); + expect(result.scanned).toBe(0); + } finally { + await sweeper.quit(); + } } - }); + ); containerTest("an aborted signal yields a partial pass", async ({ prisma, redisOptions }) => { const runStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); From 1bba0a02cd1ee4b42bf86fdc0a8816189c49910d Mon Sep 17 00:00:00 2001 From: Daniel Sutton Date: Wed, 26 Aug 2026 11:31:00 +0100 Subject: [PATCH 44/56] docs(run-store): correct two stale comments on the orphan-marker path Both survived earlier revisions of the same change and now describe behaviour the code does not have. One claimed the marker has a TTL derived from the confirm window; it is a hash field with no TTL. The other claimed the marker is cleared only for terminal runs, when it is cleared for every run the lookup returned, which is what stops a live run's stale marker from pre-authorising a later deletion. Both sit on the path that deletes a keyspace, so a reader acting on either could reopen the hole the two-sighting rule closes. --- .../run-store/src/snapshotOrphanSweeper.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/internal-packages/run-store/src/snapshotOrphanSweeper.ts b/internal-packages/run-store/src/snapshotOrphanSweeper.ts index 422217cbc84..86bb084d5c9 100644 --- a/internal-packages/run-store/src/snapshotOrphanSweeper.ts +++ b/internal-packages/run-store/src/snapshotOrphanSweeper.ts @@ -373,9 +373,6 @@ export class SnapshotOrphanSweeper { if (markedAt === undefined || Number.isNaN(markedAt)) { if (!dryRun) { - // The TTL is a multiple of the confirm window so a candidate gets several chances to be - // sighted again, while a marker left behind by a run that turned out to be alive cannot - // linger long enough to pre-authorise a later deletion. // The field carries no TTL of its own; it lives and dies with the seq hash, which the // keyspace's own completion expiry already governs. That removes the marker-lifetime knob // whose derivation was wrong in the first place. @@ -403,9 +400,12 @@ export class SnapshotOrphanSweeper { * Clears a rule 2 marker for a keyspace whose run turned out to exist after all, so a later * genuine absence still needs its own two sightings rather than inheriting a stale one. * - * Only called on a path that already found a run row, and only for terminal runs — a live run - * never reaches rule 2, so it can never hold a marker, and charging every live keyspace a round - * trip to prove that would cost more than the case is worth. + * Called for EVERY run row the lookup returned, live ones included, and it has to be: a keyspace + * marked by an earlier incomplete lookup can belong to a run that is perfectly alive, and a + * SUSPENDED run can sit that way for weeks. Leaving the marker in place would let a later genuine + * absence delete on what is really a first sighting, which is the hole the two-sighting rule + * exists to close. It costs one DEL per existing run per pass; that is the price of the guard + * being sound rather than nearly sound. */ async #clearOrphanMarker(runId: string): Promise { try { From 6e976d02ce5f61568ffcc74294f8b75735bbc7da Mon Sep 17 00:00:00 2001 From: Daniel Sutton Date: Wed, 26 Aug 2026 11:50:10 +0100 Subject: [PATCH 45/56] feat(run-store): drop the unimplemented compare dial position, cover redis-only `compare` was a name in the dial with no behaviour behind it: it wrote and read exactly as `dual-write` does, so turning it on would have looked like enabling divergence reporting and delivered plain dual-write. A dial value that silently does something other than its name is worse than a missing one. It returns with the ticket that implements the sampled dual-read and diff. `redis-only` was the thinnest tested position and the only one that cannot be rolled back, since the snapshots written while it is on exist nowhere else. It is also the only position that is a PAIR of settings, the decorator's mode and `snapshotWrites: false` on the store beneath it, and the previous single test used a store that still wrote snapshots. Every test in the new suite builds the pair, and covers the run mutation landing without its snapshot row, transitions, completions, the absent waitpoint join rows, and every read being Redis-served. One test characterises rather than endorses: a read shape the decorator does not recognise is delegated, and at this position Postgres holds nothing, so the caller gets an empty result rather than an error. Only the engine calls that method and it issues the recognised shape, so nothing is broken today. It is pinned so the terminal-cutover ticket decides deliberately whether a fall-through here should throw instead of answering empty. --- .../taskRunExecutionSnapshotStore.off.test.ts | 2 +- ...nExecutionSnapshotStore.readCohort.test.ts | 2 +- ...unExecutionSnapshotStore.redisOnly.test.ts | 321 ++++++++++++++++++ .../src/taskRunExecutionSnapshotStore.ts | 9 +- 4 files changed, 329 insertions(+), 5 deletions(-) create mode 100644 internal-packages/run-store/src/taskRunExecutionSnapshotStore.redisOnly.test.ts diff --git a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.off.test.ts b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.off.test.ts index 4f788b5ec27..437872106e7 100644 --- a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.off.test.ts +++ b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.off.test.ts @@ -87,7 +87,7 @@ describe("TaskRunExecutionSnapshotStore at mode off", () => { it("reports every other dial position as one that writes Redis", () => { const { store } = forwardingProbe(); - const modes = ["dual-write", "compare", "redis-read", "redis-only"] as const; + const modes = ["dual-write", "redis-read", "redis-only"] as const; for (const mode of modes) { const decorated = new TaskRunExecutionSnapshotStore(store, { diff --git a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.readCohort.test.ts b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.readCohort.test.ts index e33f7a9b253..58c80b4df72 100644 --- a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.readCohort.test.ts +++ b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.readCohort.test.ts @@ -28,7 +28,7 @@ const ids = Array.from({ length: 500 }, (_, n) => `run_cohort_${n}_${n * 7919}`) describe("the read cohort", () => { it("reads nothing from Redis before the read positions", () => { - for (const mode of ["off", "dual-write", "compare"] as const) { + for (const mode of ["off", "dual-write"] as const) { const store = probe(mode, 100); expect(ids.every((id) => !store.readsFromRedis(id))).toBe(true); } diff --git a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.redisOnly.test.ts b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.redisOnly.test.ts new file mode 100644 index 00000000000..b308cb20988 --- /dev/null +++ b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.redisOnly.test.ts @@ -0,0 +1,321 @@ +// `redis-only` is the terminal cutover, and it is the only dial position where Postgres stops being +// authoritative: it cannot be rolled back by turning the dial down, because the snapshots written +// while it was on exist nowhere else. It is also the only position that is a PAIR of settings, not +// one — the decorator's mode AND `snapshotWrites: false` on the store underneath it — and the two +// are set by different tickets. Every test here builds the pair, because testing the mode against a +// store that still writes snapshots would exercise a configuration that never ships. +import { describe, expect } from "vitest"; +import { containerTest } from "@internal/testcontainers"; +import { generateInternalId } from "@trigger.dev/core/v3/isomorphic"; +import { PostgresRunStore } from "./PostgresRunStore.js"; +import { RedisSnapshotStore } from "./redisSnapshotStore.js"; +import { TaskRunExecutionSnapshotStore } from "./taskRunExecutionSnapshotStore.js"; +import type { RunStore } from "./types.js"; +import { + buildCreateRunData, + seedSnapshotEnvironment, + seedSnapshotWaitpoints, + type SnapshotFixtureEnv, +} from "./testFixtures/snapshotIdFixture.js"; + +const COMPLETED_TTL_MS = 72 * 60 * 60 * 1000; + +/** The shipping pair: decorator at `redis-only` over a store that writes no snapshot rows. */ +function build(prisma: never, redisOptions: never) { + const redis = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + const reads: { method: string; source: string }[] = []; + + const decorated = new TaskRunExecutionSnapshotStore( + new PostgresRunStore({ + prisma, + readOnlyPrisma: prisma, + snapshotWrites: false, + }) as unknown as RunStore, + { + store: redis, + mode: "redis-only", + metrics: { + recordWrite: () => {}, + recordAppendFailed: () => {}, + recordRead: (method, source) => reads.push({ method, source }), + }, + } + ); + + return { decorated, redis, reads }; +} + +function birth(env: SnapshotFixtureEnv, id: string) { + return { + id, + engine: "V2" as const, + executionStatus: "RUN_CREATED" as const, + description: "Run was created", + runStatus: "PENDING" as const, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }; +} + +async function seedRun(decorated: TaskRunExecutionSnapshotStore, env: SnapshotFixtureEnv) { + const runId = generateInternalId(); + const snapshotId = generateInternalId(); + await decorated.createRun({ + data: buildCreateRunData(runId, env), + snapshot: birth(env, snapshotId), + }); + return { runId, snapshotId }; +} + +function transition(runId: string, env: SnapshotFixtureEnv, description: string) { + return { + run: { id: runId, status: "EXECUTING" as const, attemptNumber: 1 }, + snapshot: { executionStatus: "EXECUTING" as const, description }, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }; +} + +describe("redis-only: Postgres stops holding snapshots", () => { + containerTest("the run row lands but no snapshot row does", async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const { runId, snapshotId } = await seedRun(decorated, env); + + // The run itself is still Postgres-authoritative at this position. Only its snapshots move. + expect(await prisma.taskRun.count({ where: { id: runId } })).toBe(1); + expect(await prisma.taskRunExecutionSnapshot.count({ where: { runId } })).toBe(0); + + // And the snapshot is genuinely in Redis under the id the caller minted. + const head = await redis.getLatest(runId); + expect(head?.id).toBe(snapshotId); + } finally { + await redis.quit(); + } + }); + + containerTest("transitions write no snapshot row either", async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const { runId } = await seedRun(decorated, env); + + await decorated.createExecutionSnapshot(transition(runId, env, "Run started")); + await decorated.createExecutionSnapshot(transition(runId, env, "Run continued")); + + expect(await prisma.taskRunExecutionSnapshot.count({ where: { runId } })).toBe(0); + const since = await redis.getSinceCreatedAt(runId, new Date(Date.now() - 60_000), { + limit: 50, + }); + expect(since.kind).toBe("hit"); + expect(since.kind === "hit" ? since.entries.length : 0).toBeGreaterThanOrEqual(2); + } finally { + await redis.quit(); + } + }); + + containerTest( + "a completion still updates the run row while writing no snapshot", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const { runId } = await seedRun(decorated, env); + + await decorated.completeAttemptSuccess( + runId, + { + completedAt: new Date(), + outputType: "application/json", + usageDurationMs: 1, + costInCents: 0, + snapshot: { + id: generateInternalId(), + executionStatus: "FINISHED", + description: "Run completed", + runStatus: "COMPLETED_SUCCESSFULLY", + attemptNumber: 1, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }, + }, + { select: { id: true } } + ); + + // The mutation half of a nested write must still land, or the run never finishes. + const run = await prisma.taskRun.findFirstOrThrow({ where: { id: runId } }); + expect(run.status).toBe("COMPLETED_SUCCESSFULLY"); + expect(await prisma.taskRunExecutionSnapshot.count({ where: { runId } })).toBe(0); + } finally { + await redis.quit(); + } + } + ); + + containerTest( + "no completed-waitpoint join rows are written for a snapshot Postgres does not have", + async ({ prisma, redisOptions }) => { + // The join rows point at a snapshot row. With snapshot writes off there is no such row, so + // inserting them would leave links dangling at a snapshot only Redis holds. + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const { runId } = await seedRun(decorated, env); + const [wpA, wpB] = await seedSnapshotWaitpoints(prisma, env, 2); + + await decorated.createExecutionSnapshot({ + ...transition(runId, env, "Run resumed"), + completedWaitpoints: [ + { id: wpA, index: 0 }, + { id: wpB, index: 1 }, + ], + }); + + expect(await prisma.taskRunExecutionSnapshot.count({ where: { runId } })).toBe(0); + const joins = await prisma.$queryRawUnsafe<{ n: bigint }[]>( + `SELECT count(*) AS n FROM "_completedWaitpoints" WHERE "B" = ANY($1::text[])`, + [wpA, wpB] + ); + expect(Number(joins[0]!.n)).toBe(0); + } finally { + await redis.quit(); + } + } + ); +}); + +describe("redis-only: every read is served from Redis", () => { + containerTest( + "the hot read, the since window and the waitpoint lookups all come from Redis", + async ({ prisma, redisOptions }) => { + // At every earlier position a Redis miss falls back to Postgres and the caller never notices. + // Here Postgres holds nothing, so a read that fell back would answer empty rather than wrong, + // and a run would silently lose its state. Each read is asserted to be Redis-sourced. + const { decorated, redis, reads } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const { runId } = await seedRun(decorated, env); + const [wpA] = await seedSnapshotWaitpoints(prisma, env, 1); + const created = await decorated.createExecutionSnapshot({ + ...transition(runId, env, "Run resumed"), + completedWaitpoints: [{ id: wpA, index: 0 }], + }); + + const latest = await decorated.findLatestExecutionSnapshot(runId); + expect(latest!.id).toBe(created.id); + expect(latest!.completedWaitpointOrder).toEqual([wpA]); + + const window = await decorated.findManyExecutionSnapshots({ + where: { runId, isValid: true, createdAt: { gt: new Date(Date.now() - 60_000) } }, + include: { checkpoint: true }, + orderBy: { createdAt: "desc" }, + take: 50, + }); + expect(window.length).toBeGreaterThan(0); + + const withPresence = await decorated.findSnapshotCompletedWaitpointIdsWithPresence( + created.id, + undefined, + runId + ); + expect(withPresence.ids).toEqual([wpA]); + + expect(reads.length).toBeGreaterThan(0); + expect(reads.every((r) => r.source === "redis")).toBe(true); + } finally { + await redis.quit(); + } + } + ); + + containerTest( + "an unrecognised read shape falls through to a Postgres that holds nothing", + async ({ prisma, redisOptions }) => { + // CHARACTERISATION, NOT AN ENDORSEMENT. `findManyExecutionSnapshots` serves from Redis only + // for the since-window shape `matchSinceWindow` recognises; anything else delegates. At every + // dial position before this one that is harmless, because Postgres holds the same rows. Here + // it holds none, so the caller gets an EMPTY result rather than an error, and empty is a + // valid answer to this query. The same is true of the `miss` and `danglingCycle` fallbacks in + // that method: all three are safe everywhere except the one position that cannot fall back. + // + // Only the engine's own call shapes reach this method today, and it issues the since-window + // one, so nothing is broken. It is pinned here so the terminal-cutover ticket decides + // deliberately whether a fall-through at `redis-only` should throw instead of answering + // empty, rather than discovering this shape in production. + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const { runId } = await seedRun(decorated, env); + await decorated.createExecutionSnapshot(transition(runId, env, "Run started")); + + // No `createdAt` cursor, so the shape does not match and the read is delegated. + const unmatched = await decorated.findManyExecutionSnapshots({ + where: { runId, isValid: true }, + include: { checkpoint: true }, + orderBy: { createdAt: "desc" }, + take: 50, + }); + + expect(unmatched).toEqual([]); + + // The same run, asked the shape the engine actually issues, answers in full from Redis. + const matched = await decorated.findManyExecutionSnapshots({ + where: { runId, isValid: true, createdAt: { gt: new Date(Date.now() - 60_000) } }, + include: { checkpoint: true }, + orderBy: { createdAt: "desc" }, + take: 50, + }); + expect(matched.length).toBeGreaterThan(0); + } finally { + await redis.quit(); + } + } + ); + + containerTest( + "the read cohort dial cannot route a run away from Redis", + async ({ prisma, redisOptions }) => { + // readPercent is a ramp control for `redis-read`. At `redis-only` a run routed to Postgres + // would read a database that holds no snapshots at all, so the dial must be ignored here + // whatever it is set to. + const redis = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + const reads: string[] = []; + const decorated = new TaskRunExecutionSnapshotStore( + new PostgresRunStore({ + prisma: prisma as never, + readOnlyPrisma: prisma as never, + snapshotWrites: false, + }) as unknown as RunStore, + { + store: redis, + mode: "redis-only", + readPercent: 0, + metrics: { + recordWrite: () => {}, + recordAppendFailed: () => {}, + recordRead: (_m, source) => reads.push(source), + }, + } + ); + + try { + const env = await seedSnapshotEnvironment(prisma); + const { runId, snapshotId } = await seedRun(decorated, env); + + const latest = await decorated.findLatestExecutionSnapshot(runId); + + expect(latest!.id).toBe(snapshotId); + expect(reads).not.toContain("postgres"); + } finally { + await redis.quit(); + } + } + ); +}); diff --git a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.ts b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.ts index f4a46cb0414..c6bc145a91d 100644 --- a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.ts +++ b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.ts @@ -60,10 +60,13 @@ const WAITPOINT_CHUNK_SIZE = 100; * The rollout dial. Postgres stays fully written and authoritative in every position before * `redis-only`, so every earlier position rolls back losslessly by turning the dial down. * - * `compare` writes exactly as `dual-write` does. Its sampled dual-read and diff are a later ticket; - * the position is named here so the dial does not have to widen once that lands. + * A `compare` position was named here before its behaviour existed, and it read from this type as a + * real dial position while behaving in every respect exactly like `dual-write`. A dial value that + * silently does something other than its name is worse than a missing one: turning it on would have + * looked like enabling divergence reporting and delivered plain dual-write. It is added back by the + * ticket that implements the sampled dual-read and diff, at which point the name will be true. */ -export type SnapshotStoreMode = "off" | "dual-write" | "compare" | "redis-read" | "redis-only"; +export type SnapshotStoreMode = "off" | "dual-write" | "redis-read" | "redis-only"; /** * Enqueues the existing `repairSnapshot` job for a run whose append was lost. The decorator lives in From 6cc24de089ebd775d95579421cc03d820b050966 Mon Sep 17 00:00:00 2001 From: Daniel Sutton Date: Wed, 26 Aug 2026 11:55:27 +0100 Subject: [PATCH 46/56] fix(webapp): stop the dial resolver constructing its cache at module load Six webapp test shards failed on import with "At least one of max, maxSize, or ttl is required". The organisation cache was built by a top-level singleton call, so importing the module read two new environment variables. Many webapp suites mock the env module with a minimal object, so both came back undefined and the LRU constructor threw, taking the whole file down before any test ran. The cache is now built on first use rather than at import, so the module has no load-time side effect at all, and its two options carry inline defaults so a partial env cannot throw. The environment floor is read defensively for the same reason. Covered by a test that mocks the env module the way the suite that caught this does. Reintroducing either half of the fix fails it. --- .../webapp/app/v3/snapshotStoreMode.server.ts | 23 +++++++++++---- .../test/snapshotStoreModuleLoad.test.ts | 29 +++++++++++++++++++ 2 files changed, 46 insertions(+), 6 deletions(-) create mode 100644 apps/webapp/test/snapshotStoreModuleLoad.test.ts diff --git a/apps/webapp/app/v3/snapshotStoreMode.server.ts b/apps/webapp/app/v3/snapshotStoreMode.server.ts index 179f700b760..62f53cf893f 100644 --- a/apps/webapp/app/v3/snapshotStoreMode.server.ts +++ b/apps/webapp/app/v3/snapshotStoreMode.server.ts @@ -62,10 +62,15 @@ export function buildSnapshotStoreModeResolver(deps: { }; } +const DEFAULT_CACHE_MAX = 10_000; +const DEFAULT_CACHE_TTL_MS = 30_000; + function createOrgModeSource(): OrgModeSource { + // Defaults inline as well as in the schema: this must not throw when a caller supplies a partial + // env, and an LRU with neither bound set is a constructor error. const cache = new LRUCache({ - max: env.RUN_ENGINE_SNAPSHOT_STORE_ORG_MODE_CACHE_MAX, - ttl: env.RUN_ENGINE_SNAPSHOT_STORE_ORG_MODE_CACHE_TTL_MS, + max: env.RUN_ENGINE_SNAPSHOT_STORE_ORG_MODE_CACHE_MAX ?? DEFAULT_CACHE_MAX, + ttl: env.RUN_ENGINE_SNAPSHOT_STORE_ORG_MODE_CACHE_TTL_MS ?? DEFAULT_CACHE_TTL_MS, }); const inFlight = new Set(); @@ -100,15 +105,21 @@ function createOrgModeSource(): OrgModeSource { }; } -const orgModeSource = singleton("snapshotStoreOrgModeSource", createOrgModeSource); +/** Built on first use, never at import: importing this module must have no side effect. */ +function orgModeSource(): OrgModeSource { + return singleton("snapshotStoreOrgModeSource", createOrgModeSource); +} export const snapshotStoreModeResolver: SnapshotStoreModeResolver = buildSnapshotStoreModeResolver({ globalMode: () => globalFlagsRegistry.current()?.[FEATURE_FLAG.snapshotStoreMode], - orgMode: orgModeSource, - envFloor: env.RUN_ENGINE_SNAPSHOT_STORE_MODE, + orgMode: { + get: (organizationId) => orgModeSource().get(organizationId), + refresh: (organizationId) => orgModeSource().refresh(organizationId), + }, + envFloor: env.RUN_ENGINE_SNAPSHOT_STORE_MODE ?? "off", }); /** Called by the organisation flag save path so the writing process sees a dial change at once. */ export function invalidateSnapshotStoreOrgMode(organizationId: string): void { - orgModeSource.refresh(organizationId); + orgModeSource().refresh(organizationId); } diff --git a/apps/webapp/test/snapshotStoreModuleLoad.test.ts b/apps/webapp/test/snapshotStoreModuleLoad.test.ts new file mode 100644 index 00000000000..50ed8da596b --- /dev/null +++ b/apps/webapp/test/snapshotStoreModuleLoad.test.ts @@ -0,0 +1,29 @@ +import { describe, expect, it, vi } from "vitest"; + +// Reproduces the CI break. Many webapp suites mock ~/db.server and ~/env.server with minimal +// objects, so a module-load side effect that reads a new env variable takes the whole file down on +// import. This is the exact mock shape of the suite that caught it. +vi.mock("~/db.server", () => ({ prisma: {}, $replica: {} })); +vi.mock("~/env.server", () => ({ env: { SESSION_SECRET: "test-session-secret" } })); +vi.mock("~/services/logger.server", () => ({ + logger: { info: () => {}, warn: () => {}, error: () => {}, debug: () => {} }, +})); + +describe("snapshot store modules under a minimal env mock", () => { + it("imports the mode resolver without constructing anything", async () => { + await expect(import("~/v3/snapshotStoreMode.server")).resolves.toBeDefined(); + }); + + it("resolves off rather than throwing", async () => { + const { snapshotStoreModeResolver } = await import("~/v3/snapshotStoreMode.server"); + expect(snapshotStoreModeResolver.resolve()).toBe("off"); + expect(snapshotStoreModeResolver.resolve("org_anything")).toBe("off"); + }); + + it("imports the instance module and stays undecorated", async () => { + const mod = await import("~/v3/snapshotStoreInstance.server"); + const sentinel = {} as never; + expect(mod.decorateWithSnapshotStore(sentinel)).toBe(sentinel); + expect(mod.getSnapshotSweepClient()).toBeUndefined(); + }); +}); From ea6de93fc9ee96f71f6fa01bab7ced7a487ef471 Mon Sep 17 00:00:00 2001 From: Daniel Sutton Date: Wed, 26 Aug 2026 11:55:27 +0100 Subject: [PATCH 47/56] feat(webapp,run-engine): take up the store and sweeper connection contract The store now receives a pre-built client instead of building its own, so cluster mode reaches the hot path and not only the sweep. The webapp owns both sockets and closes both, with the hot-path client last so an append in flight can still land. The sweep receives the dedicated client, the confirm window that was parsed and unused, and the deadline and abort signal it now accepts. So the pass budget is enforced inside a pass rather than only bounding the lock, and a truncated pass reports itself as partial. The engine also arms an abort a minute past the budget. The deadline is the sweep's own stopping rule, checked at batch boundaries; this is the backstop for a pass that has stopped reaching one, so the signal is not decorative. --- .../webapp/app/v3/snapshotStoreInstance.server.ts | 12 ++++++++---- apps/webapp/app/v3/snapshotStoreWiring.server.ts | 15 ++++----------- internal-packages/run-engine/src/engine/index.ts | 4 ++++ 3 files changed, 16 insertions(+), 15 deletions(-) diff --git a/apps/webapp/app/v3/snapshotStoreInstance.server.ts b/apps/webapp/app/v3/snapshotStoreInstance.server.ts index 16d805652d7..7ffd5ef2d06 100644 --- a/apps/webapp/app/v3/snapshotStoreInstance.server.ts +++ b/apps/webapp/app/v3/snapshotStoreInstance.server.ts @@ -64,6 +64,7 @@ function buildClient(name: string): RedisClient { type Instance = { sweepClient: RedisClient; + hotPathClient: RedisClient; redisSnapshotStore: RedisSnapshotStore; decorate: (store: RunStore) => RunStore; }; @@ -79,16 +80,17 @@ const instance = singleton("snapshotStoreInstance", () => // transition append. It also backs the sweep's exclusion lock. const sweepClient = buildClient("sweep"); + const hotPathClient = buildClient("store"); + const redisSnapshotStore = new RedisSnapshotStore({ - // The store still builds its own single-node client, so cluster mode reaches the sweep but not - // the hot path. Hand it a pre-built client once its options accept one. - redisOptions: redisOptions(), + client: hotPathClient, completedTtlMs: env.RUN_ENGINE_SNAPSHOT_STORE_COMPLETED_TTL_MS, metrics: metrics.store, }); return { sweepClient, + hotPathClient, redisSnapshotStore, decorate: (store: RunStore) => new TaskRunExecutionSnapshotStore(store, { @@ -145,6 +147,8 @@ export async function quitSnapshotStoreClients(): Promise { await quit().catch(() => undefined); } await instance.sweepClient.quit().catch(() => undefined); - // Last: an append in flight must still land. + // Last: an append in flight must still land. The store's own quit() returns early on a + // caller-supplied client, so closing the socket is ours. await instance.redisSnapshotStore.quit().catch(() => undefined); + await instance.hotPathClient.quit().catch(() => undefined); } diff --git a/apps/webapp/app/v3/snapshotStoreWiring.server.ts b/apps/webapp/app/v3/snapshotStoreWiring.server.ts index 0a0148099ec..06a9114664a 100644 --- a/apps/webapp/app/v3/snapshotStoreWiring.server.ts +++ b/apps/webapp/app/v3/snapshotStoreWiring.server.ts @@ -23,27 +23,20 @@ export function registerSnapshotStoreWiring(): boolean { }); const sweeper = new SnapshotOrphanSweeper({ - redisOptions: { - keyPrefix: "engine:", - host: env.RUN_ENGINE_SNAPSHOT_STORE_REDIS_HOST ?? undefined, - port: env.RUN_ENGINE_SNAPSHOT_STORE_REDIS_PORT ?? undefined, - username: env.RUN_ENGINE_SNAPSHOT_STORE_REDIS_USERNAME ?? undefined, - password: env.RUN_ENGINE_SNAPSHOT_STORE_REDIS_PASSWORD ?? undefined, - ...(env.RUN_ENGINE_SNAPSHOT_STORE_REDIS_TLS_DISABLED === "true" ? {} : { tls: {} }), - }, + // Its own connection, so a scan of every master can never stall a transition append. + client: sweepClient, // The undecorated router: rule 2 asks Postgres whether a run row exists, and must never be // able to ask Redis whether Redis is an orphan. runStore: runStoreWithoutSnapshotDecorator, completedTtlMs: env.RUN_ENGINE_SNAPSHOT_STORE_COMPLETED_TTL_MS, orphanAgeMs: env.RUN_ENGINE_SNAPSHOT_STORE_ORPHAN_AGE_MS, + confirmOrphanAfterMs: env.RUN_ENGINE_SNAPSHOT_STORE_CONFIRM_ORPHAN_AFTER_MS, }); setSnapshotSweepRunner( buildSnapshotSweepRunner({ client: sweepClient, - // The sweep does not yet accept a deadline, so the budget is not enforced inside a pass. The - // fenced lock is what keeps two passes apart; its TTL covers the expected pass duration. - sweep: async () => ({ ...(await sweeper.sweep()) }), + sweep: async ({ deadline, signal }) => ({ ...(await sweeper.sweep({ deadline, signal })) }), lockTtlMs: env.RUN_ENGINE_SNAPSHOT_STORE_GC_SWEEP_BUDGET_MS + 3_600_000, }) ); diff --git a/internal-packages/run-engine/src/engine/index.ts b/internal-packages/run-engine/src/engine/index.ts index ee9a21b1d36..3a5ad4021f9 100644 --- a/internal-packages/run-engine/src/engine/index.ts +++ b/internal-packages/run-engine/src/engine/index.ts @@ -2961,6 +2961,9 @@ export class RunEngine { const budgetMs = this.options.snapshotStore?.sweepBudgetMs ?? 10_800_000; const controller = new AbortController(); + // The deadline is the sweep's own stopping rule; this is the backstop for a pass that has + // stopped reaching a batch boundary, so the signal is not merely decorative. + const abortAt = globalThis.setTimeout(() => controller.abort(), budgetMs + 60_000); let outcome = "failed"; let counts: Record | undefined; @@ -2974,6 +2977,7 @@ export class RunEngine { } catch (error) { this.logger.error("sweepSnapshotOrphans threw", { error }); } finally { + globalThis.clearTimeout(abortAt); this.snapshotSweepPassCounter?.add(1, { outcome }); for (const [field, value] of Object.entries(counts ?? {})) { if (typeof value === "number") { From dd927b3b7d92e76dab417cc42b9292f826599c49 Mon Sep 17 00:00:00 2001 From: Daniel Sutton Date: Wed, 26 Aug 2026 12:30:01 +0100 Subject: [PATCH 48/56] fix(webapp): decouple the dial resolver from the run-store type, and probe a CI-only failure The dial positions are declared locally rather than imported, so this module no longer depends on the run-store package's build output to typecheck. An assertion keeps the two in step and fails loudly if either gains or loses a position. That is worth doing regardless, but it does not explain the failure it was prompted by. This module typechecks clean locally, on a clean rebuild of run-store from the same source, under both source and built resolution, and under the same command CI runs. It fails in CI with three errors that are only mutually consistent if the imported type resolves to never there. So this also carries a temporary probe whose reported type names which case CI is in. It comes out in the next commit once the answer is in the log. --- .../webapp/app/v3/snapshotStoreMode.server.ts | 28 ++++++++++++++++--- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/apps/webapp/app/v3/snapshotStoreMode.server.ts b/apps/webapp/app/v3/snapshotStoreMode.server.ts index 62f53cf893f..12b10056dc0 100644 --- a/apps/webapp/app/v3/snapshotStoreMode.server.ts +++ b/apps/webapp/app/v3/snapshotStoreMode.server.ts @@ -10,7 +10,27 @@ import { globalFlagsRegistry } from "~/v3/globalFlagsRegistry.server"; /** A cached "this organisation has no override", distinct from "not cached". */ export const NO_OVERRIDE = "__none__" as const; -type CachedOrgMode = SnapshotStoreMode | typeof NO_OVERRIDE; +/** + * The dial positions, declared here rather than imported, so this module does not depend on the + * run-store package's build output to typecheck. The assertion below fails if the two ever diverge. + */ +type DialMode = "off" | "dual-write" | "compare" | "redis-read" | "redis-only"; + +/** Only the write positions are settable per organisation: reads are global. */ +type OrgDialMode = "off" | "dual-write" | "compare"; + +type AssertSame = [A] extends [B] ? ([B] extends [A] ? true : never) : never; +const _dialMatchesRunStore: AssertSame = true; +void _dialMatchesRunStore; + +// Temporary probe. This module typechecks locally and fails in CI with errors that are only +// consistent with SnapshotStoreMode resolving to `never` there. The reported type here names which +// it is, so the CI log answers it outright. Remove once the cause is known. +type ProbeIsNever = [SnapshotStoreMode] extends [never] ? "RESOLVED_NEVER" : "RESOLVED_UNION"; +const _probe: ProbeIsNever = "RESOLVED_UNION"; +void _probe; + +type CachedOrgMode = OrgDialMode | typeof NO_OVERRIDE; /** * What to cache for one organisation's blob value. An unparseable or absent value caches @@ -30,12 +50,12 @@ type OrgModeSource = { }; export function buildSnapshotStoreModeResolver(deps: { - globalMode: () => SnapshotStoreMode | undefined; + globalMode: () => DialMode | undefined; orgMode: OrgModeSource; - envFloor: SnapshotStoreMode; + envFloor: DialMode; }): SnapshotStoreModeResolver { return { - resolve(organizationId?: string): SnapshotStoreMode { + resolve(organizationId?: string): DialMode { const global = deps.globalMode() ?? deps.envFloor; if (!organizationId) { return global; From 3624cd3289d0e7a011150c8fb9136d0119828b87 Mon Sep 17 00:00:00 2001 From: Daniel Sutton Date: Wed, 26 Aug 2026 12:58:44 +0100 Subject: [PATCH 49/56] chore(webapp): probe which dial positions CI resolves The probe deliberately fails: the compiler names the union members in the message, so the CI log states exactly which positions it sees. Local output lists all five. Comes out in the next commit. --- apps/webapp/app/v3/snapshotStoreMode.server.ts | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/apps/webapp/app/v3/snapshotStoreMode.server.ts b/apps/webapp/app/v3/snapshotStoreMode.server.ts index 12b10056dc0..20a371825af 100644 --- a/apps/webapp/app/v3/snapshotStoreMode.server.ts +++ b/apps/webapp/app/v3/snapshotStoreMode.server.ts @@ -23,12 +23,11 @@ type AssertSame = [A] extends [B] ? ([B] extends [A] ? true : never) : nev const _dialMatchesRunStore: AssertSame = true; void _dialMatchesRunStore; -// Temporary probe. This module typechecks locally and fails in CI with errors that are only -// consistent with SnapshotStoreMode resolving to `never` there. The reported type here names which -// it is, so the CI log answers it outright. Remove once the cause is known. -type ProbeIsNever = [SnapshotStoreMode] extends [never] ? "RESOLVED_NEVER" : "RESOLVED_UNION"; -const _probe: ProbeIsNever = "RESOLVED_UNION"; -void _probe; +// Temporary probe. It is a union in CI, not never, but a narrower one than here. Assigning a bogus +// literal makes the compiler expand the alias, so the CI log names the exact members it sees. +type RevealMembers = { [K in T]: true }; +const _probeMembers: RevealMembers = {}; +void _probeMembers; type CachedOrgMode = OrgDialMode | typeof NO_OVERRIDE; From 3277ac30ae3b7c4da3ad30549c0151de3f10ab5b Mon Sep 17 00:00:00 2001 From: Daniel Sutton Date: Wed, 26 Aug 2026 13:05:26 +0100 Subject: [PATCH 50/56] fix(webapp): drop the compare dial position, and remove the probes The base branch removed compare from the dial as an unimplemented position. CI builds the pull request's merge ref against the current base, so it saw four positions while this branch still had five, and reported it as three type errors that named neither cause. My clone was simply stale. So compare goes from the environment schema, both flag catalog entries, the resolver's local unions and the tests. The per-organisation key is now off and dual-write, which is every write position there is. The drift assertion added while chasing this stays. It is what turned an obscure failure into a statement that two definitions disagree, and it will catch the next divergence the same way. Both temporary probes come out. --- apps/webapp/app/env.server.ts | 2 +- apps/webapp/app/v3/featureFlags.ts | 10 ++-------- apps/webapp/app/v3/snapshotStoreMode.server.ts | 10 ++-------- apps/webapp/test/snapshotStoreFlags.test.ts | 6 +++--- apps/webapp/test/snapshotStoreMode.test.ts | 4 ++-- 5 files changed, 10 insertions(+), 22 deletions(-) diff --git a/apps/webapp/app/env.server.ts b/apps/webapp/app/env.server.ts index 80af0951798..411dee2cc13 100644 --- a/apps/webapp/app/env.server.ts +++ b/apps/webapp/app/env.server.ts @@ -1306,7 +1306,7 @@ const EnvironmentSchema = z // Execution-snapshot store. MODE here is only the FLOOR: the operational dial is the // snapshotStoreMode feature flag, so it can move without a deploy. RUN_ENGINE_SNAPSHOT_STORE_MODE: z - .enum(["off", "dual-write", "compare", "redis-read", "redis-only"]) + .enum(["off", "dual-write", "redis-read", "redis-only"]) .default("off"), RUN_ENGINE_SNAPSHOT_STORE_COMPLETED_TTL_MS: z.coerce .number() diff --git a/apps/webapp/app/v3/featureFlags.ts b/apps/webapp/app/v3/featureFlags.ts index 8406fc11a61..8caecf428c5 100644 --- a/apps/webapp/app/v3/featureFlags.ts +++ b/apps/webapp/app/v3/featureFlags.ts @@ -161,14 +161,8 @@ export const FeatureFlagCatalog = { [FEATURE_FLAG.additionalApiKeysEnabled]: z.boolean(), [FEATURE_FLAG.additionalApiKeyIssuanceEnabled]: z.boolean(), [FEATURE_FLAG.additionalApiKeyLookupEnabled]: z.boolean(), - [FEATURE_FLAG.snapshotStoreMode]: z.enum([ - "off", - "dual-write", - "compare", - "redis-read", - "redis-only", - ]), - [FEATURE_FLAG.snapshotStoreOrgMode]: z.enum(["off", "dual-write", "compare"]), + [FEATURE_FLAG.snapshotStoreMode]: z.enum(["off", "dual-write", "redis-read", "redis-only"]), + [FEATURE_FLAG.snapshotStoreOrgMode]: z.enum(["off", "dual-write"]), }; export type FeatureFlagKey = keyof typeof FeatureFlagCatalog; diff --git a/apps/webapp/app/v3/snapshotStoreMode.server.ts b/apps/webapp/app/v3/snapshotStoreMode.server.ts index 20a371825af..cbc80d5c6fb 100644 --- a/apps/webapp/app/v3/snapshotStoreMode.server.ts +++ b/apps/webapp/app/v3/snapshotStoreMode.server.ts @@ -14,21 +14,15 @@ export const NO_OVERRIDE = "__none__" as const; * The dial positions, declared here rather than imported, so this module does not depend on the * run-store package's build output to typecheck. The assertion below fails if the two ever diverge. */ -type DialMode = "off" | "dual-write" | "compare" | "redis-read" | "redis-only"; +type DialMode = "off" | "dual-write" | "redis-read" | "redis-only"; /** Only the write positions are settable per organisation: reads are global. */ -type OrgDialMode = "off" | "dual-write" | "compare"; +type OrgDialMode = "off" | "dual-write"; type AssertSame = [A] extends [B] ? ([B] extends [A] ? true : never) : never; const _dialMatchesRunStore: AssertSame = true; void _dialMatchesRunStore; -// Temporary probe. It is a union in CI, not never, but a narrower one than here. Assigning a bogus -// literal makes the compiler expand the alias, so the CI log names the exact members it sees. -type RevealMembers = { [K in T]: true }; -const _probeMembers: RevealMembers = {}; -void _probeMembers; - type CachedOrgMode = OrgDialMode | typeof NO_OVERRIDE; /** diff --git a/apps/webapp/test/snapshotStoreFlags.test.ts b/apps/webapp/test/snapshotStoreFlags.test.ts index 9cbce38c7de..db43bc111c8 100644 --- a/apps/webapp/test/snapshotStoreFlags.test.ts +++ b/apps/webapp/test/snapshotStoreFlags.test.ts @@ -6,8 +6,8 @@ import { } from "~/v3/featureFlags"; describe("snapshot store dial catalog", () => { - it("accepts all five positions globally", () => { - for (const value of ["off", "dual-write", "compare", "redis-read", "redis-only"]) { + it("accepts all four positions globally", () => { + for (const value of ["off", "dual-write", "redis-read", "redis-only"]) { expect(FeatureFlagCatalog.snapshotStoreMode.safeParse(value).success).toBe(true); } }); @@ -17,7 +17,7 @@ describe("snapshot store dial catalog", () => { }); it("accepts only write positions per organisation", () => { - for (const value of ["off", "dual-write", "compare"]) { + for (const value of ["off", "dual-write"]) { expect(FeatureFlagCatalog.snapshotStoreOrgMode.safeParse(value).success).toBe(true); } // Reads are global, so an org at a read position would read state its own writes never created. diff --git a/apps/webapp/test/snapshotStoreMode.test.ts b/apps/webapp/test/snapshotStoreMode.test.ts index 0c46918bbb7..e01fa3c8dbd 100644 --- a/apps/webapp/test/snapshotStoreMode.test.ts +++ b/apps/webapp/test/snapshotStoreMode.test.ts @@ -102,11 +102,11 @@ describe("snapshot store mode resolver", () => { it("does not consult the organisation source when no organisation is supplied", () => { const get = vi.fn(() => undefined); const r = buildSnapshotStoreModeResolver({ - globalMode: () => "compare", + globalMode: () => "redis-read", orgMode: { get, refresh: () => {} }, envFloor: "off", }); - expect(r.resolve()).toBe("compare"); + expect(r.resolve()).toBe("redis-read"); expect(get).not.toHaveBeenCalled(); }); }); From ab27e4ecde697c0969c0dbe724277ab3623e92da Mon Sep 17 00:00:00 2001 From: Daniel Sutton Date: Wed, 26 Aug 2026 13:49:35 +0100 Subject: [PATCH 51/56] fix(webapp,run-engine): address review findings on the snapshot store wiring The sweep handler caught its own failure, logged it and returned, which acknowledges the job. The dead-letter path is what re-enqueues the next occurrence, and only a throw reaches it, so one transient failure would have stopped the sweep for good. It now rethrows after the outcome is recorded. The save guard only covered the global dial, so an organisation could be set past off with no connection configured and silently do nothing, which is the exact silence the guard exists to prevent. It now covers both keys and runs on the organisation routes as well. A companion check refuses the per-organisation key on a global save, where nothing reads it. Invalidation read the replica immediately after a primary write, so replica lag could re-cache the value the write had just replaced and hold it for the whole cache window. That is worse than not invalidating at all. Invalidation now drops the entry and re-reads from the primary; the background warm-up keeps the replica. A pass cancelled at shutdown reported itself as failed. It now reports aborted. Metric attributes are bounded at the adapter and in the engine's option type. The values were already bounded in practice, but both sides typed them as plain strings, so an unrecognised value now collapses to a single bucket rather than minting a time series. Also drops mocks from the boot test, which the injected dependencies made unnecessary, and replaces a dial position in the run-store test that no longer exists. --- .../app/routes/admin.api.v1.feature-flags.ts | 10 ++- ...i.v1.orgs.$organizationId.feature-flags.ts | 8 ++ ...i.v2.orgs.$organizationId.feature-flags.ts | 8 ++ .../webapp/app/routes/admin.feature-flags.tsx | 10 ++- .../app/v3/snapshotStoreFlagGuard.server.ts | 22 ++++- .../app/v3/snapshotStoreMetrics.server.ts | 80 +++++++++++++++++-- .../webapp/app/v3/snapshotStoreMode.server.ts | 58 +++++++++----- .../app/v3/snapshotSweepRunner.server.ts | 3 + apps/webapp/test/snapshotStoreBoot.test.ts | 31 ++++--- .../test/snapshotStoreFlagGuard.test.ts | 38 ++++++++- .../run-engine/src/engine/index.ts | 9 ++- .../run-engine/src/engine/types.ts | 22 ++++- ...xecutionSnapshotStore.modeResolver.test.ts | 2 +- 13 files changed, 248 insertions(+), 53 deletions(-) diff --git a/apps/webapp/app/routes/admin.api.v1.feature-flags.ts b/apps/webapp/app/routes/admin.api.v1.feature-flags.ts index 34bee6fe575..6caac2d78d2 100644 --- a/apps/webapp/app/routes/admin.api.v1.feature-flags.ts +++ b/apps/webapp/app/routes/admin.api.v1.feature-flags.ts @@ -10,7 +10,10 @@ import { withoutDerivedKeys, } from "~/v3/featureFlags.server"; import { validatePartialFeatureFlags } from "~/v3/featureFlags"; -import { snapshotStoreFlagSaveError } from "~/v3/snapshotStoreFlagGuard.server"; +import { + globalOnlySnapshotStoreFlagError, + snapshotStoreFlagSaveError, +} from "~/v3/snapshotStoreFlagGuard.server"; export async function action({ request }: ActionFunctionArgs) { await requireAdminApiRequest(request); @@ -31,6 +34,11 @@ export async function action({ request }: ActionFunctionArgs) { ); } + const globalOnlyError = globalOnlySnapshotStoreFlagError(body as Record); + if (globalOnlyError) { + return json({ error: globalOnlyError }, { status: 400 }); + } + const snapshotStoreError = snapshotStoreFlagSaveError(body as Record, { redisHostConfigured: !!env.RUN_ENGINE_SNAPSHOT_STORE_REDIS_HOST, }); diff --git a/apps/webapp/app/routes/admin.api.v1.orgs.$organizationId.feature-flags.ts b/apps/webapp/app/routes/admin.api.v1.orgs.$organizationId.feature-flags.ts index 3f4327e4547..c66f608d5d8 100644 --- a/apps/webapp/app/routes/admin.api.v1.orgs.$organizationId.feature-flags.ts +++ b/apps/webapp/app/routes/admin.api.v1.orgs.$organizationId.feature-flags.ts @@ -6,6 +6,7 @@ import { env } from "~/env.server"; import { prisma } from "~/db.server"; import { requireAdminApiRequest } from "~/services/personalAccessToken.server"; import { controlPlaneResolver } from "~/v3/runOpsMigration/controlPlaneResolver.server"; +import { snapshotStoreFlagSaveError } from "~/v3/snapshotStoreFlagGuard.server"; import { invalidateSnapshotStoreOrgMode } from "~/v3/snapshotStoreMode.server"; import { selectMintBaselineSource, stampMintKindFlip } from "~/v3/runOpsMigration/mintFlipGrace"; import { validatePartialFeatureFlags, withoutOrgForbiddenSnapshotKeys } from "~/v3/featureFlags"; @@ -77,6 +78,13 @@ export async function action({ request, params }: ActionFunctionArgs) { const requestedFlags = withoutOrgForbiddenSnapshotKeys(rawRequestedFlags); + const snapshotStoreError = snapshotStoreFlagSaveError(requestedFlags, { + redisHostConfigured: !!env.RUN_ENGINE_SNAPSHOT_STORE_REDIS_HOST, + }); + if (snapshotStoreError) { + return json({ error: snapshotStoreError }, { status: 400 }); + } + // Seed the flip baseline from the current GLOBAL mint flags so an org's FIRST per-org override // is graced from the currently-effective global kind, not the hardcoded default "cuid". const globalFlags = (await getGlobalFlags()) as Record; diff --git a/apps/webapp/app/routes/admin.api.v2.orgs.$organizationId.feature-flags.ts b/apps/webapp/app/routes/admin.api.v2.orgs.$organizationId.feature-flags.ts index 5a98fd4a52e..773a2b6376c 100644 --- a/apps/webapp/app/routes/admin.api.v2.orgs.$organizationId.feature-flags.ts +++ b/apps/webapp/app/routes/admin.api.v2.orgs.$organizationId.feature-flags.ts @@ -6,6 +6,7 @@ import { env } from "~/env.server"; import { prisma } from "~/db.server"; import { requireUser } from "~/services/session.server"; import { controlPlaneResolver } from "~/v3/runOpsMigration/controlPlaneResolver.server"; +import { snapshotStoreFlagSaveError } from "~/v3/snapshotStoreFlagGuard.server"; import { invalidateSnapshotStoreOrgMode } from "~/v3/snapshotStoreMode.server"; import { selectMintBaselineSource, stampMintKindFlip } from "~/v3/runOpsMigration/mintFlipGrace"; import { flags as getGlobalFlags } from "~/v3/featureFlags.server"; @@ -146,6 +147,13 @@ export async function action({ request, params }: ActionFunctionArgs) { const requestedFlags = withoutOrgForbiddenSnapshotKeys(rawRequestedFlags); + const snapshotStoreError = snapshotStoreFlagSaveError(requestedFlags, { + redisHostConfigured: !!env.RUN_ENGINE_SNAPSHOT_STORE_REDIS_HOST, + }); + if (snapshotStoreError) { + return json({ error: snapshotStoreError }, { status: 400 }); + } + // Seed the flip baseline from the current GLOBAL mint flags so an org's FIRST per-org override // is graced from the currently-effective global kind, not the hardcoded default "cuid". const globalFlags = (await getGlobalFlags()) as Record; diff --git a/apps/webapp/app/routes/admin.feature-flags.tsx b/apps/webapp/app/routes/admin.feature-flags.tsx index 49788195ade..bd3fec0c3de 100644 --- a/apps/webapp/app/routes/admin.feature-flags.tsx +++ b/apps/webapp/app/routes/admin.feature-flags.tsx @@ -17,7 +17,10 @@ import { lockedFlagsInPayload, validatePartialFeatureFlags, } from "~/v3/featureFlags"; -import { snapshotStoreFlagSaveError } from "~/v3/snapshotStoreFlagGuard.server"; +import { + globalOnlySnapshotStoreFlagError, + snapshotStoreFlagSaveError, +} from "~/v3/snapshotStoreFlagGuard.server"; import { flags as getGlobalFlags, replaceGlobalFeatureFlags } from "~/v3/featureFlags.server"; import { featuresForRequest } from "~/features.server"; import { Button } from "~/components/primitives/Buttons"; @@ -130,6 +133,11 @@ export const action = dashboardAction( ); } + const globalOnlyError = globalOnlySnapshotStoreFlagError(parsed.data.flags); + if (globalOnlyError) { + return json({ error: globalOnlyError }, { status: 400 }); + } + const snapshotStoreError = snapshotStoreFlagSaveError(parsed.data.flags, { redisHostConfigured: !!env.RUN_ENGINE_SNAPSHOT_STORE_REDIS_HOST, }); diff --git a/apps/webapp/app/v3/snapshotStoreFlagGuard.server.ts b/apps/webapp/app/v3/snapshotStoreFlagGuard.server.ts index 8ee224cd85a..8e69e82e367 100644 --- a/apps/webapp/app/v3/snapshotStoreFlagGuard.server.ts +++ b/apps/webapp/app/v3/snapshotStoreFlagGuard.server.ts @@ -13,10 +13,26 @@ export function snapshotStoreFlagSaveError( return undefined; } - const mode = requested[FEATURE_FLAG.snapshotStoreMode]; - if (typeof mode === "string" && mode !== "off") { - return `Cannot set ${FEATURE_FLAG.snapshotStoreMode} to "${mode}": RUN_ENGINE_SNAPSHOT_STORE_REDIS_HOST is not configured in this deployment, so the snapshot store is never constructed and the flag would have no effect.`; + // Both keys, because either one past `off` is equally silent without a connection. + for (const key of [FEATURE_FLAG.snapshotStoreMode, FEATURE_FLAG.snapshotStoreOrgMode] as const) { + const value = requested[key]; + if (typeof value === "string" && value !== "off") { + return `Cannot set ${key} to "${value}": RUN_ENGINE_SNAPSHOT_STORE_REDIS_HOST is not configured in this deployment, so the snapshot store is never constructed and the flag would have no effect.`; + } } return undefined; } + +/** + * Refuses an organisation-only key on a global save. Nothing reads the global row for it, so a + * value saved there is inert, and an inert control an operator can set is worse than no control. + */ +export function globalOnlySnapshotStoreFlagError( + requested: Record +): string | undefined { + if (FEATURE_FLAG.snapshotStoreOrgMode in requested) { + return `${FEATURE_FLAG.snapshotStoreOrgMode} is per-organisation only; nothing reads it from the global flags, so setting it here would have no effect.`; + } + return undefined; +} diff --git a/apps/webapp/app/v3/snapshotStoreMetrics.server.ts b/apps/webapp/app/v3/snapshotStoreMetrics.server.ts index e30282bec36..31f8c98c93c 100644 --- a/apps/webapp/app/v3/snapshotStoreMetrics.server.ts +++ b/apps/webapp/app/v3/snapshotStoreMetrics.server.ts @@ -3,6 +3,61 @@ import type { DecoratorMetrics, SnapshotStoreMetrics } from "@internal/run-store export type SnapshotSweepCounts = Record; +// Metric attributes must be bounded: every one of these is a time series. The store and the +// decorator type their outcome strings loosely, so anything unrecognised collapses to "other" +// rather than minting a series. +const APPEND_OUTCOMES = ["written", "duplicate", "forked", "skippedNoKeyspace"] as const; +const APPEND_TTLS = ["none", "completion", "reapplied"] as const; +const WRITE_OUTCOMES = ["written", "staged", "post_expiry", "skipped", "failed"] as const; +const READ_SOURCES = ["redis", "postgres"] as const; +const SWEEP_OUTCOMES = [ + "completed", + "partial", + "skipped_locked", + "failed", + "unbound", + "aborted", +] as const; +const SWEEP_FIELDS = [ + "scanned", + "expired", + "deleted", + "skipped", + "pendingDeletion", + "nodes", +] as const; + +const WRITE_SITES = [ + "createRun", + "createCancelledRun", + "completeAttemptSuccess", + "expireRun", + "expireParkedRun", + "rescheduleRun", + "lockRunToWorker", + "createExecutionSnapshot", + "runInTransaction", +] as const; +const READ_METHODS = [ + "findLatestExecutionSnapshot", + "findExecutionSnapshot", + "findManyExecutionSnapshots", + "findSnapshotCompletedWaitpointIds", + "findSnapshotCompletedWaitpointIdsWithPresence", +] as const; +const SNAPSHOT_OPS = [ + "append", + "getById", + "getLatest", + "getSince", + "getSinceCreatedAt", + "getSnapshotWaitpointIds", +] as const; + +function bounded(value: string, allowed: readonly string[]): string { + return allowed.includes(value) ? value : "other"; +} + /** * Every instrument is created inside this function. At module scope they would register on every * boot, including deployments with no snapshot-store Redis configured. @@ -23,18 +78,25 @@ export function createSnapshotStoreMetrics(meter: Meter) { const sweepCounts = meter.createHistogram("run_engine.snapshot_store.sweep_counts"); const store: SnapshotStoreMetrics = { - recordAppend: (outcome, ttl) => appendTotal.add(1, { outcome, ttl }), + recordAppend: (outcome, ttl) => + appendTotal.add(1, { + outcome: bounded(outcome, APPEND_OUTCOMES), + ttl: bounded(ttl, APPEND_TTLS), + }), recordEntryBytes: (bytes) => entryBytes.record(bytes), recordCycleKeyBytes: (bytes) => cycleKeyBytes.record(bytes), recordCycleCount: (count) => cycleCount.record(count), recordSkippedNoKeyspace: () => skippedNoKeyspace.add(1), recordCycleMismatch: () => cycleMismatch.add(1), - recordLatency: (op, ms) => opLatency.record(ms, { op }), + recordLatency: (op, ms) => opLatency.record(ms, { op: bounded(op, SNAPSHOT_OPS) }), }; const decorator: DecoratorMetrics = { recordWrite: (site, outcome) => { - appendTotal.add(1, { site, outcome }); + appendTotal.add(1, { + site: bounded(site, WRITE_SITES), + outcome: bounded(outcome, WRITE_OUTCOMES), + }); if (outcome === "post_expiry") { postExpiryWrite.add(1); } @@ -42,18 +104,22 @@ export function createSnapshotStoreMetrics(meter: Meter) { flushStaged.add(1); } }, - recordAppendFailed: (site) => appendFailed.add(1, { site }), - recordRead: (method, source) => readSource.add(1, { method, source }), + recordAppendFailed: (site) => appendFailed.add(1, { site: bounded(site, WRITE_SITES) }), + recordRead: (method, source) => + readSource.add(1, { + method: bounded(method, READ_METHODS), + source: bounded(source, READ_SOURCES), + }), }; /** One emitter per pass, so a pass that throws is distinguishable from one that succeeded. */ function recordSweepPass(outcome: string, counts?: SnapshotSweepCounts): void { - sweepPass.add(1, { outcome }); + sweepPass.add(1, { outcome: bounded(outcome, SWEEP_OUTCOMES) }); if (!counts) { return; } for (const [field, value] of Object.entries(counts)) { - if (typeof value === "number") { + if (typeof value === "number" && SWEEP_FIELDS.includes(field as never)) { sweepCounts.record(value, { field }); } } diff --git a/apps/webapp/app/v3/snapshotStoreMode.server.ts b/apps/webapp/app/v3/snapshotStoreMode.server.ts index cbc80d5c6fb..f9b27dc2373 100644 --- a/apps/webapp/app/v3/snapshotStoreMode.server.ts +++ b/apps/webapp/app/v3/snapshotStoreMode.server.ts @@ -1,6 +1,6 @@ import { LRUCache } from "lru-cache"; import type { SnapshotStoreMode, SnapshotStoreModeResolver } from "@internal/run-store"; -import { $replica } from "~/db.server"; +import { $replica, prisma } from "~/db.server"; import { env } from "~/env.server"; import { logger } from "~/services/logger.server"; import { singleton } from "~/utils/singleton"; @@ -40,11 +40,16 @@ type OrgModeSource = { get(organizationId: string): CachedOrgMode | undefined; /** Fire-and-forget, de-duplicated per organisation, never throws. */ refresh(organizationId: string): void; + /** Re-reads from the primary after a write, so replica lag cannot re-cache the old value. */ + invalidate(organizationId: string): void; }; +/** What resolution needs. Invalidation is a save-path concern, not a read-path one. */ +type ResolverOrgSource = Pick; + export function buildSnapshotStoreModeResolver(deps: { globalMode: () => DialMode | undefined; - orgMode: OrgModeSource; + orgMode: ResolverOrgSource; envFloor: DialMode; }): SnapshotStoreModeResolver { return { @@ -89,33 +94,42 @@ function createOrgModeSource(): OrgModeSource { return { get: (organizationId) => cache.get(organizationId), + invalidate: (organizationId) => { + // Drop first, so a resolve between now and the re-read falls back rather than serving a + // value the write just replaced. + cache.delete(organizationId); + void load(organizationId, prisma); + }, refresh: (organizationId) => { if (inFlight.has(organizationId)) { return; } inFlight.add(organizationId); - void $replica.organization - .findFirst({ where: { id: organizationId }, select: { featureFlags: true } }) - .then((row) => { - // Only the narrow per-org key. The blob is never passed as `overrides` for the global - // key, where a parsing override would win outright. - const raw = (row?.featureFlags as Record | null | undefined)?.[ - FEATURE_FLAG.snapshotStoreOrgMode - ]; - cache.set(organizationId, cachedOrgModeFor(raw)); - }) - .catch((error) => { - logger.warn("snapshotStoreMode: organisation override refresh failed", { - organizationId, - error, - }); - }) - .finally(() => { - inFlight.delete(organizationId); - }); + void load(organizationId, $replica).finally(() => { + inFlight.delete(organizationId); + }); }, }; + + function load(organizationId: string, client: typeof prisma | typeof $replica) { + return client.organization + .findFirst({ where: { id: organizationId }, select: { featureFlags: true } }) + .then((row) => { + // Only the narrow per-org key. The blob is never passed as `overrides` for the global + // key, where a parsing override would win outright. + const raw = (row?.featureFlags as Record | null | undefined)?.[ + FEATURE_FLAG.snapshotStoreOrgMode + ]; + cache.set(organizationId, cachedOrgModeFor(raw)); + }) + .catch((error) => { + logger.warn("snapshotStoreMode: organisation override read failed", { + organizationId, + error, + }); + }); + } } /** Built on first use, never at import: importing this module must have no side effect. */ @@ -134,5 +148,5 @@ export const snapshotStoreModeResolver: SnapshotStoreModeResolver = buildSnapsho /** Called by the organisation flag save path so the writing process sees a dial change at once. */ export function invalidateSnapshotStoreOrgMode(organizationId: string): void { - orgModeSource().refresh(organizationId); + orgModeSource().invalidate(organizationId); } diff --git a/apps/webapp/app/v3/snapshotSweepRunner.server.ts b/apps/webapp/app/v3/snapshotSweepRunner.server.ts index a3da059f449..c39c86dca8d 100644 --- a/apps/webapp/app/v3/snapshotSweepRunner.server.ts +++ b/apps/webapp/app/v3/snapshotSweepRunner.server.ts @@ -44,6 +44,9 @@ export function buildSnapshotSweepRunner(deps: { } return { outcome: "completed", counts }; } catch (error) { + if (signal.aborted) { + return { outcome: "aborted" }; + } logger.error("snapshot orphan sweep pass failed", { error }); return { outcome: "failed" }; } finally { diff --git a/apps/webapp/test/snapshotStoreBoot.test.ts b/apps/webapp/test/snapshotStoreBoot.test.ts index 735e2d2c1d7..f56b6c639da 100644 --- a/apps/webapp/test/snapshotStoreBoot.test.ts +++ b/apps/webapp/test/snapshotStoreBoot.test.ts @@ -1,31 +1,37 @@ -import { describe, expect, it, vi } from "vitest"; +import { describe, expect, it } from "vitest"; import { assertSnapshotStoreBoot } from "~/v3/snapshotStoreBoot.server"; +type Recorded = { logs: string[]; warnings: string[]; pings: number }; + function deps(overrides: Partial[0]> = {}) { - return { + const recorded: Recorded = { logs: [], warnings: [], pings: 0 }; + const base = { mode: "off" as const, hostConfigured: false, completedTtlMs: 1, orphanAgeMs: 1, - ping: async () => true, + ping: async () => { + recorded.pings += 1; + return true; + }, repairBound: () => true, - log: vi.fn(), - warn: vi.fn(), - ...overrides, + log: (message: string) => recorded.logs.push(message), + warn: (message: string) => recorded.warnings.push(message), }; + return { ...base, ...overrides, recorded }; } describe("assertSnapshotStoreBoot", () => { it("passes and logs when the dial is off and nothing is configured", async () => { const d = deps(); await expect(assertSnapshotStoreBoot(d)).resolves.toBeUndefined(); - expect(d.log).toHaveBeenCalled(); + expect(d.recorded.logs).toHaveLength(1); }); it("does not probe reachability at off", async () => { - const ping = vi.fn(async () => false); - await expect(assertSnapshotStoreBoot(deps({ ping }))).resolves.toBeUndefined(); - expect(ping).not.toHaveBeenCalled(); + const d = deps(); + await expect(assertSnapshotStoreBoot(d)).resolves.toBeUndefined(); + expect(d.recorded.pings).toBe(0); }); it("refuses a dial past off with no host", async () => { @@ -54,7 +60,7 @@ describe("assertSnapshotStoreBoot", () => { it("boots on an unreachable endpoint below redis-only, loudly", async () => { const d = deps({ mode: "dual-write", hostConfigured: true, ping: async () => false }); await expect(assertSnapshotStoreBoot(d)).resolves.toBeUndefined(); - expect(d.warn).toHaveBeenCalled(); + expect(d.recorded.warnings.length).toBeGreaterThan(0); }); it("refuses an unreachable endpoint at redis-only", async () => { @@ -68,7 +74,6 @@ describe("assertSnapshotStoreBoot", () => { it("warns at redis-only because this build still writes Postgres snapshots", async () => { const d = deps({ mode: "redis-only", hostConfigured: true }); await expect(assertSnapshotStoreBoot(d)).resolves.toBeUndefined(); - const warnings = d.warn.mock.calls.flat().join(" "); - expect(warnings).toMatch(/Postgres/i); + expect(d.recorded.warnings.join(" ")).toMatch(/Postgres/i); }); }); diff --git a/apps/webapp/test/snapshotStoreFlagGuard.test.ts b/apps/webapp/test/snapshotStoreFlagGuard.test.ts index 543a1c7b958..f13e2c76f0c 100644 --- a/apps/webapp/test/snapshotStoreFlagGuard.test.ts +++ b/apps/webapp/test/snapshotStoreFlagGuard.test.ts @@ -1,5 +1,8 @@ import { describe, expect, it } from "vitest"; -import { snapshotStoreFlagSaveError } from "~/v3/snapshotStoreFlagGuard.server"; +import { + globalOnlySnapshotStoreFlagError, + snapshotStoreFlagSaveError, +} from "~/v3/snapshotStoreFlagGuard.server"; describe("snapshotStoreFlagSaveError", () => { it("refuses a flip past off when no host is configured", () => { @@ -43,4 +46,37 @@ describe("snapshotStoreFlagSaveError", () => { snapshotStoreFlagSaveError({ snapshotStoreMode: 3 }, { redisHostConfigured: false }) ).toBeUndefined(); }); + + it("refuses a per-organisation flip past off when no host is configured", () => { + // Same silence as the global key: without a connection the store is never constructed. + expect( + snapshotStoreFlagSaveError( + { snapshotStoreOrgMode: "dual-write" }, + { redisHostConfigured: false } + ) + ).toMatch(/RUN_ENGINE_SNAPSHOT_STORE_REDIS_HOST/); + }); + + it("allows a per-organisation off with no host", () => { + expect( + snapshotStoreFlagSaveError({ snapshotStoreOrgMode: "off" }, { redisHostConfigured: false }) + ).toBeUndefined(); + }); +}); + +describe("globalOnlySnapshotStoreFlagError", () => { + it("refuses the per-organisation key on a global save", () => { + // Nothing reads it from the global row, so a value saved there is an inert control. + expect(globalOnlySnapshotStoreFlagError({ snapshotStoreOrgMode: "dual-write" })).toMatch( + /per-organisation only/ + ); + }); + + it("allows the global key", () => { + expect(globalOnlySnapshotStoreFlagError({ snapshotStoreMode: "dual-write" })).toBeUndefined(); + }); + + it("ignores unrelated payloads", () => { + expect(globalOnlySnapshotStoreFlagError({ runOpsMintKind: "cuid" })).toBeUndefined(); + }); }); diff --git a/internal-packages/run-engine/src/engine/index.ts b/internal-packages/run-engine/src/engine/index.ts index 3a5ad4021f9..c001e4f9855 100644 --- a/internal-packages/run-engine/src/engine/index.ts +++ b/internal-packages/run-engine/src/engine/index.ts @@ -98,6 +98,7 @@ import type { HeartbeatTimeouts, ReportableQueue, RunEngineOptions, + SnapshotSweepOutcome, TriggerParams, } from "./types.js"; import { createTtlWorkerCatalog } from "./ttlWorkerCatalog.js"; @@ -2964,8 +2965,8 @@ export class RunEngine { // The deadline is the sweep's own stopping rule; this is the backstop for a pass that has // stopped reaching a batch boundary, so the signal is not merely decorative. const abortAt = globalThis.setTimeout(() => controller.abort(), budgetMs + 60_000); - let outcome = "failed"; - let counts: Record | undefined; + let outcome: SnapshotSweepOutcome = "failed"; + let counts: Partial> | undefined; try { const result = await runSweep({ @@ -2976,6 +2977,10 @@ export class RunEngine { counts = result.counts; } catch (error) { this.logger.error("sweepSnapshotOrphans threw", { error }); + // Rethrow after the finally records the outcome. Acknowledging here would skip the + // dead-letter path, and that path is what re-enqueues the next occurrence, so one transient + // failure would stop the sweep for good. + throw error; } finally { globalThis.clearTimeout(abortAt); this.snapshotSweepPassCounter?.add(1, { outcome }); diff --git a/internal-packages/run-engine/src/engine/types.ts b/internal-packages/run-engine/src/engine/types.ts index 47596a544a2..c3dc90ddcdc 100644 --- a/internal-packages/run-engine/src/engine/types.ts +++ b/internal-packages/run-engine/src/engine/types.ts @@ -50,6 +50,23 @@ export type CrossSeamGuardHook = (input: { routeKind: "MANUAL" | "DATETIME" | "RESUME_TOKEN" | "IDEMPOTENCY_REUSE" | "RUN"; }) => Promise; +export type SnapshotSweepOutcome = + | "completed" + | "partial" + | "skipped_locked" + | "failed" + | "unbound" + | "aborted"; + +export type SnapshotSweepCountField = + | "scanned" + | "expired" + | "deleted" + | "skipped" + | "pendingDeletion" + | "nodes" + | "partial"; + export type RunEngineOptions = { prisma: PrismaClient; readOnlyPrisma?: PrismaReplicaClient; @@ -171,9 +188,10 @@ export type RunEngineOptions = { * runs, because a pass needs a run store and its own Redis client and the engine opens neither. */ snapshotStore?: { + /** Bounded on purpose: both fields become metric attributes, so each value is a time series. */ runSweep?: (opts: { deadline: number; signal: AbortSignal }) => Promise<{ - outcome: string; - counts?: Record; + outcome: SnapshotSweepOutcome; + counts?: Partial>; }>; /** Cron. Absent or empty falls back to the catalog default. */ sweepSchedule?: string; diff --git a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.modeResolver.test.ts b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.modeResolver.test.ts index 2175de9f097..37db6f2bae6 100644 --- a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.modeResolver.test.ts +++ b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.modeResolver.test.ts @@ -26,7 +26,7 @@ describe("TaskRunExecutionSnapshotStore mode resolution", () => { }); it("falls back to the static mode when no resolver is supplied", () => { - expect(storeWith({ mode: "compare" }).mode).toBe("compare"); + expect(storeWith({ mode: "redis-read" }).mode).toBe("redis-read"); }); it("defaults to off with neither", () => { From f944a54aa7a71d8a2de8eec3ef4a7901d514b8ca Mon Sep 17 00:00:00 2001 From: Daniel Sutton Date: Wed, 26 Aug 2026 14:50:29 +0100 Subject: [PATCH 52/56] fix(webapp,run-engine): bound the sweep count fields, and guard the org cache against a stale read Two review findings, plus a correction of my own. The engine recorded whatever count keys a pass returned, and each key is a metric attribute, so an unrecognised one would mint a time series. Keys are now filtered against a fixed list at the point of record, matching the allowlist the webapp adapter already applies. A replica read that started before an invalidation could land after the primary read and put the superseded value back in the cache, holding it for the rest of the window. Each organisation now carries a generation that invalidation increments, and a load whose generation is no longer current discards its own result. The correction: an earlier commit made the sweep failure path rethrow, on the stated grounds that only the dead-letter path re-enqueues the next cron occurrence. That is wrong. The worker reschedules a cron job on the acknowledge path as well, so returning normally always continued the chain. Throwing only added a dead-letter entry for every transient failure. Both rethrows are reverted and the reason is recorded at each site, because the claim reads plausibly and would otherwise be reintroduced. --- .../webapp/app/v3/snapshotStoreMode.server.ts | 20 ++++++++++++++++--- apps/webapp/test/snapshotSweepRunner.test.ts | 2 ++ .../run-engine/src/engine/index.ts | 11 +++++----- .../run-engine/src/engine/types.ts | 10 ++++++++++ 4 files changed, 35 insertions(+), 8 deletions(-) diff --git a/apps/webapp/app/v3/snapshotStoreMode.server.ts b/apps/webapp/app/v3/snapshotStoreMode.server.ts index f9b27dc2373..e5b4b199c4e 100644 --- a/apps/webapp/app/v3/snapshotStoreMode.server.ts +++ b/apps/webapp/app/v3/snapshotStoreMode.server.ts @@ -91,14 +91,20 @@ function createOrgModeSource(): OrgModeSource { ttl: env.RUN_ENGINE_SNAPSHOT_STORE_ORG_MODE_CACHE_TTL_MS ?? DEFAULT_CACHE_TTL_MS, }); const inFlight = new Set(); + // A replica read that started before an invalidation can land after the primary read and put the + // superseded value back. A per-organisation generation lets a stale load discard its own result. + const generations = new Map(); + const generationOf = (organizationId: string) => generations.get(organizationId) ?? 0; return { get: (organizationId) => cache.get(organizationId), invalidate: (organizationId) => { // Drop first, so a resolve between now and the re-read falls back rather than serving a // value the write just replaced. + const generation = generationOf(organizationId) + 1; + generations.set(organizationId, generation); cache.delete(organizationId); - void load(organizationId, prisma); + void load(organizationId, prisma, generation); }, refresh: (organizationId) => { if (inFlight.has(organizationId)) { @@ -106,13 +112,17 @@ function createOrgModeSource(): OrgModeSource { } inFlight.add(organizationId); - void load(organizationId, $replica).finally(() => { + void load(organizationId, $replica, generationOf(organizationId)).finally(() => { inFlight.delete(organizationId); }); }, }; - function load(organizationId: string, client: typeof prisma | typeof $replica) { + function load( + organizationId: string, + client: typeof prisma | typeof $replica, + generation: number + ) { return client.organization .findFirst({ where: { id: organizationId }, select: { featureFlags: true } }) .then((row) => { @@ -121,6 +131,10 @@ function createOrgModeSource(): OrgModeSource { const raw = (row?.featureFlags as Record | null | undefined)?.[ FEATURE_FLAG.snapshotStoreOrgMode ]; + // A newer invalidation happened while this read was in flight, so its answer is stale. + if (generation < generationOf(organizationId)) { + return; + } cache.set(organizationId, cachedOrgModeFor(raw)); }) .catch((error) => { diff --git a/apps/webapp/test/snapshotSweepRunner.test.ts b/apps/webapp/test/snapshotSweepRunner.test.ts index 6c8f0ed0a53..1be447c55fc 100644 --- a/apps/webapp/test/snapshotSweepRunner.test.ts +++ b/apps/webapp/test/snapshotSweepRunner.test.ts @@ -77,6 +77,8 @@ redisTest("reports failed and still releases its own lock", async ({ redisOption lockTtlMs: 60_000, }); + // Resolving is deliberate: the worker reschedules a cron job on acknowledge as well as on the + // dead-letter path, so a failure needs no throw to keep the chain alive. expect((await runner(opts())).outcome).toBe("failed"); expect(await client.get(LOCK_KEY)).toBeNull(); } finally { diff --git a/internal-packages/run-engine/src/engine/index.ts b/internal-packages/run-engine/src/engine/index.ts index c001e4f9855..f310a7bd2f7 100644 --- a/internal-packages/run-engine/src/engine/index.ts +++ b/internal-packages/run-engine/src/engine/index.ts @@ -93,6 +93,7 @@ import { } from "./controlPlaneResolver.js"; import { TtlSystem } from "./systems/ttlSystem.js"; import { WaitpointSystem } from "./systems/waitpointSystem.js"; +import { SNAPSHOT_SWEEP_COUNT_FIELDS } from "./types.js"; import type { EngineWorker, HeartbeatTimeouts, @@ -2976,16 +2977,16 @@ export class RunEngine { outcome = result.outcome; counts = result.counts; } catch (error) { + // Deliberately not rethrown. Both the acknowledge path and the dead-letter path reschedule a + // cron job, so returning here continues the chain; throwing would only add a dead-letter + // entry for every transient blip. The outcome metric is the signal. this.logger.error("sweepSnapshotOrphans threw", { error }); - // Rethrow after the finally records the outcome. Acknowledging here would skip the - // dead-letter path, and that path is what re-enqueues the next occurrence, so one transient - // failure would stop the sweep for good. - throw error; } finally { globalThis.clearTimeout(abortAt); this.snapshotSweepPassCounter?.add(1, { outcome }); for (const [field, value] of Object.entries(counts ?? {})) { - if (typeof value === "number") { + // Each field is a metric attribute, so an unrecognised key would mint a time series. + if (typeof value === "number" && SNAPSHOT_SWEEP_COUNT_FIELDS.includes(field as never)) { this.snapshotSweepCountsHistogram?.record(value, { field }); } } diff --git a/internal-packages/run-engine/src/engine/types.ts b/internal-packages/run-engine/src/engine/types.ts index c3dc90ddcdc..509a8c14023 100644 --- a/internal-packages/run-engine/src/engine/types.ts +++ b/internal-packages/run-engine/src/engine/types.ts @@ -58,6 +58,16 @@ export type SnapshotSweepOutcome = | "unbound" | "aborted"; +export const SNAPSHOT_SWEEP_COUNT_FIELDS = [ + "scanned", + "expired", + "deleted", + "skipped", + "pendingDeletion", + "nodes", + "partial", +] as const; + export type SnapshotSweepCountField = | "scanned" | "expired" From ba146e1bda2372831b14574a715b3418680547a4 Mon Sep 17 00:00:00 2001 From: Daniel Sutton Date: Wed, 26 Aug 2026 15:10:37 +0100 Subject: [PATCH 53/56] fix(run-engine): derive the sweep count field type from its list The type was exported and nothing outside the package consumed it, which the dead-code gate reports as an unused export. Deriving it from the list that the runtime filter already uses removes both the export and the duplication, so the filter and the type cannot drift apart. --- internal-packages/run-engine/src/engine/types.ts | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/internal-packages/run-engine/src/engine/types.ts b/internal-packages/run-engine/src/engine/types.ts index 509a8c14023..cb47f0f9f11 100644 --- a/internal-packages/run-engine/src/engine/types.ts +++ b/internal-packages/run-engine/src/engine/types.ts @@ -68,14 +68,8 @@ export const SNAPSHOT_SWEEP_COUNT_FIELDS = [ "partial", ] as const; -export type SnapshotSweepCountField = - | "scanned" - | "expired" - | "deleted" - | "skipped" - | "pendingDeletion" - | "nodes" - | "partial"; +/** Derived from the list above, so the runtime filter and the type cannot drift apart. */ +type SnapshotSweepCountField = (typeof SNAPSHOT_SWEEP_COUNT_FIELDS)[number]; export type RunEngineOptions = { prisma: PrismaClient; From 22c5a87bea5911cee6053f61edb99eaa9c431c55 Mon Sep 17 00:00:00 2001 From: Daniel Sutton Date: Wed, 26 Aug 2026 16:06:18 +0100 Subject: [PATCH 54/56] fix(webapp,run-engine): split the append and write counters, derive the sweep delivery window Two findings from a second reviewer. The append counter was incremented by both layers: the store's append with {outcome, ttl}, and the decorator's write with {site, outcome}. So one logical write counted twice, and the series carried inconsistent attribute keys, which makes any sum or grouping over the name meaningless. The decorator now has its own counter. A source assertion keeps them apart, since merging them again reads like a tidy-up. The sweep job's delivery window was a fixed value while the pass budget is configurable, so raising the budget past that value would put the runner's lock TTL above the window and let a redelivery arrive while the previous pass still held the fence. The window is now derived from the budget, with the ordering covered for a range of budgets rather than only the default. --- .../app/v3/snapshotStoreMetrics.server.ts | 6 ++++- apps/webapp/test/snapshotStoreMetrics.test.ts | 14 +++++++++++ .../run-engine/src/engine/index.ts | 14 +++++++++-- .../src/engine/snapshotSweepSchedule.test.ts | 24 ++++++++++++++++++- .../src/engine/snapshotSweepSchedule.ts | 12 ++++++++++ 5 files changed, 66 insertions(+), 4 deletions(-) diff --git a/apps/webapp/app/v3/snapshotStoreMetrics.server.ts b/apps/webapp/app/v3/snapshotStoreMetrics.server.ts index 31f8c98c93c..c6990f7bac7 100644 --- a/apps/webapp/app/v3/snapshotStoreMetrics.server.ts +++ b/apps/webapp/app/v3/snapshotStoreMetrics.server.ts @@ -63,7 +63,11 @@ function bounded(value: string, allowed: readonly string[]): string { * boot, including deployments with no snapshot-store Redis configured. */ export function createSnapshotStoreMetrics(meter: Meter) { + // Two layers, two counters. Sharing one would count a single logical write twice and mix + // {outcome, ttl} points with {site, outcome} points under one name, so no sum or grouping over it + // would mean anything. const appendTotal = meter.createCounter("run_engine.snapshot_store.append_total"); + const writeTotal = meter.createCounter("run_engine.snapshot_store.write_total"); const appendFailed = meter.createCounter("run_engine.snapshot_store.append_failed"); const flushStaged = meter.createCounter("run_engine.snapshot_store.flush_staged"); const readSource = meter.createCounter("run_engine.snapshot_store.read_source"); @@ -93,7 +97,7 @@ export function createSnapshotStoreMetrics(meter: Meter) { const decorator: DecoratorMetrics = { recordWrite: (site, outcome) => { - appendTotal.add(1, { + writeTotal.add(1, { site: bounded(site, WRITE_SITES), outcome: bounded(outcome, WRITE_OUTCOMES), }); diff --git a/apps/webapp/test/snapshotStoreMetrics.test.ts b/apps/webapp/test/snapshotStoreMetrics.test.ts index 597d959578c..e57bdc711b7 100644 --- a/apps/webapp/test/snapshotStoreMetrics.test.ts +++ b/apps/webapp/test/snapshotStoreMetrics.test.ts @@ -33,4 +33,18 @@ describe("snapshotStoreMetrics module shape", () => { expect(source).not.toMatch(/compare_divergence/); expect(source).not.toMatch(/\btrimmed\b/); }); + + it("gives the two layers separate counters", () => { + // Sharing one would count a single logical write twice and mix {outcome, ttl} points with + // {site, outcome} points under one name. + const appendBlock = source.slice( + source.indexOf("recordAppend:"), + source.indexOf("recordWrite:") + ); + const writeBlock = source.slice(source.indexOf("recordWrite:")); + expect(appendBlock).toMatch(/appendTotal\.add/); + expect(appendBlock).not.toMatch(/writeTotal\.add/); + expect(writeBlock).toMatch(/writeTotal\.add/); + expect(writeBlock).not.toMatch(/appendTotal\.add/); + }); }); diff --git a/internal-packages/run-engine/src/engine/index.ts b/internal-packages/run-engine/src/engine/index.ts index f310a7bd2f7..39ca85949cc 100644 --- a/internal-packages/run-engine/src/engine/index.ts +++ b/internal-packages/run-engine/src/engine/index.ts @@ -103,7 +103,11 @@ import type { TriggerParams, } from "./types.js"; import { createTtlWorkerCatalog } from "./ttlWorkerCatalog.js"; -import { resolveSnapshotSweepCron } from "./snapshotSweepSchedule.js"; +import { + DEFAULT_SNAPSHOT_SWEEP_BUDGET_MS, + resolveSnapshotSweepCron, + snapshotSweepVisibilityTimeoutMs, +} from "./snapshotSweepSchedule.js"; import { workerCatalog } from "./workerCatalog.js"; import pMap from "p-map"; @@ -261,6 +265,12 @@ export class RunEngine { ...workerCatalog, sweepSnapshotOrphans: { ...workerCatalog.sweepSnapshotOrphans, + // Derived, not fixed. The runner's lock TTL is the budget plus an hour, so a hardcoded + // timeout would sit below the lock once the budget is raised, inverting the ordering the + // fence depends on. + visibilityTimeoutMs: snapshotSweepVisibilityTimeoutMs( + options.snapshotStore?.sweepBudgetMs + ), cron: resolveSnapshotSweepCron({ hasRunner: !!options.snapshotStore?.runSweep, schedule: options.snapshotStore?.sweepSchedule, @@ -2961,7 +2971,7 @@ export class RunEngine { return; } - const budgetMs = this.options.snapshotStore?.sweepBudgetMs ?? 10_800_000; + const budgetMs = this.options.snapshotStore?.sweepBudgetMs ?? DEFAULT_SNAPSHOT_SWEEP_BUDGET_MS; const controller = new AbortController(); // The deadline is the sweep's own stopping rule; this is the backstop for a pass that has // stopped reaching a batch boundary, so the signal is not merely decorative. diff --git a/internal-packages/run-engine/src/engine/snapshotSweepSchedule.test.ts b/internal-packages/run-engine/src/engine/snapshotSweepSchedule.test.ts index 65c505f92f7..c20f6ea5492 100644 --- a/internal-packages/run-engine/src/engine/snapshotSweepSchedule.test.ts +++ b/internal-packages/run-engine/src/engine/snapshotSweepSchedule.test.ts @@ -1,5 +1,9 @@ import { describe, expect, it } from "vitest"; -import { resolveSnapshotSweepCron } from "./snapshotSweepSchedule.js"; +import { + DEFAULT_SNAPSHOT_SWEEP_BUDGET_MS, + resolveSnapshotSweepCron, + snapshotSweepVisibilityTimeoutMs, +} from "./snapshotSweepSchedule.js"; const FALLBACK = "0 */6 * * *"; @@ -40,3 +44,21 @@ describe("the unconfigured deployment", () => { ).toBeUndefined(); }); }); + +describe("snapshotSweepVisibilityTimeoutMs", () => { + // The runner's lock TTL is the budget plus an hour. The delivery window has to stay above it, or + // a redelivery arrives while the previous pass still holds the fence. + const lockTtl = (budget: number) => budget + 60 * 60 * 1000; + + it("stays above the lock TTL at the default budget", () => { + expect(snapshotSweepVisibilityTimeoutMs()).toBeGreaterThan( + lockTtl(DEFAULT_SNAPSHOT_SWEEP_BUDGET_MS) + ); + }); + + it("stays above the lock TTL for a raised budget", () => { + for (const budget of [60_000, 10_800_000, 43_200_000, 86_400_000]) { + expect(snapshotSweepVisibilityTimeoutMs(budget)).toBeGreaterThan(lockTtl(budget)); + } + }); +}); diff --git a/internal-packages/run-engine/src/engine/snapshotSweepSchedule.ts b/internal-packages/run-engine/src/engine/snapshotSweepSchedule.ts index 1da24e5d224..20d967650c7 100644 --- a/internal-packages/run-engine/src/engine/snapshotSweepSchedule.ts +++ b/internal-packages/run-engine/src/engine/snapshotSweepSchedule.ts @@ -12,3 +12,15 @@ export function resolveSnapshotSweepCron(opts: { } return opts.schedule?.trim() ? opts.schedule : opts.fallback; } + +/** Default budget for one sweep pass, when the caller supplies none. */ +export const DEFAULT_SNAPSHOT_SWEEP_BUDGET_MS = 10_800_000; + +/** + * Keeps the delivery window strictly above the runner's lock TTL, which is the budget plus an hour. + * Two hours of headroom, so a pass that overruns its budget still holds a lock that outlives the + * delivery it belongs to. + */ +export function snapshotSweepVisibilityTimeoutMs(budgetMs?: number): number { + return (budgetMs ?? DEFAULT_SNAPSHOT_SWEEP_BUDGET_MS) + 2 * 60 * 60 * 1000; +} From 663b3446047772c26d0f2212a84288de55846528 Mon Sep 17 00:00:00 2001 From: Daniel Sutton Date: Wed, 26 Aug 2026 16:56:22 +0100 Subject: [PATCH 55/56] fix(run-store): resolve the fatal birth-append refusal per organisation Every write entry point resolves the rollout position for the run own organisation, but the decision about whether a lost run-birth append is fatal still read the global position. An organisation whose own position keeps Postgres authoritative could therefore have run creation failed by a transient Redis fault. It now reads the same per-organisation position that decided to append in the first place. --- ...nExecutionSnapshotStore.modeResolver.test.ts | 12 ++++++++++++ .../src/taskRunExecutionSnapshotStore.ts | 17 +++++++++++++++-- 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.modeResolver.test.ts b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.modeResolver.test.ts index 37db6f2bae6..b1f425e929e 100644 --- a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.modeResolver.test.ts +++ b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.modeResolver.test.ts @@ -57,4 +57,16 @@ describe("TaskRunExecutionSnapshotStore mode resolution", () => { const store = storeWith({ modeResolver: resolverOf({ org_a: "off" }, "redis-read") }); expect(store.writesRedisForTest()).toBe(true); }); + + it("resolves the fatal-birth decision per organisation, not globally", () => { + // A lost birth append is fatal only where Postgres holds nothing. An organisation still on a + // dual-write position must not have its run creation failed by the global position. + const store = storeWith({ + modeResolver: resolverOf({ org_dual: "dual-write" }, "redis-only"), + }); + + expect(store.modeForTest("org_dual")).toBe("dual-write"); + expect(store.modeForTest("org_other")).toBe("redis-only"); + expect(store.modeForTest()).toBe("redis-only"); + }); }); diff --git a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.ts b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.ts index 61a8276ce05..302401449da 100644 --- a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.ts +++ b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.ts @@ -155,9 +155,14 @@ export class TaskRunExecutionSnapshotStore extends DelegatingRunStore { return this.modeResolver?.resolve() ?? this.#staticMode; } + /** The resolved position for one organisation. Falls back to the global answer when unknown. */ + protected modeFor(organizationId?: string): SnapshotStoreMode { + return this.modeResolver?.resolve(organizationId) ?? this.#staticMode; + } + /** True in every position that appends to Redis, for this entry's organisation. */ protected writesRedisFor(organizationId?: string): boolean { - return (this.modeResolver?.resolve(organizationId) ?? this.#staticMode) !== "off"; + return this.modeFor(organizationId) !== "off"; } /** Test seam for the per-organisation predicate. Not for production callers. */ @@ -165,6 +170,11 @@ export class TaskRunExecutionSnapshotStore extends DelegatingRunStore { return this.writesRedisFor(organizationId); } + /** Test seam for the resolved position. Not for production callers. */ + modeForTest(organizationId?: string): SnapshotStoreMode { + return this.modeFor(organizationId); + } + /** * The staging facade. Two writes share one Postgres transaction here, and the Redis half of each * cannot run until that transaction commits: a rollback would otherwise leave Redis holding a @@ -499,7 +509,10 @@ export class TaskRunExecutionSnapshotStore extends DelegatingRunStore { error, }); - if (this.mode === "redis-only") { + // The same organisation dial that decided to append decides whether a lost birth is + // fatal. Using the global position here would fail run creation for an organisation whose + // own position still has Postgres authoritative. + if (this.modeFor(entry.organizationId) === "redis-only") { throw error; } return; From e7d040a4a862b0ecb31d02cd946c8da7798fddeb Mon Sep 17 00:00:00 2001 From: Daniel Sutton Date: Wed, 26 Aug 2026 16:56:22 +0100 Subject: [PATCH 56/56] fix(webapp): close the snapshot store clients on shutdown The snapshot store connections and the orphan sweeper now close on SIGTERM and SIGINT, matching the other Redis-backed singletons. --- .../webapp/app/v3/snapshotStoreWiring.server.ts | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/apps/webapp/app/v3/snapshotStoreWiring.server.ts b/apps/webapp/app/v3/snapshotStoreWiring.server.ts index 06a9114664a..9cf96b0e35c 100644 --- a/apps/webapp/app/v3/snapshotStoreWiring.server.ts +++ b/apps/webapp/app/v3/snapshotStoreWiring.server.ts @@ -1,11 +1,16 @@ import { SnapshotOrphanSweeper } from "@internal/run-store"; import { env } from "~/env.server"; import { logger } from "~/services/logger.server"; +import { signalsEmitter } from "~/services/signals.server"; import { engine } from "./runEngine.server"; import { runStoreWithoutSnapshotDecorator } from "./runStore.server"; import { buildSnapshotSweepRunner } from "./snapshotSweepRunner.server"; import { setSnapshotRepairEnqueuer, setSnapshotSweepRunner } from "./snapshotStoreBindings.server"; -import { getSnapshotSweepClient, registerSnapshotStoreQuit } from "./snapshotStoreInstance.server"; +import { + getSnapshotSweepClient, + quitSnapshotStoreClients, + registerSnapshotStoreQuit, +} from "./snapshotStoreInstance.server"; /** * The third module: it imports both sides, so neither the run store nor the engine has to import @@ -43,6 +48,16 @@ export function registerSnapshotStoreWiring(): boolean { registerSnapshotStoreQuit(() => sweeper.quit()); + // Close the sweeper and all three connections on the way out, the same way the other Redis-backed + // singletons do. `quitSnapshotStoreClients` is async and the signals emitter swallows listener + // rejections, so discard the promise explicitly rather than handing it a floating one. The caller + // wraps this function in `singleton`, so the listeners are registered once per process. + const onShutdown = (): void => { + void quitSnapshotStoreClients(); + }; + signalsEmitter.on("SIGTERM", onShutdown); + signalsEmitter.on("SIGINT", onShutdown); + logger.info("snapshot store wiring registered"); return true; }