diff --git a/internal-packages/redis/src/index.ts b/internal-packages/redis/src/index.ts index 6fb30b9a4bf..622efe613fc 100644 --- a/internal-packages/redis/src/index.ts +++ b/internal-packages/redis/src/index.ts @@ -1,7 +1,24 @@ -import { Redis, type RedisOptions } from "ioredis"; +import { type Cluster, Redis, type RedisOptions } from "ioredis"; import { Logger } from "@trigger.dev/core/logger"; -export { Redis, type Callback, type RedisOptions, type Result, type RedisCommander } from "ioredis"; +export { + Redis, + Cluster, + type Callback, + type RedisOptions, + type ClusterNode, + type ClusterOptions, + type Result, + type RedisCommander, +} from "ioredis"; + +/** + * Either endpoint shape. A component that only issues key-addressed commands works against both, so + * it should accept this rather than pin itself to a standalone connection. Commands with no key — + * SCAN above all — do NOT fan out across a cluster, so anything that issues one must iterate + * `cluster.nodes("master")` itself. + */ +export type RedisClient = Redis | Cluster; /** * Reply-error -> reconnect mapping. Without this hook, an ElastiCache diff --git a/internal-packages/run-engine/src/engine/tests/finalRunStatusParity.test.ts b/internal-packages/run-engine/src/engine/tests/finalRunStatusParity.test.ts new file mode 100644 index 00000000000..50eba9d3a04 --- /dev/null +++ b/internal-packages/run-engine/src/engine/tests/finalRunStatusParity.test.ts @@ -0,0 +1,20 @@ +// The snapshot sweeper needs to know which run statuses are terminal, and it cannot import that +// list: run-engine depends on run-store, not the other way round. So the list is duplicated, and +// this is the only thing that keeps the copy honest. +// +// Without it, a status added here and not there makes the sweeper treat a finished run as live and +// never apply its completion expiry. A status removed here and not there makes it treat a live run +// as finished. The second one reaps state a run is still using. +import { describe, expect, it } from "vitest"; +import { FINAL_RUN_STATUSES } from "@internal/run-store"; +import { getFinalRunStatuses } from "../statuses.js"; + +describe("terminal run statuses", () => { + it("match between the engine and the snapshot sweeper", () => { + expect([...FINAL_RUN_STATUSES].sort()).toEqual([...getFinalRunStatuses()].sort()); + }); + + it("are not empty, so the comparison cannot pass vacuously", () => { + expect(FINAL_RUN_STATUSES.length).toBeGreaterThan(0); + }); +}); diff --git a/internal-packages/run-engine/src/engine/tests/helpers/decoratedStore.ts b/internal-packages/run-engine/src/engine/tests/helpers/decoratedStore.ts new file mode 100644 index 00000000000..c74aa86db92 --- /dev/null +++ b/internal-packages/run-engine/src/engine/tests/helpers/decoratedStore.ts @@ -0,0 +1,75 @@ +// Builds the snapshot-store decorator over a real PostgresRunStore, for injection through the +// engine's `store` option — the seam runStoreInjectability.test.ts already proves. +// +// The point of injecting it is that the engine suites keep their own assertions: the same flows, +// the same expectations, a different store underneath. +import { + PostgresRunStore, + RedisSnapshotStore, + TaskRunExecutionSnapshotStore, + type SnapshotFaultInjector, + type SnapshotRepairEnqueuer, + type SnapshotStoreMode, +} from "@internal/run-store"; +import type { PrismaClient } from "@trigger.dev/database"; +import type { RedisOptions } from "@internal/redis"; + +const COMPLETED_TTL_MS = 72 * 60 * 60 * 1000; + +export type DecoratedStoreHarness = { + store: TaskRunExecutionSnapshotStore; + redis: RedisSnapshotStore; + /** Every read the decorator served, and which store answered it. */ + reads: { method: string; source: "redis" | "postgres" }[]; + /** Every append outcome, keyed by the write site that produced it. */ + writes: { site: string; outcome: string }[]; + /** Runs handed to the repair job because their append was lost. */ + repairs: { runId: string; snapshotId: string; executionStatus: string }[]; + quit(): Promise; +}; + +export function buildDecoratedStore(opts: { + prisma: PrismaClient; + redisOptions: RedisOptions; + mode: SnapshotStoreMode; + readPercent?: number; + faults?: SnapshotFaultInjector; + onAppendFailure?: SnapshotRepairEnqueuer; +}): DecoratedStoreHarness { + const redis = new RedisSnapshotStore({ + redisOptions: opts.redisOptions, + completedTtlMs: COMPLETED_TTL_MS, + }); + + const reads: DecoratedStoreHarness["reads"] = []; + const writes: DecoratedStoreHarness["writes"] = []; + const repairs: DecoratedStoreHarness["repairs"] = []; + + const store = new TaskRunExecutionSnapshotStore( + new PostgresRunStore({ prisma: opts.prisma as never, readOnlyPrisma: opts.prisma as never }), + { + store: redis, + mode: opts.mode, + readPercent: opts.readPercent ?? 100, + ...(opts.faults && { faults: opts.faults }), + onAppendFailure: async (args) => { + repairs.push(args); + await opts.onAppendFailure?.(args); + }, + metrics: { + recordWrite: (site, outcome) => writes.push({ site, outcome }), + recordAppendFailed: () => {}, + recordRead: (method, source) => reads.push({ method, source }), + }, + } + ); + + return { + store, + redis, + reads, + writes, + repairs, + quit: () => redis.quit(), + }; +} diff --git a/internal-packages/run-engine/src/engine/tests/snapshotStoreChaos.test.ts b/internal-packages/run-engine/src/engine/tests/snapshotStoreChaos.test.ts new file mode 100644 index 00000000000..ea3447acc56 --- /dev/null +++ b/internal-packages/run-engine/src/engine/tests/snapshotStoreChaos.test.ts @@ -0,0 +1,402 @@ +// The correctness spine: kill the process at each write boundary and prove the run still converges. +// +// The write protocol's whole claim is that whatever a crash leaves behind is a state the existing +// stall-and-repair machinery heals. That claim is not checkable by reading the code, so each test +// here injects a fault at one named boundary and then asserts three things: the run converges, it +// does not hang, and it burns at most one attempt number per crash. +// +// The bound is PER CRASH, not a flat one. The plan records that TLC refuted a flat bound of one in +// seven states, and that the property which holds is pgAttempt - maxLoggedAttempt <= crashCount. +import { assertNonNullable, containerTest } from "@internal/testcontainers"; +import { trace } from "@internal/tracing"; +import { generateInternalId, RunId } from "@trigger.dev/core/v3/isomorphic"; +import { + InjectedSnapshotFault, + type SnapshotFaultBoundary, + type SnapshotFaultInjector, +} from "@internal/run-store"; +import { setTimeout } from "timers/promises"; +import { RunEngine } from "../index.js"; +import { buildDecoratedStore, type DecoratedStoreHarness } from "./helpers/decoratedStore.js"; +import { setupAuthenticatedEnvironment, setupBackgroundWorker } from "./setup.js"; + +vi.setConfig({ testTimeout: 60_000 }); + +/** + * A local stand-in for the shared fault harness being built alongside this ticket. Its surface is + * the agreed one — arm, disarm, hook, fired — so swapping the import in costs no test-body change. + * + * `fired` is the guard against a silent pass. A boundary can be armed and never reached, in which + * case the test would go green having proved nothing, so every test asserts its boundary fired. + */ +function createFaultInjector(opts: { error: (boundary: SnapshotFaultBoundary) => Error }) { + const armed = new Map(); + const counts = new Map(); + + return { + arm(boundary: SnapshotFaultBoundary, opts?: { times?: number; runId?: string }) { + armed.set(boundary, { times: opts?.times ?? 1, ...(opts?.runId && { runId: opts.runId }) }); + }, + disarm(boundary: SnapshotFaultBoundary) { + armed.delete(boundary); + }, + fired(boundary: SnapshotFaultBoundary): number { + return counts.get(boundary) ?? 0; + }, + hook: ((boundary, context) => { + const entry = armed.get(boundary); + if (!entry) return; + if (entry.runId && context.runId !== entry.runId) return; + + counts.set(boundary, (counts.get(boundary) ?? 0) + 1); + entry.times -= 1; + if (entry.times <= 0) armed.delete(boundary); + + throw opts.error(boundary); + }) satisfies SnapshotFaultInjector, + }; +} + +function engineOptions( + prisma: any, + redisOptions: any, + harness: DecoratedStoreHarness, + heartbeatMs: number +) { + return { + prisma, + store: harness.store, + worker: { redis: redisOptions, workers: 1, tasksPerWorker: 10, pollIntervalMs: 100 }, + queue: { + redis: redisOptions, + retryOptions: { maxTimeoutInMs: 50 }, + masterQueueConsumersDisabled: true, + processWorkerQueueDebounceMs: 50, + }, + runLock: { redis: redisOptions }, + machines: { + defaultMachine: "small-1x" as const, + machines: { + "small-1x": { name: "small-1x" as const, cpu: 0.5, memory: 0.5, centsPerMs: 0.0001 }, + }, + baseCostInCents: 0.0001, + }, + heartbeatTimeoutsMs: { PENDING_EXECUTING: heartbeatMs }, + tracer: trace.getTracer("test", "0.0.0"), + }; +} + +const triggerArgs = (taskIdentifier: string, environment: any) => ({ + number: 1, + friendlyId: RunId.generate().friendlyId, + environment, + taskIdentifier, + payload: "{}", + payloadType: "application/json", + context: {}, + traceContext: {}, + traceId: `t_${generateInternalId().slice(-12)}`, + spanId: `s_${generateInternalId().slice(-12)}`, + workerQueue: "main", + queue: `task/${taskIdentifier}`, + isTest: false, + tags: [], +}); + +describe("snapshot store crash boundaries", () => { + containerTest( + "afterPgBeforeRedis: the run converges and burns at most one attempt", + async ({ prisma, redisOptions }) => { + const faults = createFaultInjector({ error: (b) => new InjectedSnapshotFault(b) }); + const harness = buildDecoratedStore({ + prisma, + redisOptions, + mode: "redis-read", + readPercent: 100, + faults: faults.hook, + }); + const engine = new RunEngine(engineOptions(prisma, redisOptions, harness, 200) as never); + + try { + const environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + await setupBackgroundWorker(engine, environment, "chaos-task"); + const run = await engine.trigger(triggerArgs("chaos-task", environment), prisma); + await setTimeout(500); + + const dequeued = await engine.dequeueFromWorkerQueue({ + consumerId: "chaos", + workerQueue: "main", + }); + expect(dequeued.length).toBe(1); + + // Crash between the Postgres commit and the Redis append of ONE transition. + faults.arm("afterPgBeforeRedis", { times: 1, runId: run.id }); + const attempt = await engine.startRunAttempt({ + runId: dequeued[0]!.run.id, + snapshotId: dequeued[0]!.snapshot.id, + }); + faults.disarm("afterPgBeforeRedis"); + + // The boundary was actually reached. Without this the test could pass having proved nothing. + expect(faults.fired("afterPgBeforeRedis")).toBe(1); + + // Postgres committed the attempt bump; the run is not stuck and not lost. + expect(attempt.run.attemptNumber).toBe(1); + const pgRun = await prisma.taskRun.findFirstOrThrow({ where: { id: run.id } }); + expect(pgRun.attemptNumber).toBe(1); + + // The gap handed the run to the repair job rather than failing the caller. + expect(harness.repairs).toHaveLength(1); + expect(harness.repairs[0]!.runId).toBe(run.id); + + // Reads still resolve: the run's state machine is readable, so nothing hangs. + const data = await engine.getRunExecutionData({ runId: run.id }); + assertNonNullable(data); + + // The bound: one crash costs at most one attempt number. + expect(pgRun.attemptNumber! - 1).toBeLessThanOrEqual(1); + } finally { + await engine.quit(); + await harness.quit(); + } + } + ); + + containerTest( + "afterRedisBirthBeforePg: no run is created, and the next trigger succeeds", + async ({ prisma, redisOptions }) => { + const faults = createFaultInjector({ error: (b) => new InjectedSnapshotFault(b) }); + const harness = buildDecoratedStore({ + prisma, + redisOptions, + mode: "redis-read", + readPercent: 100, + faults: faults.hook, + }); + const engine = new RunEngine(engineOptions(prisma, redisOptions, harness, 200) as never); + + try { + const environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + await setupBackgroundWorker(engine, environment, "chaos-task"); + + faults.arm("afterRedisBirthBeforePg", { times: 1 }); + await expect( + engine.trigger(triggerArgs("chaos-task", environment), prisma) + ).rejects.toBeInstanceOf(InjectedSnapshotFault); + expect(faults.fired("afterRedisBirthBeforePg")).toBe(1); + + // The harmless state: no run row, so nothing can ever read a run that has no snapshot. + const runsAfterCrash = await prisma.taskRun.count({ + where: { runtimeEnvironmentId: environment.id }, + }); + expect(runsAfterCrash).toBe(0); + + // A crashed birth must not poison the path: the next trigger runs to completion. + const run = await engine.trigger(triggerArgs("chaos-task", environment), prisma); + await setTimeout(500); + const dequeued = await engine.dequeueFromWorkerQueue({ + consumerId: "chaos", + workerQueue: "main", + }); + const attempt = await engine.startRunAttempt({ + runId: dequeued[0]!.run.id, + snapshotId: dequeued[0]!.snapshot.id, + }); + await engine.completeRunAttempt({ + runId: run.id, + snapshotId: attempt.snapshot.id, + completion: { + ok: true, + id: run.id, + output: `{"done":true}`, + outputType: "application/json", + }, + }); + + const finished = await prisma.taskRun.findFirstOrThrow({ where: { id: run.id } }); + expect(finished.status).toBe("COMPLETED_SUCCESSFULLY"); + expect(finished.attemptNumber).toBe(1); + } finally { + await engine.quit(); + await harness.quit(); + } + } + ); + + containerTest( + "midFlushRetry: a crash during a retry still converges through the repair job", + async ({ prisma, redisOptions }) => { + const faults = createFaultInjector({ error: (b) => new InjectedSnapshotFault(b) }); + // A dead port makes attempt 0 fail FOR REAL, which is the only way the retry boundary is + // reachable: an injected fault at attempt 0 is treated as a dead process and skips the + // retries entirely. Arming midFlushRetry alone would fire nothing and pass for the wrong + // reason, which is what the fired() assertion below catches. + const harness = buildDecoratedStore({ + prisma, + redisOptions: { ...(redisOptions as object), port: 1, retryStrategy: () => null } as never, + mode: "dual-write", + faults: faults.hook, + }); + const engine = new RunEngine(engineOptions(prisma, redisOptions, harness, 200) as never); + + try { + const environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + await setupBackgroundWorker(engine, environment, "chaos-task"); + + faults.arm("midFlushRetry", { times: 1 }); + const run = await engine.trigger(triggerArgs("chaos-task", environment), prisma); + await setTimeout(500); + + // The birth append failed for real and, before redis-only, that is survivable: Postgres is + // authoritative and the run exists. + const pgRun = await prisma.taskRun.findFirstOrThrow({ where: { id: run.id } }); + expect(pgRun.id).toBe(run.id); + + const dequeued = await engine.dequeueFromWorkerQueue({ + consumerId: "chaos", + workerQueue: "main", + }); + expect(dequeued.length).toBe(1); + + const attempt = await engine.startRunAttempt({ + runId: dequeued[0]!.run.id, + snapshotId: dequeued[0]!.snapshot.id, + }); + + // The retry boundary was genuinely reached, not merely armed. + expect(faults.fired("midFlushRetry")).toBeGreaterThanOrEqual(1); + + // The run converges regardless: Postgres holds every snapshot at this dial position. + expect(attempt.run.attemptNumber).toBe(1); + const data = await engine.getRunExecutionData({ runId: run.id }); + assertNonNullable(data); + expect(data.snapshot.executionStatus).toBe("EXECUTING"); + } finally { + await engine.quit(); + await harness.quit(); + } + } + ); + + containerTest( + "a crash-stalled run rejects a stale snapshot rather than hanging", + async ({ prisma, redisOptions }) => { + const faults = createFaultInjector({ error: (b) => new InjectedSnapshotFault(b) }); + const harness = buildDecoratedStore({ + prisma, + redisOptions, + mode: "redis-read", + readPercent: 100, + faults: faults.hook, + }); + const engine = new RunEngine(engineOptions(prisma, redisOptions, harness, 200) as never); + + try { + const environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + await setupBackgroundWorker(engine, environment, "chaos-task"); + const run = await engine.trigger(triggerArgs("chaos-task", environment), prisma); + await setTimeout(500); + + faults.arm("afterPgBeforeRedis", { times: 1, runId: run.id }); + const dequeued = await engine.dequeueFromWorkerQueue({ + consumerId: "chaos", + workerQueue: "main", + }); + faults.disarm("afterPgBeforeRedis"); + expect(faults.fired("afterPgBeforeRedis")).toBe(1); + + // Postgres advanced; the Redis head did not. Reads come from Redis, so the caller now holds + // a snapshot id that no longer matches what the read store reports as latest. + // + // The contract is that this SURFACES rather than corrupts: the next operation to validate + // against latest rejects with a stale-snapshot error, which is the same answer a caller gets + // from an ordinary lost race. It does not hang, and it does not silently execute against the + // wrong state. + await expect( + engine.startRunAttempt({ + runId: dequeued[0]!.run.id, + snapshotId: dequeued[0]!.snapshot.id, + }) + ).rejects.toThrow(/Snapshot changed/); + + // The run is still readable and still has a coherent state machine. + const data = await engine.getRunExecutionData({ runId: run.id }); + assertNonNullable(data); + + // The gap was handed to the repair job, which is the compensator the protocol names. + expect(harness.repairs.length).toBeGreaterThanOrEqual(1); + expect(harness.repairs.some((r) => r.runId === run.id)).toBe(true); + + // And no attempt was burned by the rejection itself: the bound is per crash, and the + // rejected call never reached the attempt bump. + const pgRun = await prisma.taskRun.findFirstOrThrow({ where: { id: run.id } }); + expect(pgRun.attemptNumber ?? 0).toBeLessThanOrEqual(faults.fired("afterPgBeforeRedis")); + } finally { + await engine.quit(); + await harness.quit(); + } + } + ); + + containerTest( + "two crashes cost at most two attempts, and the divergence does not amplify", + async ({ prisma, redisOptions }) => { + const faults = createFaultInjector({ error: (b) => new InjectedSnapshotFault(b) }); + // dual-write, so reads still come from Postgres and the run can be driven forward through the + // normal API. That isolates the property under test — how many attempts two crashes cost — + // from the stale-read rejection the previous test covers. + const harness = buildDecoratedStore({ + prisma, + redisOptions, + mode: "dual-write", + faults: faults.hook, + }); + const engine = new RunEngine(engineOptions(prisma, redisOptions, harness, 200) as never); + + try { + const environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + await setupBackgroundWorker(engine, environment, "chaos-task"); + const run = await engine.trigger(triggerArgs("chaos-task", environment), prisma); + await setTimeout(500); + + faults.arm("afterPgBeforeRedis", { times: 2, runId: run.id }); + + const dequeued = await engine.dequeueFromWorkerQueue({ + consumerId: "chaos", + workerQueue: "main", + }); + const attempt = await engine.startRunAttempt({ + runId: dequeued[0]!.run.id, + snapshotId: dequeued[0]!.snapshot.id, + }); + await engine.completeRunAttempt({ + runId: run.id, + snapshotId: attempt.snapshot.id, + completion: { + ok: true, + id: run.id, + output: `{"done":true}`, + outputType: "application/json", + }, + }); + + const crashes = faults.fired("afterPgBeforeRedis"); + expect(crashes).toBeGreaterThanOrEqual(1); + + const pgRun = await prisma.taskRun.findFirstOrThrow({ where: { id: run.id } }); + + // pgAttempt - maxLoggedAttempt <= crashCount. Each crash costs at most one attempt, and the + // divergence does not amplify: two crashes never cost three. A flat bound of one was + // refuted by the model check, so the assertion is against the crash count, not a constant. + expect((pgRun.attemptNumber ?? 0) - 1).toBeLessThanOrEqual(crashes); + + // Postgres holds every snapshot at this dial position, so the run still converges. + expect(pgRun.status).toBe("COMPLETED_SUCCESSFULLY"); + expect(harness.repairs.length).toBe(crashes); + } finally { + await engine.quit(); + await harness.quit(); + } + } + ); +}); diff --git a/internal-packages/run-engine/src/engine/tests/snapshotStoreReadGate.test.ts b/internal-packages/run-engine/src/engine/tests/snapshotStoreReadGate.test.ts new file mode 100644 index 00000000000..17c186d074d --- /dev/null +++ b/internal-packages/run-engine/src/engine/tests/snapshotStoreReadGate.test.ts @@ -0,0 +1,304 @@ +// The read gate: the engine's own snapshot flows, run against the decorator with reads served from +// Redis. Same flows, same expectations, different store underneath — the point is that nothing in +// the engine has to know, so no existing suite is modified to make this pass. +import { assertNonNullable, containerTest } from "@internal/testcontainers"; +import { trace } from "@internal/tracing"; +import { setTimeout } from "timers/promises"; +import { generateInternalId, RunId } from "@trigger.dev/core/v3/isomorphic"; +import { RunEngine } from "../index.js"; +import { buildDecoratedStore, type DecoratedStoreHarness } from "./helpers/decoratedStore.js"; +import { setupAuthenticatedEnvironment, setupBackgroundWorker } from "./setup.js"; + +vi.setConfig({ testTimeout: 60_000 }); + +function engineOptions(prisma: any, redisOptions: any, harness: DecoratedStoreHarness) { + return { + prisma, + store: harness.store, + worker: { redis: redisOptions, workers: 1, tasksPerWorker: 10, pollIntervalMs: 100 }, + queue: { + redis: redisOptions, + masterQueueConsumersDisabled: true, + processWorkerQueueDebounceMs: 50, + }, + runLock: { redis: redisOptions }, + machines: { + defaultMachine: "small-1x" as const, + machines: { + "small-1x": { name: "small-1x" as const, cpu: 0.5, memory: 0.5, centsPerMs: 0.0001 }, + }, + baseCostInCents: 0.0001, + }, + tracer: trace.getTracer("test", "0.0.0"), + }; +} + +const triggerArgs = (taskIdentifier: string, environment: any, n: number) => ({ + number: n, + // A real minted friendly id: the engine converts it back with RunId.fromFriendlyId, which + // rejects anything that is not the prefix plus a cuid body. + friendlyId: RunId.generate().friendlyId, + environment, + taskIdentifier, + payload: "{}", + payloadType: "application/json", + context: {}, + traceContext: {}, + traceId: `t_gate_${n}`, + spanId: `s_gate_${n}`, + workerQueue: "main", + queue: `task/${taskIdentifier}`, + isTest: false, + tags: [], +}); + +describe("snapshot store read gate", () => { + containerTest( + "drives a run to completion with every snapshot read served from Redis", + async ({ prisma, redisOptions }) => { + const harness = buildDecoratedStore({ + prisma, + redisOptions, + mode: "redis-read", + readPercent: 100, + }); + const engine = new RunEngine(engineOptions(prisma, redisOptions, harness) as never); + + try { + const environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + const taskIdentifier = "gate-task"; + await setupBackgroundWorker(engine, environment, taskIdentifier); + + const run = await engine.trigger(triggerArgs(taskIdentifier, environment, 1), prisma); + await setTimeout(500); + + const dequeued = await engine.dequeueFromWorkerQueue({ + consumerId: "gate_consumer", + workerQueue: "main", + }); + expect(dequeued.length).toBe(1); + + const attempt = await engine.startRunAttempt({ + runId: dequeued[0]!.run.id, + snapshotId: dequeued[0]!.snapshot.id, + }); + expect(attempt.run.status).toBe("EXECUTING"); + + await engine.completeRunAttempt({ + runId: run.id, + snapshotId: attempt.snapshot.id, + completion: { + ok: true, + id: run.id, + output: `{"done":true}`, + outputType: "application/json", + }, + }); + + const finished = await prisma.taskRun.findFirstOrThrow({ where: { id: run.id } }); + expect(finished.status).toBe("COMPLETED_SUCCESSFULLY"); + + // The gate: the engine read its snapshots, and Redis is what answered. + const fromRedis = harness.reads.filter((r) => r.source === "redis"); + expect(fromRedis.length).toBeGreaterThan(0); + expect(harness.reads.filter((r) => r.source === "postgres")).toEqual([]); + } finally { + await engine.quit(); + await harness.quit(); + } + } + ); + + containerTest( + "serves getRunExecutionData from Redis at every step", + async ({ prisma, redisOptions }) => { + const harness = buildDecoratedStore({ + prisma, + redisOptions, + mode: "redis-read", + readPercent: 100, + }); + const engine = new RunEngine(engineOptions(prisma, redisOptions, harness) as never); + + try { + const environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + const taskIdentifier = "gate-task"; + await setupBackgroundWorker(engine, environment, taskIdentifier); + + const run = await engine.trigger(triggerArgs(taskIdentifier, environment, 2), prisma); + await setTimeout(500); + + const queued = await engine.getRunExecutionData({ runId: run.id }); + assertNonNullable(queued); + expect(queued.snapshot.executionStatus).toBe("QUEUED"); + + const dequeued = await engine.dequeueFromWorkerQueue({ + consumerId: "gate_consumer", + workerQueue: "main", + }); + const pending = await engine.getRunExecutionData({ runId: run.id }); + assertNonNullable(pending); + expect(pending.snapshot.executionStatus).toBe("PENDING_EXECUTING"); + + await engine.startRunAttempt({ + runId: dequeued[0]!.run.id, + snapshotId: dequeued[0]!.snapshot.id, + }); + const executing = await engine.getRunExecutionData({ runId: run.id }); + assertNonNullable(executing); + expect(executing.snapshot.executionStatus).toBe("EXECUTING"); + expect(executing.run.attemptNumber).toBe(1); + } finally { + await engine.quit(); + await harness.quit(); + } + } + ); + + containerTest( + "keeps the environment boundary on a snapshot read", + async ({ prisma, redisOptions }) => { + const harness = buildDecoratedStore({ + prisma, + redisOptions, + mode: "redis-read", + readPercent: 100, + }); + const engine = new RunEngine(engineOptions(prisma, redisOptions, harness) as never); + + try { + const environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + const taskIdentifier = "gate-task"; + await setupBackgroundWorker(engine, environment, taskIdentifier); + + const run = await engine.trigger(triggerArgs(taskIdentifier, environment, 3), prisma); + await setTimeout(500); + + // Scoped to its own environment the run reads normally. + const own = await engine.getRunExecutionData({ + runId: run.id, + environmentId: environment.id, + }); + assertNonNullable(own); + + // Scoped to any other environment the run must not leak across the tenant boundary. The + // assertion is parity rather than a fixed shape: whatever Postgres answers for this call, + // Redis has to answer the same, or the boundary behaves differently once reads move over. + const foreignEnvironmentId = generateInternalId(); + + const viaRedis = await engine + .getRunExecutionData({ runId: run.id, environmentId: foreignEnvironmentId }) + .catch((error: unknown) => ({ threw: (error as Error).constructor.name })); + + const postgresOnly = buildDecoratedStore({ prisma, redisOptions, mode: "off" }); + const engineOff = new RunEngine(engineOptions(prisma, redisOptions, postgresOnly) as never); + let viaPostgres: unknown; + try { + viaPostgres = await engineOff + .getRunExecutionData({ runId: run.id, environmentId: foreignEnvironmentId }) + .catch((error: unknown) => ({ threw: (error as Error).constructor.name })); + } finally { + await engineOff.quit(); + await postgresOnly.quit(); + } + + expect(viaRedis).toEqual(viaPostgres); + // And whatever that shape is, it must not be the run's data. + expect(viaRedis).not.toMatchObject({ run: { id: run.id } }); + } finally { + await engine.quit(); + await harness.quit(); + } + } + ); + + containerTest( + "serves a since-window wider than the cap from Redis", + async ({ prisma, redisOptions }) => { + const harness = buildDecoratedStore({ + prisma, + redisOptions, + mode: "redis-read", + readPercent: 100, + }); + const engine = new RunEngine(engineOptions(prisma, redisOptions, harness) as never); + + try { + const environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + const taskIdentifier = "gate-task"; + await setupBackgroundWorker(engine, environment, taskIdentifier); + + const run = await engine.trigger(triggerArgs(taskIdentifier, environment, 4), prisma); + await setTimeout(500); + + const first = await engine.getRunExecutionData({ runId: run.id }); + assertNonNullable(first); + + // More transitions than the 50-cap, so the window is exercised at its boundary. + for (let i = 0; i < 60; i++) { + await harness.store.createExecutionSnapshot({ + run: { id: run.id, status: "PENDING", attemptNumber: null }, + snapshot: { executionStatus: "QUEUED", description: `filler ${i}` }, + environmentId: environment.id, + environmentType: environment.type, + projectId: environment.project.id, + organizationId: environment.organization.id, + }); + } + + const since = await engine.getSnapshotsSince({ + runId: run.id, + snapshotId: first.snapshot.id, + }); + assertNonNullable(since); + + // The newest 50, ascending — the same window Postgres would have produced. + expect(since.length).toBe(50); + expect(since[since.length - 1]!.snapshot.description).toBe("filler 59"); + expect(harness.reads.some((r) => r.source === "redis")).toBe(true); + } finally { + await engine.quit(); + await harness.quit(); + } + } + ); + + containerTest( + "falls back to Postgres for a pre-cutover run", + async ({ prisma, redisOptions }) => { + // A run created while the dial was off has no keyspace. Turning reads on must not lose it. + const off = buildDecoratedStore({ prisma, redisOptions, mode: "off" }); + const engineOff = new RunEngine(engineOptions(prisma, redisOptions, off) as never); + + let runId: string; + let environment: any; + try { + environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + await setupBackgroundWorker(engineOff, environment, "gate-task"); + const run = await engineOff.trigger(triggerArgs("gate-task", environment, 5), prisma); + runId = run.id; + await setTimeout(500); + } finally { + await engineOff.quit(); + await off.quit(); + } + + const on = buildDecoratedStore({ + prisma, + redisOptions, + mode: "redis-read", + readPercent: 100, + }); + const engineOn = new RunEngine(engineOptions(prisma, redisOptions, on) as never); + try { + const data = await engineOn.getRunExecutionData({ runId }); + assertNonNullable(data); + expect(data.snapshot.executionStatus).toBe("QUEUED"); + expect(on.reads.some((r) => r.source === "postgres")).toBe(true); + } finally { + await engineOn.quit(); + await on.quit(); + } + } + ); +}); diff --git a/internal-packages/run-store/src/PostgresRunStore.snapshotId.test.ts b/internal-packages/run-store/src/PostgresRunStore.snapshotId.test.ts new file mode 100644 index 00000000000..07948670b28 --- /dev/null +++ b/internal-packages/run-store/src/PostgresRunStore.snapshotId.test.ts @@ -0,0 +1,150 @@ +// A caller-supplied snapshot id must survive into Postgres, so the decorator can own the id and both +// stores hold the same one under dual-write. Absent, Prisma's @default(cuid()) still supplies it. +import { describe, expect } from "vitest"; +import { postgresTest } from "@internal/testcontainers"; +import { generateInternalId } from "@trigger.dev/core/v3/isomorphic"; +import { PostgresRunStore } from "./PostgresRunStore.js"; +import { setupSnapshotIdFixture } from "./testFixtures/snapshotIdFixture.js"; + +describe("PostgresRunStore caller-supplied snapshot id", () => { + postgresTest("completeAttemptSuccess writes the supplied id", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const { run, env } = await setupSnapshotIdFixture(prisma); + const id = generateInternalId(); + + await store.completeAttemptSuccess( + run.id, + { + completedAt: new Date(), + outputType: "application/json", + usageDurationMs: 1, + costInCents: 0, + snapshot: { + id, + executionStatus: "FINISHED", + description: "Run completed", + runStatus: "COMPLETED_SUCCESSFULLY", + attemptNumber: 1, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }, + }, + { select: { id: true } } + ); + + const snapshot = await prisma.taskRunExecutionSnapshot.findFirst({ where: { id } }); + expect(snapshot).not.toBeNull(); + expect(snapshot!.runId).toBe(run.id); + }); + + postgresTest("expireRun writes the supplied id", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const { run, env } = await setupSnapshotIdFixture(prisma); + const id = generateInternalId(); + + await store.expireRun( + run.id, + { + error: { type: "STRING_ERROR", raw: "expired" }, + completedAt: new Date(), + expiredAt: new Date(), + snapshot: { + id, + engine: "V2", + executionStatus: "FINISHED", + description: "Run expired", + runStatus: "EXPIRED", + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }, + }, + { select: { id: true } } + ); + + expect(await prisma.taskRunExecutionSnapshot.findFirst({ where: { id } })).not.toBeNull(); + }); + + postgresTest("expireParkedRun writes the supplied id", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const { run, env } = await setupSnapshotIdFixture(prisma, { status: "PENDING_VERSION" }); + const id = generateInternalId(); + + const result = await store.expireParkedRun(run.id, { + error: { type: "STRING_ERROR", raw: "expired" }, + completedAt: new Date(), + expiredAt: new Date(), + statusReason: "VERSION_NEVER_ARRIVED", + snapshot: { + id, + engine: "V2", + executionStatus: "FINISHED", + description: "Parked run expired", + runStatus: "EXPIRED", + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }, + }); + + expect(result.count).toBe(1); + expect(await prisma.taskRunExecutionSnapshot.findFirst({ where: { id } })).not.toBeNull(); + }); + + postgresTest("rescheduleRun writes the supplied id", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const { run, env } = await setupSnapshotIdFixture(prisma, { status: "DELAYED" }); + const id = generateInternalId(); + + await store.rescheduleRun(run.id, { + delayUntil: new Date(Date.now() + 60_000), + snapshot: { + id, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }, + }); + + expect(await prisma.taskRunExecutionSnapshot.findFirst({ where: { id } })).not.toBeNull(); + }); + + postgresTest("createExecutionSnapshot writes the supplied id", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const { run, env } = await setupSnapshotIdFixture(prisma); + const id = generateInternalId(); + + const created = await store.createExecutionSnapshot({ + id, + run: { id: run.id, status: "EXECUTING", attemptNumber: 1 }, + snapshot: { executionStatus: "EXECUTING", description: "Run started" }, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }); + + expect(created.id).toBe(id); + }); + + postgresTest("an absent id still gets a generated one", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const { run, env } = await setupSnapshotIdFixture(prisma); + + const created = await store.createExecutionSnapshot({ + run: { id: run.id, status: "EXECUTING", attemptNumber: 1 }, + snapshot: { executionStatus: "EXECUTING", description: "Run started" }, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }); + + expect(created.id).toMatch(/^c[a-z0-9]{24}$/); + }); +}); diff --git a/internal-packages/run-store/src/PostgresRunStore.snapshotTimestamps.test.ts b/internal-packages/run-store/src/PostgresRunStore.snapshotTimestamps.test.ts new file mode 100644 index 00000000000..8026f2f99ec --- /dev/null +++ b/internal-packages/run-store/src/PostgresRunStore.snapshotTimestamps.test.ts @@ -0,0 +1,132 @@ +// The caller's instant must land in BOTH timestamp columns, on BOTH schema variants. +// +// `updatedAt` is declared `@updatedAt`, which Prisma manages itself, so whether an explicit value +// survives a create is a property of the client rather than of the schema. The two variants are +// separately generated clients over separately declared schemas, so agreeing declarations are not +// evidence that they agree in behaviour. This asserts it on each. +// +// It matters because the decorator writes one instant to both stores. If Prisma overrode it here, +// Postgres and Redis would hold different values for a column the comparator checks for equality, +// on every snapshot. +import { heteroPostgresTest, heteroRunOpsPostgresTest } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import type { RunOpsPrismaClient } from "@internal/run-ops-database"; +import { generateInternalId } from "@trigger.dev/core/v3/isomorphic"; +import { describe, expect } from "vitest"; +import { PostgresRunStore } from "./PostgresRunStore.js"; + +type AnyClient = PrismaClient | RunOpsPrismaClient; + +/** Five minutes in the past, so a column default could never coincide with it. */ +const STAMP = new Date(Date.now() - 5 * 60 * 1000); + +async function writeSnapshot( + prisma: AnyClient, + schemaVariant: "legacy" | "dedicated", + suffix: string +) { + const scope = + schemaVariant === "dedicated" + ? { + environmentId: `env_${suffix}`, + projectId: `proj_${suffix}`, + organizationId: `org_${suffix}`, + } + : await seedLegacyScope(prisma as PrismaClient, suffix); + + const store = new PostgresRunStore({ + prisma: prisma as never, + readOnlyPrisma: prisma as never, + schemaVariant, + }); + + const runId = generateInternalId(); + const id = generateInternalId(); + + await (prisma as PrismaClient).taskRun.create({ + data: { + id: runId, + engine: "V2", + status: "PENDING", + friendlyId: `run_${suffix}`, + runtimeEnvironmentId: scope.environmentId, + environmentType: "DEVELOPMENT", + organizationId: scope.organizationId, + projectId: scope.projectId, + taskIdentifier: "my-task", + payload: "{}", + payloadType: "application/json", + traceContext: {}, + traceId: `trace_${suffix}`, + spanId: `span_${suffix}`, + queue: "task/my-task", + isTest: false, + taskEventStore: "taskEvent", + depth: 0, + } as never, + }); + + await store.createExecutionSnapshot({ + id, + createdAt: STAMP, + run: { id: runId, status: "EXECUTING", attemptNumber: 1 }, + snapshot: { executionStatus: "EXECUTING", description: "Run started" }, + environmentId: scope.environmentId, + environmentType: "DEVELOPMENT", + projectId: scope.projectId, + organizationId: scope.organizationId, + }); + + return (prisma as PrismaClient).taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); +} + +async function seedLegacyScope(prisma: PrismaClient, suffix: string) { + const organization = await prisma.organization.create({ + data: { title: `Org ${suffix}`, slug: `org-${suffix}` }, + }); + const project = await prisma.project.create({ + data: { + name: `Project ${suffix}`, + slug: `project-${suffix}`, + externalRef: `proj_${suffix}`, + organizationId: organization.id, + }, + }); + const environment = await prisma.runtimeEnvironment.create({ + data: { + type: "DEVELOPMENT", + slug: `dev-${suffix}`, + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_dev_${suffix}`, + pkApiKey: `pk_dev_${suffix}`, + shortcode: `short_${suffix}`, + }, + }); + return { + environmentId: environment.id, + projectId: project.id, + organizationId: organization.id, + }; +} + +describe("snapshot timestamps are the caller's, on both schema variants", () => { + heteroPostgresTest("legacy client honours the supplied instant", async ({ prisma14 }) => { + const row = await writeSnapshot(prisma14, "legacy", "tsleg"); + + expect(row.createdAt.toISOString()).toBe(STAMP.toISOString()); + // The one Prisma manages. If it overrode the value, the two stores would disagree here on + // every snapshot. + expect(row.updatedAt.toISOString()).toBe(STAMP.toISOString()); + }); + + heteroRunOpsPostgresTest( + "dedicated client honours the supplied instant", + async ({ prisma17 }) => { + const row = await writeSnapshot(prisma17, "dedicated", "tsded"); + + expect(row.createdAt.toISOString()).toBe(STAMP.toISOString()); + expect(row.updatedAt.toISOString()).toBe(STAMP.toISOString()); + } + ); +}); diff --git a/internal-packages/run-store/src/PostgresRunStore.snapshotWrites.test.ts b/internal-packages/run-store/src/PostgresRunStore.snapshotWrites.test.ts new file mode 100644 index 00000000000..dda0f483bf5 --- /dev/null +++ b/internal-packages/run-store/src/PostgresRunStore.snapshotWrites.test.ts @@ -0,0 +1,312 @@ +// snapshotWrites: false is the redis-only dial position. Every run mutation still lands; no snapshot +// row is written and no completed-waitpoint join row is inserted. The default stays true, so nothing +// changes for any existing caller. +import { describe, expect } from "vitest"; +import { postgresTest } from "@internal/testcontainers"; +import { generateInternalId } from "@trigger.dev/core/v3/isomorphic"; +import { PostgresRunStore } from "./PostgresRunStore.js"; +import { + buildCreateRunData, + seedSnapshotEnvironment, + seedSnapshotWorker, + setupSnapshotIdFixture, +} from "./testFixtures/snapshotIdFixture.js"; + +describe("PostgresRunStore snapshotWrites flag", () => { + postgresTest("defaults to writing snapshots", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const { run, env } = await setupSnapshotIdFixture(prisma); + + await store.completeAttemptSuccess( + run.id, + { + completedAt: new Date(), + outputType: "application/json", + usageDurationMs: 1, + costInCents: 0, + snapshot: { + executionStatus: "FINISHED", + description: "Run completed", + runStatus: "COMPLETED_SUCCESSFULLY", + attemptNumber: 1, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }, + }, + { select: { id: true } } + ); + + expect(await prisma.taskRunExecutionSnapshot.count({ where: { runId: run.id } })).toBe(1); + }); + + postgresTest("writes the run mutation but no snapshot when off", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma, snapshotWrites: false }); + const { run, env } = await setupSnapshotIdFixture(prisma); + + await store.completeAttemptSuccess( + run.id, + { + completedAt: new Date(), + outputType: "application/json", + usageDurationMs: 1, + costInCents: 0, + snapshot: { + executionStatus: "FINISHED", + description: "Run completed", + runStatus: "COMPLETED_SUCCESSFULLY", + attemptNumber: 1, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }, + }, + { select: { id: true } } + ); + + const updated = await prisma.taskRun.findFirstOrThrow({ where: { id: run.id } }); + expect(updated.status).toBe("COMPLETED_SUCCESSFULLY"); + expect(await prisma.taskRunExecutionSnapshot.count({ where: { runId: run.id } })).toBe(0); + }); + + postgresTest("createRun writes the run but no snapshot when off", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma, snapshotWrites: false }); + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + + await store.createRun({ + data: buildCreateRunData(runId, env), + snapshot: { + id: generateInternalId(), + engine: "V2", + executionStatus: "RUN_CREATED", + description: "Run was created", + runStatus: "PENDING", + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }, + }); + + expect(await prisma.taskRun.count({ where: { id: runId } })).toBe(1); + expect(await prisma.taskRunExecutionSnapshot.count({ where: { runId } })).toBe(0); + }); + + postgresTest("createCancelledRun writes the run but no snapshot when off", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma, snapshotWrites: false }); + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + + await store.createCancelledRun({ + data: { + ...buildCreateRunData(runId, env), + status: "CANCELED", + error: { type: "STRING_ERROR", raw: "cancelled" }, + completedAt: new Date(), + updatedAt: new Date(), + attemptNumber: 0, + }, + snapshot: { + id: generateInternalId(), + engine: "V2", + executionStatus: "FINISHED", + description: "Run was cancelled", + runStatus: "CANCELED", + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }, + }); + + expect(await prisma.taskRun.count({ where: { id: runId } })).toBe(1); + expect(await prisma.taskRunExecutionSnapshot.count({ where: { runId } })).toBe(0); + }); + + postgresTest("expireRun writes the run but no snapshot when off", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma, snapshotWrites: false }); + const { run, env } = await setupSnapshotIdFixture(prisma); + + await store.expireRun( + run.id, + { + error: { type: "STRING_ERROR", raw: "expired" }, + completedAt: new Date(), + expiredAt: new Date(), + snapshot: { + engine: "V2", + executionStatus: "FINISHED", + description: "Run expired", + runStatus: "EXPIRED", + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }, + }, + { select: { id: true } } + ); + + expect((await prisma.taskRun.findFirstOrThrow({ where: { id: run.id } })).status).toBe( + "EXPIRED" + ); + expect(await prisma.taskRunExecutionSnapshot.count({ where: { runId: run.id } })).toBe(0); + }); + + postgresTest("expireParkedRun writes the run but no snapshot when off", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma, snapshotWrites: false }); + const { run, env } = await setupSnapshotIdFixture(prisma, { status: "PENDING_VERSION" }); + + const result = await store.expireParkedRun(run.id, { + error: { type: "STRING_ERROR", raw: "expired" }, + completedAt: new Date(), + expiredAt: new Date(), + statusReason: "VERSION_NEVER_ARRIVED", + snapshot: { + engine: "V2", + executionStatus: "FINISHED", + description: "Parked run expired", + runStatus: "EXPIRED", + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }, + }); + + expect(result.count).toBe(1); + expect(await prisma.taskRunExecutionSnapshot.count({ where: { runId: run.id } })).toBe(0); + }); + + postgresTest("rescheduleRun writes the run but no snapshot when off", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma, snapshotWrites: false }); + const { run, env } = await setupSnapshotIdFixture(prisma, { status: "DELAYED" }); + const delayUntil = new Date(Date.now() + 60_000); + + await store.rescheduleRun(run.id, { + delayUntil, + snapshot: { + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }, + }); + + const updated = await prisma.taskRun.findFirstOrThrow({ where: { id: run.id } }); + expect(updated.delayUntil?.toISOString()).toBe(delayUntil.toISOString()); + expect(await prisma.taskRunExecutionSnapshot.count({ where: { runId: run.id } })).toBe(0); + }); + + postgresTest("lockRunToWorker writes the lock but no snapshot when off", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma, snapshotWrites: false }); + const { run, env } = await setupSnapshotIdFixture(prisma); + const { workerId, taskId } = await seedSnapshotWorker(prisma, env); + + await store.lockRunToWorker(run.id, { + lockedAt: new Date(), + lockedById: taskId, + lockedToVersionId: workerId, + lockedQueueId: undefined, + startedAt: new Date(), + baseCostInCents: 0, + machinePreset: "small-1x", + taskVersion: "1.0.0", + snapshot: { + id: generateInternalId(), + previousSnapshotId: generateInternalId(), + attemptNumber: 1, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + completedWaitpointIds: [], + completedWaitpointOrder: [], + }, + }); + + expect((await prisma.taskRun.findFirstOrThrow({ where: { id: run.id } })).status).toBe( + "DEQUEUED" + ); + expect(await prisma.taskRunExecutionSnapshot.count({ where: { runId: run.id } })).toBe(0); + }); + + postgresTest("createExecutionSnapshot echoes the input when off", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma, snapshotWrites: false }); + const { run, env } = await setupSnapshotIdFixture(prisma); + const id = generateInternalId(); + + const echoed = await store.createExecutionSnapshot({ + id, + run: { id: run.id, status: "EXECUTING", attemptNumber: 2 }, + snapshot: { executionStatus: "EXECUTING", description: "Run started" }, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }); + + expect(echoed.id).toBe(id); + expect(echoed.runId).toBe(run.id); + expect(echoed.executionStatus).toBe("EXECUTING"); + expect(echoed.attemptNumber).toBe(2); + expect(echoed.isValid).toBe(true); + expect(echoed.checkpoint).toBeNull(); + expect(await prisma.taskRunExecutionSnapshot.count({ where: { runId: run.id } })).toBe(0); + }); + + postgresTest("the echoed row rewrites a DEQUEUED run status", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma, snapshotWrites: false }); + const { run, env } = await setupSnapshotIdFixture(prisma); + + const echoed = await store.createExecutionSnapshot({ + id: generateInternalId(), + run: { id: run.id, status: "DEQUEUED", attemptNumber: 1 }, + snapshot: { executionStatus: "PENDING_EXECUTING", description: "Run was dequeued" }, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }); + + expect(echoed.runStatus).toBe("PENDING"); + }); + + postgresTest("the echoed row reports an errored snapshot as invalid", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma, snapshotWrites: false }); + const { run, env } = await setupSnapshotIdFixture(prisma); + + const echoed = await store.createExecutionSnapshot({ + id: generateInternalId(), + run: { id: run.id, status: "EXECUTING", attemptNumber: 1 }, + snapshot: { executionStatus: "EXECUTING", description: "Stale write" }, + error: "snapshot is not the latest", + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }); + + expect(echoed.isValid).toBe(false); + expect(echoed.error).toBe("snapshot is not the latest"); + }); + + postgresTest("createExecutionSnapshot needs an id when off", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma, snapshotWrites: false }); + const { run, env } = await setupSnapshotIdFixture(prisma); + + await expect( + store.createExecutionSnapshot({ + run: { id: run.id, status: "EXECUTING", attemptNumber: 1 }, + snapshot: { executionStatus: "EXECUTING", description: "Run started" }, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }) + ).rejects.toThrow(/snapshotWrites is off/); + }); +}); diff --git a/internal-packages/run-store/src/PostgresRunStore.ts b/internal-packages/run-store/src/PostgresRunStore.ts index 33604d01148..7af040eb99c 100644 --- a/internal-packages/run-store/src/PostgresRunStore.ts +++ b/internal-packages/run-store/src/PostgresRunStore.ts @@ -118,6 +118,13 @@ export type PostgresRunStoreOptions = { maxWait?: number; /** Env-driven P2028-at-acquisition retry config, threaded from the app boundary (IoC). */ transactionStartRetry?: TransactionStartRetryConfig; + /** + * When false the store writes no execution-snapshot rows: every nested `executionSnapshots.create` + * is omitted and `createExecutionSnapshot` echoes its input instead of inserting. Only the + * redis-only dial position sets this, once the Redis store is the sole snapshot writer. + * Defaults to true, so the store behaves exactly as it always has. + */ + snapshotWrites?: boolean; }; // A caller sub-select for a relation: `{ select?, include? }` or `true` for a bare `key: true`. @@ -638,6 +645,7 @@ export class PostgresRunStore implements RunStore { private readonly prisma: RunOpsCapableClient; private readonly readOnlyPrisma: RunOpsCapableClient; private readonly schemaVariant: RunStoreSchemaVariant; + private readonly snapshotWrites: boolean; private readonly maxWait?: number; private readonly transactionStartRetry?: TransactionStartRetryConfig; @@ -650,6 +658,22 @@ export class PostgresRunStore implements RunStore { this.schemaVariant = options.schemaVariant ?? "legacy"; this.maxWait = options.maxWait; this.transactionStartRetry = options.transactionStartRetry; + this.snapshotWrites = options.snapshotWrites ?? true; + } + + /** + * Wraps a nested snapshot create so a single flag removes it everywhere. Prisma treats an absent + * key and `undefined` alike, so spreading an empty object drops the nested write entirely rather + * than sending an empty one. + */ + #nestedSnapshot(create: Prisma.TaskRunExecutionSnapshotUncheckedCreateWithoutRunInput): + | { + executionSnapshots: { + create: Prisma.TaskRunExecutionSnapshotUncheckedCreateWithoutRunInput; + }; + } + | Record { + return this.snapshotWrites ? { executionSnapshots: { create } } : {}; } // The writer handle in read-client form, so the routing layer can honor a caller-passed client @@ -726,6 +750,8 @@ export class PostgresRunStore implements RunStore { const snapshotCreate = { id: params.snapshot.id, + createdAt: params.snapshot.createdAt, + updatedAt: params.snapshot.createdAt, engine: params.snapshot.engine, executionStatus: params.snapshot.executionStatus, description: params.snapshot.description, @@ -743,7 +769,7 @@ export class PostgresRunStore implements RunStore { const run = (await this.#writeClientWithoutTransaction(tx).taskRun.create({ data: { ...params.data, - executionSnapshots: { create: snapshotCreate }, + ...this.#nestedSnapshot(snapshotCreate), }, })) as TaskRun; return { ...run, associatedWaitpoint: null }; @@ -755,7 +781,7 @@ export class PostgresRunStore implements RunStore { const run = (await c.taskRun.create({ data: { ...params.data, - executionSnapshots: { create: snapshotCreate }, + ...this.#nestedSnapshot(snapshotCreate), }, })) as TaskRun; @@ -772,9 +798,7 @@ export class PostgresRunStore implements RunStore { }, data: { ...params.data, - executionSnapshots: { - create: snapshotCreate, - }, + ...this.#nestedSnapshot(snapshotCreate), associatedWaitpoint: params.associatedWaitpoint ? { create: params.associatedWaitpoint, @@ -813,23 +837,26 @@ export class PostgresRunStore implements RunStore { ): Promise { const client = tx ?? this.prisma; + const snapshotCreate = { + id: params.snapshot.id, + createdAt: params.snapshot.createdAt, + updatedAt: params.snapshot.createdAt, + engine: params.snapshot.engine, + executionStatus: params.snapshot.executionStatus, + description: params.snapshot.description, + runStatus: params.snapshot.runStatus, + environmentId: params.snapshot.environmentId, + environmentType: params.snapshot.environmentType, + projectId: params.snapshot.projectId, + organizationId: params.snapshot.organizationId, + workerId: params.snapshot.workerId, + runnerId: params.snapshot.runnerId, + }; + return client.taskRun.create({ data: { ...params.data, - executionSnapshots: { - create: { - engine: params.snapshot.engine, - executionStatus: params.snapshot.executionStatus, - description: params.snapshot.description, - runStatus: params.snapshot.runStatus, - environmentId: params.snapshot.environmentId, - environmentType: params.snapshot.environmentType, - projectId: params.snapshot.projectId, - organizationId: params.snapshot.organizationId, - workerId: params.snapshot.workerId, - runnerId: params.snapshot.runnerId, - }, - }, + ...this.#nestedSnapshot(snapshotCreate), }, }); } @@ -923,20 +950,21 @@ export class PostgresRunStore implements RunStore { outputType: data.outputType, usageDurationMs: data.usageDurationMs, costInCents: data.costInCents, - executionSnapshots: { - create: { - executionStatus: data.snapshot.executionStatus, - description: data.snapshot.description, - runStatus: data.snapshot.runStatus, - attemptNumber: data.snapshot.attemptNumber, - environmentId: data.snapshot.environmentId, - environmentType: data.snapshot.environmentType, - projectId: data.snapshot.projectId, - organizationId: data.snapshot.organizationId, - workerId: data.snapshot.workerId, - runnerId: data.snapshot.runnerId, - }, - }, + ...this.#nestedSnapshot({ + id: data.snapshot.id, + createdAt: data.snapshot.createdAt, + updatedAt: data.snapshot.createdAt, + executionStatus: data.snapshot.executionStatus, + description: data.snapshot.description, + runStatus: data.snapshot.runStatus, + attemptNumber: data.snapshot.attemptNumber, + environmentId: data.snapshot.environmentId, + environmentType: data.snapshot.environmentType, + projectId: data.snapshot.projectId, + organizationId: data.snapshot.organizationId, + workerId: data.snapshot.workerId, + runnerId: data.snapshot.runnerId, + }), }, { select: args.select } ) as Promise>; @@ -1129,18 +1157,19 @@ export class PostgresRunStore implements RunStore { completedAt: data.completedAt, expiredAt: data.expiredAt, error: data.error as Prisma.InputJsonValue, - executionSnapshots: { - create: { - engine: data.snapshot.engine, - executionStatus: data.snapshot.executionStatus, - description: data.snapshot.description, - runStatus: data.snapshot.runStatus, - environmentId: data.snapshot.environmentId, - environmentType: data.snapshot.environmentType, - projectId: data.snapshot.projectId, - organizationId: data.snapshot.organizationId, - }, - }, + ...this.#nestedSnapshot({ + id: data.snapshot.id, + createdAt: data.snapshot.createdAt, + updatedAt: data.snapshot.createdAt, + engine: data.snapshot.engine, + executionStatus: data.snapshot.executionStatus, + description: data.snapshot.description, + runStatus: data.snapshot.runStatus, + environmentId: data.snapshot.environmentId, + environmentType: data.snapshot.environmentType, + projectId: data.snapshot.projectId, + organizationId: data.snapshot.organizationId, + }), }, { select: args.select } ) as Promise>; @@ -1260,42 +1289,46 @@ export class PostgresRunStore implements RunStore { cliVersion: data.cliVersion ?? undefined, maxDurationInSeconds: data.maxDurationInSeconds ?? undefined, maxAttempts: data.maxAttempts ?? undefined, - executionSnapshots: { - create: { - id: data.snapshot.id, - engine: "V2", - executionStatus: "PENDING_EXECUTING", - description: "Run was dequeued for execution", - runStatus: "PENDING", - attemptNumber: data.snapshot.attemptNumber ?? undefined, - previousSnapshotId: data.snapshot.previousSnapshotId, - environmentId: data.snapshot.environmentId, - environmentType: data.snapshot.environmentType, - projectId: data.snapshot.projectId, - organizationId: data.snapshot.organizationId, - checkpointId: data.snapshot.checkpointId ?? undefined, - batchId: data.snapshot.batchId ?? undefined, - // Completed-waitpoint links are inserted FK-free after create (below) for BOTH schemas. - completedWaitpointOrder: data.snapshot.completedWaitpointOrder, - workerId: data.snapshot.workerId ?? undefined, - runnerId: data.snapshot.runnerId ?? undefined, - }, - }, + ...this.#nestedSnapshot({ + id: data.snapshot.id, + createdAt: data.snapshot.createdAt, + updatedAt: data.snapshot.createdAt, + engine: "V2", + executionStatus: "PENDING_EXECUTING", + description: "Run was dequeued for execution", + runStatus: "PENDING", + attemptNumber: data.snapshot.attemptNumber ?? undefined, + previousSnapshotId: data.snapshot.previousSnapshotId, + environmentId: data.snapshot.environmentId, + environmentType: data.snapshot.environmentType, + projectId: data.snapshot.projectId, + organizationId: data.snapshot.organizationId, + checkpointId: data.snapshot.checkpointId ?? undefined, + batchId: data.snapshot.batchId ?? undefined, + // Completed-waitpoint links are inserted FK-free after create (below) for BOTH schemas. + completedWaitpointOrder: data.snapshot.completedWaitpointOrder, + workerId: data.snapshot.workerId ?? undefined, + runnerId: data.snapshot.runnerId ?? undefined, + }), }, }); - if (dedicated) { - await this.#connectCompletedWaitpoints( - prisma, - data.snapshot.id, - data.snapshot.completedWaitpointIds - ); - } else { - await this.#connectCompletedWaitpointsLegacy( - prisma, - data.snapshot.id, - data.snapshot.completedWaitpointIds - ); + // The join rows link to the snapshot row above. With snapshot writes off there is no such row, + // so inserting them would leave dangling links for a snapshot that only the Redis store holds. + if (this.snapshotWrites) { + if (dedicated) { + await this.#connectCompletedWaitpoints( + prisma, + data.snapshot.id, + data.snapshot.completedWaitpointIds + ); + } else { + await this.#connectCompletedWaitpointsLegacy( + prisma, + data.snapshot.id, + data.snapshot.completedWaitpointIds + ); + } } return result; @@ -1363,18 +1396,19 @@ export class PostgresRunStore implements RunStore { completedAt: data.completedAt, expiredAt: data.expiredAt, error: data.error as Prisma.InputJsonValue, - executionSnapshots: { - create: { - engine: data.snapshot.engine, - executionStatus: data.snapshot.executionStatus, - description: data.snapshot.description, - runStatus: data.snapshot.runStatus, - environmentId: data.snapshot.environmentId, - environmentType: data.snapshot.environmentType, - projectId: data.snapshot.projectId, - organizationId: data.snapshot.organizationId, - }, - }, + ...this.#nestedSnapshot({ + id: data.snapshot.id, + createdAt: data.snapshot.createdAt, + updatedAt: data.snapshot.createdAt, + engine: data.snapshot.engine, + executionStatus: data.snapshot.executionStatus, + description: data.snapshot.description, + runStatus: data.snapshot.runStatus, + environmentId: data.snapshot.environmentId, + environmentType: data.snapshot.environmentType, + projectId: data.snapshot.projectId, + organizationId: data.snapshot.organizationId, + }), }, }); } catch (error) { @@ -1435,21 +1469,21 @@ export class PostgresRunStore implements RunStore { data: { delayUntil: data.delayUntil, ...(data.queueTimestamp !== undefined && { queueTimestamp: data.queueTimestamp }), - ...(data.snapshot && { - executionSnapshots: { - create: { - engine: "V2", - executionStatus: data.snapshot.executionStatus ?? "DELAYED", - description: - data.snapshot.description ?? "Delayed run was rescheduled to a future date", - runStatus: data.snapshot.runStatus ?? "DELAYED", - environmentId: data.snapshot.environmentId, - environmentType: data.snapshot.environmentType, - projectId: data.snapshot.projectId, - organizationId: data.snapshot.organizationId, - }, - }, - }), + ...(data.snapshot && + this.#nestedSnapshot({ + id: data.snapshot.id, + createdAt: data.snapshot.createdAt, + updatedAt: data.snapshot.createdAt, + engine: "V2", + executionStatus: data.snapshot.executionStatus ?? "DELAYED", + description: + data.snapshot.description ?? "Delayed run was rescheduled to a future date", + runStatus: data.snapshot.runStatus ?? "DELAYED", + environmentId: data.snapshot.environmentId, + environmentType: data.snapshot.environmentType, + projectId: data.snapshot.projectId, + organizationId: data.snapshot.organizationId, + })), }, }); } @@ -1969,6 +2003,8 @@ export class PostgresRunStore implements RunStore { prisma: PrismaClientOrTransaction ): Promise> { const { + id, + createdAt, run, snapshot, previousSnapshotId, @@ -1984,10 +2020,65 @@ export class PostgresRunStore implements RunStore { error, } = input; + // Left possibly-undefined ON PURPOSE. Prisma omits an undefined key, so the column keeps taking + // whatever it took before this method was touched: the schema declares `completedWaitpointOrder + // String[]` with no default and the column is nullable, so an omitted key stores NULL, not `{}`. + // Defaulting here would send `{}` instead and change what a live write stores. + // + // The redis-only echo below DOES need a concrete array, because it returns the row shape to the + // caller and that field is not nullable in the payload type. That default belongs to the echo, + // not to the write, so the two are kept apart. + const completedWaitpointOrder = completedWaitpoints + ?.filter((c) => c.index !== undefined) + .sort((a, b) => a.index! - b.index!) + .map((w) => w.id); + + // Redis-only: no row is written and the decorator owns the document. Echo the input in the shape + // the caller expects, so every caller of this method keeps working while Postgres holds nothing. + if (!this.snapshotWrites) { + if (!id) { + throw new Error( + "PostgresRunStore.createExecutionSnapshot: snapshotWrites is off, so the caller must supply the snapshot id" + ); + } + + const now = createdAt ?? new Date(); + return { + id, + engine: "V2", + executionStatus: snapshot.executionStatus, + description: snapshot.description, + previousSnapshotId: previousSnapshotId ?? null, + runId: run.id, + runStatus: run.status === "DEQUEUED" ? "PENDING" : run.status, + attemptNumber: run.attemptNumber ?? null, + batchId: batchId ?? null, + environmentId, + environmentType, + projectId, + organizationId, + checkpointId: checkpointId ?? null, + workerId: workerId ?? null, + runnerId: runnerId ?? null, + metadata: snapshot.metadata ?? null, + completedWaitpointOrder: completedWaitpointOrder ?? [], + isValid: !error, + error: error ?? null, + createdAt: now, + updatedAt: now, + checkpoint: null, + } as unknown as Prisma.TaskRunExecutionSnapshotGetPayload<{ + include: { checkpoint: true }; + }>; + } + const dedicated = this.schemaVariant === "dedicated"; const newSnapshot = await prisma.taskRunExecutionSnapshot.create({ data: { + id, + createdAt, + updatedAt: createdAt, engine: "V2", executionStatus: snapshot.executionStatus, description: snapshot.description, @@ -2007,10 +2098,7 @@ export class PostgresRunStore implements RunStore { metadata: snapshot.metadata ?? undefined, // Completed-waitpoint links are inserted FK-free after create (below) for BOTH schemas, so a // cross-DB (NEW-resident) token can be recorded without a Prisma `connect` existence check. - completedWaitpointOrder: completedWaitpoints - ?.filter((c) => c.index !== undefined) - .sort((a, b) => a.index! - b.index!) - .map((w) => w.id), + completedWaitpointOrder, isValid: !error, error, }, diff --git a/internal-packages/run-store/src/delegatingRunStore.forwarding.test.ts b/internal-packages/run-store/src/delegatingRunStore.forwarding.test.ts new file mode 100644 index 00000000000..081e9efe1d4 --- /dev/null +++ b/internal-packages/run-store/src/delegatingRunStore.forwarding.test.ts @@ -0,0 +1,188 @@ +// Every declared parameter must actually reach the delegate. +// +// The compiler cannot check this. A forwarder that omits a trailing OPTIONAL argument compiles +// cleanly, and the effect is silent: `findLatestExecutionSnapshot` would stop applying its tenant +// scope, and `upsertWaitpointTag` would stop applying its residency hint, so a write would land on +// the wrong database. Both of those shipped in this file before this test existed. +// +// So this reads the source of the base against the source of the interface and asserts that each +// forward passes exactly the parameters its signature declares, in order. Source-level, because +// that is the only place the property is visible. +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; + +const dir = join(import.meta.dirname); +const interfaceSource = readFileSync(join(dir, "types.ts"), "utf8"); +const baseSource = readFileSync(join(dir, "delegatingRunStore.ts"), "utf8"); + +/** Replaces comments and string bodies with spaces, so neither can shift a brace depth. */ +function blank(text: string): string { + let out = ""; + let i = 0; + while (i < text.length) { + const two = text.slice(i, i + 2); + if (two === "//") { + const end = text.indexOf("\n", i); + const stop = end === -1 ? text.length : end; + out += " ".repeat(stop - i); + i = stop; + } else if (two === "/*") { + const end = text.indexOf("*/", i + 2); + const stop = end === -1 ? text.length : end + 2; + out += text.slice(i, stop).replace(/[^\n]/g, " "); + i = stop; + } else if (text[i] === '"' || text[i] === "'" || text[i] === "`") { + const quote = text[i]; + let j = i + 1; + while (j < text.length && text[j] !== quote) j += text[j] === "\\" ? 2 : 1; + out += quote + " ".repeat(Math.max(0, j - i - 1)) + (text[j] ?? ""); + i = j + 1; + } else { + out += text[i]; + i += 1; + } + } + return out; +} + +function interfaceBody(source: string): string { + const blanked = blank(source); + const decl = "export interface RunStore {"; + const start = blanked.indexOf(decl) + decl.length; + let depth = 1; + let end = start; + while (depth > 0 && end < blanked.length) { + if (blanked[end] === "{") depth += 1; + else if (blanked[end] === "}") depth -= 1; + if (depth > 0) end += 1; + } + return blanked.slice(start, end); +} + +/** Splits a balanced parameter list on top-level commas. */ +function splitParams(signature: string): string[] { + // A generic member reads `name(...)`, so the parameter list starts after the + // balanced angle block, not at the first parenthesis. + let searchFrom = 0; + const angle = signature.indexOf("<"); + const paren = signature.indexOf("("); + if (angle !== -1 && angle < paren) { + let angleDepth = 0; + for (let i = angle; i < signature.length; i++) { + if (signature[i] === "<") angleDepth += 1; + else if (signature[i] === ">") { + angleDepth -= 1; + if (angleDepth === 0) { + searchFrom = i; + break; + } + } + } + } + + const open = signature.indexOf("(", searchFrom); + let depth = 0; + let close = open; + for (let i = open; i < signature.length; i++) { + if ("([{<".includes(signature[i]!)) depth += 1; + else if (")]}>".includes(signature[i]!)) { + depth -= 1; + if (depth === 0) { + close = i; + break; + } + } + } + const inner = signature.slice(open + 1, close); + const parts: string[] = []; + let level = 0; + let current = ""; + for (const ch of inner) { + if ("([{<".includes(ch)) level += 1; + else if (")]}>".includes(ch)) level -= 1; + if (ch === "," && level === 0) { + parts.push(current); + current = ""; + } else { + current += ch; + } + } + if (current.trim()) parts.push(current); + return parts; +} + +function paramNames(signature: string): string[] { + return splitParams(signature) + .map((p) => /^\s*([A-Za-z_$][\w$]*)\s*\??\s*:/.exec(p)?.[1]) + .filter((n): n is string => Boolean(n)); +} + +/** Member name to its declared parameter names, for members with a single signature. */ +function declaredParams(): Map { + const body = interfaceBody(interfaceSource); + const spans: string[] = []; + let level = 0; + let from = 0; + for (let i = 0; i < body.length; i++) { + const ch = body[i]!; + if ("{([".includes(ch)) level += 1; + else if ("})]".includes(ch)) level -= 1; + else if (ch === ";" && level === 0) { + spans.push(body.slice(from, i)); + from = i + 1; + } + } + + const seen = new Map(); + for (const span of spans) { + const match = /^\s*(?:readonly\s+)?([A-Za-z_$][\w$]*)\s*\??\s*[(<]/.exec(span); + if (!match) continue; + const name = match[1]!; + seen.set(name, [...(seen.get(name) ?? []), paramNames(span)]); + } + + // Overloaded members forward through a cast and apply the whole argument list, so they are not + // subject to this check. + return new Map( + [...seen].filter(([, sigs]) => sigs.length === 1).map(([n, sigs]) => [n, sigs[0]!]) + ); +} + +describe("the pass-through forwards every declared parameter", () => { + const declared = declaredParams(); + + it("parsed the interface, so a parse failure cannot pass this suite", () => { + expect(declared.size).toBeGreaterThan(50); + expect(declared.get("expireParkedRun")).toEqual(["runId", "data", "tx"]); + expect(declared.get("findLatestExecutionSnapshot")).toEqual([ + "runId", + "client", + "environmentId", + ]); + }); + + it("passes exactly the declared parameters, in order, for every single-signature member", () => { + const wrong: string[] = []; + + for (const [name, params] of declared) { + const forward = new RegExp(`return this\\.delegate\\.${name}\\(([^;]*)\\);`).exec(baseSource); + + if (!forward) { + wrong.push(`${name}: no forward found`); + continue; + } + + const passed = forward[1]! + .split(",") + .map((a) => a.trim()) + .filter(Boolean); + + if (passed.join(",") !== params.join(",")) { + wrong.push(`${name}: declares (${params.join(", ")}) but forwards (${passed.join(", ")})`); + } + } + + expect(wrong).toEqual([]); + }); +}); diff --git a/internal-packages/run-store/src/delegatingRunStore.test.ts b/internal-packages/run-store/src/delegatingRunStore.test.ts new file mode 100644 index 00000000000..7fdcd9ac07b --- /dev/null +++ b/internal-packages/run-store/src/delegatingRunStore.test.ts @@ -0,0 +1,118 @@ +// What this suite covers, and why it is now small. +// +// `DelegatingRunStore` restates every interface signature and forwards its arguments by name, so +// most of what a pass-through can get wrong is a compile error rather than a test failure: +// +// member of the interface missing -> `implements RunStore`, TS2420 +// public member the interface lacks -> the parity assertion in the base +// forwarded to the wrong delegate member -> argument types do not match, TS2345/TS2322 +// arguments reordered -> same +// +// Three things remain invisible to the compiler, and they are what is left here. +// +// First, the seven overloaded members. TypeScript cannot express one body that satisfies an overload +// set, so their single implementation forwards through a cast, and the cast is exactly where a +// wrong-member forward would stop being a type error. +// +// Second, a dropped OPTIONAL argument. Omitting a trailing `tx` compiles cleanly and silently stops +// forwarding the caller's transaction. +// +// Third, whether a data property is read live or captured once at construction. Both typecheck; only +// one is correct. +// +// No database is involved in whether a pass-through passes through, so none is started. Behaviour +// against a real store is covered by the container suites for the decorator built on this base. +import { describe, expect, it } from "vitest"; +import { DelegatingRunStore } from "./delegatingRunStore.js"; +import { RUN_STORE_METHOD_NAMES, RUN_STORE_PROPERTY_NAMES } from "./runStoreMethodNames.js"; +import type { RunStore } from "./types.js"; + +/** + * The members whose implementation forwards through a cast, because they are overloaded. These are + * the only methods where the compiler is not already checking the forward. + */ +const OVERLOADED_MEMBERS = [ + "finalizeRun", + "findRun", + "findRunOrThrow", + "findRunOnPrimary", + "findRunOrThrowOnPrimary", + "findRuns", + "findRunsByIds", +] as const; + +type ProbedCall = { name: string; args: unknown[] }; + +/** + * Records what was called and answers with a per-member sentinel, so a forward to the wrong member + * returns the wrong value rather than merely returning something. + */ +function forwardingProbe(): { store: RunStore; calls: ProbedCall[] } { + const calls: ProbedCall[] = []; + + const store = new Proxy({} as Record, { + get(_target, prop: string) { + if ((RUN_STORE_PROPERTY_NAMES as readonly string[]).includes(prop)) { + return `property:${prop}`; + } + return (...args: unknown[]) => { + calls.push({ name: prop, args }); + return `result:${prop}`; + }; + }, + }); + + return { store: store as unknown as RunStore, calls }; +} + +describe("DelegatingRunStore", () => { + it("forwards every method to the member of the same name", () => { + const { store, calls } = forwardingProbe(); + const base = new DelegatingRunStore(store) as unknown as Record< + string, + (...args: unknown[]) => unknown + >; + + for (const name of RUN_STORE_METHOD_NAMES) { + expect(base[name]()).toBe(`result:${name}`); + } + + expect(calls.map((c) => c.name)).toEqual([...RUN_STORE_METHOD_NAMES]); + }); + + it("covers every overloaded member, so the list cannot rot", () => { + // If a member gains or loses overloads, the cast set changes and this suite should follow. + for (const name of OVERLOADED_MEMBERS) { + expect(RUN_STORE_METHOD_NAMES).toContain(name); + } + }); + + it("forwards arguments untouched through an overloaded member's cast", () => { + const { store, calls } = forwardingProbe(); + const base = new DelegatingRunStore(store) as unknown as Record< + string, + (...args: unknown[]) => unknown + >; + const args = ["first", { second: true }, undefined, 4]; + + for (const name of OVERLOADED_MEMBERS) { + base[name](...args); + } + + // The overloaded implementations apply the whole argument list, so every argument survives, + // including a trailing optional the typed members would legitimately drop. + for (const call of calls) { + expect(call.args).toEqual(args); + } + expect(calls.map((c) => c.name)).toEqual([...OVERLOADED_MEMBERS]); + }); + + it("reads a data property live, so a delegate that changes is not cached", () => { + const store = { primaryReadClient: "first" } as unknown as RunStore; + const base = new DelegatingRunStore(store); + + expect(base.primaryReadClient).toBe("first" as unknown); + (store as unknown as Record).primaryReadClient = "second"; + expect(base.primaryReadClient).toBe("second" as unknown); + }); +}); diff --git a/internal-packages/run-store/src/delegatingRunStore.ts b/internal-packages/run-store/src/delegatingRunStore.ts new file mode 100644 index 00000000000..c7fe3225c76 --- /dev/null +++ b/internal-packages/run-store/src/delegatingRunStore.ts @@ -0,0 +1,750 @@ +// A pass-through over another RunStore. +// +// It exists so a decorator can override the handful of methods it cares about and inherit the rest. +// +// Every member restates the interface signature and forwards its arguments BY NAME, so the +// forwarding is itself type-checked: a body that called the wrong delegate method, or dropped an +// argument, does not compile. That is the whole point of the shape. An untyped forwarder would let +// both mistakes through, because a pass-through has no other behaviour to catch them. +// +// Seven members are overloaded. Their overloads are declared so callers keep the full contract, and +// their single implementation signature is the one place a cast appears: TypeScript cannot express +// one body that satisfies an overload set without it. +// +// Keeping this in step with the interface is not a matter of memory. `implements RunStore` rejects a +// member that is missing, and the assertion at the foot of the file rejects one the interface never +// declared. + +import type { + BatchTaskRun, + BatchTaskRunItemStatus, + Prisma, + PrismaClientOrTransaction, + TaskRun, + TaskRunStatus, + WaitpointTag, +} from "@trigger.dev/database"; +import type { TaskRunError } from "@trigger.dev/core/v3/schemas"; +import type { Residency } from "@trigger.dev/core/v3/isomorphic"; +import type { + ClearIdempotencyKeyInput, + CompletionSnapshotInput, + CreateBatchTaskRunData, + CreateCancelledRunInput, + CreateExecutionSnapshotInput, + CreateFailedRunInput, + CreateRunInput, + ExpireSnapshotInput, + FinalizeRunData, + ForWaitpointCompletionContext, + IdempotencyKeyRunMatch, + LockRunData, + PromotePendingVersionArgs, + ReadClient, + RescheduleSnapshotInput, + RewriteDebouncedRunData, + RunStore, + TaskRunWithWaitpoint, + WaitpointColocationOptions, +} from "./types.js"; + +export class DelegatingRunStore implements RunStore { + constructor(protected readonly delegate: RunStore) {} + + runInTransaction( + runId: string | undefined, + fn: (store: RunStore, tx: PrismaClientOrTransaction) => Promise + ): Promise { + return this.delegate.runInTransaction(runId, fn); + } + + createRun(params: CreateRunInput, tx?: PrismaClientOrTransaction): Promise { + return this.delegate.createRun(params, tx); + } + + createCancelledRun( + params: CreateCancelledRunInput, + tx?: PrismaClientOrTransaction + ): Promise { + return this.delegate.createCancelledRun(params, tx); + } + + createFailedRun( + params: CreateFailedRunInput, + tx?: PrismaClientOrTransaction + ): Promise { + return this.delegate.createFailedRun(params, tx); + } + + startAttempt( + runId: string, + data: { attemptNumber: number; executedAt?: Date; isWarmStart: boolean }, + args: { select: S }, + tx?: PrismaClientOrTransaction + ): Promise> { + return this.delegate.startAttempt(runId, data, args, tx); + } + + completeAttemptSuccess( + runId: string, + data: { + completedAt: Date; + output?: string; + outputType: string; + usageDurationMs: number; + costInCents: number; + snapshot: CompletionSnapshotInput; + }, + args: { select: S }, + tx?: PrismaClientOrTransaction + ): Promise> { + return this.delegate.completeAttemptSuccess(runId, data, args, tx); + } + + recordRetryOutcome( + runId: string, + data: { machinePreset?: string; usageDurationMs: number; costInCents: number }, + args: { select: S }, + tx?: PrismaClientOrTransaction + ): Promise> { + return this.delegate.recordRetryOutcome(runId, data, args, tx); + } + + requeueRun( + runId: string, + args: { select: S }, + tx?: PrismaClientOrTransaction + ): Promise> { + return this.delegate.requeueRun(runId, args, tx); + } + + recordBulkActionMembership( + runId: string, + bulkActionId: string, + tx?: PrismaClientOrTransaction + ): Promise { + return this.delegate.recordBulkActionMembership(runId, bulkActionId, tx); + } + + cancelRun( + runId: string, + data: { + completedAt?: Date; + error: TaskRunError; + bulkActionId?: string; + usageDurationMs?: number; + costInCents?: number; + }, + args: { select: S }, + tx?: PrismaClientOrTransaction + ): Promise> { + return this.delegate.cancelRun(runId, data, args, tx); + } + + failRunPermanently( + runId: string, + data: { + status: TaskRunStatus; + completedAt: Date; + error: TaskRunError; + usageDurationMs: number; + costInCents: number; + }, + args: { select: S }, + tx?: PrismaClientOrTransaction + ): Promise> { + return this.delegate.failRunPermanently(runId, data, args, tx); + } + + finalizeRun( + runId: string, + data: FinalizeRunData, + args: { select: S }, + tx?: PrismaClientOrTransaction + ): Promise>; + finalizeRun( + runId: string, + data: FinalizeRunData, + args: { include: I }, + tx?: PrismaClientOrTransaction + ): Promise>; + finalizeRun( + runId: string, + data: FinalizeRunData, + tx?: PrismaClientOrTransaction + ): Promise; + finalizeRun(...args: unknown[]): unknown { + return (this.delegate.finalizeRun as (...a: unknown[]) => unknown).apply(this.delegate, args); + } + + expireRun( + runId: string, + data: { + error: TaskRunError; + completedAt: Date; + expiredAt: Date; + snapshot: ExpireSnapshotInput; + }, + args: { select: S }, + tx?: PrismaClientOrTransaction + ): Promise> { + return this.delegate.expireRun(runId, data, args, tx); + } + + expireRunsBatch( + runIds: string[], + data: { error: TaskRunError; now: Date }, + tx?: PrismaClientOrTransaction + ): Promise { + return this.delegate.expireRunsBatch(runIds, data, tx); + } + + lockRunToWorker( + runId: string, + data: LockRunData, + tx?: PrismaClientOrTransaction + ): Promise> { + return this.delegate.lockRunToWorker(runId, data, tx); + } + + parkPendingVersion( + runId: string, + data: { statusReason: string }, + args: { select: S }, + tx?: PrismaClientOrTransaction + ): Promise> { + return this.delegate.parkPendingVersion(runId, data, args, tx); + } + + promotePendingVersionRuns( + runId: string, + args?: PromotePendingVersionArgs, + tx?: PrismaClientOrTransaction + ): Promise<{ count: number }> { + return this.delegate.promotePendingVersionRuns(runId, args, tx); + } + + expireParkedRun( + runId: string, + data: { + error: TaskRunError; + completedAt: Date; + expiredAt: Date; + statusReason: string; + snapshot: ExpireSnapshotInput; + }, + tx?: PrismaClientOrTransaction + ): Promise<{ count: number }> { + return this.delegate.expireParkedRun(runId, data, tx); + } + + suspendForCheckpoint( + runId: string, + args: { include: I }, + tx?: PrismaClientOrTransaction + ): Promise> { + return this.delegate.suspendForCheckpoint(runId, args, tx); + } + + resumeFromCheckpoint( + runId: string, + args: { select: S }, + tx?: PrismaClientOrTransaction + ): Promise> { + return this.delegate.resumeFromCheckpoint(runId, args, tx); + } + + rescheduleRun( + runId: string, + data: { delayUntil: Date; queueTimestamp?: Date; snapshot?: RescheduleSnapshotInput }, + tx?: PrismaClientOrTransaction + ): Promise { + return this.delegate.rescheduleRun(runId, data, tx); + } + + enqueueDelayedRun( + runId: string, + data: { queuedAt: Date }, + tx?: PrismaClientOrTransaction + ): Promise { + return this.delegate.enqueueDelayedRun(runId, data, tx); + } + + rewriteDebouncedRun( + runId: string, + data: RewriteDebouncedRunData, + tx?: PrismaClientOrTransaction + ): Promise { + return this.delegate.rewriteDebouncedRun(runId, data, tx); + } + + updateMetadata( + runId: string, + data: { + metadata: string | null; + metadataType?: string; + metadataVersion: { increment: number }; + updatedAt: Date; + }, + options: { expectedMetadataVersion?: number }, + tx?: PrismaClientOrTransaction + ): Promise<{ count: number }> { + return this.delegate.updateMetadata(runId, data, options, tx); + } + + clearIdempotencyKey( + params: ClearIdempotencyKeyInput, + tx?: PrismaClientOrTransaction + ): Promise<{ count: number }> { + return this.delegate.clearIdempotencyKey(params, tx); + } + + pushTags( + runId: string, + tags: string[], + where: { runtimeEnvironmentId: string }, + tx?: PrismaClientOrTransaction + ): Promise<{ updatedAt: Date }> { + return this.delegate.pushTags(runId, tags, where, tx); + } + + pushRealtimeStream( + runId: string, + streamId: string, + tx?: PrismaClientOrTransaction + ): Promise { + return this.delegate.pushRealtimeStream(runId, streamId, tx); + } + + get primaryReadClient(): ReadClient { + return this.delegate.primaryReadClient; + } + + findRun( + where: Prisma.TaskRunWhereInput, + args: { select: S }, + client?: ReadClient + ): Promise | null>; + findRun( + where: Prisma.TaskRunWhereInput, + args: { include: I }, + client?: ReadClient + ): Promise | null>; + findRun(where: Prisma.TaskRunWhereInput, client?: ReadClient): Promise; + findRun(...args: unknown[]): unknown { + return (this.delegate.findRun as (...a: unknown[]) => unknown).apply(this.delegate, args); + } + + findRunOrThrow( + where: Prisma.TaskRunWhereInput, + args: { select: S }, + client?: ReadClient + ): Promise>; + findRunOrThrow( + where: Prisma.TaskRunWhereInput, + args: { include: I }, + client?: ReadClient + ): Promise>; + findRunOrThrow(where: Prisma.TaskRunWhereInput, client?: ReadClient): Promise; + findRunOrThrow(...args: unknown[]): unknown { + return (this.delegate.findRunOrThrow as (...a: unknown[]) => unknown).apply( + this.delegate, + args + ); + } + + findRunOnPrimary( + where: Prisma.TaskRunWhereInput, + args: { select: S } + ): Promise | null>; + findRunOnPrimary( + where: Prisma.TaskRunWhereInput, + args: { include: I } + ): Promise | null>; + findRunOnPrimary(where: Prisma.TaskRunWhereInput): Promise; + findRunOnPrimary(...args: unknown[]): unknown { + return (this.delegate.findRunOnPrimary as (...a: unknown[]) => unknown).apply( + this.delegate, + args + ); + } + + findRunOrThrowOnPrimary( + where: Prisma.TaskRunWhereInput, + args: { select: S } + ): Promise>; + findRunOrThrowOnPrimary( + where: Prisma.TaskRunWhereInput, + args: { include: I } + ): Promise>; + findRunOrThrowOnPrimary(where: Prisma.TaskRunWhereInput): Promise; + findRunOrThrowOnPrimary(...args: unknown[]): unknown { + return (this.delegate.findRunOrThrowOnPrimary as (...a: unknown[]) => unknown).apply( + this.delegate, + args + ); + } + + findRuns( + args: { + where: Prisma.TaskRunWhereInput; + select: S; + orderBy?: Prisma.TaskRunOrderByWithRelationInput | Prisma.TaskRunOrderByWithRelationInput[]; + take?: number; + skip?: number; + cursor?: Prisma.TaskRunWhereUniqueInput; + }, + client?: ReadClient + ): Promise[]>; + findRuns( + args: { + where: Prisma.TaskRunWhereInput; + include: I; + orderBy?: Prisma.TaskRunOrderByWithRelationInput | Prisma.TaskRunOrderByWithRelationInput[]; + take?: number; + skip?: number; + cursor?: Prisma.TaskRunWhereUniqueInput; + }, + client?: ReadClient + ): Promise[]>; + findRuns( + args: { + where: Prisma.TaskRunWhereInput; + orderBy?: Prisma.TaskRunOrderByWithRelationInput | Prisma.TaskRunOrderByWithRelationInput[]; + take?: number; + skip?: number; + cursor?: Prisma.TaskRunWhereUniqueInput; + }, + client?: ReadClient + ): Promise; + findRuns(...args: unknown[]): unknown { + return (this.delegate.findRuns as (...a: unknown[]) => unknown).apply(this.delegate, args); + } + + findRunsByIds( + ids: string[], + args: { select: S }, + client?: ReadClient + ): Promise>>; + findRunsByIds( + ids: string[], + args: { include: I }, + client?: ReadClient + ): Promise>>; + findRunsByIds(ids: string[], client?: ReadClient): Promise>; + findRunsByIds(...args: unknown[]): unknown { + return (this.delegate.findRunsByIds as (...a: unknown[]) => unknown).apply(this.delegate, args); + } + + findRunsByIdempotencyKeys( + args: { runtimeEnvironmentId: string; taskIdentifier: string; idempotencyKeys: string[] }, + client?: ReadClient + ): Promise { + return this.delegate.findRunsByIdempotencyKeys(args, client); + } + + createBatchTaskRunItem( + data: { batchTaskRunId: string; taskRunId: string; status: BatchTaskRunItemStatus }, + tx?: PrismaClientOrTransaction + ): Promise { + return this.delegate.createBatchTaskRunItem(data, tx); + } + + findLatestExecutionSnapshot( + runId: string, + client?: ReadClient, + // When set, scopes the read to this environment (tenant boundary); a run in another env reads as + // not-found. Omit to read regardless of environment (internal callers). + environmentId?: string + ): Promise | null> { + return this.delegate.findLatestExecutionSnapshot(runId, client, environmentId); + } + + findExecutionSnapshot( + args: Prisma.SelectSubset, + client?: ReadClient + ): Promise | null> { + return this.delegate.findExecutionSnapshot(args, client); + } + + findManyExecutionSnapshots( + args: Prisma.SelectSubset, + client?: ReadClient + ): Promise[]> { + return this.delegate.findManyExecutionSnapshots(args, client); + } + + createExecutionSnapshot( + input: CreateExecutionSnapshotInput, + tx?: PrismaClientOrTransaction + ): Promise> { + return this.delegate.createExecutionSnapshot(input, tx); + } + + findSnapshotCompletedWaitpointIds( + snapshotId: string, + client?: ReadClient, + runId?: string + ): Promise { + return this.delegate.findSnapshotCompletedWaitpointIds(snapshotId, client, runId); + } + + findSnapshotCompletedWaitpointIdsWithPresence( + snapshotId: string, + client?: ReadClient, + runId?: string + ): Promise<{ present: boolean; ids: string[] }> { + return this.delegate.findSnapshotCompletedWaitpointIdsWithPresence(snapshotId, client, runId); + } + + findWaitpointConnectedRunIds(waitpointId: string, client?: ReadClient): Promise { + return this.delegate.findWaitpointConnectedRunIds(waitpointId, client); + } + + findWaitpointCompletedSnapshotIds(waitpointId: string, client?: ReadClient): Promise { + return this.delegate.findWaitpointCompletedSnapshotIds(waitpointId, client); + } + + blockRunWithWaitpointEdges(params: { + runId: string; + waitpointIds: string[]; + projectId: string; + spanIdToComplete?: string; + batchId?: string; + batchIndex?: number; + tx?: PrismaClientOrTransaction; + }): Promise { + return this.delegate.blockRunWithWaitpointEdges(params); + } + + countPendingWaitpoints( + waitpointIds: string[], + client?: ReadClient, + runId?: string + ): Promise { + return this.delegate.countPendingWaitpoints(waitpointIds, client, runId); + } + + countPendingWaitpointsWithPresence( + waitpointIds: string[], + client?: ReadClient + ): Promise<{ pendingIds: string[]; presentIds: string[] }> { + return this.delegate.countPendingWaitpointsWithPresence(waitpointIds, client); + } + + createWaitpoint( + args: Prisma.SelectSubset, + tx?: PrismaClientOrTransaction, + opts?: WaitpointColocationOptions + ): Promise> { + return this.delegate.createWaitpoint(args, tx, opts); + } + + upsertWaitpoint( + args: Prisma.SelectSubset, + tx?: PrismaClientOrTransaction, + opts?: WaitpointColocationOptions + ): Promise> { + return this.delegate.upsertWaitpoint(args, tx, opts); + } + + findWaitpoint( + args: Prisma.SelectSubset, + client?: ReadClient, + opts?: WaitpointColocationOptions + ): Promise | null> { + return this.delegate.findWaitpoint(args, client, opts); + } + + findWaitpointOnPrimary( + args: Prisma.SelectSubset + ): Promise | null> { + return this.delegate.findWaitpointOnPrimary(args); + } + + findManyWaitpoints( + args: Prisma.SelectSubset, + client?: ReadClient, + runId?: string + ): Promise[]> { + return this.delegate.findManyWaitpoints(args, client, runId); + } + + updateWaitpoint( + args: Prisma.SelectSubset, + tx?: PrismaClientOrTransaction, + opts?: WaitpointColocationOptions + ): Promise> { + return this.delegate.updateWaitpoint(args, tx, opts); + } + + updateManyWaitpoints( + args: Prisma.WaitpointUpdateManyArgs, + tx?: PrismaClientOrTransaction + ): Promise { + return this.delegate.updateManyWaitpoints(args, tx); + } + + forWaitpointCompletion( + waitpointId: string, + context: ForWaitpointCompletionContext + ): Promise { + return this.delegate.forWaitpointCompletion(waitpointId, context); + } + + findManyTaskRunWaitpoints( + args: Prisma.SelectSubset, + client?: ReadClient + ): Promise[]> { + return this.delegate.findManyTaskRunWaitpoints(args, client); + } + + deleteManyTaskRunWaitpoints( + args: Prisma.TaskRunWaitpointDeleteManyArgs, + tx?: PrismaClientOrTransaction + ): Promise { + return this.delegate.deleteManyTaskRunWaitpoints(args, tx); + } + + findTaskRunAttempt( + args: Prisma.SelectSubset, + client?: ReadClient + ): Promise | null> { + return this.delegate.findTaskRunAttempt(args, client); + } + + createTaskRunCheckpoint( + args: Prisma.SelectSubset, + ownerRunId?: string, + tx?: PrismaClientOrTransaction + ): Promise> { + return this.delegate.createTaskRunCheckpoint(args, ownerRunId, tx); + } + + createBatchTaskRun( + data: CreateBatchTaskRunData, + tx?: PrismaClientOrTransaction + ): Promise { + return this.delegate.createBatchTaskRun(data, tx); + } + + updateBatchTaskRun( + args: { + where: Prisma.BatchTaskRunWhereUniqueInput; + data: Prisma.BatchTaskRunUpdateInput; + select: S; + }, + tx?: PrismaClientOrTransaction + ): Promise> { + return this.delegate.updateBatchTaskRun(args, tx); + } + + findBatchTaskRunById( + id: string, + args?: { include?: T }, + client?: ReadClient + ): Promise | null> { + return this.delegate.findBatchTaskRunById(id, args, client); + } + + findBatchTaskRunByFriendlyId( + friendlyId: string, + environmentId: string, + args?: { include?: T }, + client?: ReadClient + ): Promise | null> { + return this.delegate.findBatchTaskRunByFriendlyId(friendlyId, environmentId, args, client); + } + + findBatchTaskRunByIdempotencyKey( + environmentId: string, + idempotencyKey: string, + args?: { include?: T }, + client?: ReadClient + ): Promise | null> { + return this.delegate.findBatchTaskRunByIdempotencyKey( + environmentId, + idempotencyKey, + args, + client + ); + } + + updateManyBatchTaskRun( + args: Prisma.BatchTaskRunUpdateManyArgs, + tx?: PrismaClientOrTransaction + ): Promise { + return this.delegate.updateManyBatchTaskRun(args, tx); + } + + countBatchTaskRunItems( + where: { batchTaskRunId: string; status?: BatchTaskRunItemStatus }, + client?: ReadClient + ): Promise { + return this.delegate.countBatchTaskRunItems(where, client); + } + + updateManyBatchTaskRunItems( + args: Prisma.BatchTaskRunItemUpdateManyArgs, + tx?: PrismaClientOrTransaction + ): Promise { + return this.delegate.updateManyBatchTaskRunItems(args, tx); + } + + findManyBatchTaskRunItems( + where: { taskRunId?: string; batchTaskRunId?: string }, + args?: { include?: I }, + client?: ReadClient + ): Promise[]> { + return this.delegate.findManyBatchTaskRunItems(where, args, client); + } + + findBatchTaskRunItem( + where: { batchTaskRunId: string; taskRunId?: string }, + args?: { include?: I }, + client?: ReadClient + ): Promise | null> { + return this.delegate.findBatchTaskRunItem(where, args, client); + } + + upsertWaitpointTag( + data: { environmentId: string; name: string; projectId: string; id?: string }, + tx?: PrismaClientOrTransaction, + // A tag has no owning run to co-locate with; when no minted `id` pins it by id-shape, a + // minted-new env's tags read this residency (NEW) so they land with the env's tokens/runs + // instead of defaulting to LEGACY. Single-store impls ignore it. + residency?: Residency + ): Promise { + return this.delegate.upsertWaitpointTag(data, tx, residency); + } + + findManyWaitpointTags( + args: { + where: Prisma.WaitpointTagWhereInput; + orderBy?: + | Prisma.WaitpointTagOrderByWithRelationInput + | Prisma.WaitpointTagOrderByWithRelationInput[]; + take?: number; + skip?: number; + }, + client?: ReadClient + ): Promise { + return this.delegate.findManyWaitpointTags(args, client); + } +} + +// `implements` above rejects a member of the interface that is missing here. It says nothing about a +// member that should not exist, so the reverse direction is asserted too: a public member this class +// declares and the interface does not is a build failure. +// +// `protected delegate` is correctly absent from `keyof`, so the constructor parameter does not trip +// this. +type _ClassDeclaresNoExtraMembers = [Exclude] extends [ + never, +] + ? true + : never; +const _classParity: _ClassDeclaresNoExtraMembers = true; +void _classParity; diff --git a/internal-packages/run-store/src/index.ts b/internal-packages/run-store/src/index.ts index a4109ab0104..07be12da94e 100644 --- a/internal-packages/run-store/src/index.ts +++ b/internal-packages/run-store/src/index.ts @@ -5,3 +5,8 @@ export * from "./readReplicaClient.js"; export * from "./redisSnapshotStore.js"; export * from "./routingStoreMetrics.js"; export * from "./snapshotComparator.js"; +export * from "./delegatingRunStore.js"; +export * from "./taskRunExecutionSnapshotStore.js"; +export * from "./snapshotEntry.js"; +export * from "./snapshotFaultInjection.js"; +export * from "./snapshotOrphanSweeper.js"; diff --git a/internal-packages/run-store/src/redisSnapshotStore.sinceCreatedAt.test.ts b/internal-packages/run-store/src/redisSnapshotStore.sinceCreatedAt.test.ts new file mode 100644 index 00000000000..f1394be597e --- /dev/null +++ b/internal-packages/run-store/src/redisSnapshotStore.sinceCreatedAt.test.ts @@ -0,0 +1,193 @@ +// getExecutionSnapshotsSince resolves its cursor to a createdAt before it asks for the window, so +// the snapshot id is gone by then and getSince cannot serve it. This read takes the cursor instead, +// and has to agree with the Postgres read it stands in for — same-millisecond blind spot included. +import { describe, expect } from "vitest"; +import { redisTest } from "@internal/testcontainers"; +import { RedisSnapshotStore } from "./redisSnapshotStore.js"; +import type { SnapshotEntryInput } from "./redisSnapshotStore.js"; + +const COMPLETED_TTL_MS = 72 * 60 * 60 * 1000; + +function entry(runId: string, id: string, createdAt: string): SnapshotEntryInput { + return { + id, + engine: "V2", + executionStatus: "EXECUTING", + description: "d", + runId, + runStatus: "EXECUTING", + createdAt, + environmentId: "env_1", + environmentType: "DEVELOPMENT", + projectId: "proj_1", + organizationId: "org_1", + }; +} + +const at = (seconds: number) => new Date(Date.UTC(2026, 0, 1, 0, 0, seconds)).toISOString(); + +async function seed( + store: RedisSnapshotStore, + runId: string, + stamps: { id: string; createdAt: string }[] +): Promise { + for (const [index, stamp] of stamps.entries()) { + await store.append({ + entry: entry(runId, stamp.id, stamp.createdAt), + kind: index === 0 ? "birth" : "transition", + isTerminal: false, + }); + } +} + +describe("getSinceCreatedAt", () => { + redisTest( + "returns only entries newer than the cursor, oldest first", + async ({ redisOptions }) => { + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + try { + const runId = "run_window"; + await seed( + store, + runId, + [0, 1, 2, 3, 4].map((n) => ({ id: `snap_${n}`, createdAt: at(n) })) + ); + + const result = await store.getSinceCreatedAt(runId, at(1)); + + expect(result.kind).toBe("hit"); + if (result.kind !== "hit") return; + // Ascending, matching what the engine hands its caller after its own reverse(). + expect(result.entries.map((e) => e.id)).toEqual(["snap_2", "snap_3", "snap_4"]); + } finally { + await store.quit(); + } + } + ); + + redisTest("misses when the run has no keyspace", async ({ redisOptions }) => { + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + try { + // A miss is the coexistence path: the caller falls back to Postgres for a pre-cutover run. + expect((await store.getSinceCreatedAt("run_absent", at(0))).kind).toBe("miss"); + } finally { + await store.quit(); + } + }); + + redisTest("returns an empty hit when nothing is newer", async ({ redisOptions }) => { + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + try { + const runId = "run_nothing_newer"; + await seed(store, runId, [{ id: "snap_0", createdAt: at(0) }]); + + const result = await store.getSinceCreatedAt(runId, at(5)); + + // A hit, not a miss: Redis owns this run, so the caller must not fall back and re-read + // Postgres for a window it already answered. + expect(result.kind).toBe("hit"); + if (result.kind !== "hit") return; + expect(result.entries).toEqual([]); + } finally { + await store.quit(); + } + }); + + redisTest("drops a same-millisecond neighbour, as Postgres does", async ({ redisOptions }) => { + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + try { + const runId = "run_same_ms"; + const shared = at(1); + await seed(store, runId, [ + { id: "snap_0", createdAt: at(0) }, + { id: "snap_1a", createdAt: shared }, + { id: "snap_1b", createdAt: shared }, + { id: "snap_2", createdAt: at(2) }, + ]); + + const result = await store.getSinceCreatedAt(runId, shared); + + // Postgres serves this window with `createdAt: { gt: cursor }`, which drops both same-ms + // entries. Returning snap_1b here would be more correct than Postgres and would therefore + // read as divergence in compare mode. + expect(result.kind).toBe("hit"); + if (result.kind !== "hit") return; + expect(result.entries.map((e) => e.id)).toEqual(["snap_2"]); + } finally { + await store.quit(); + } + }); + + redisTest("caps the window at the limit, keeping the newest", async ({ redisOptions }) => { + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + try { + const runId = "run_capped"; + await seed( + store, + runId, + Array.from({ length: 60 }, (_, n) => ({ id: `snap_${n}`, createdAt: at(n) })) + ); + + const result = await store.getSinceCreatedAt(runId, at(0), { limit: 50 }); + + expect(result.kind).toBe("hit"); + if (result.kind !== "hit") return; + expect(result.entries).toHaveLength(50); + // The engine takes the NEWEST 50 and reverses, so the window ends at the newest entry. + expect(result.entries[result.entries.length - 1]!.id).toBe("snap_59"); + expect(result.entries[0]!.id).toBe("snap_10"); + } finally { + await store.quit(); + } + }); + + redisTest("scans no further than the answer", async ({ redisOptions }) => { + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + try { + const runId = "run_deep_history"; + await seed( + store, + runId, + Array.from({ length: 400 }, (_, n) => ({ id: `snap_${n}`, createdAt: at(n) })) + ); + + const started = Date.now(); + const result = await store.getSinceCreatedAt(runId, at(394), { limit: 50 }); + const elapsed = Date.now() - started; + + expect(result.kind).toBe("hit"); + if (result.kind !== "hit") return; + expect(result.entries.map((e) => e.id)).toEqual([ + "snap_395", + "snap_396", + "snap_397", + "snap_398", + "snap_399", + ]); + // The walk stops at the cursor rather than reading the run's history. The bound is generous + // on purpose: it fails on a full scan of 400 entries, not on ordinary timing noise. + expect(elapsed).toBeLessThan(1_000); + } finally { + await store.quit(); + } + }); + + redisTest("scopes the window to an environment", async ({ redisOptions }) => { + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + try { + const runId = "run_env_scoped"; + await seed(store, runId, [ + { id: "snap_0", createdAt: at(0) }, + { id: "snap_1", createdAt: at(1) }, + ]); + + const foreign = await store.getSinceCreatedAt(runId, at(0), { environmentId: "env_other" }); + + expect(foreign.kind).toBe("hit"); + if (foreign.kind !== "hit") return; + expect(foreign.entries).toEqual([]); + } finally { + await store.quit(); + } + }); +}); diff --git a/internal-packages/run-store/src/redisSnapshotStore.ts b/internal-packages/run-store/src/redisSnapshotStore.ts index 2964959c4fe..2339ab0dd5e 100644 --- a/internal-packages/run-store/src/redisSnapshotStore.ts +++ b/internal-packages/run-store/src/redisSnapshotStore.ts @@ -1,7 +1,7 @@ import { createRedisClient, type Callback, - type Redis, + type RedisClient, type RedisOptions, type Result, } from "@internal/redis"; @@ -29,6 +29,19 @@ export function deriveOrder(completedWaitpoints: CompletedWaitpointRef[]): strin .map((w) => w.id); } +/** + * The COMPLETE distinct set of completed-waitpoint ids, including those with no batch index. + * + * This is deliberately not `deriveOrder` deduped. `order` is the index oracle and carries only + * batch-indexed ids, because its positions ARE the indexes. A wait with no batch index (every + * `wait.for`, every single `triggerAndWait`, every token) has no position and is absent from it, + * while Postgres records it in the completed-waitpoint join like any other. Reading the id set back + * from `order` therefore loses exactly those waits, and a run resumed from Redis loses their results. + */ +export function deriveDistinctIds(completedWaitpoints: CompletedWaitpointRef[]): string[] { + return [...new Set(completedWaitpoints.map((w) => w.id))]; +} + // isValid is derived, never stored, so the entry JSON stays byte-identical to the caller's document. export function isValidFor(entry: { error?: unknown }): boolean { return !entry.error; @@ -162,6 +175,12 @@ export type SnapshotRead = { raw: string; cycle?: CompletedWaitpointsPointer; completedWaitpointIds?: WaitpointIds; + /** + * The entry points at a cycle key that no longer exists, so its waitpoints are unreachable rather + * than absent. A caller must not treat this as an empty set: it has to fall back to Postgres, + * which still holds the join rows. + */ + danglingCycle?: boolean; }; export type AppendResult = @@ -186,8 +205,20 @@ export type SnapshotStoreMetrics = { recordLatency(op: string, ms: number): void; }; -export type RedisSnapshotStoreOptions = { - redisOptions: RedisOptions; +/** + * How the store reaches Redis. Exactly one of the two, enforced by the type rather than a runtime + * check: `never` on the opposite member makes both "neither" and "both" a compile error. + * + * `client` exists because production points at a Valkey/Redis CLUSTER, and cluster topology is not + * this package's business. Every command the store issues is key-addressed and every key carries a + * `{runId}` hashtag, so one slot serves a whole run and both endpoint shapes behave identically. + * A caller-supplied client is owned by the caller: `quit()` leaves it open. + */ +export type RedisSnapshotStoreConnection = + | { client: RedisClient; redisOptions?: never } + | { client?: never; redisOptions: RedisOptions }; + +export type RedisSnapshotStoreOptions = RedisSnapshotStoreConnection & { completedTtlMs: number; sinceLimit?: number; highWater?: { entryBytes?: number; cycleKeyBytes?: number; cycleCount?: number }; @@ -195,13 +226,25 @@ export type RedisSnapshotStoreOptions = { logger?: Logger; }; +/** + * Both window scripts return four leading slots before the first row: the id-cursor variant's + * `sinceRaw`, the head's order, the head's distinct set, and the head's dangling flag. Rows follow + * in four-element groups, so the head row is the group at this offset. + * + * Named because the offset drifted out of the comments describing it twice, and the second drift + * arrived in the change that fixed the first. + */ +const WINDOW_HEAD_ROW_INDEX = 4; + const SKIPPED = "skipped"; const FORKED = "forked"; const WRITTEN = "written"; const DUPLICATE = "duplicate"; export class RedisSnapshotStore { - private readonly redis: Redis; + private readonly redis: RedisClient; + /** Only a client this class opened may be closed by it. */ + private readonly ownsClient: boolean; private readonly logger: Logger; private readonly completedTtlMs: number; private readonly sinceLimit: number; @@ -215,15 +258,22 @@ export class RedisSnapshotStore { this.sinceLimit = options.sinceLimit ?? 50; this.metrics = options.metrics; this.highWater = options.highWater ?? {}; - this.redis = createRedisClient(options.redisOptions, { - onError: (error) => this.logger.error("RedisSnapshotStore redis client error", { error }), - }); + this.ownsClient = options.client === undefined; + this.redis = + options.client ?? + createRedisClient(options.redisOptions, { + onError: (error) => this.logger.error("RedisSnapshotStore redis client error", { error }), + }); this.#registerCommands(); } async quit(): Promise { // Idempotent and error-swallowing: every test calls this in a `finally`, and a double quit() // (or one after a failed connect) must never mask the real assertion failure. + // + // An injected client is the caller's. Closing it here would take down a connection shared with + // the sweeper or with another component, so a borrowed client is left open. + if (!this.ownsClient) return; if (!this.#quit) { this.#quit = this.redis.quit().then( () => undefined, @@ -253,7 +303,19 @@ export class RedisSnapshotStore { completedWaitpoints: CompletedWaitpointRef[]; records?: CompletedWaitpointRecord[]; } - | { kind: "carryForward"; cycleSeq: number }; + | { + kind: "carryForward"; + cycleSeq: number; + /** + * The same refs a `new` cycle would carry. A carry the store refuses falls back to + * minting inside the same call, and it cannot do that without them: with no refs there is + * nothing to mint from, so the entry is written with no pointer, as before. + * + * Every production caller supplies them. Omitting them gives up the fallback. + */ + completedWaitpoints?: CompletedWaitpointRef[]; + records?: CompletedWaitpointRecord[]; + }; }): Promise { if (args.entry.completedWaitpoints !== undefined) { throw new Error( @@ -270,17 +332,29 @@ export class RedisSnapshotStore { let cycleMode = "none"; let cycleSeqIn = "0"; let orderJson = ""; + let distinctJson = ""; let records = ""; let orderCount = "0"; if (args.cycle?.kind === "new") { const order = deriveOrder(args.cycle.completedWaitpoints); cycleMode = "new"; orderJson = JSON.stringify(order); + distinctJson = JSON.stringify(deriveDistinctIds(args.cycle.completedWaitpoints)); records = args.cycle.records ? JSON.stringify(args.cycle.records) : ""; orderCount = String(order.length); } else if (args.cycle?.kind === "carryForward") { cycleMode = "carry"; cycleSeqIn = String(args.cycle.cycleSeq); + + // Carried for the refusal path only. The script uses these solely when it declines the + // pointer and mints a replacement, and can only do that when the caller supplied them. + if (args.cycle.completedWaitpoints) { + const order = deriveOrder(args.cycle.completedWaitpoints); + orderJson = JSON.stringify(order); + records = args.cycle.records ? JSON.stringify(args.cycle.records) : ""; + orderCount = String(order.length); + distinctJson = JSON.stringify(deriveDistinctIds(args.cycle.completedWaitpoints)); + } } const reply = (await this.redis.appendSnapshotEntry( @@ -300,7 +374,8 @@ export class RedisSnapshotStore { records, orderCount, args.expectedCur ?? "", - args.expectedCur !== undefined ? "1" : "0" + args.expectedCur !== undefined ? "1" : "0", + distinctJson )) as string[]; return this.#interpretAppend(reply, raw, orderJson, records, args.entry.runId); @@ -407,7 +482,17 @@ export class RedisSnapshotStore { return this.#timed("getSnapshotWaitpointIds", async () => { const k = snapshotKeys(runId); const reply = await this.redis.readSnapshotWaitpointIds(k.e, k.idx, k.cur, k.seq, snapshotId); - return decodeWaitpointIds(reply[0] === "1", reply[1] ?? ""); + // A dangling pointer means this entry's waitpoints are unreachable, not absent. Reporting + // `present: false` is what sends the caller to Postgres, which still holds the join rows. + if (reply[3] === "1") { + this.metrics?.recordCycleMismatch(); + this.logger.warn("RedisSnapshotStore snapshot points at a cycle key that is gone", { + runId, + snapshotId, + }); + return { present: false, distinctIds: [], order: [] }; + } + return decodeWaitpointIds(reply[0] === "1", reply[1] ?? "", reply[2] ?? ""); }); } @@ -441,11 +526,13 @@ export class RedisSnapshotStore { } const headOrder = reply[1] ?? ""; + const headDistinct = reply[2] ?? ""; + const headDangling = reply[3] ?? ""; const rows: SnapshotRead[] = []; - // Tracks whether the Lua-chosen head row (always the first, i === 2) itself survives the + // Tracks whether the Lua-chosen head row (always the first, WINDOW_HEAD_ROW_INDEX) survives the // env filter below -- headOrder must never be attributed to a different, surviving row. let headSurvived = false; - for (let i = 2; i + 3 < reply.length; i += 4) { + for (let i = WINDOW_HEAD_ROW_INDEX; i + 3 < reply.length; i += 4) { // orderKnown is false here: headOrder covers only the head row, resolved separately below. const decoded = this.#decode( [reply[i], reply[i + 1], reply[i + 2], reply[i + 3], ""], @@ -455,15 +542,97 @@ export class RedisSnapshotStore { ); if (decoded) { rows.push(decoded); - if (i === 2) headSurvived = true; + if (i === WINDOW_HEAD_ROW_INDEX) headSurvived = true; + } + } + + rows.reverse(); + const head = headSurvived ? rows[rows.length - 1] : undefined; + const headWaitpointIds = decodeWaitpointIds( + head !== undefined, + head ? headOrder : "", + head ? headDistinct : "" + ); + if (head) { + head.completedWaitpointIds = headWaitpointIds; + // A head whose cycle key has expired carries an empty order that means "unknown", not + // "none". The caller cannot distinguish those, so it has to be told, or it resumes a batch + // with every position lost. This is what makes the decorator's Postgres fallback reachable + // on the since-window path as well as the hot read. + if (headDangling === "1") { + head.danglingCycle = true; + } + if (head.cycle) { + this.#checkCycleMismatch(runId, head.cycle.count, headWaitpointIds.order.length); + } + } + return { kind: "hit", entries: rows, headWaitpointIds }; + }); + } + + /** + * The same window as {@link getSince}, addressed by a createdAt cursor instead of a snapshot id. + * + * `getExecutionSnapshotsSince` resolves its cursor to a createdAt before it asks for the window, + * so the snapshot id is gone by the time this call is made and `getSince` cannot serve it. The + * cursor is exclusive and keeps Postgres's same-millisecond blind spot, so the two reads agree. + */ + async getSinceCreatedAt( + runId: string, + createdAt: Date | string, + opts?: { environmentId?: string; limit?: number } + ): Promise { + return this.#timed("getSinceCreatedAt", async () => { + const k = snapshotKeys(runId); + const limit = opts?.limit ?? this.sinceLimit; + const cursor = typeof createdAt === "string" ? createdAt : createdAt.toISOString(); + + const reply = await this.redis.readSnapshotsSinceCreatedAt( + k.e, + k.idx, + k.cur, + k.seq, + cursor, + String(limit) + ); + if (reply === null) return { kind: "miss" }; + + const headOrder = reply[1] ?? ""; + const headDistinct = reply[2] ?? ""; + const headDangling = reply[3] ?? ""; + const rows: SnapshotRead[] = []; + // Tracks whether the Lua-chosen head row (always the first, WINDOW_HEAD_ROW_INDEX) survives the env filter, + // so headOrder is never attributed to a different, surviving row. + let headSurvived = false; + for (let i = WINDOW_HEAD_ROW_INDEX; i + 3 < reply.length; i += 4) { + const decoded = this.#decode( + [reply[i], reply[i + 1], reply[i + 2], reply[i + 3], ""], + opts?.environmentId, + runId, + false + ); + if (decoded) { + rows.push(decoded); + if (i === WINDOW_HEAD_ROW_INDEX) headSurvived = true; } } rows.reverse(); const head = headSurvived ? rows[rows.length - 1] : undefined; - const headWaitpointIds = decodeWaitpointIds(head !== undefined, head ? headOrder : ""); + const headWaitpointIds = decodeWaitpointIds( + head !== undefined, + head ? headOrder : "", + head ? headDistinct : "" + ); if (head) { head.completedWaitpointIds = headWaitpointIds; + // A head whose cycle key has expired carries an empty order that means "unknown", not + // "none". The caller cannot distinguish those, so it has to be told, or it resumes a batch + // with every position lost. This is what makes the decorator's Postgres fallback reachable + // on the since-window path as well as the hot read. + if (headDangling === "1") { + head.danglingCycle = true; + } if (head.cycle) { this.#checkCycleMismatch(runId, head.cycle.count, headWaitpointIds.order.length); } @@ -494,7 +663,7 @@ export class RedisSnapshotStore { orderKnown: boolean ): SnapshotRead | null { if (!reply || reply.length === 0) return null; - const [id, raw, seqStr, pointer, orderJson] = reply; + const [id, raw, seqStr, pointer, orderJson, distinctJson, dangling] = reply; const entry = JSON.parse(raw) as Record; if (environmentId !== undefined && entry.environmentId !== environmentId) return null; const read: SnapshotRead = { @@ -507,8 +676,16 @@ export class RedisSnapshotStore { if (pointer) { const [cs, count] = pointer.split(":"); read.cycle = { cycleSeq: Number(cs), count: Number(count) }; + if (dangling === "1") { + read.danglingCycle = true; + this.metrics?.recordCycleMismatch(); + this.logger.warn("RedisSnapshotStore entry points at a cycle key that is gone", { + runId, + snapshotId: id, + }); + } if (orderKnown) { - const ids = decodeWaitpointIds(true, orderJson); + const ids = decodeWaitpointIds(true, orderJson, distinctJson ?? ""); read.completedWaitpointIds = ids; this.#checkCycleMismatch(runId, Number(count), ids.order.length); } @@ -530,6 +707,24 @@ export class RedisSnapshotStore { if not cs then return '' end return redis.call('HGET', wpKey(cs), 'order') or '' end + -- A pointer whose cycle key is GONE. Not the same as having no pointer: this entry should + -- have waitpoints and cannot produce them, so a read must refuse rather than answer empty. + -- Reachable by eviction, and by the completion TTL, which is applied to every key for a run + -- at the same moment but lets them expire independently. + local function danglingFor(pointer) + if not pointer then return '0' end + local cs = string.match(pointer, '^(%d+):') + if not cs then return '0' end + if redis.call('EXISTS', wpKey(cs)) == 0 then return '1' end + return '0' + end + -- The complete id set, which is NOT the order deduped: order holds only batch-indexed ids. + local function distinctFor(pointer) + if not pointer then return '' end + local cs = string.match(pointer, '^(%d+):') + if not cs then return '' end + return redis.call('HGET', wpKey(cs), 'distinct') or '' + end `; this.redis.defineCommand("appendSnapshotEntry", { @@ -549,6 +744,9 @@ export class RedisSnapshotStore { local orderCount = ARGV[11] local expectedCur = ARGV[12] local casEnabled = ARGV[13] == '1' + -- The COMPLETE distinct id set. Not the order deduped: order omits every id with no batch + -- index, and those ids still have to come back on a read. + local distinctJson = ARGV[14] -- Liveness is TWO anchors: e and seq. All keys get the same PEXPIRE but expire independently -- (or seq can vanish under maxmemory eviction while e survives), so checking e alone lets a @@ -579,27 +777,43 @@ export class RedisSnapshotStore { local cycleSeq = 0 local mismatch = 0 - if cycleMode == 'new' then - -- The STORE mints cycleSeq, so the sequence is dense by construction and the terminal - -- PEXPIRE loop from 1..c is correct. - cycleSeq = redis.call('HINCRBY', seqKey, 'c', 1) - redis.call('HSET', wpKey(cycleSeq), 'order', orderJson, 'count', orderCount) + + -- The STORE mints cycleSeq, so the sequence is dense by construction and the terminal + -- PEXPIRE loop from 1..c is correct. + local function mintCycle() + local minted = redis.call('HINCRBY', seqKey, 'c', 1) + redis.call('HSET', wpKey(minted), 'order', orderJson, 'count', orderCount, 'distinct', distinctJson) if records ~= '' then - redis.call('HSET', wpKey(cycleSeq), 'records', records) + redis.call('HSET', wpKey(minted), 'records', records) else -- A new cycle owns the whole key: a lost seq counter can re-mint a cycleSeq whose key -- still holds another cycle's records, and order/count stay mutually consistent so the -- mismatch check cannot see it. No-op on a fresh key. - redis.call('HDEL', wpKey(cycleSeq), 'records') + redis.call('HDEL', wpKey(minted), 'records') end + return minted + end + + if cycleMode == 'new' then + cycleSeq = mintCycle() elseif cycleMode == 'carry' then - -- Attach a pointer only if this incarnation actually minted the cycle. seq can be - -- evicted while a wp: key survives, so a bare key-exists check would adopt a dead + -- Attach the CARRIED pointer only if this incarnation actually minted that cycle. seq can + -- be evicted while a wp: key survives, so a bare key-exists check would adopt a dead -- incarnation's order and records under a consistent count, invisibly. local minted = tonumber(redis.call('HGET', seqKey, 'c') or '0') local c = redis.call('HGET', wpKey(cycleSeqIn), 'count') if not c or minted < cycleSeqIn then + -- Refusing the pointer is right. Writing the entry WITHOUT one is not: it becomes the + -- head with no waitpoints, and a read of it answers present-with-nothing, which is the + -- one answer that tells the engine's repair it need not look. Mint a fresh cycle from + -- the refs the caller carried, in this same atomic call, so the entry always has a + -- pointer that can be trusted. The mismatch is still reported, for the metric. mismatch = 1 + -- Only possible when the caller carried the refs. With none there is nothing to mint + -- from, and the entry is written with no pointer, which is the older behaviour. + if distinctJson ~= '' then + cycleSeq = mintCycle() + end else cycleSeq = cycleSeqIn orderCount = c @@ -653,7 +867,7 @@ export class RedisSnapshotStore { local vals = redis.call('HMGET', eKey, id, id .. '#s', id .. '#c') if not vals[1] then return nil end -- Coerce every element: a Lua false TRUNCATES the returned array at that position. - return { id, vals[1], vals[2] or '', vals[3] or '', orderFor(vals[3]) } + return { id, vals[1], vals[2] or '', vals[3] or '', orderFor(vals[3]), distinctFor(vals[3]), danglingFor(vals[3]) } `, }); @@ -665,7 +879,7 @@ export class RedisSnapshotStore { if not cur then return nil end local vals = redis.call('HMGET', eKey, cur, cur .. '#s', cur .. '#c') if not vals[1] then return nil end - return { cur, vals[1], vals[2] or '', vals[3] or '', orderFor(vals[3]) } + return { cur, vals[1], vals[2] or '', vals[3] or '', orderFor(vals[3]), distinctFor(vals[3]), danglingFor(vals[3]) } `, }); @@ -678,7 +892,78 @@ export class RedisSnapshotStore { return { '0', '' } end local pointer = redis.call('HGET', eKey, id .. '#c') - return { '1', orderFor(pointer) } + return { '1', orderFor(pointer), distinctFor(pointer), danglingFor(pointer) } + `, + }); + + this.redis.defineCommand("readSnapshotsSinceCreatedAt", { + numberOfKeys: 4, + lua: ` + ${PRELUDE} + local cursor = ARGV[1] + local limit = tonumber(ARGV[2]) + + -- A run with no keyspace is a MISS, so the caller falls back to Postgres. A run that has one + -- and nothing newer is an empty HIT, so it does not fall back for a window it owns. + -- + -- Both anchors, for the reason the append script gives: keys expire independently, and an + -- index lost to eviction while the entry hash survives would otherwise report an empty HIT + -- on every poll for the rest of the run's life, with Postgres holding the transitions. + if redis.call('EXISTS', eKey) == 0 or redis.call('EXISTS', idxKey) == 0 then return nil end + + -- STRICTLY greater than the cursor, and same-millisecond entries are dropped. Postgres + -- serves this window with createdAt > cursor and drops them too; a Redis read that is more + -- correct than the Postgres read shows up as divergence in compare mode. + -- + -- createdAt is always toISOString() output, one fixed-width UTC format, so a lexicographic + -- compare is a chronological compare. Walking newest-first lets the scan stop at the first + -- entry at or before the cursor, which makes its length the length of the ANSWER rather + -- than the length of the run's history. + local out = { '', '', '', '' } + local headId = nil + local offset = 0 + local page = limit + local done = false + + while not done do + local ids = redis.call('ZREVRANGE', idxKey, offset, offset + page - 1) + if #ids == 0 then break end + + for i = 1, #ids do + local id = ids[i] + local vals = redis.call('HMGET', eKey, id, id .. '#s', id .. '#c') + if vals[1] then + local createdAt = cjson.decode(vals[1])['createdAt'] + if not createdAt or createdAt <= cursor then + done = true + break + end + if not headId then headId = id end + out[#out + 1] = id + out[#out + 1] = vals[1] + out[#out + 1] = vals[2] or '' + out[#out + 1] = vals[3] or '' + if (#out - 2) / 4 >= limit then + done = true + break + end + end + end + + offset = offset + page + end + + if headId then + local headPointer = redis.call('HGET', eKey, headId .. '#c') + out[2] = orderFor(headPointer) + out[3] = distinctFor(headPointer) + -- The head's cycle key can expire while its entry survives: the completion TTL is applied + -- per key. Without this flag the head returns an EMPTY order and the caller cannot tell + -- that from a head that genuinely had no indexed waitpoints, so a batched resume loses + -- every position instead of falling back to Postgres. + out[4] = danglingFor(headPointer) + end + return out `, }); @@ -708,7 +993,7 @@ export class RedisSnapshotStore { -- The head is the newest SURVIVING entry, and it is the only one whose cycle key is read. -- Deriving the order after the loop keeps it paired with the row it is attached to: a row -- dropped for a missing body must not donate its cycle data to the next one. - local out = { sinceRaw, '' } + local out = { sinceRaw, '', '', '' } local headId = nil for i = 1, #ids do local id = ids[i] @@ -722,7 +1007,14 @@ export class RedisSnapshotStore { end end if headId then - out[2] = orderFor(redis.call('HGET', eKey, headId .. '#c')) + local headPointer = redis.call('HGET', eKey, headId .. '#c') + out[2] = orderFor(headPointer) + out[3] = distinctFor(headPointer) + -- The head's cycle key can expire while its entry survives: the completion TTL is applied + -- per key. Without this flag the head returns an EMPTY order and the caller cannot tell + -- that from a head that genuinely had no indexed waitpoints, so a batched resume loses + -- every position instead of falling back to Postgres. + out[4] = danglingFor(headPointer) end return out `, @@ -730,9 +1022,26 @@ export class RedisSnapshotStore { } } -export function decodeWaitpointIds(present: boolean, orderJson: string): WaitpointIds { +export function decodeWaitpointIds( + present: boolean, + orderJson: string, + distinctJson = "" +): WaitpointIds { const order: string[] = orderJson === "" ? [] : (JSON.parse(orderJson) as string[]); - return { present, distinctIds: [...new Set(order)], order }; + + // The complete set is stored separately, because `order` omits every id with no batch index, so + // deduping the order to recover it silently drops every wait that has none. + // + // A cycle key always holds both fields, written by one command, so a missing `distinct` beside a + // NON-EMPTY `order` means the invariant is broken. Reconstructing from the order there would be + // the same lossy shortcut this field exists to remove, and the loss would be silent. Report the + // entry as not present instead, which sends the caller to Postgres. + if (distinctJson === "" && order.length > 0) { + return { present: false, distinctIds: [], order: [] }; + } + + const distinctIds: string[] = distinctJson === "" ? [] : (JSON.parse(distinctJson) as string[]); + return { present, distinctIds, order }; } declare module "@internal/redis" { @@ -755,6 +1064,7 @@ declare module "@internal/redis" { orderCount: string, expectedCur: string, casEnabled: string, + distinctJson: string, callback?: Callback ): Result; readSnapshotById( @@ -780,6 +1090,15 @@ declare module "@internal/redis" { id: string, callback?: Callback ): Result; + readSnapshotsSinceCreatedAt( + eKey: string, + idxKey: string, + curKey: string, + seqKey: string, + createdAtCursor: string, + limit: string, + callback?: Callback + ): Result; readSnapshotsSince( eKey: string, idxKey: string, diff --git a/internal-packages/run-store/src/runStoreMethodNames.ts b/internal-packages/run-store/src/runStoreMethodNames.ts new file mode 100644 index 00000000000..2b10bd766d0 --- /dev/null +++ b/internal-packages/run-store/src/runStoreMethodNames.ts @@ -0,0 +1,115 @@ +// The member names of RunStore, as data. +// +// The decorator suites enumerate this to drive one call per member. Member PRESENCE is not proved +// here: that is the compiler's job, through `implements RunStore` on the pass-through base and the +// assertions at the foot of this file. +// +import type { RunStore } from "./types.js"; + +// Every method the RunStore interface declares. The forwarding probe enumerates this to drive one +// call per member; member PRESENCE is proved by the compiler, in the assertions at the foot of this +// file and by `implements RunStore` on the generated class. +export const RUN_STORE_METHOD_NAMES = [ + "runInTransaction", + "createRun", + "createCancelledRun", + "createFailedRun", + "startAttempt", + "completeAttemptSuccess", + "recordRetryOutcome", + "requeueRun", + "recordBulkActionMembership", + "cancelRun", + "failRunPermanently", + "finalizeRun", + "expireRun", + "expireRunsBatch", + "lockRunToWorker", + "parkPendingVersion", + "promotePendingVersionRuns", + "expireParkedRun", + "suspendForCheckpoint", + "resumeFromCheckpoint", + "rescheduleRun", + "enqueueDelayedRun", + "rewriteDebouncedRun", + "updateMetadata", + "clearIdempotencyKey", + "pushTags", + "pushRealtimeStream", + "findRun", + "findRunOrThrow", + "findRunOnPrimary", + "findRunOrThrowOnPrimary", + "findRuns", + "findRunsByIds", + "findRunsByIdempotencyKeys", + "createBatchTaskRunItem", + "findLatestExecutionSnapshot", + "findExecutionSnapshot", + "findManyExecutionSnapshots", + "createExecutionSnapshot", + "findSnapshotCompletedWaitpointIds", + "findSnapshotCompletedWaitpointIdsWithPresence", + "findWaitpointConnectedRunIds", + "findWaitpointCompletedSnapshotIds", + "blockRunWithWaitpointEdges", + "countPendingWaitpoints", + "countPendingWaitpointsWithPresence", + "createWaitpoint", + "upsertWaitpoint", + "findWaitpoint", + "findWaitpointOnPrimary", + "findManyWaitpoints", + "updateWaitpoint", + "updateManyWaitpoints", + "forWaitpointCompletion", + "findManyTaskRunWaitpoints", + "deleteManyTaskRunWaitpoints", + "findTaskRunAttempt", + "createTaskRunCheckpoint", + "createBatchTaskRun", + "updateBatchTaskRun", + "findBatchTaskRunById", + "findBatchTaskRunByFriendlyId", + "findBatchTaskRunByIdempotencyKey", + "updateManyBatchTaskRun", + "countBatchTaskRunItems", + "updateManyBatchTaskRunItems", + "findManyBatchTaskRunItems", + "findBatchTaskRunItem", + "upsertWaitpointTag", + "findManyWaitpointTags", +] as const; + +// Data properties the base exposes as getters over the delegate, not as forwarders. +export const RUN_STORE_PROPERTY_NAMES = ["primaryReadClient"] as const; + +// --------------------------------------------------------------------------- +// Parity with the interface, checked by the compiler. +// +// The lists above are produced by parsing types.ts. These assertions compare them +// against `keyof RunStore`, which the compiler derives from the interface itself, +// so a name this generator failed to parse, or invented, is a build failure rather +// than a silent gap. Both directions are checked: a missing name and an extra one. +// --------------------------------------------------------------------------- + +type RunStoreMemberName = + | (typeof RUN_STORE_METHOD_NAMES)[number] + | (typeof RUN_STORE_PROPERTY_NAMES)[number]; + +/** Fails when the interface declares a member the generator did not emit. */ +type _EveryInterfaceMemberIsListed = [Exclude] extends [never] + ? true + : never; +const _everyInterfaceMemberIsListed: _EveryInterfaceMemberIsListed = true; +void _everyInterfaceMemberIsListed; + +/** Fails when the generator emitted a name the interface does not declare. */ +type _EveryListedNameIsOnTheInterface = [Exclude] extends [ + never, +] + ? true + : never; +const _everyListedNameIsOnTheInterface: _EveryListedNameIsOnTheInterface = true; +void _everyListedNameIsOnTheInterface; diff --git a/internal-packages/run-store/src/snapshotEntry.parity.test.ts b/internal-packages/run-store/src/snapshotEntry.parity.test.ts new file mode 100644 index 00000000000..635160fd831 --- /dev/null +++ b/internal-packages/run-store/src/snapshotEntry.parity.test.ts @@ -0,0 +1,522 @@ +// The entry is built from a write site's input while Postgres builds the row from the same input by +// a different code path. This suite is the only thing that keeps those two paths equal, so it covers +// every one of the ten physical snapshot-create sites in PostgresRunStore. +import { describe, expect } from "vitest"; +import { postgresTest } from "@internal/testcontainers"; +import { generateInternalId } from "@trigger.dev/core/v3/isomorphic"; +import { PostgresRunStore } from "./PostgresRunStore.js"; +import type { SnapshotEntryInput } from "./redisSnapshotStore.js"; +import { + entryFromCompletion, + entryFromCreateExecutionSnapshot, + entryFromCreateRun, + entryFromExpire, + entryFromLock, + entryFromReschedule, +} from "./snapshotEntry.js"; +import { + buildCreateRunData, + seedSnapshotEnvironment, + seedSnapshotWorker, + setupSnapshotIdFixture, + type SnapshotFixtureEnv, +} from "./testFixtures/snapshotIdFixture.js"; + +/** + * NOTE ON createdAt. An earlier version of this suite built the expected entry with + * `createdAt: row.createdAt`, reading the value off the row it was checking and then asserting the + * two matched. That can never fail, and it hid a real divergence: seven of the eight write sites + * stamped the entry from the app clock while Postgres stamped its own column default, so the two + * stores held different instants for one snapshot. + * + * Every case now mints ONE instant, passes it to the store call, and gives the builder the same + * value. The row must carry it because the write site forwards it. A write site that stops + * forwarding the caller's instant fails here. + * + * Compares only what the entry claims. The Redis model carries no `updatedAt` and no join rows, and + * it holds `createdAt` as an ISO string, so those are checked separately or not at all. + */ +function assertParity(entry: SnapshotEntryInput, row: Record) { + expect(row.id).toBe(entry.id); + expect(row.runId).toBe(entry.runId); + expect(row.engine).toBe(entry.engine); + expect(row.executionStatus).toBe(entry.executionStatus); + expect(row.description).toBe(entry.description); + expect(row.runStatus).toBe(entry.runStatus); + expect(row.environmentId).toBe(entry.environmentId); + expect(row.environmentType).toBe(entry.environmentType); + expect(row.projectId).toBe(entry.projectId); + expect(row.organizationId).toBe(entry.organizationId); + expect(row.attemptNumber ?? undefined).toBe(entry.attemptNumber ?? undefined); + expect(row.previousSnapshotId ?? undefined).toBe(entry.previousSnapshotId ?? undefined); + expect(row.batchId ?? undefined).toBe(entry.batchId ?? undefined); + expect(row.checkpointId ?? undefined).toBe(entry.checkpointId ?? undefined); + expect(row.workerId ?? undefined).toBe(entry.workerId ?? undefined); + expect(row.runnerId ?? undefined).toBe(entry.runnerId ?? undefined); + expect(row.isValid).toBe(entry.error === undefined); + expect((row.createdAt as Date).toISOString()).toBe(entry.createdAt); + // Write-once rows: both columns hold the one instant, so a Prisma-stamped updatedAt would drift. + expect((row.updatedAt as Date).toISOString()).toBe(entry.createdAt); +} + +/** Five minutes in the past, so a database default could never coincide with it. */ +const independentStamp = new Date(Date.now() - 5 * 60 * 1000); + +function birthSnapshot(id: string, env: SnapshotFixtureEnv) { + return { + id, + createdAt: independentStamp, + engine: "V2" as const, + executionStatus: "RUN_CREATED" as const, + description: "Run was created", + runStatus: "PENDING" as const, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }; +} + +describe("entry to Postgres row parity", () => { + postgresTest("createRun, legacy schema", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + const id = generateInternalId(); + const snapshot = birthSnapshot(id, env); + + await store.createRun({ data: buildCreateRunData(runId, env), snapshot }); + + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); + assertParity(entryFromCreateRun({ id, runId, createdAt: independentStamp }, snapshot), row); + }); + + postgresTest("createRun with an associated waitpoint, legacy schema", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + const id = generateInternalId(); + const snapshot = birthSnapshot(id, env); + + await store.createRun({ + data: buildCreateRunData(runId, env), + snapshot, + associatedWaitpoint: { + id: generateInternalId(), + friendlyId: `waitpoint_${runId.slice(-12)}`, + type: "RUN", + status: "PENDING", + idempotencyKey: generateInternalId(), + userProvidedIdempotencyKey: false, + projectId: env.projectId, + environmentId: env.id, + }, + }); + + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); + assertParity(entryFromCreateRun({ id, runId, createdAt: independentStamp }, snapshot), row); + }); + + postgresTest("createRun carries the worker and runner ids", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const env = await seedSnapshotEnvironment(prisma); + const { workerId } = await seedSnapshotWorker(prisma, env); + const runId = generateInternalId(); + const id = generateInternalId(); + const snapshot = { ...birthSnapshot(id, env), workerId, runnerId: "runner_1" }; + + await store.createRun({ data: buildCreateRunData(runId, env), snapshot }); + + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); + assertParity(entryFromCreateRun({ id, runId, createdAt: independentStamp }, snapshot), row); + }); + + postgresTest("createCancelledRun", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + const id = generateInternalId(); + const snapshot = { + ...birthSnapshot(id, env), + executionStatus: "FINISHED" as const, + description: "Run was cancelled", + runStatus: "CANCELED" as const, + }; + + await store.createCancelledRun({ + data: { + ...buildCreateRunData(runId, env), + status: "CANCELED", + error: { type: "STRING_ERROR", raw: "cancelled" }, + completedAt: new Date(), + updatedAt: new Date(), + attemptNumber: 0, + }, + snapshot, + }); + + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); + assertParity(entryFromCreateRun({ id, runId, createdAt: independentStamp }, snapshot), row); + }); + + postgresTest("completeAttemptSuccess", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const { run, env } = await setupSnapshotIdFixture(prisma); + const id = generateInternalId(); + const snapshot = { + id, + createdAt: independentStamp, + executionStatus: "FINISHED" as const, + description: "Run completed", + runStatus: "COMPLETED_SUCCESSFULLY" as const, + attemptNumber: 1, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }; + + await store.completeAttemptSuccess( + run.id, + { + completedAt: new Date(), + outputType: "application/json", + usageDurationMs: 1, + costInCents: 0, + snapshot, + }, + { select: { id: true } } + ); + + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); + assertParity( + entryFromCompletion({ id, runId: run.id, createdAt: independentStamp }, snapshot), + row + ); + }); + + postgresTest("expireRun", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const { run, env } = await setupSnapshotIdFixture(prisma); + const id = generateInternalId(); + const snapshot = { + id, + createdAt: independentStamp, + engine: "V2" as const, + executionStatus: "FINISHED" as const, + description: "Run expired", + runStatus: "EXPIRED" as const, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }; + + await store.expireRun( + run.id, + { + error: { type: "STRING_ERROR", raw: "expired" }, + completedAt: new Date(), + expiredAt: new Date(), + snapshot, + }, + { select: { id: true } } + ); + + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); + assertParity( + entryFromExpire({ id, runId: run.id, createdAt: independentStamp }, snapshot), + row + ); + }); + + postgresTest("expireParkedRun", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const { run, env } = await setupSnapshotIdFixture(prisma, { status: "PENDING_VERSION" }); + const id = generateInternalId(); + const snapshot = { + id, + createdAt: independentStamp, + engine: "V2" as const, + executionStatus: "FINISHED" as const, + description: "Parked run expired", + runStatus: "EXPIRED" as const, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }; + + const result = await store.expireParkedRun(run.id, { + error: { type: "STRING_ERROR", raw: "expired" }, + completedAt: new Date(), + expiredAt: new Date(), + statusReason: "VERSION_NEVER_ARRIVED", + snapshot, + }); + + expect(result.count).toBe(1); + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); + assertParity( + entryFromExpire({ id, runId: run.id, createdAt: independentStamp }, snapshot), + row + ); + }); + + postgresTest("rescheduleRun with every default applied", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const { run, env } = await setupSnapshotIdFixture(prisma, { status: "DELAYED" }); + const id = generateInternalId(); + const snapshot = { + id, + createdAt: independentStamp, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }; + + await store.rescheduleRun(run.id, { + delayUntil: new Date(Date.now() + 60_000), + snapshot, + }); + + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); + assertParity( + entryFromReschedule({ id, runId: run.id, createdAt: independentStamp }, snapshot), + row + ); + }); + + postgresTest("rescheduleRun with every value supplied", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const { run, env } = await setupSnapshotIdFixture(prisma, { status: "DELAYED" }); + const id = generateInternalId(); + const snapshot = { + id, + createdAt: independentStamp, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + executionStatus: "QUEUED" as const, + runStatus: "PENDING" as const, + description: "custom reschedule", + }; + + await store.rescheduleRun(run.id, { + delayUntil: new Date(Date.now() + 60_000), + snapshot, + }); + + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); + assertParity( + entryFromReschedule({ id, runId: run.id, createdAt: independentStamp }, snapshot), + row + ); + }); + + postgresTest("lockRunToWorker", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const { run, env } = await setupSnapshotIdFixture(prisma); + const { workerId, taskId } = await seedSnapshotWorker(prisma, env); + const previous = await store.createExecutionSnapshot({ + run: { id: run.id, status: "PENDING", attemptNumber: null }, + snapshot: { executionStatus: "QUEUED", description: "Run was queued" }, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }); + + const id = generateInternalId(); + const snapshot = { + id, + createdAt: independentStamp, + previousSnapshotId: previous.id, + attemptNumber: 1, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + completedWaitpointIds: [], + completedWaitpointOrder: [], + }; + + await store.lockRunToWorker(run.id, { + lockedAt: new Date(), + lockedById: taskId, + lockedToVersionId: workerId, + lockedQueueId: undefined, + startedAt: new Date(), + baseCostInCents: 0, + machinePreset: "small-1x", + taskVersion: "1.0.0", + snapshot, + }); + + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); + assertParity(entryFromLock({ id, runId: run.id, createdAt: independentStamp }, snapshot), row); + }); + + postgresTest("createExecutionSnapshot, the standalone site", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const { run, env } = await setupSnapshotIdFixture(prisma); + const id = generateInternalId(); + const input = { + id, + createdAt: independentStamp, + run: { id: run.id, status: "EXECUTING" as const, attemptNumber: 2 }, + snapshot: { executionStatus: "EXECUTING" as const, description: "Run started" }, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }; + + const created = await store.createExecutionSnapshot(input); + + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); + expect(created.id).toBe(id); + assertParity( + entryFromCreateExecutionSnapshot({ id, runId: run.id, createdAt: independentStamp }, input), + row + ); + }); + + postgresTest("createExecutionSnapshot rewrites a DEQUEUED run status", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const { run, env } = await setupSnapshotIdFixture(prisma); + const id = generateInternalId(); + const input = { + id, + createdAt: independentStamp, + run: { id: run.id, status: "DEQUEUED" as const, attemptNumber: 1 }, + snapshot: { executionStatus: "PENDING_EXECUTING" as const, description: "Run was dequeued" }, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }; + + await store.createExecutionSnapshot(input); + + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); + expect(row.runStatus).toBe("PENDING"); + assertParity( + entryFromCreateExecutionSnapshot({ id, runId: run.id, createdAt: independentStamp }, input), + row + ); + }); + + postgresTest("createExecutionSnapshot with an error is invalid in both", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const { run, env } = await setupSnapshotIdFixture(prisma); + const id = generateInternalId(); + const input = { + id, + createdAt: independentStamp, + run: { id: run.id, status: "EXECUTING" as const, attemptNumber: 1 }, + snapshot: { executionStatus: "EXECUTING" as const, description: "Stale write" }, + error: "snapshot is not the latest", + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }; + + await store.createExecutionSnapshot(input); + + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); + expect(row.isValid).toBe(false); + assertParity( + entryFromCreateExecutionSnapshot({ id, runId: run.id, createdAt: independentStamp }, input), + row + ); + }); +}); + +// The clock-provenance guard. Independent of the builders above: it asserts that what Postgres +// stores is the instant the CALLER supplied, not one the database chose. Without this, a snapshot +// has two different creation times depending on which store answers, the compared field can never +// reach zero divergence, and the since-window cursor resolved from one store misfilters the window +// walked in the other. +describe("createdAt provenance", () => { + postgresTest("Postgres stores the caller's instant, not its own", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const { run, env } = await setupSnapshotIdFixture(prisma); + const id = generateInternalId(); + // Far enough from now that a database default could never coincide with it. + const stamp = new Date(Date.now() - 5 * 60 * 1000); + + await store.createExecutionSnapshot({ + id, + createdAt: stamp, + run: { id: run.id, status: "EXECUTING", attemptNumber: 1 }, + snapshot: { executionStatus: "EXECUTING", description: "Run started" }, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }); + + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); + expect(row.createdAt.toISOString()).toBe(stamp.toISOString()); + expect(row.updatedAt.toISOString()).toBe(stamp.toISOString()); + }); + + postgresTest("an absent instant still takes the database default", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const { run, env } = await setupSnapshotIdFixture(prisma); + const id = generateInternalId(); + const before = new Date(Date.now() - 1000); + + // Mode off supplies nothing, so Postgres must behave exactly as it always has. This is what + // keeps the merge test true. + await store.createExecutionSnapshot({ + id, + run: { id: run.id, status: "EXECUTING", attemptNumber: 1 }, + snapshot: { executionStatus: "EXECUTING", description: "Run started" }, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }); + + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); + expect(row.createdAt.getTime()).toBeGreaterThan(before.getTime()); + }); + + postgresTest("a nested write site stores the caller's instant too", async ({ prisma }) => { + const store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const { run, env } = await setupSnapshotIdFixture(prisma); + const id = generateInternalId(); + const stamp = new Date(Date.now() - 5 * 60 * 1000); + + await store.expireRun( + run.id, + { + error: { type: "STRING_ERROR", raw: "expired" }, + completedAt: new Date(), + expiredAt: new Date(), + snapshot: { + id, + createdAt: stamp, + engine: "V2", + executionStatus: "FINISHED", + description: "Run expired", + runStatus: "EXPIRED", + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }, + }, + { select: { id: true } } + ); + + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { id } }); + expect(row.createdAt.toISOString()).toBe(stamp.toISOString()); + expect(row.updatedAt.toISOString()).toBe(stamp.toISOString()); + }); +}); diff --git a/internal-packages/run-store/src/snapshotEntry.test.ts b/internal-packages/run-store/src/snapshotEntry.test.ts new file mode 100644 index 00000000000..cc624d690a9 --- /dev/null +++ b/internal-packages/run-store/src/snapshotEntry.test.ts @@ -0,0 +1,185 @@ +// These mappings are values Postgres derives rather than receives. If either side changes and the +// other does not, dual-write silently stores two different documents for one snapshot. The parity +// suite next to this file checks the same thing against a real Postgres row; this one pins the +// rules on their own, so a failure says which rule broke. +import { describe, expect, it } from "vitest"; +import { + entryFromCompletion, + entryFromCreateExecutionSnapshot, + entryFromCreateRun, + entryFromExpire, + entryFromLock, + entryFromReschedule, + isTerminalEntry, +} from "./snapshotEntry.js"; + +const ctx = { id: "snap_1", runId: "run_1", createdAt: new Date("2026-08-24T00:00:00.000Z") }; +const scope = { + environmentId: "env_1", + environmentType: "DEVELOPMENT" as const, + projectId: "proj_1", + organizationId: "org_1", +}; + +describe("snapshotEntry derived values", () => { + it("rewrites a DEQUEUED run status to PENDING", () => { + const entry = entryFromCreateExecutionSnapshot(ctx, { + run: { id: "run_1", status: "DEQUEUED", attemptNumber: 1 }, + snapshot: { executionStatus: "PENDING_EXECUTING", description: "d" }, + ...scope, + }); + + expect(entry.runStatus).toBe("PENDING"); + }); + + it("keeps every other run status unchanged", () => { + const entry = entryFromCreateExecutionSnapshot(ctx, { + run: { id: "run_1", status: "EXECUTING", attemptNumber: 1 }, + snapshot: { executionStatus: "EXECUTING", description: "d" }, + ...scope, + }); + + expect(entry.runStatus).toBe("EXECUTING"); + }); + + it("applies the lock site's hard-coded values", () => { + const entry = entryFromLock(ctx, { + id: "snap_1", + previousSnapshotId: "snap_0", + attemptNumber: 2, + completedWaitpointIds: [], + completedWaitpointOrder: [], + ...scope, + }); + + expect(entry.executionStatus).toBe("PENDING_EXECUTING"); + expect(entry.description).toBe("Run was dequeued for execution"); + expect(entry.runStatus).toBe("PENDING"); + expect(entry.engine).toBe("V2"); + expect(entry.previousSnapshotId).toBe("snap_0"); + expect(entry.attemptNumber).toBe(2); + }); + + it("applies the reschedule defaults", () => { + const entry = entryFromReschedule(ctx, { ...scope }); + + expect(entry.executionStatus).toBe("DELAYED"); + expect(entry.runStatus).toBe("DELAYED"); + expect(entry.description).toBe("Delayed run was rescheduled to a future date"); + }); + + it("prefers a supplied reschedule value over the default", () => { + const entry = entryFromReschedule(ctx, { + ...scope, + executionStatus: "QUEUED", + runStatus: "PENDING", + description: "custom", + }); + + expect(entry.executionStatus).toBe("QUEUED"); + expect(entry.runStatus).toBe("PENDING"); + expect(entry.description).toBe("custom"); + }); + + it("sets engine V2 on a completion, which Postgres leaves to the column default", () => { + const entry = entryFromCompletion(ctx, { + executionStatus: "FINISHED", + description: "Run completed", + runStatus: "COMPLETED_SUCCESSFULLY", + attemptNumber: 1, + ...scope, + }); + + expect(entry.engine).toBe("V2"); + }); + + it("carries a null completion attemptNumber through as null", () => { + const entry = entryFromCompletion(ctx, { + executionStatus: "FINISHED", + description: "Run completed", + runStatus: "COMPLETED_SUCCESSFULLY", + attemptNumber: null, + ...scope, + }); + + expect(entry.attemptNumber).toBeNull(); + }); + + it("omits an absent optional rather than writing undefined into the document", () => { + const entry = entryFromExpire(ctx, { + engine: "V2", + executionStatus: "FINISHED", + description: "Run expired", + runStatus: "EXPIRED", + ...scope, + }); + + expect(Object.keys(entry)).not.toContain("workerId"); + expect(Object.keys(entry)).not.toContain("attemptNumber"); + expect(JSON.parse(JSON.stringify(entry))).toEqual(entry); + }); + + it("reports a FINISHED entry as terminal and any other as not", () => { + const finished = entryFromCompletion(ctx, { + executionStatus: "FINISHED", + description: "Run completed", + runStatus: "COMPLETED_SUCCESSFULLY", + attemptNumber: 1, + ...scope, + }); + const running = entryFromCreateExecutionSnapshot(ctx, { + run: { id: "run_1", status: "EXECUTING", attemptNumber: 1 }, + snapshot: { executionStatus: "EXECUTING", description: "d" }, + ...scope, + }); + + expect(isTerminalEntry(finished)).toBe(true); + expect(isTerminalEntry(running)).toBe(false); + }); + + it("serialises createdAt as an ISO string", () => { + const entry = entryFromReschedule(ctx, { ...scope }); + + expect(entry.createdAt).toBe("2026-08-24T00:00:00.000Z"); + }); + + it("carries the birth site's worker and runner ids", () => { + const entry = entryFromCreateRun(ctx, { + engine: "V2", + executionStatus: "RUN_CREATED", + description: "Run was created", + runStatus: "PENDING", + workerId: "worker_1", + runnerId: "runner_1", + ...scope, + }); + + expect(entry.workerId).toBe("worker_1"); + expect(entry.runnerId).toBe("runner_1"); + expect(entry.executionStatus).toBe("RUN_CREATED"); + }); + + it("never sets the reserved completedWaitpoints field", () => { + const built = [ + entryFromReschedule(ctx, { ...scope }), + entryFromLock(ctx, { + id: "snap_1", + previousSnapshotId: "snap_0", + completedWaitpointIds: ["w_1"], + completedWaitpointOrder: ["w_1"], + ...scope, + }), + entryFromCreateExecutionSnapshot(ctx, { + run: { id: "run_1", status: "EXECUTING", attemptNumber: 1 }, + snapshot: { executionStatus: "EXECUTING", description: "d" }, + completedWaitpoints: [{ id: "w_1", index: 0 }], + ...scope, + }), + ]; + + // The append script mints the pointer as a sidecar field, and rejects an entry that carries one. + for (const entry of built) { + expect(entry.completedWaitpoints).toBeUndefined(); + } + }); +}); diff --git a/internal-packages/run-store/src/snapshotEntry.ts b/internal-packages/run-store/src/snapshotEntry.ts new file mode 100644 index 00000000000..748a95f6d1a --- /dev/null +++ b/internal-packages/run-store/src/snapshotEntry.ts @@ -0,0 +1,168 @@ +// Builds the Redis entry for each execution-snapshot write site, from that site's own INPUT. +// +// Not from the delegate's return value: no nested write site includes the snapshot in what it +// returns. `createRun` returns the run, `expireParkedRun` returns a count, and the rest return a +// selected `TaskRun`. That means every value Postgres derives rather than receives has to be +// reproduced here, and snapshotEntry.parity.test.ts is what keeps the two sides from drifting. +import type { TaskRunStatus } from "@trigger.dev/database"; +import type { SnapshotEntryInput } from "./redisSnapshotStore.js"; +import type { + CompletionSnapshotInput, + CreateExecutionSnapshotInput, + CreateRunSnapshotInput, + ExpireSnapshotInput, + LockSnapshotInput, + RescheduleSnapshotInput, +} from "./types.js"; + +export type EntryBuildContext = { id: string; runId: string; createdAt: Date }; + +/** + * PostgresRunStore.#createExecutionSnapshot rewrites DEQUEUED to PENDING, because older runners + * reject DEQUEUED on a snapshot. Every site that can carry that status must rewrite it identically. + */ +function snapshotRunStatus(status: TaskRunStatus): string { + return status === "DEQUEUED" ? "PENDING" : status; +} + +function base(ctx: EntryBuildContext) { + return { + id: ctx.id, + runId: ctx.runId, + createdAt: ctx.createdAt.toISOString(), + engine: "V2" as const, + }; +} + +export function entryFromCreateRun( + ctx: EntryBuildContext, + snapshot: CreateRunSnapshotInput +): SnapshotEntryInput { + return { + ...base(ctx), + executionStatus: snapshot.executionStatus, + description: snapshot.description, + runStatus: snapshotRunStatus(snapshot.runStatus), + environmentId: snapshot.environmentId, + environmentType: snapshot.environmentType, + projectId: snapshot.projectId, + organizationId: snapshot.organizationId, + ...(snapshot.workerId !== undefined && { workerId: snapshot.workerId }), + ...(snapshot.runnerId !== undefined && { runnerId: snapshot.runnerId }), + }; +} + +/** + * `completeAttemptSuccess` writes no `engine` column, so Postgres applies the schema default of + * `V2`. The entry states it, because SnapshotEntryInput requires the field. + */ +export function entryFromCompletion( + ctx: EntryBuildContext, + snapshot: CompletionSnapshotInput +): SnapshotEntryInput { + return { + ...base(ctx), + executionStatus: snapshot.executionStatus, + description: snapshot.description, + runStatus: snapshotRunStatus(snapshot.runStatus), + attemptNumber: snapshot.attemptNumber, + environmentId: snapshot.environmentId, + environmentType: snapshot.environmentType, + projectId: snapshot.projectId, + organizationId: snapshot.organizationId, + ...(snapshot.workerId !== undefined && { workerId: snapshot.workerId }), + ...(snapshot.runnerId !== undefined && { runnerId: snapshot.runnerId }), + }; +} + +/** Serves both `expireRun` and `expireParkedRun`; the two write identical snapshot columns. */ +export function entryFromExpire( + ctx: EntryBuildContext, + snapshot: ExpireSnapshotInput +): SnapshotEntryInput { + return { + ...base(ctx), + executionStatus: snapshot.executionStatus, + description: snapshot.description, + runStatus: snapshotRunStatus(snapshot.runStatus), + environmentId: snapshot.environmentId, + environmentType: snapshot.environmentType, + projectId: snapshot.projectId, + organizationId: snapshot.organizationId, + }; +} + +/** PostgresRunStore.rescheduleRun supplies these three defaults inline, so the entry repeats them. */ +export function entryFromReschedule( + ctx: EntryBuildContext, + snapshot: RescheduleSnapshotInput +): SnapshotEntryInput { + return { + ...base(ctx), + executionStatus: snapshot.executionStatus ?? "DELAYED", + description: snapshot.description ?? "Delayed run was rescheduled to a future date", + runStatus: snapshotRunStatus(snapshot.runStatus ?? "DELAYED"), + environmentId: snapshot.environmentId, + environmentType: snapshot.environmentType, + projectId: snapshot.projectId, + organizationId: snapshot.organizationId, + }; +} + +/** PostgresRunStore.#lockRunToWorker hard-codes the status, description and run status. */ +export function entryFromLock( + ctx: EntryBuildContext, + snapshot: LockSnapshotInput +): SnapshotEntryInput { + return { + ...base(ctx), + executionStatus: "PENDING_EXECUTING", + description: "Run was dequeued for execution", + runStatus: "PENDING", + previousSnapshotId: snapshot.previousSnapshotId, + ...(snapshot.attemptNumber !== undefined && { attemptNumber: snapshot.attemptNumber }), + ...(snapshot.batchId !== undefined && { batchId: snapshot.batchId }), + ...(snapshot.checkpointId !== undefined && { checkpointId: snapshot.checkpointId }), + environmentId: snapshot.environmentId, + environmentType: snapshot.environmentType, + projectId: snapshot.projectId, + organizationId: snapshot.organizationId, + ...(snapshot.workerId !== undefined && { workerId: snapshot.workerId }), + ...(snapshot.runnerId !== undefined && { runnerId: snapshot.runnerId }), + }; +} + +export function entryFromCreateExecutionSnapshot( + ctx: EntryBuildContext, + input: CreateExecutionSnapshotInput +): SnapshotEntryInput { + return { + ...base(ctx), + executionStatus: input.snapshot.executionStatus, + description: input.snapshot.description, + runStatus: snapshotRunStatus(input.run.status), + ...(input.run.attemptNumber !== undefined && + input.run.attemptNumber !== null && { attemptNumber: input.run.attemptNumber }), + ...(input.previousSnapshotId !== undefined && { previousSnapshotId: input.previousSnapshotId }), + ...(input.batchId !== undefined && { batchId: input.batchId }), + environmentId: input.environmentId, + environmentType: input.environmentType, + projectId: input.projectId, + organizationId: input.organizationId, + ...(input.checkpointId !== undefined && { checkpointId: input.checkpointId }), + ...(input.workerId !== undefined && { workerId: input.workerId }), + ...(input.runnerId !== undefined && { runnerId: input.runnerId }), + ...(input.snapshot.metadata !== undefined && + input.snapshot.metadata !== null && { metadata: input.snapshot.metadata }), + ...(input.error !== undefined && { error: input.error }), + }; +} + +/** + * A terminal entry is what makes the append script apply the completion TTL. FINISHED is the only + * terminal execution status; the run-level status is not consulted, because a run reaches its + * terminal state through a FINISHED snapshot in every path. + */ +export function isTerminalEntry(entry: SnapshotEntryInput): boolean { + return entry.executionStatus === "FINISHED"; +} diff --git a/internal-packages/run-store/src/snapshotFaultInjection.ts b/internal-packages/run-store/src/snapshotFaultInjection.ts new file mode 100644 index 00000000000..83abf43b179 --- /dev/null +++ b/internal-packages/run-store/src/snapshotFaultInjection.ts @@ -0,0 +1,45 @@ +// Test-only seam for the execution-snapshot write protocol. +// +// The protocol's correctness claim is about crashes: whatever the write order leaves behind at each +// boundary must be a state the existing stall-and-repair machinery heals. Proving that needs a crash +// at an exact point, which is what an injector gives. Production never sets one, so each boundary +// costs one optional call. + +/** The three points a crash can land between the two stores' writes. */ +export type SnapshotFaultBoundary = + /** A transition: Postgres has committed and the Redis append has not started. */ + | "afterPgBeforeRedis" + /** A birth: the Redis append has landed and the Postgres insert has not started. */ + | "afterRedisBirthBeforePg" + /** Inside the append retry loop, after at least one attempt has failed. */ + | "midFlushRetry"; + +export type SnapshotFaultInjector = ( + boundary: SnapshotFaultBoundary, + context: { runId: string; snapshotId: string } +) => void; + +/** + * Thrown by a test injector. The write path tells this apart from a real append failure, because an + * injected fault models a process that died rather than a call that failed. The two write paths then + * do different things with it, and both differ from a real failure: + * + * - A transition skips its remaining retries, hands the run to the repair job, and does NOT rethrow. + * Postgres has already committed, so the caller must not see an error. + * - A birth rethrows, so the Postgres insert never runs and the crash leaves an orphaned keyspace + * with no run row, which is the harmless state that ordering exists to produce. + * - A real append failure is retried, and only then handed to the repair job. + */ +export class InjectedSnapshotFault extends Error { + readonly boundary: SnapshotFaultBoundary; + + constructor(boundary: SnapshotFaultBoundary) { + super(`injected snapshot fault at ${boundary}`); + this.name = "InjectedSnapshotFault"; + this.boundary = boundary; + } +} + +export function isInjectedFault(error: unknown): error is InjectedSnapshotFault { + return error instanceof InjectedSnapshotFault; +} diff --git a/internal-packages/run-store/src/snapshotOrphanSweeper.cluster.test.ts b/internal-packages/run-store/src/snapshotOrphanSweeper.cluster.test.ts new file mode 100644 index 00000000000..90df7bfcc1d --- /dev/null +++ b/internal-packages/run-store/src/snapshotOrphanSweeper.cluster.test.ts @@ -0,0 +1,160 @@ +// Production points at a Valkey/Redis CLUSTER. SCAN carries no key, so a cluster cannot route it: +// one connection iterates ONE node's keyspace and then returns a completed cursor. A single-client +// sweep would therefore report {scanned, expired, deleted, skipped} looking exactly like a clean +// pass, having examined roughly 1/N of the keyspace, and the rest would leak with nothing left to +// revisit it. Both sweep rules close unbounded leaks, and TRI-13453 gates the rollout dial on an +// OBSERVED sweep pass, so a false green here is the worst failure this component has. +// +// Everything the sweep does after the scan is key-addressed and a cluster client routes it without +// help, so the node list is the whole of the exposure. These tests pin that decision directly. +// +// There is no Redis-cluster container fixture in the repo (@internal/testcontainers ships slot +// arithmetic, not a cluster), so the cluster cases drive a real ioredis `Cluster` object that has +// never connected and assert which method the code reaches for. That is a test of our branch, not +// a simulation of Redis. A true multi-node integration test wants a cluster fixture, and the ticket +// building the cluster client is the one placed to add it. +import { describe, expect, it } from "vitest"; +import { Cluster, Redis } from "@internal/redis"; +import { containerTest } from "@internal/testcontainers"; +import { generateInternalId } from "@trigger.dev/core/v3/isomorphic"; +import { PostgresRunStore } from "./PostgresRunStore.js"; +import { clientPrefixOf, scanTargetsOf, SnapshotOrphanSweeper } from "./snapshotOrphanSweeper.js"; + +/** An ioredis Cluster that is never connected. `lazyConnect` keeps the constructor from dialling. */ +function offlineCluster(options?: { keyPrefix?: string }): Cluster { + return new Cluster([{ host: "127.0.0.1", port: 7000 }], { + lazyConnect: true, + redisOptions: options?.keyPrefix ? { keyPrefix: options.keyPrefix } : undefined, + }); +} + +describe("scanTargetsOf", () => { + it("returns the one connection for a standalone client", () => { + const client = new Redis({ lazyConnect: true, port: 65000 }); + try { + expect(scanTargetsOf(client)).toEqual([client]); + } finally { + client.disconnect(); + } + }); + + it("returns every master for a cluster, and never a replica", async () => { + const cluster = offlineCluster(); + const masters = [ + new Redis({ lazyConnect: true, port: 65001 }), + new Redis({ lazyConnect: true, port: 65002 }), + new Redis({ lazyConnect: true, port: 65003 }), + ]; + const replica = new Redis({ lazyConnect: true, port: 65004 }); + + const asked: string[] = []; + // The assertion that matters: the sweep asks for "master" specifically. Asking for "all" would + // scan replicas too and act twice on one keyspace. + (cluster as unknown as { nodes: (role: string) => Redis[] }).nodes = (role: string) => { + asked.push(role); + return role === "master" ? masters : [...masters, replica]; + }; + + expect(scanTargetsOf(cluster)).toEqual(masters); + expect(asked).toEqual(["master"]); + expect(scanTargetsOf(cluster)).not.toContain(replica); + + for (const client of [...masters, replica]) client.disconnect(); + cluster.disconnect(); + }); + + it("resolves the node list per call, so a failover is picked up", () => { + const cluster = offlineCluster(); + let generation = 0; + (cluster as unknown as { nodes: () => Redis[] }).nodes = () => { + generation += 1; + return Array.from( + { length: generation }, + (_v, i) => new Redis({ lazyConnect: true, port: 65100 + i }) + ); + }; + + expect(scanTargetsOf(cluster)).toHaveLength(1); + expect(scanTargetsOf(cluster)).toHaveLength(2); + cluster.disconnect(); + }); +}); + +describe("clientPrefixOf", () => { + it("reads the top-level keyPrefix on a standalone client", () => { + const client = new Redis({ lazyConnect: true, port: 65000, keyPrefix: "engine:" }); + try { + expect(clientPrefixOf(client)).toBe("engine:"); + } finally { + client.disconnect(); + } + }); + + it("reads the nested redisOptions.keyPrefix on a cluster", () => { + // On a Cluster the prefix lives under redisOptions. Reading the top level yields "", every + // SCAN MATCH then misses, and the pass reports a clean sweep of nothing. + const cluster = offlineCluster({ keyPrefix: "engine:" }); + try { + expect(clientPrefixOf(cluster)).toBe("engine:"); + } finally { + cluster.disconnect(); + } + }); + + it("is empty when no prefix is configured, for either shape", () => { + const client = new Redis({ lazyConnect: true, port: 65000 }); + const cluster = offlineCluster(); + try { + expect(clientPrefixOf(client)).toBe(""); + expect(clientPrefixOf(cluster)).toBe(""); + } finally { + client.disconnect(); + cluster.disconnect(); + } + }); +}); + +describe("SweepResult.nodes", () => { + containerTest( + "a standalone pass reports the one connection it covered", + async ({ prisma, redisOptions }) => { + const runStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const sweeper = new SnapshotOrphanSweeper({ + redisOptions, + runStore, + completedTtlMs: 72 * 60 * 60 * 1000, + }); + + try { + const result = await sweeper.sweep({ dryRun: true }); + // Without this field a pass that covered one node of six is indistinguishable from a + // complete one, which is exactly the false green the fan-out exists to prevent. + expect(result.nodes).toBe(1); + } finally { + await sweeper.quit(); + } + } + ); +}); + +describe("client ownership", () => { + containerTest("quit() leaves a borrowed client open", async ({ prisma, redisOptions }) => { + const runStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const client = new Redis(redisOptions); + const sweeper = new SnapshotOrphanSweeper({ + client, + runStore, + completedTtlMs: 72 * 60 * 60 * 1000, + }); + + try { + await sweeper.quit(); + // The store and the sweep can share one cluster client. If quit() closed a client it did not + // open, the first component to shut down would take the other one's connection with it. + await client.set(`ownership:${generateInternalId()}`, "1"); + expect(await client.ping()).toBe("PONG"); + } finally { + client.disconnect(); + } + }); +}); diff --git a/internal-packages/run-store/src/snapshotOrphanSweeper.confirm.test.ts b/internal-packages/run-store/src/snapshotOrphanSweeper.confirm.test.ts new file mode 100644 index 00000000000..40c90bc4774 --- /dev/null +++ b/internal-packages/run-store/src/snapshotOrphanSweeper.confirm.test.ts @@ -0,0 +1,360 @@ +// Rule 2 deletes a whole keyspace on the strength of "findRunsByIds returned no row for it". The +// catch in #sweepBatch covers a lookup that THROWS; it cannot see a lookup that succeeds and is +// incomplete, and a row that exists but did not come back reads exactly like a run that never +// existed. `findRunsByIds` partitions ids by residency and asks each store only for its own, and +// with no client passed it reads each store's replica — both sound today, but neither is something +// this delete path can verify. +// +// A false negative leaks keys, which is bounded and recoverable. A false positive destroys a live +// run's execution state. So deletion requires two sightings across the confirm window, and these +// tests pin that: a single pass never deletes, however old the keyspace. +import { describe, expect } from "vitest"; +import { containerTest } from "@internal/testcontainers"; +import { createRedisClient } from "@internal/redis"; +import { generateInternalId } from "@trigger.dev/core/v3/isomorphic"; +import { PostgresRunStore } from "./PostgresRunStore.js"; +import { RedisSnapshotStore, snapshotKeys } from "./redisSnapshotStore.js"; +import { entryFromCreateRun } from "./snapshotEntry.js"; +import { SnapshotOrphanSweeper } from "./snapshotOrphanSweeper.js"; +import type { RunStore } from "./types.js"; +import { + buildCreateRunData, + seedSnapshotEnvironment, + type SnapshotFixtureEnv, +} from "./testFixtures/snapshotIdFixture.js"; + +const COMPLETED_TTL_MS = 72 * 60 * 60 * 1000; +const ORPHAN_AGE_MS = 60 * 60 * 1000; + +function birthEntry(runId: string, env: SnapshotFixtureEnv, createdAt: Date) { + const snapshot = { + id: generateInternalId(), + engine: "V2" as const, + executionStatus: "RUN_CREATED" as const, + description: "Run was created", + runStatus: "PENDING" as const, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }; + return entryFromCreateRun({ id: snapshot.id, runId, createdAt }, snapshot); +} + +describe("rule 2 requires a second sighting", () => { + containerTest( + "one pass marks an orphan and deletes nothing, however old the keyspace", + async ({ prisma, redisOptions }) => { + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + const runStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const sweeper = new SnapshotOrphanSweeper({ + redisOptions, + runStore: runStore as unknown as RunStore, + completedTtlMs: COMPLETED_TTL_MS, + orphanAgeMs: ORPHAN_AGE_MS, + confirmOrphanAfterMs: 0, + }); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + // A thousand times the age gate. Age is not what holds the deletion back. + const ancient = new Date(Date.now() - 1000 * ORPHAN_AGE_MS); + await store.append({ + entry: birthEntry(runId, env, ancient), + kind: "birth", + isTerminal: false, + }); + + const first = await sweeper.sweep(); + + expect(first.deleted).toBe(0); + expect(first.pendingDeletion).toBe(1); + expect(await probe.exists(snapshotKeys(runId).e)).toBe(1); + + const second = await sweeper.sweep(); + + expect(second.deleted).toBe(1); + expect(await probe.exists(snapshotKeys(runId).e)).toBe(0); + } finally { + await Promise.all([store.quit(), sweeper.quit(), probe.quit().catch(() => {})]); + } + } + ); + + containerTest( + "a marked keyspace is not deleted until the confirm window has passed", + async ({ prisma, redisOptions }) => { + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + const runStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const sweeper = new SnapshotOrphanSweeper({ + redisOptions, + runStore: runStore as unknown as RunStore, + completedTtlMs: COMPLETED_TTL_MS, + orphanAgeMs: ORPHAN_AGE_MS, + // An hour. Passes minutes apart must not convert a candidate. + confirmOrphanAfterMs: 60 * 60 * 1000, + }); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + await store.append({ + entry: birthEntry(runId, env, new Date(Date.now() - 2 * ORPHAN_AGE_MS)), + kind: "birth", + isTerminal: false, + }); + + await sweeper.sweep(); + const second = await sweeper.sweep(); + const third = await sweeper.sweep(); + + expect(second.deleted).toBe(0); + expect(second.pendingDeletion).toBe(1); + expect(third.deleted).toBe(0); + expect(await probe.exists(snapshotKeys(runId).e)).toBe(1); + } finally { + await Promise.all([store.quit(), sweeper.quit(), probe.quit().catch(() => {})]); + } + } + ); + + containerTest( + "a transient miss followed by a found run does not delete, and does not leave the keyspace pre-authorised", + async ({ prisma, redisOptions }) => { + // The case the guard exists for. Pass 1 gets an incomplete answer and marks the keyspace. + // Pass 2 sees the run, so it must clear the mark: were the mark to survive, a LATER genuine + // absence would delete on its own first sighting and the two-sighting rule would be gone. + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + const real = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + + let lie = true; + const flaky = { + ...real, + findRunsByIds: (...args: unknown[]) => + lie + ? // Succeeds and is incomplete: exactly what the catch cannot see. + Promise.resolve(new Map()) + : (real.findRunsByIds as (...rest: unknown[]) => Promise>).apply( + real, + args + ), + } as unknown as RunStore; + + const sweeper = new SnapshotOrphanSweeper({ + redisOptions, + runStore: flaky, + completedTtlMs: COMPLETED_TTL_MS, + orphanAgeMs: ORPHAN_AGE_MS, + confirmOrphanAfterMs: 0, + }); + + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + await store.append({ + entry: birthEntry(runId, env, new Date(Date.now() - 2 * ORPHAN_AGE_MS)), + kind: "birth", + isTerminal: false, + }); + // The run is alive and terminal in Postgres the whole time. Only the lookup lies. + await prisma.taskRun.create({ + data: { ...buildCreateRunData(runId, env), status: "COMPLETED_SUCCESSFULLY" }, + }); + + const first = await sweeper.sweep(); + expect(first.deleted).toBe(0); + expect(first.pendingDeletion).toBe(1); + + lie = false; + const second = await sweeper.sweep(); + expect(second.deleted).toBe(0); + expect(await probe.exists(snapshotKeys(runId).e)).toBe(1); + + // The run vanishes for real. With the mark cleared this is a first sighting again. + lie = true; + const third = await sweeper.sweep(); + expect(third.deleted).toBe(0); + expect(third.pendingDeletion).toBe(1); + expect(await probe.exists(snapshotKeys(runId).e)).toBe(1); + + const fourth = await sweeper.sweep(); + expect(fourth.deleted).toBe(1); + } finally { + await Promise.all([store.quit(), sweeper.quit(), probe.quit().catch(() => {})]); + } + } + ); +}); + +describe("the marker cannot expire out from under a candidate", () => { + containerTest( + "a marked keyspace still carries its mark after the whole keyspace is re-read", + async ({ prisma, redisOptions }) => { + // The marker used to be a key with its own TTL derived from the confirm window, which could + // be shorter than the interval between passes: the marker written at T was gone by + // T+interval, every pass wrote a fresh one, and rule 2 deleted nothing while reporting clean. + // It is now a field on the run's `seq` hash, so it lives exactly as long as the keyspace and + // there is no lifetime left to misconfigure. + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + const runStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const sweeper = new SnapshotOrphanSweeper({ + redisOptions, + runStore: runStore as unknown as RunStore, + completedTtlMs: COMPLETED_TTL_MS, + orphanAgeMs: ORPHAN_AGE_MS, + confirmOrphanAfterMs: 0, + }); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + await store.append({ + entry: birthEntry(runId, env, new Date(Date.now() - 2 * ORPHAN_AGE_MS)), + kind: "birth", + isTerminal: false, + }); + + expect((await sweeper.sweep()).pendingDeletion).toBe(1); + + // The mark is a field on seq, and it carries no expiry of its own. + expect(await probe.hget(snapshotKeys(runId).seq, "orph")).not.toBeNull(); + expect(await probe.pttl(snapshotKeys(runId).seq)).toBe(-1); + + expect((await sweeper.sweep()).deleted).toBe(1); + // And it went with the keyspace rather than outliving it. + expect(await probe.exists(snapshotKeys(runId).seq)).toBe(0); + } finally { + await Promise.all([store.quit(), sweeper.quit(), probe.quit().catch(() => {})]); + } + } + ); +}); + +describe("a run seen alive clears its marker", () => { + containerTest( + "a lie, then a LIVE run, then a lie again does not delete", + async ({ prisma, redisOptions }) => { + // The hole a long marker lifetime opens. Only terminal runs used to clear the marker, so a + // keyspace marked by an incomplete lookup and then seen ALIVE kept its mark. A later genuine + // absence would then find a mature marker and delete on what is really a first sighting. + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + const real = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + + let lie = true; + const flaky = { + ...real, + findRunsByIds: (...args: unknown[]) => + lie + ? Promise.resolve(new Map()) + : (real.findRunsByIds as (...rest: unknown[]) => Promise>).apply( + real, + args + ), + } as unknown as RunStore; + + const sweeper = new SnapshotOrphanSweeper({ + redisOptions, + runStore: flaky, + completedTtlMs: COMPLETED_TTL_MS, + orphanAgeMs: ORPHAN_AGE_MS, + confirmOrphanAfterMs: 0, + }); + + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + await store.append({ + entry: birthEntry(runId, env, new Date(Date.now() - 2 * ORPHAN_AGE_MS)), + kind: "birth", + isTerminal: false, + }); + // EXECUTING, not terminal. This run never reaches rule 1, so rule 1 cannot be what clears + // the mark; a SUSPENDED run can legitimately sit here for weeks. + await prisma.taskRun.create({ + data: { ...buildCreateRunData(runId, env), status: "EXECUTING" }, + }); + + expect((await sweeper.sweep()).pendingDeletion).toBe(1); + + lie = false; + const seenAlive = await sweeper.sweep(); + expect(seenAlive.skipped).toBeGreaterThan(0); + expect(seenAlive.deleted).toBe(0); + + lie = true; + const afterAlive = await sweeper.sweep(); + // A first sighting again, because being seen alive cleared the mark. + expect(afterAlive.deleted).toBe(0); + expect(afterAlive.pendingDeletion).toBe(1); + expect(await probe.exists(snapshotKeys(runId).e)).toBe(1); + } finally { + await Promise.all([store.quit(), sweeper.quit(), probe.quit().catch(() => {})]); + } + } + ); +}); + +describe("a pass can stop inside a budget", () => { + containerTest( + "an already-passed deadline yields a partial pass", + async ({ prisma, redisOptions }) => { + const runStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const sweeper = new SnapshotOrphanSweeper({ + redisOptions, + runStore: runStore as unknown as RunStore, + completedTtlMs: COMPLETED_TTL_MS, + }); + + try { + const result = await sweeper.sweep({ deadline: Date.now() - 1 }); + + // redis-worker redelivers a job that outlives its visibility timeout, and nothing extends it, + // so a pass that cannot stop on its own runs concurrently with itself. + expect(result.partial).toBe(true); + expect(result.scanned).toBe(0); + } finally { + await sweeper.quit(); + } + } + ); + + containerTest("an aborted signal yields a partial pass", async ({ prisma, redisOptions }) => { + const runStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const sweeper = new SnapshotOrphanSweeper({ + redisOptions, + runStore: runStore as unknown as RunStore, + completedTtlMs: COMPLETED_TTL_MS, + }); + const controller = new AbortController(); + controller.abort(); + + try { + const result = await sweeper.sweep({ signal: controller.signal }); + expect(result.partial).toBe(true); + } finally { + await sweeper.quit(); + } + }); + + containerTest("a pass with budget to spare is not partial", async ({ prisma, redisOptions }) => { + const runStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const sweeper = new SnapshotOrphanSweeper({ + redisOptions, + runStore: runStore as unknown as RunStore, + completedTtlMs: COMPLETED_TTL_MS, + }); + + try { + const result = await sweeper.sweep({ deadline: Date.now() + 60_000 }); + expect(result.partial).toBe(false); + } finally { + await sweeper.quit(); + } + }); +}); diff --git a/internal-packages/run-store/src/snapshotOrphanSweeper.test.ts b/internal-packages/run-store/src/snapshotOrphanSweeper.test.ts new file mode 100644 index 00000000000..7d7eb2aca0d --- /dev/null +++ b/internal-packages/run-store/src/snapshotOrphanSweeper.test.ts @@ -0,0 +1,469 @@ +// The sweep deletes whole keyspaces, so most of these tests are about what it must NOT touch: a live +// run, a young orphan, and any batch whose Postgres lookup did not come back. +import { describe, expect } from "vitest"; +import { containerTest } from "@internal/testcontainers"; +import { createRedisClient } from "@internal/redis"; +import { generateInternalId } from "@trigger.dev/core/v3/isomorphic"; +import { PostgresRunStore } from "./PostgresRunStore.js"; +import { RedisSnapshotStore, snapshotKeys } from "./redisSnapshotStore.js"; +import { entryFromCreateRun } from "./snapshotEntry.js"; +import { SnapshotOrphanSweeper } from "./snapshotOrphanSweeper.js"; +import type { RunStore } from "./types.js"; +import { + buildCreateRunData, + seedSnapshotEnvironment, + type SnapshotFixtureEnv, +} from "./testFixtures/snapshotIdFixture.js"; + +const COMPLETED_TTL_MS = 72 * 60 * 60 * 1000; +const ORPHAN_AGE_MS = 60 * 60 * 1000; + +function birthEntry(runId: string, env: SnapshotFixtureEnv, createdAt: Date, terminal = false) { + const snapshot = { + id: generateInternalId(), + engine: "V2" as const, + executionStatus: terminal ? ("FINISHED" as const) : ("RUN_CREATED" as const), + description: "Run was created", + runStatus: terminal ? ("CANCELED" as const) : ("PENDING" as const), + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }; + return entryFromCreateRun({ id: snapshot.id, runId, createdAt }, snapshot); +} + +describe("SnapshotOrphanSweeper", () => { + containerTest( + "rule 1 expires a terminal run whose keyspace never got one", + async ({ prisma, redisOptions }) => { + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + const runStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const sweeper = new SnapshotOrphanSweeper({ + redisOptions, + runStore: runStore as unknown as RunStore, + completedTtlMs: COMPLETED_TTL_MS, + orphanAgeMs: ORPHAN_AGE_MS, + // Rule 2 needs two sightings. These cases are about WHICH keyspaces it picks, not about + // the confirm window, so the window is zero and they sweep twice. The window itself has + // its own tests below. + confirmOrphanAfterMs: 0, + }); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + // Non-terminal append, so no expiry is ever set — the lost-TTL-set case. + await store.append({ + entry: birthEntry(runId, env, new Date()), + kind: "birth", + isTerminal: false, + }); + await prisma.taskRun.create({ + data: { ...buildCreateRunData(runId, env), status: "COMPLETED_SUCCESSFULLY" }, + }); + + const keys = snapshotKeys(runId); + expect(await probe.pttl(keys.e)).toBe(-1); + + const result = await sweeper.sweep(); + + expect(result.expired).toBe(1); + expect(result.deleted).toBe(0); + for (const key of [keys.e, keys.idx, keys.cur, keys.seq]) { + const ttl = await probe.pttl(key); + expect(ttl).toBeGreaterThan(0); + expect(ttl).toBeLessThanOrEqual(COMPLETED_TTL_MS); + } + } finally { + await Promise.all([store.quit(), sweeper.quit(), probe.quit().catch(() => {})]); + } + } + ); + + containerTest( + "rule 1 leaves a keyspace that already has an expiry alone", + async ({ prisma, redisOptions }) => { + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + const runStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const sweeper = new SnapshotOrphanSweeper({ + redisOptions, + runStore: runStore as unknown as RunStore, + completedTtlMs: COMPLETED_TTL_MS, + orphanAgeMs: ORPHAN_AGE_MS, + // Rule 2 needs two sightings. These cases are about WHICH keyspaces it picks, not about + // the confirm window, so the window is zero and they sweep twice. The window itself has + // its own tests below. + confirmOrphanAfterMs: 0, + }); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + // A healthy terminal append sets the completion TTL itself. + await store.append({ + entry: birthEntry(runId, env, new Date(), true), + kind: "birth", + isTerminal: true, + }); + await prisma.taskRun.create({ + data: { ...buildCreateRunData(runId, env), status: "CANCELED" }, + }); + + const before = await probe.pttl(snapshotKeys(runId).e); + const result = await sweeper.sweep(); + + expect(result.expired).toBe(0); + expect(result.skipped).toBe(1); + const after = await probe.pttl(snapshotKeys(runId).e); + // Not extended: the sweep must not keep resetting a countdown that is already running. + expect(after).toBeLessThanOrEqual(before); + } finally { + await Promise.all([store.quit(), sweeper.quit(), probe.quit().catch(() => {})]); + } + } + ); + + containerTest( + "rule 2 deletes a keyspace with no run row, cycle keys included", + async ({ prisma, redisOptions }) => { + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + const runStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const sweeper = new SnapshotOrphanSweeper({ + redisOptions, + runStore: runStore as unknown as RunStore, + completedTtlMs: COMPLETED_TTL_MS, + orphanAgeMs: ORPHAN_AGE_MS, + // Rule 2 needs two sightings. These cases are about WHICH keyspaces it picks, not about + // the confirm window, so the window is zero and they sweep twice. The window itself has + // its own tests below. + confirmOrphanAfterMs: 0, + }); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + const old = new Date(Date.now() - 2 * ORPHAN_AGE_MS); + + // The crashed birth: an entry, no Postgres run, non-terminal so no expiry. + await store.append({ + entry: birthEntry(runId, env, old), + kind: "birth", + isTerminal: false, + }); + await store.append({ + entry: birthEntry(runId, env, old), + kind: "transition", + isTerminal: false, + cycle: { kind: "new", completedWaitpoints: [{ id: "w_1", index: 0 }] }, + }); + + const cyclesBefore = await probe.keys(`snap:{${runId}}:wp:*`); + expect(cyclesBefore.length).toBeGreaterThan(0); + + await sweeper.sweep(); + const result = await sweeper.sweep(); + + expect(result.deleted).toBe(1); + const keys = snapshotKeys(runId); + for (const key of [keys.e, keys.idx, keys.cur, keys.seq]) { + expect(await probe.exists(key)).toBe(0); + } + expect(await probe.keys(`snap:{${runId}}:wp:*`)).toEqual([]); + } finally { + await Promise.all([store.quit(), sweeper.quit(), probe.quit().catch(() => {})]); + } + } + ); + + containerTest("rule 2 spares a young orphan", async ({ prisma, redisOptions }) => { + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + const runStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const sweeper = new SnapshotOrphanSweeper({ + redisOptions, + runStore: runStore as unknown as RunStore, + completedTtlMs: COMPLETED_TTL_MS, + orphanAgeMs: ORPHAN_AGE_MS, + confirmOrphanAfterMs: 0, + }); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + // Written just now: the Postgres insert of a healthy birth may still be in flight. + await store.append({ + entry: birthEntry(runId, env, new Date()), + kind: "birth", + isTerminal: false, + }); + + const result = await sweeper.sweep(); + + expect(result.deleted).toBe(0); + expect(result.skipped).toBe(1); + expect(await probe.exists(snapshotKeys(runId).e)).toBe(1); + } finally { + await Promise.all([store.quit(), sweeper.quit(), probe.quit().catch(() => {})]); + } + }); + + containerTest( + "never touches a live run, however old its keyspace", + async ({ prisma, redisOptions }) => { + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + const runStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const sweeper = new SnapshotOrphanSweeper({ + redisOptions, + runStore: runStore as unknown as RunStore, + completedTtlMs: COMPLETED_TTL_MS, + orphanAgeMs: ORPHAN_AGE_MS, + // Rule 2 needs two sightings. These cases are about WHICH keyspaces it picks, not about + // the confirm window, so the window is zero and they sweep twice. The window itself has + // its own tests below. + confirmOrphanAfterMs: 0, + }); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + const ancient = new Date(Date.now() - 90 * 24 * 60 * 60 * 1000); + + // A run waiting on an untimed token can sit non-terminal for weeks. Reaping it would drop + // live state, which is the failure this rule exists to avoid. + await store.append({ + entry: birthEntry(runId, env, ancient), + kind: "birth", + isTerminal: false, + }); + await prisma.taskRun.create({ + data: { ...buildCreateRunData(runId, env), status: "WAITING_TO_RESUME" }, + }); + + const result = await sweeper.sweep(); + + expect(result.deleted).toBe(0); + expect(result.expired).toBe(0); + expect(result.skipped).toBe(1); + expect(await probe.exists(snapshotKeys(runId).e)).toBe(1); + expect(await probe.pttl(snapshotKeys(runId).e)).toBe(-1); + } finally { + await Promise.all([store.quit(), sweeper.quit(), probe.quit().catch(() => {})]); + } + } + ); + + containerTest("a dry run reports but changes nothing", async ({ prisma, redisOptions }) => { + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + const runStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const sweeper = new SnapshotOrphanSweeper({ + redisOptions, + runStore: runStore as unknown as RunStore, + completedTtlMs: COMPLETED_TTL_MS, + orphanAgeMs: ORPHAN_AGE_MS, + confirmOrphanAfterMs: 0, + }); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + try { + const env = await seedSnapshotEnvironment(prisma); + const orphan = generateInternalId(); + const terminal = generateInternalId(); + const old = new Date(Date.now() - 2 * ORPHAN_AGE_MS); + + await store.append({ entry: birthEntry(orphan, env, old), kind: "birth", isTerminal: false }); + await store.append({ + entry: birthEntry(terminal, env, new Date()), + kind: "birth", + isTerminal: false, + }); + await prisma.taskRun.create({ + data: { ...buildCreateRunData(terminal, env), status: "COMPLETED_SUCCESSFULLY" }, + }); + + const result = await sweeper.sweep({ dryRun: true }); + + // A dry pass writes no marker, so an unconfirmed rule 2 candidate reports as pending rather + // than as a deletion. That is what a real pass would do at this instant, which is the honest + // answer for a preview: nothing is confirmed yet, so nothing would be deleted yet. + expect(result.deleted).toBe(0); + expect(result.pendingDeletion).toBe(1); + expect(result.expired).toBe(1); + expect(await probe.exists(snapshotKeys(orphan).e)).toBe(1); + expect(await probe.pttl(snapshotKeys(terminal).e)).toBe(-1); + } finally { + await Promise.all([store.quit(), sweeper.quit(), probe.quit().catch(() => {})]); + } + }); + + containerTest( + "skips a batch whose run lookup failed, and deletes nothing", + async ({ prisma, redisOptions }) => { + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + const failing = { + findRunsByIds: async () => { + throw new Error("run lookup unavailable"); + }, + } as unknown as RunStore; + const sweeper = new SnapshotOrphanSweeper({ + redisOptions, + runStore: failing, + completedTtlMs: COMPLETED_TTL_MS, + orphanAgeMs: ORPHAN_AGE_MS, + // Rule 2 needs two sightings. These cases are about WHICH keyspaces it picks, not about + // the confirm window, so the window is zero and they sweep twice. The window itself has + // its own tests below. + confirmOrphanAfterMs: 0, + }); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + const old = new Date(Date.now() - 2 * ORPHAN_AGE_MS); + await store.append({ + entry: birthEntry(runId, env, old), + kind: "birth", + isTerminal: false, + }); + + // A failed lookup says nothing about whether the run exists, and rule 2 deletes a whole + // keyspace. The sweep must resolve rather than throw, and must reap nothing. + const result = await sweeper.sweep(); + + expect(result.deleted).toBe(0); + expect(result.skipped).toBeGreaterThan(0); + expect(await probe.exists(snapshotKeys(runId).e)).toBe(1); + } finally { + await Promise.all([store.quit(), sweeper.quit(), probe.quit().catch(() => {})]); + } + } + ); + + containerTest( + "discovers and reaps a keyspace whose entries are all invalid", + async ({ prisma, redisOptions }) => { + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + const runStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const sweeper = new SnapshotOrphanSweeper({ + redisOptions, + runStore: runStore as unknown as RunStore, + completedTtlMs: COMPLETED_TTL_MS, + orphanAgeMs: ORPHAN_AGE_MS, + // Rule 2 needs two sightings. These cases are about WHICH keyspaces it picks, not about + // the confirm window, so the window is zero and they sweep twice. The window itself has + // its own tests below. + confirmOrphanAfterMs: 0, + }); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + const old = new Date(Date.now() - 2 * ORPHAN_AGE_MS); + + // The append script writes `cur` and indexes the entry only when it is valid, so a keyspace + // whose entries all carry an error has neither. A sweep that discovers keyspaces by their + // `cur` key would never see this one, and neither rule would ever apply to it. + await store.append({ + entry: { ...birthEntry(runId, env, old), error: "stale write" }, + kind: "birth", + isTerminal: false, + }); + + const keys = snapshotKeys(runId); + expect(await probe.exists(keys.e)).toBe(1); + expect(await probe.exists(keys.cur)).toBe(0); + + await sweeper.sweep(); + const result = await sweeper.sweep(); + + expect(result.deleted).toBe(1); + expect(await probe.exists(keys.e)).toBe(0); + expect(await probe.exists(keys.seq)).toBe(0); + } finally { + await Promise.all([store.quit(), sweeper.quit(), probe.quit().catch(() => {})]); + } + } + ); + + containerTest( + "still finds keyspaces when the client carries a key prefix", + async ({ prisma, redisOptions }) => { + // ioredis prepends its keyPrefix to keys for ordinary commands, but NOT to a SCAN MATCH + // pattern, and it returns matched keys with the prefix still on them. The engine sets a + // prefix on every other Redis client it builds, so a sweep that ignored this would match + // nothing and report a clean pass: a safety net that silently protects nothing. + const prefixed = { ...(redisOptions as object), keyPrefix: "engine:" } as never; + const store = new RedisSnapshotStore({ + redisOptions: prefixed, + completedTtlMs: COMPLETED_TTL_MS, + }); + const runStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const sweeper = new SnapshotOrphanSweeper({ + redisOptions: prefixed, + runStore: runStore as unknown as RunStore, + completedTtlMs: COMPLETED_TTL_MS, + orphanAgeMs: ORPHAN_AGE_MS, + // Rule 2 needs two sightings. These cases are about WHICH keyspaces it picks, not about + // the confirm window, so the window is zero and they sweep twice. The window itself has + // its own tests below. + confirmOrphanAfterMs: 0, + }); + const probe = createRedisClient(prefixed, { onError: () => {} }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + const old = new Date(Date.now() - 2 * ORPHAN_AGE_MS); + + await store.append({ + entry: birthEntry(runId, env, old), + kind: "birth", + isTerminal: false, + cycle: { kind: "new", completedWaitpoints: [{ id: "w_1", index: 0 }] }, + }); + + await sweeper.sweep(); + const result = await sweeper.sweep(); + + expect(result.scanned).toBe(1); + expect(result.deleted).toBe(1); + expect(await probe.exists(snapshotKeys(runId).e)).toBe(0); + expect(await probe.exists(`snap:{${runId}}:wp:1`)).toBe(0); + } finally { + await Promise.all([store.quit(), sweeper.quit(), probe.quit().catch(() => {})]); + } + } + ); + + containerTest("processes every keyspace across batches", async ({ prisma, redisOptions }) => { + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + const runStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const sweeper = new SnapshotOrphanSweeper({ + redisOptions, + runStore: runStore as unknown as RunStore, + completedTtlMs: COMPLETED_TTL_MS, + orphanAgeMs: ORPHAN_AGE_MS, + confirmOrphanAfterMs: 0, + }); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + try { + const env = await seedSnapshotEnvironment(prisma); + const old = new Date(Date.now() - 2 * ORPHAN_AGE_MS); + const orphans = Array.from({ length: 5 }, () => generateInternalId()); + for (const runId of orphans) { + await store.append({ + entry: birthEntry(runId, env, old), + kind: "birth", + isTerminal: false, + }); + } + + await sweeper.sweep({ batchSize: 2 }); + const result = await sweeper.sweep({ batchSize: 2 }); + + expect(result.deleted).toBe(5); + for (const runId of orphans) { + expect(await probe.exists(snapshotKeys(runId).e)).toBe(0); + } + } finally { + await Promise.all([store.quit(), sweeper.quit(), probe.quit().catch(() => {})]); + } + }); +}); diff --git a/internal-packages/run-store/src/snapshotOrphanSweeper.ts b/internal-packages/run-store/src/snapshotOrphanSweeper.ts new file mode 100644 index 00000000000..86bb084d5c9 --- /dev/null +++ b/internal-packages/run-store/src/snapshotOrphanSweeper.ts @@ -0,0 +1,567 @@ +// Reaps snapshot keyspaces that no healthy path will ever clean up. +// +// Two rules, because neither can see what the other leaves behind: +// +// 1. The run is terminal in Postgres but its keyspace never got the completion expiry — a +// terminal append whose TTL-set was lost. Applying the expiry now reaps it on the same +// schedule a healthy terminal append would have. +// 2. The keyspace has no Postgres run row at all, and is older than a threshold — a crashed +// birth. It is non-terminal so it carries no expiry, and it has no run row, so rule 1 can +// never match it. Without this rule that leak has no bound. +// +// Nothing schedules this. The engine's worker is what has to run it, and run-store cannot reach the +// engine, so the wiring belongs to the ticket that owns production construction. +import { + Cluster, + createRedisClient, + type Redis, + type RedisClient, + type RedisOptions, +} from "@internal/redis"; +import { Logger } from "@trigger.dev/core/logger"; +import type { TaskRunStatus } from "@trigger.dev/database"; +import { snapshotKeys } from "./redisSnapshotStore.js"; +import type { RunStore } from "./types.js"; + +/** + * Mirrors the engine's `finalStatuses`. run-store cannot import from run-engine — the dependency + * runs the other way — so the list is duplicated and a parity test in run-engine asserts the copy + * stays equal to the original. + */ +export const FINAL_RUN_STATUSES: readonly TaskRunStatus[] = [ + "CANCELED", + "INTERRUPTED", + "COMPLETED_SUCCESSFULLY", + "COMPLETED_WITH_ERRORS", + "SYSTEM_FAILURE", + "CRASHED", + "EXPIRED", + "TIMED_OUT", +]; + +const FINAL = new Set(FINAL_RUN_STATUSES); + +/** Comfortably above run-creation latency, so a birth in flight is never mistaken for an orphan. */ +const DEFAULT_ORPHAN_AGE_MS = 24 * 60 * 60 * 1000; + +/** + * The keyspace prefix, owned by `snapshotKeys` in the store rather than configurable here. A sweep + * that could be pointed at a different prefix would be a fiction: the store writes `snap:` keys + * unconditionally, so there is no other keyspace to point it at. + */ +const SNAPSHOT_KEYSPACE_PREFIX = "snap:"; +const DEFAULT_BATCH_SIZE = 1000; + +/** + * How long a rule 2 candidate must have been marked before it may be deleted. It has to exceed the + * interval between passes, or a candidate is never sighted twice and never converts. + */ +const DEFAULT_ORPHAN_CONFIRM_MS = 60 * 60 * 1000; + +export type SweepResult = { + /** Keyspaces examined. */ + scanned: number; + /** Rule 1: terminal runs whose keyspace was given the completion expiry. */ + expired: number; + /** Rule 2: keyspaces with no run row, deleted. */ + deleted: number; + /** Left alone: a live run, a young orphan, or a batch whose Postgres lookup failed. */ + skipped: number; + /** + * Rule 2 candidates that were marked but not deleted, because deletion needs a second sighting + * in a later pass. A number that never converts to `deleted` means the confirm window is longer + * than the interval between passes, or the marker TTL is shorter than it. + */ + pendingDeletion: number; + /** + * Connections the pass iterated: every master of a cluster, or 1 standalone. Reported because the + * failure this component cannot tolerate is a false green, and a pass that covered one node of + * six is indistinguishable from a complete one by any other field here. TRI-13453 gates the dial + * on an observed sweep pass, so the observation has to carry its own coverage. + */ + nodes: number; + /** True when the pass stopped early on its deadline or abort signal, so coverage is incomplete. */ + partial: boolean; +}; + +/** + * Exactly one of `redisOptions` or `client`, enforced by the type rather than a runtime check. + * With `redisOptions` the sweep opens its OWN connection, which is the preferred shape: a long + * scan can then never stall a hot-path client. `client` exists for a caller that has already built + * a client and wants the sweep to use it; a borrowed client is left open by `quit()`. + * + * What the sweep needs is a connection of its OWN, not one it built itself. A caller pointing at a + * cluster should build a SECOND, sweep-dedicated cluster client and pass it here: that keeps a long + * scan off the hot path just as well as `redisOptions` does. Handing over the client the snapshot + * store is using is the case to avoid. + */ +export type SnapshotOrphanSweeperConnection = + | { client: RedisClient; redisOptions?: never } + | { client?: never; redisOptions: RedisOptions }; + +export type SnapshotOrphanSweeperOptions = SnapshotOrphanSweeperConnection & { + /** + * Resolved through the run store, not a raw client. Under the run-ops split a run row can live on + * either database, and only the store knows which — a raw lookup would report a live run as an + * orphan and delete its keyspace. + */ + runStore: RunStore; + completedTtlMs: number; + orphanAgeMs?: number; + /** + * How long a rule 2 candidate must stay marked before the sweep will delete it. Defaults to one + * hour. + * + * Set it at or below the interval between passes, or the second sighting arrives too early to + * count and every candidate needs three passes instead of two. It does NOT need to exceed the + * interval; the constraint people reach for ("longer than the interval") is the wrong one and + * only costs latency. + * + * There is no marker-lifetime constraint to satisfy alongside it. The marker is a field on the + * run's `seq` hash, so it lives exactly as long as the keyspace it describes: it cannot expire + * out from under a candidate that is still waiting for its second sighting, and it cannot outlive + * a keyspace that was deleted. + */ + confirmOrphanAfterMs?: number; + logger?: Logger; +}; + +export class SnapshotOrphanSweeper { + readonly #redis: RedisClient; + /** Only a client this class opened may be closed by it. */ + readonly #ownsClient: boolean; + readonly #runStore: RunStore; + readonly #completedTtlMs: number; + readonly #orphanAgeMs: number; + readonly #confirmOrphanAfterMs: number; + /** + * The ioredis client-level prefix, which is NOT the keyspace prefix. ioredis prepends it to keys + * for ordinary commands, but it does not prepend it to a SCAN MATCH pattern, and it does return + * matched keys with it still attached. Unhandled, a prefixed client makes the sweep match nothing + * and report a clean pass, which is the worst outcome for a safety net. + */ + readonly #clientPrefix: string; + readonly #logger: Logger; + #quit?: Promise; + + constructor(options: SnapshotOrphanSweeperOptions) { + this.#logger = options.logger ?? new Logger("SnapshotOrphanSweeper", "debug"); + this.#runStore = options.runStore; + this.#completedTtlMs = options.completedTtlMs; + this.#orphanAgeMs = options.orphanAgeMs ?? DEFAULT_ORPHAN_AGE_MS; + this.#confirmOrphanAfterMs = options.confirmOrphanAfterMs ?? DEFAULT_ORPHAN_CONFIRM_MS; + this.#ownsClient = options.client === undefined; + this.#redis = + options.client ?? + createRedisClient(options.redisOptions, { + onError: (error) => + this.#logger.error("SnapshotOrphanSweeper redis client error", { error }), + }); + this.#clientPrefix = clientPrefixOf(this.#redis); + } + + async quit(): Promise { + // A borrowed client belongs to the caller; closing it here would take down a connection the + // snapshot store may still be using. + if (!this.#ownsClient) return; + if (!this.#quit) { + this.#quit = this.#redis.quit().then( + () => undefined, + () => undefined + ); + } + await this.#quit; + } + + /** + * One full pass over the keyspace. `dryRun` reports what it would do and changes nothing. + * + * `deadline` and `signal` let the caller stop a pass cleanly instead of having it killed + * mid-cursor. The scheduler needs this: redis-worker moves a dequeued item's score to + * `now + visibilityTimeoutMs` and nothing extends it, so a pass that outlives its timeout is + * redelivered and runs concurrently with itself. A pass that stops inside its budget cannot. + * + * Whichever way it stops, `partial` comes back true. Reporting a truncated pass as a full one is + * the same false green as under-scanning a cluster: TRI-13453 gates the dial on an OBSERVED + * sweep pass, so the observation has to say how much of the keyspace it actually reached. + */ + async sweep(opts?: { + batchSize?: number; + dryRun?: boolean; + /** Epoch ms. The pass stops at the next batch boundary once passed. */ + deadline?: number; + signal?: AbortSignal; + }): Promise { + const batchSize = opts?.batchSize ?? DEFAULT_BATCH_SIZE; + const dryRun = opts?.dryRun ?? false; + const result: SweepResult = { + scanned: 0, + expired: 0, + deleted: 0, + skipped: 0, + pendingDeletion: 0, + nodes: 0, + partial: false, + }; + + // Checked at batch boundaries only. Stopping mid-batch would leave a run half-acted-on, and a + // batch is bounded work, so the boundary is both the safe and the timely place. + const outOfBudget = () => + opts?.signal?.aborted === true || + (opts?.deadline !== undefined && Date.now() >= opts.deadline); + + // SCAN carries no key, so a cluster cannot route it: one connection iterates ONE node's + // keyspace and then reports a completed cursor. A single-client sweep against a cluster would + // therefore return a clean-looking result having examined roughly 1/N of the keyspace, and the + // rest would leak with nothing to revisit it. Both rules are unbounded leaks when missed, so + // the pass fans out over every master and only reports done when all of them are done. + const nodes = this.#scanTargets(); + result.nodes = nodes.length; + + for (const node of nodes) { + if (outOfBudget()) { + result.partial = true; + break; + } + + let cursor = "0"; + do { + // Match on the entry hash, not on `cur`. The append script writes `cur` only when the entry + // is valid, so a keyspace whose entries are all invalid would never be discovered and would + // leak with no expiry, which is the same unbounded leak rule 2 exists to close. `e` is + // written by every append. + const [next, keys] = await node.scan( + cursor, + "MATCH", + `${this.#clientPrefix}${SNAPSHOT_KEYSPACE_PREFIX}{*}:e`, + "COUNT", + batchSize + ); + cursor = next; + + const runIds = [...new Set(keys.map((key) => this.#runIdFrom(key)).filter(isString))]; + if (runIds.length === 0) continue; + + await this.#sweepBatch(runIds, dryRun, result); + + if (outOfBudget()) { + // A cursor mid-iteration means this node is not finished, so the pass is not either. + result.partial = true; + break; + } + } while (cursor !== "0"); + + if (result.partial) break; + } + + this.#logger.log("SnapshotOrphanSweeper pass complete", { ...result, dryRun }); + return result; + } + + #scanTargets(): Redis[] { + return scanTargetsOf(this.#redis); + } + + async #sweepBatch(runIds: string[], dryRun: boolean, result: SweepResult): Promise { + result.scanned += runIds.length; + + let rows: Map; + try { + rows = (await this.#runStore.findRunsByIds(runIds, { + select: { id: true, status: true }, + })) as unknown as Map; + } catch (error) { + // Never reap on an unknown answer. A lookup that failed says nothing about whether the run + // exists, and rule 2 deletes a whole keyspace. + this.#logger.error("SnapshotOrphanSweeper skipped a batch after a failed run lookup", { + count: runIds.length, + error, + }); + result.skipped += runIds.length; + return; + } + + // Every run that EXISTS clears its rule 2 marker, live ones included. It has to be every one, + // not just the terminal ones: a keyspace marked by an incomplete lookup, then seen alive, then + // missed again would otherwise present a mature marker on what is really a first sighting, and + // the two-sighting rule would be gone exactly when it was needed. This costs one DEL per + // existing run per pass, which is the price of the guard being sound rather than nearly sound. + const present = runIds.filter((runId) => rows.has(runId)); + if (!dryRun && present.length > 0) { + // Individual commands, never one pipeline: these keys span runs, so they span cluster slots. + await Promise.all(present.map((runId) => this.#clearOrphanMarker(runId))); + } + + for (const runId of runIds) { + const run = rows.get(runId); + + if (!run) { + await this.#applyRuleTwo(runId, dryRun, result); + continue; + } + + if (!FINAL.has(run.status)) { + // A live run. A SUSPENDED run can legitimately wait for weeks, so this is never touched. + result.skipped += 1; + continue; + } + + await this.#applyRuleOne(runId, dryRun, result); + } + } + + /** Rule 1: a terminal run whose keyspace never received the completion expiry. */ + async #applyRuleOne(runId: string, dryRun: boolean, result: SweepResult): Promise { + const keys = await this.#allKeys(runId); + if (keys.length === 0) { + result.skipped += 1; + return; + } + + const ttls = await Promise.all(keys.map((key) => this.#redis.pttl(key))); + // -1 is "exists, no expiry". Anything already counting down was set by a healthy append. + if (!ttls.some((ttl) => ttl === -1)) { + result.skipped += 1; + return; + } + + if (!dryRun) { + const pipeline = this.#redis.pipeline(); + for (const key of keys) { + pipeline.pexpire(key, this.#completedTtlMs); + } + await pipeline.exec(); + } + + result.expired += 1; + } + + /** + * Rule 2: a keyspace with no run row at all, past the age threshold. + * + * TWO SIGHTINGS ARE REQUIRED. The `catch` in #sweepBatch covers a lookup that THROWS, but it + * cannot see a lookup that succeeds and is incomplete: a row that exists but did not come back + * reads exactly like a run that never existed, and the response to that is deleting a live run's + * execution state. `findRunsByIds` routes through RoutingRunStore.#findRunsByIdSet, which + * partitions ids by residency and asks each store only for its own — and with no client passed it + * reads each store's REPLICA. Both are sound today (id classification is authoritative for runs, + * and replica lag is nowhere near the 24h age gate), but each is an assumption held somewhere + * else in the codebase, not something this delete path can check. + * + * The asymmetry decides it: a false negative leaks keys, which is bounded and recoverable, while + * a false positive destroys live state. So an absent row marks the keyspace and returns; only a + * candidate still absent in a LATER pass is deleted. Any transient incomplete answer, whatever + * its cause, has to occur twice across the confirm window to do damage. + */ + async #applyRuleTwo(runId: string, dryRun: boolean, result: SweepResult): Promise { + const keys = await this.#allKeys(runId); + if (keys.length === 0) { + result.skipped += 1; + return; + } + + const age = await this.#newestEntryAgeMs(runId); + if (age === undefined || age < this.#orphanAgeMs) { + // Either the keyspace carries no readable timestamp, or a birth may still be in flight. + result.skipped += 1; + return; + } + + const seqKey = snapshotKeys(runId).seq; + const markedAtRaw = await this.#redis.hget(seqKey, ORPHAN_MARKER_FIELD); + const markedAt = markedAtRaw === null ? undefined : Number(markedAtRaw); + + if (markedAt === undefined || Number.isNaN(markedAt)) { + if (!dryRun) { + // The field carries no TTL of its own; it lives and dies with the seq hash, which the + // keyspace's own completion expiry already governs. That removes the marker-lifetime knob + // whose derivation was wrong in the first place. + await this.#redis.hset(seqKey, ORPHAN_MARKER_FIELD, String(Date.now())); + } + result.pendingDeletion += 1; + return; + } + + if (Date.now() - markedAt < this.#confirmOrphanAfterMs) { + result.pendingDeletion += 1; + return; + } + + if (!dryRun) { + // One slot: every key here carries the same `{runId}` hash tag. The marker is a field on + // `seq`, which is in `keys`, so it goes with the keyspace rather than needing its own entry. + await this.#redis.del(...keys); + } + + result.deleted += 1; + } + + /** + * Clears a rule 2 marker for a keyspace whose run turned out to exist after all, so a later + * genuine absence still needs its own two sightings rather than inheriting a stale one. + * + * Called for EVERY run row the lookup returned, live ones included, and it has to be: a keyspace + * marked by an earlier incomplete lookup can belong to a run that is perfectly alive, and a + * SUSPENDED run can sit that way for weeks. Leaving the marker in place would let a later genuine + * absence delete on what is really a first sighting, which is the hole the two-sighting rule + * exists to close. It costs one DEL per existing run per pass; that is the price of the guard + * being sound rather than nearly sound. + */ + async #clearOrphanMarker(runId: string): Promise { + try { + await this.#redis.hdel(snapshotKeys(runId).seq, ORPHAN_MARKER_FIELD); + } catch { + // Best effort. A marker that outlives its usefulness expires on its own TTL. + } + } + + /** + * Every key for one run: the four core keys plus each wait-cycle key. + * + * The cycle keys are enumerated from the `c` high-water field on the seq hash, which the append + * script mints densely with HINCRBY, so 1..high covers every wp key that was ever written. This + * is the same source the store's own terminal-expiry loop uses. + * + * It deliberately does NOT use `KEYS`. That command iterates the whole database and blocks while + * it does, and a hash tag routes a key without scoping the scan, so one sweep pass over a batch + * would issue a full keyspace scan per run. + * + * The trade-off: if the seq hash is evicted while a wp key survives, `high` reads 0 and that + * orphaned cycle key is left behind. That is the right way to be wrong here. Leaving one small + * key costs bytes, where scanning the keyspace to find it costs every hot-path client latency on + * every pass. + */ + async #allKeys(runId: string): Promise { + const core = snapshotKeys(runId); + + const high = Number((await this.#redis.hget(core.seq, "c")) ?? "0"); + const cycles: string[] = []; + for (let n = 1; n <= high; n++) { + cycles.push(`${SNAPSHOT_KEYSPACE_PREFIX}{${runId}}:wp:${n}`); + } + + const candidates = [core.e, core.idx, core.cur, core.seq, ...cycles]; + + // One round trip for the whole set, rather than one per candidate. Cluster-safe: every key here + // carries the same `{runId}` hash tag, so the whole pipeline lands in one slot on one node. + // The same holds for the pexpire pipeline and the multi-key DEL above. + const pipeline = this.#redis.pipeline(); + for (const key of candidates) { + pipeline.exists(key); + } + const replies = await pipeline.exec(); + + return candidates.filter((_key, index) => replies?.[index]?.[1] === 1); + } + + /** + * Age of the newest entry, so a keyspace still being written to is never treated as an orphan. + * The newest is the right end: an old first entry says nothing about whether the run is dead. + */ + async #newestEntryAgeMs(runId: string): Promise { + const core = snapshotKeys(runId); + + const newest = await this.#redis.zrevrange(core.idx, 0, 0); + const id = newest[0]; + + const raw = id + ? await this.#redis.hget(core.e, id) + : // The index holds valid entries only, so an all-invalid keyspace has an empty index. Fall + // back to the newest instant in the entry hash, or that keyspace is never old enough to + // reap and the leak survives the scan fix above. + await this.#newestRawFromEntries(core.e); + + if (!raw) return undefined; + + try { + const createdAt = (JSON.parse(raw) as { createdAt?: string }).createdAt; + if (!createdAt) return undefined; + const parsed = Date.parse(createdAt); + return Number.isNaN(parsed) ? undefined : Date.now() - parsed; + } catch { + return undefined; + } + } + + /** + * The newest entry document in the hash, by its own createdAt. Only reached for a keyspace with + * no index, which is rare, so the whole-hash read is acceptable where it would not be on the + * indexed path. + */ + async #newestRawFromEntries(eKey: string): Promise { + const all = await this.#redis.hgetall(eKey); + let newestRaw: string | undefined; + let newestAt = -Infinity; + + for (const [field, raw] of Object.entries(all)) { + // Sidecar fields hang off the entry ids as `#s` and `#c`; skip them. + if (field.includes("#")) continue; + try { + const at = Date.parse((JSON.parse(raw) as { createdAt?: string }).createdAt ?? ""); + if (!Number.isNaN(at) && at > newestAt) { + newestAt = at; + newestRaw = raw; + } + } catch { + continue; + } + } + + return newestRaw; + } + + /** + * The run id is whatever sits inside the hash tag, so a client prefix on the returned key does not + * need stripping: `engine:snap:{run_x}:e` and `snap:{run_x}:e` both yield `run_x`. + */ + #runIdFrom(key: string): string | undefined { + const open = key.indexOf("{"); + const close = key.indexOf("}", open + 1); + if (open === -1 || close === -1 || close === open + 1) return undefined; + return key.slice(open + 1, close); + } +} + +/** + * Every connection a pass must iterate to cover the whole keyspace: each master of a cluster, or + * the one standalone connection. Replicas are excluded — they hold the same keys as their master, + * so scanning them would double-count and act on one keyspace twice. + * + * Module-level and exported so the fan-out decision can be pinned on its own. It is the whole of + * the defect this guards against: everything the sweep does AFTER the scan is key-addressed and a + * cluster client routes it correctly without help, so the node list is the only place a cluster + * can silently cost the pass coverage. + * + * Resolved per pass, never cached: cluster topology changes under failover and resharding, and a + * stale node list is the same silent under-scan this exists to prevent. + */ + +/** + * Rule 2's "seen absent once" marker is a FIELD on the run's `seq` hash, not a key of its own. + * + * As a separate key its removal depended on the deleting call site remembering to append it to the + * DEL, which is the kind of contract a later edit breaks with no test noticing: a marker outliving + * its keyspace would let a recreated keyspace be deleted on what is really a first sighting. `seq` + * is already in `#allKeys`, so as a field the marker cannot outlive the keyspace at all. + */ +const ORPHAN_MARKER_FIELD = "orph"; + +export function scanTargetsOf(client: RedisClient): Redis[] { + return client instanceof Cluster ? client.nodes("master") : [client]; +} + +/** + * The ioredis client-level prefix for either endpoint shape. On a Cluster it lives on the nested + * `redisOptions`, not on the top-level options, and reading the wrong one yields "" — which makes + * every SCAN MATCH miss and the pass report a clean sweep of nothing. + */ +export function clientPrefixOf(client: RedisClient): string { + if (client instanceof Cluster) { + return (client.options.redisOptions?.keyPrefix as string | undefined) ?? ""; + } + return (client.options.keyPrefix as string | undefined) ?? ""; +} + +function isString(value: string | undefined): value is string { + return typeof value === "string"; +} diff --git a/internal-packages/run-store/src/snapshotReadShapes.test.ts b/internal-packages/run-store/src/snapshotReadShapes.test.ts new file mode 100644 index 00000000000..258731a60b5 --- /dev/null +++ b/internal-packages/run-store/src/snapshotReadShapes.test.ts @@ -0,0 +1,151 @@ +// A matcher that is too loose is the dangerous failure: it answers a query Redis cannot actually +// serve, and the caller gets a wrong answer rather than a slow one. So most of these tests are +// about what must NOT match. +import { describe, expect, it } from "vitest"; +import { matchSinceCursorLookup, matchSinceWindow } from "./snapshotReadShapes.js"; + +const cursorArgs = { + where: { id: "snap_1", runId: "run_1" }, + select: { createdAt: true }, +}; + +const windowArgs = { + where: { runId: "run_1", isValid: true, createdAt: { gt: new Date("2026-08-24T00:00:00Z") } }, + include: { checkpoint: true }, + orderBy: { createdAt: "desc" }, + take: 50, +}; + +describe("matchSinceCursorLookup", () => { + it("matches the engine's since-cursor lookup", () => { + expect(matchSinceCursorLookup(cursorArgs)).toEqual({ id: "snap_1", runId: "run_1" }); + }); + + it("carries an environment scope when present", () => { + expect( + matchSinceCursorLookup({ + where: { ...cursorArgs.where, environmentId: "env_1" }, + select: { createdAt: true }, + }) + ).toEqual({ id: "snap_1", runId: "run_1", environmentId: "env_1" }); + }); + + it("ignores keys explicitly set to undefined", () => { + expect( + matchSinceCursorLookup({ + where: { ...cursorArgs.where, environmentId: undefined }, + select: { createdAt: true }, + }) + ).toEqual({ id: "snap_1", runId: "run_1" }); + }); + + it("refuses a selection of anything but createdAt", () => { + expect( + matchSinceCursorLookup({ where: cursorArgs.where, select: { description: true } }) + ).toBeUndefined(); + expect( + matchSinceCursorLookup({ + where: cursorArgs.where, + select: { createdAt: true, description: true }, + }) + ).toBeUndefined(); + }); + + it("refuses a where with no run id, because there is no keyspace to look in", () => { + expect( + matchSinceCursorLookup({ where: { id: "snap_1" }, select: { createdAt: true } }) + ).toBeUndefined(); + }); + + it("refuses an unknown where key", () => { + expect( + matchSinceCursorLookup({ + where: { ...cursorArgs.where, isValid: true }, + select: { createdAt: true }, + }) + ).toBeUndefined(); + }); + + it("refuses an unknown top-level key", () => { + expect( + matchSinceCursorLookup({ ...cursorArgs, orderBy: { createdAt: "desc" } }) + ).toBeUndefined(); + }); + + it("refuses anything that is not an argument object", () => { + expect(matchSinceCursorLookup(undefined)).toBeUndefined(); + expect(matchSinceCursorLookup(null)).toBeUndefined(); + expect(matchSinceCursorLookup("where")).toBeUndefined(); + expect(matchSinceCursorLookup([cursorArgs])).toBeUndefined(); + }); +}); + +describe("matchSinceWindow", () => { + it("matches the engine's window query", () => { + expect(matchSinceWindow(windowArgs)).toEqual({ + runId: "run_1", + createdAt: new Date("2026-08-24T00:00:00Z"), + take: 50, + }); + }); + + it("carries an environment scope when present", () => { + expect( + matchSinceWindow({ + ...windowArgs, + where: { ...windowArgs.where, environmentId: "env_1" }, + }) + ).toMatchObject({ environmentId: "env_1" }); + }); + + it("refuses a query that also wants the completed waitpoints", () => { + // The engine omits them on purpose to avoid an N x M read. An include that asks for them is a + // different query, and answering it from this path would return them empty. + expect( + matchSinceWindow({ + ...windowArgs, + include: { checkpoint: true, completedWaitpoints: true }, + }) + ).toBeUndefined(); + }); + + it("refuses ascending order", () => { + expect(matchSinceWindow({ ...windowArgs, orderBy: { createdAt: "asc" } })).toBeUndefined(); + }); + + it("refuses a window that does not filter to valid entries", () => { + expect( + matchSinceWindow({ ...windowArgs, where: { ...windowArgs.where, isValid: false } }) + ).toBeUndefined(); + }); + + it("refuses a cursor that is not a strict greater-than on a Date", () => { + expect( + matchSinceWindow({ + ...windowArgs, + where: { ...windowArgs.where, createdAt: { gte: new Date() } }, + }) + ).toBeUndefined(); + expect( + matchSinceWindow({ + ...windowArgs, + where: { ...windowArgs.where, createdAt: { gt: "2026-08-24T00:00:00Z" } }, + }) + ).toBeUndefined(); + }); + + it("refuses a missing take", () => { + const { take: _dropped, ...withoutTake } = windowArgs; + expect(matchSinceWindow(withoutTake)).toBeUndefined(); + }); + + it("refuses an unknown where key", () => { + expect( + matchSinceWindow({ ...windowArgs, where: { ...windowArgs.where, batchId: "batch_1" } }) + ).toBeUndefined(); + }); + + it("refuses an unknown top-level key", () => { + expect(matchSinceWindow({ ...windowArgs, skip: 10 })).toBeUndefined(); + }); +}); diff --git a/internal-packages/run-store/src/snapshotReadShapes.ts b/internal-packages/run-store/src/snapshotReadShapes.ts new file mode 100644 index 00000000000..31dda17f7f0 --- /dev/null +++ b/internal-packages/run-store/src/snapshotReadShapes.ts @@ -0,0 +1,95 @@ +// Shape matchers for the two generic Prisma-args snapshot reads. +// +// `findExecutionSnapshot` and `findManyExecutionSnapshots` take arbitrary Prisma arguments, and a +// key-value store cannot answer an arbitrary query. Only three production call sites exist, all in +// the engine's executionSnapshotSystem, and both generic ones send a single fixed shape. So these +// matchers recognise exactly those shapes and return undefined for anything else, which sends the +// call to Postgres. +// +// Each matcher rejects an argument object carrying any key it does not know about. A query that has +// drifted must fall through and be answered correctly by Postgres, never answered approximately +// from Redis. + +type Unknown = Record; + +function isPlainObject(value: unknown): value is Unknown { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +/** True when `value` has exactly `allowed` keys, ignoring keys explicitly set to undefined. */ +function hasOnlyKeys(value: Unknown, allowed: string[]): boolean { + const present = Object.keys(value).filter((k) => value[k] !== undefined); + return present.every((k) => allowed.includes(k)); +} + +function isString(value: unknown): value is string { + return typeof value === "string"; +} + +export type SinceCursorLookup = { id: string; runId: string; environmentId?: string }; + +/** + * Step 1 of `getExecutionSnapshotsSince`: resolve a known snapshot id to its createdAt. + * + * { where: { id, runId, environmentId? }, select: { createdAt: true } } + */ +export function matchSinceCursorLookup(args: unknown): SinceCursorLookup | undefined { + if (!isPlainObject(args) || !hasOnlyKeys(args, ["where", "select"])) return undefined; + + const { where, select } = args; + if (!isPlainObject(where) || !isPlainObject(select)) return undefined; + if (!hasOnlyKeys(where, ["id", "runId", "environmentId"])) return undefined; + if (!hasOnlyKeys(select, ["createdAt"]) || select.createdAt !== true) return undefined; + if (!isString(where.id) || !isString(where.runId)) return undefined; + if (where.environmentId !== undefined && !isString(where.environmentId)) return undefined; + + return { + id: where.id, + runId: where.runId, + ...(isString(where.environmentId) && { environmentId: where.environmentId }), + }; +} + +export type SinceWindow = { + runId: string; + createdAt: Date; + take: number; + environmentId?: string; +}; + +/** + * Step 2 of `getExecutionSnapshotsSince`: the capped window after a createdAt cursor. + * + * { where: { runId, isValid: true, createdAt: { gt }, environmentId? }, + * include: { checkpoint: true }, orderBy: { createdAt: "desc" }, take: N } + * + * The engine deliberately omits completedWaitpoints from the include to avoid an N x M read, so an + * include asking for them is a different query and is not matched. + */ +export function matchSinceWindow(args: unknown): SinceWindow | undefined { + if (!isPlainObject(args) || !hasOnlyKeys(args, ["where", "include", "orderBy", "take"])) { + return undefined; + } + + const { where, include, orderBy, take } = args; + if (!isPlainObject(where) || !isPlainObject(include) || !isPlainObject(orderBy)) return undefined; + if (typeof take !== "number") return undefined; + + if (!hasOnlyKeys(where, ["runId", "isValid", "createdAt", "environmentId"])) return undefined; + if (!isString(where.runId) || where.isValid !== true) return undefined; + if (where.environmentId !== undefined && !isString(where.environmentId)) return undefined; + + if (!hasOnlyKeys(include, ["checkpoint"]) || include.checkpoint !== true) return undefined; + if (!hasOnlyKeys(orderBy, ["createdAt"]) || orderBy.createdAt !== "desc") return undefined; + + const cursor = where.createdAt; + if (!isPlainObject(cursor) || !hasOnlyKeys(cursor, ["gt"])) return undefined; + if (!(cursor.gt instanceof Date)) return undefined; + + return { + runId: where.runId, + createdAt: cursor.gt, + take, + ...(isString(where.environmentId) && { environmentId: where.environmentId }), + }; +} diff --git a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.births.test.ts b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.births.test.ts new file mode 100644 index 00000000000..12ab76a0f26 --- /dev/null +++ b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.births.test.ts @@ -0,0 +1,285 @@ +// A birth writes Redis FIRST. The order is proved by crashing between the two writes and observing +// which side survived: an orphaned key with no run row is the harmless state, and a run with no +// snapshot at all is the one the order exists to prevent. +import { describe, expect } from "vitest"; +import { containerTest } from "@internal/testcontainers"; +import { generateInternalId } from "@trigger.dev/core/v3/isomorphic"; +import { PostgresRunStore } from "./PostgresRunStore.js"; +import { RedisSnapshotStore } from "./redisSnapshotStore.js"; +import { InjectedSnapshotFault } from "./snapshotFaultInjection.js"; +import { + TaskRunExecutionSnapshotStore, + type SnapshotStoreMode, +} from "./taskRunExecutionSnapshotStore.js"; +import type { RunStore } from "./types.js"; +import { + buildCreateRunData, + seedSnapshotEnvironment, + type SnapshotFixtureEnv, +} from "./testFixtures/snapshotIdFixture.js"; + +const COMPLETED_TTL_MS = 72 * 60 * 60 * 1000; + +function build( + prisma: never, + redisOptions: never, + opts?: { + mode?: SnapshotStoreMode; + faults?: ConstructorParameters[1]["faults"]; + unreachableRedis?: boolean; + } +) { + // An unreachable port makes every append throw for real, which is the failure the retry loop and + // the mode-dependent refusal are about. A fault injector cannot stand in: an injected fault means + // "the process died", and the two are handled differently on purpose. + const redis = new RedisSnapshotStore({ + redisOptions: opts?.unreachableRedis + ? ({ ...(redisOptions as object), port: 1, retryStrategy: () => null } as never) + : redisOptions, + completedTtlMs: COMPLETED_TTL_MS, + }); + + const decorated = new TaskRunExecutionSnapshotStore( + new PostgresRunStore({ prisma, readOnlyPrisma: prisma }) as unknown as RunStore, + { + store: redis, + mode: opts?.mode ?? "dual-write", + ...(opts?.faults && { faults: opts.faults }), + } + ); + + return { decorated, redis }; +} + +function birthSnapshot(id: string, env: SnapshotFixtureEnv) { + return { + id, + engine: "V2" as const, + executionStatus: "RUN_CREATED" as const, + description: "Run was created", + runStatus: "PENDING" as const, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }; +} + +function cancelledData(runId: string, env: SnapshotFixtureEnv) { + return { + ...buildCreateRunData(runId, env), + status: "CANCELED" as const, + error: { type: "STRING_ERROR", raw: "cancelled" } as never, + completedAt: new Date(), + updatedAt: new Date(), + attemptNumber: 0 as const, + }; +} + +describe("birth write ordering", () => { + containerTest("writes Redis then Postgres", async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + const snapshotId = generateInternalId(); + + await decorated.createRun({ + data: buildCreateRunData(runId, env), + snapshot: birthSnapshot(snapshotId, env), + }); + + const read = await redis.getLatest(runId); + expect(read).not.toBeNull(); + expect(read!.entry.id).toBe(snapshotId); + expect(read!.entry.executionStatus).toBe("RUN_CREATED"); + expect(await prisma.taskRunExecutionSnapshot.count({ where: { id: snapshotId } })).toBe(1); + } finally { + await redis.quit(); + } + }); + + containerTest("mints an id when the caller supplies none", async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + const { id: _omitted, ...withoutId } = birthSnapshot(generateInternalId(), env); + + await decorated.createRun({ data: buildCreateRunData(runId, env), snapshot: withoutId }); + + const read = await redis.getLatest(runId); + expect(read).not.toBeNull(); + // The same minted id must reach both stores, or the comparator chases a difference that is + // not real. + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ where: { runId } }); + expect(read!.entry.id).toBe(row.id); + } finally { + await redis.quit(); + } + }); + + containerTest( + "a crash after the Redis append leaves an orphan key and no run", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never, { + faults: (boundary) => { + if (boundary === "afterRedisBirthBeforePg") throw new InjectedSnapshotFault(boundary); + }, + }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + + await expect( + decorated.createRun({ + data: buildCreateRunData(runId, env), + snapshot: birthSnapshot(generateInternalId(), env), + }) + ).rejects.toBeInstanceOf(InjectedSnapshotFault); + + // The harmless state: a keyspace nothing can reach, and no run that lacks a snapshot. + expect(await redis.getLatest(runId)).not.toBeNull(); + expect(await prisma.taskRun.count({ where: { id: runId } })).toBe(0); + } finally { + await redis.quit(); + } + } + ); + + containerTest( + "creates the run anyway when the birth append fails before redis-only", + { timeout: 60_000 }, + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never, { + mode: "dual-write", + unreachableRedis: true, + }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + const snapshotId = generateInternalId(); + + // Postgres is authoritative in every position before redis-only, so a Redis outage must not + // stop runs being created. + await decorated.createRun({ + data: buildCreateRunData(runId, env), + snapshot: birthSnapshot(snapshotId, env), + }); + + expect(await prisma.taskRun.count({ where: { id: runId } })).toBe(1); + expect(await prisma.taskRunExecutionSnapshot.count({ where: { id: snapshotId } })).toBe(1); + } finally { + await redis.quit(); + } + } + ); + + containerTest( + "refuses to create the run when the birth append fails at redis-only", + { timeout: 60_000 }, + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never, { + mode: "redis-only", + unreachableRedis: true, + }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + + // At redis-only Postgres writes no snapshot, so a run created without its Redis birth would + // have no snapshot anywhere. Failing before the run row exists lets the caller retry clean. + await expect( + decorated.createRun({ + data: buildCreateRunData(runId, env), + snapshot: birthSnapshot(generateInternalId(), env), + }) + ).rejects.toThrow(); + + expect(await prisma.taskRun.count({ where: { id: runId } })).toBe(0); + } finally { + await redis.quit(); + } + } + ); + + containerTest("createCancelledRun writes Redis first", async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + const snapshotId = generateInternalId(); + + await decorated.createCancelledRun({ + data: cancelledData(runId, env), + snapshot: { + ...birthSnapshot(snapshotId, env), + executionStatus: "FINISHED", + description: "Run was cancelled", + runStatus: "CANCELED", + }, + }); + + const read = await redis.getLatest(runId); + expect(read!.entry.id).toBe(snapshotId); + expect(read!.entry.executionStatus).toBe("FINISHED"); + expect(await prisma.taskRunExecutionSnapshot.count({ where: { id: snapshotId } })).toBe(1); + } finally { + await redis.quit(); + } + }); + + containerTest( + "a born-terminal run gets the completion expiry immediately", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + + await decorated.createCancelledRun({ + data: cancelledData(runId, env), + snapshot: { + ...birthSnapshot(generateInternalId(), env), + executionStatus: "FINISHED", + description: "Run was cancelled", + runStatus: "CANCELED", + }, + }); + + // A born-terminal run never transitions again, so the completion TTL has to be applied by + // the birth itself or the keyspace never expires. + const nonTerminal = generateInternalId(); + await decorated.createRun({ + data: buildCreateRunData(nonTerminal, env), + snapshot: birthSnapshot(generateInternalId(), env), + }); + + const terminal = await redis.getLatest(runId); + const alive = await redis.getLatest(nonTerminal); + expect(terminal).not.toBeNull(); + expect(alive).not.toBeNull(); + } finally { + await redis.quit(); + } + } + ); + + containerTest("writes nothing to Redis at mode off", async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never, { mode: "off" }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + + await decorated.createRun({ + data: buildCreateRunData(runId, env), + snapshot: birthSnapshot(generateInternalId(), env), + }); + + expect(await prisma.taskRun.count({ where: { id: runId } })).toBe(1); + expect(await redis.getLatest(runId)).toBeNull(); + } finally { + await redis.quit(); + } + }); +}); diff --git a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.off.test.ts b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.off.test.ts new file mode 100644 index 00000000000..437872106e7 --- /dev/null +++ b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.off.test.ts @@ -0,0 +1,100 @@ +// Mode off is the merge-test position: the decorator must be indistinguishable from its delegate and +// must not touch Redis at all. A Redis store whose every member throws proves the second half, and +// enumerating the generated name list proves the first for every method rather than a chosen few. +import { describe, expect, it } from "vitest"; +import { RUN_STORE_METHOD_NAMES } from "./runStoreMethodNames.js"; +import type { RedisSnapshotStore } from "./redisSnapshotStore.js"; +import { TaskRunExecutionSnapshotStore } from "./taskRunExecutionSnapshotStore.js"; +import type { RunStore } from "./types.js"; + +function explodingRedisStore(): RedisSnapshotStore { + return new Proxy({} as RedisSnapshotStore, { + get(_target, prop) { + return () => { + throw new Error(`the Redis store must not be called at mode off, but ${String(prop)} was`); + }; + }, + }); +} + +/** + * Records what the decorator forwarded, and answers with a per-member sentinel. No database is + * involved in whether mode off is a pass-through, so none is started; the behavioural suites for + * every other mode run against a real Postgres and a real Redis. + */ +function forwardingProbe(): { store: RunStore; calls: string[] } { + const calls: string[] = []; + + const store = new Proxy({} as Record, { + get(_target, prop: string) { + return (...args: unknown[]) => { + calls.push(prop); + return `result:${prop}`; + }; + }, + }); + + return { store: store as unknown as RunStore, calls }; +} + +describe("TaskRunExecutionSnapshotStore at mode off", () => { + it("defaults to mode off", () => { + const { store } = forwardingProbe(); + + const decorated = new TaskRunExecutionSnapshotStore(store, { store: explodingRedisStore() }); + + expect(decorated.mode).toBe("off"); + }); + + it("forwards every method to the delegate and never calls Redis", async () => { + const { store, calls } = forwardingProbe(); + const decorated = new TaskRunExecutionSnapshotStore(store, { + store: explodingRedisStore(), + mode: "off", + }) as unknown as Record unknown>; + + for (const name of RUN_STORE_METHOD_NAMES) { + if (name === "runInTransaction") continue; + expect(await decorated[name]("arg-one", "arg-two")).toBe(`result:${name}`); + } + + expect(calls).toEqual(RUN_STORE_METHOD_NAMES.filter((n) => n !== "runInTransaction")); + }); + + it("hands the delegate's own store to a transaction callback", async () => { + const inner = forwardingProbe().store; + let seen: unknown; + const delegate = { + runInTransaction: async ( + _runId: string | undefined, + fn: (store: RunStore, tx: unknown) => Promise + ) => { + await fn(inner, "tx"); + }, + } as unknown as RunStore; + + const decorated = new TaskRunExecutionSnapshotStore(delegate, { + store: explodingRedisStore(), + mode: "off", + }); + + await decorated.runInTransaction("run_1", async (store) => { + seen = store; + }); + + expect(seen).toBe(inner); + }); + + it("reports every other dial position as one that writes Redis", () => { + const { store } = forwardingProbe(); + const modes = ["dual-write", "redis-read", "redis-only"] as const; + + for (const mode of modes) { + const decorated = new TaskRunExecutionSnapshotStore(store, { + store: explodingRedisStore(), + mode, + }); + expect(decorated.mode).toBe(mode); + } + }); +}); diff --git a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.readCohort.test.ts b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.readCohort.test.ts new file mode 100644 index 00000000000..58c80b4df72 --- /dev/null +++ b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.readCohort.test.ts @@ -0,0 +1,104 @@ +// The read cohort is pure arithmetic on the run id, so it needs no containers. Keeping it out of the +// container-backed suite also keeps that suite small enough to run reliably. +import { describe, expect, it } from "vitest"; +import { RedisSnapshotStore } from "./redisSnapshotStore.js"; +import { + TaskRunExecutionSnapshotStore, + type SnapshotStoreMode, +} from "./taskRunExecutionSnapshotStore.js"; +import type { RunStore } from "./types.js"; + +type CohortProbe = { readsFromRedis(runId: string): boolean }; + +function probe(mode: SnapshotStoreMode, readPercent: number): CohortProbe { + // lazyConnect keeps the client from dialling anything: no read in this suite reaches the store. + const redis = new RedisSnapshotStore({ + redisOptions: { host: "127.0.0.1", port: 1, lazyConnect: true, retryStrategy: () => null }, + completedTtlMs: 1, + }); + + return new TaskRunExecutionSnapshotStore({} as RunStore, { + store: redis, + mode, + readPercent, + }) as unknown as CohortProbe; +} + +const ids = Array.from({ length: 500 }, (_, n) => `run_cohort_${n}_${n * 7919}`); + +describe("the read cohort", () => { + it("reads nothing from Redis before the read positions", () => { + for (const mode of ["off", "dual-write"] as const) { + const store = probe(mode, 100); + expect(ids.every((id) => !store.readsFromRedis(id))).toBe(true); + } + }); + + it("reads everything from Redis at 100 percent", () => { + for (const mode of ["redis-read", "redis-only"] as const) { + const store = probe(mode, 100); + expect(ids.every((id) => store.readsFromRedis(id))).toBe(true); + } + }); + + it("reads nothing from Redis at 0 percent", () => { + const store = probe("redis-read", 0); + expect(ids.every((id) => !store.readsFromRedis(id))).toBe(true); + }); + + it("ignores the dial at redis-only, whatever it is set to", () => { + // Postgres holds no snapshot rows at that position, so a run routed away from Redis reads + // nothing at all. The percentage is meaningful only while both stores hold the data. + for (const percent of [0, 1, 50, 99]) { + const store = probe("redis-only", percent); + expect(ids.every((id) => store.readsFromRedis(id))).toBe(true); + } + }); + + it("gives one run the same answer every time", () => { + // A run that changed store between two reads of one poll could show the log going backwards. + const store = probe("redis-read", 50); + + for (const id of ids.slice(0, 50)) { + const first = store.readsFromRedis(id); + for (let i = 0; i < 5; i++) { + expect(store.readsFromRedis(id)).toBe(first); + } + } + }); + + it("gives two instances of the same dial the same answer", () => { + // The cohort must not depend on process state, or a redeploy reshuffles every in-flight run. + const first = probe("redis-read", 50); + const second = probe("redis-read", 50); + + for (const id of ids.slice(0, 50)) { + expect(second.readsFromRedis(id)).toBe(first.readsFromRedis(id)); + } + }); + + it("spreads a population across the dial", () => { + const store = probe("redis-read", 50); + const enabled = ids.filter((id) => store.readsFromRedis(id)).length; + + // A wide band: this asserts the hash spreads at all, not that it is uniform. + expect(enabled).toBeGreaterThan(150); + expect(enabled).toBeLessThan(350); + }); + + it("grows the cohort monotonically as the dial rises", () => { + const at = (percent: number) => { + const store = probe("redis-read", percent); + return new Set(ids.filter((id) => store.readsFromRedis(id))); + }; + + const ten = at(10); + const fifty = at(50); + const ninety = at(90); + + // Raising the dial must only ever add runs. A run that fell out on the way up would flip back to + // Postgres mid-flight, which is the thing the stable hash exists to prevent. + expect([...ten].every((id) => fifty.has(id))).toBe(true); + expect([...fifty].every((id) => ninety.has(id))).toBe(true); + }); +}); diff --git a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.reads.test.ts b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.reads.test.ts new file mode 100644 index 00000000000..a458e9070ed --- /dev/null +++ b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.reads.test.ts @@ -0,0 +1,420 @@ +// Reads served from Redis must be indistinguishable from the Postgres reads they replace: the same +// payload shape, the same tenant boundary, the same fallback when Redis does not hold the answer. +import { describe, expect } from "vitest"; +import { containerTest } from "@internal/testcontainers"; +import { generateInternalId } from "@trigger.dev/core/v3/isomorphic"; +import { PostgresRunStore } from "./PostgresRunStore.js"; +import { RedisSnapshotStore } from "./redisSnapshotStore.js"; +import { + TaskRunExecutionSnapshotStore, + type SnapshotStoreMode, +} from "./taskRunExecutionSnapshotStore.js"; +import type { RunStore } from "./types.js"; +import { + buildCreateRunData, + seedSnapshotEnvironment, + type SnapshotFixtureEnv, +} from "./testFixtures/snapshotIdFixture.js"; + +const COMPLETED_TTL_MS = 72 * 60 * 60 * 1000; + +function build( + prisma: never, + redisOptions: never, + opts?: { mode?: SnapshotStoreMode; readPercent?: number } +) { + const redis = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + const reads: { method: string; source: string }[] = []; + + const decorated = new TaskRunExecutionSnapshotStore( + new PostgresRunStore({ prisma, readOnlyPrisma: prisma }) as unknown as RunStore, + { + store: redis, + mode: opts?.mode ?? "redis-read", + readPercent: opts?.readPercent ?? 100, + metrics: { + recordWrite: () => {}, + recordAppendFailed: () => {}, + recordRead: (method, source) => reads.push({ method, source }), + }, + } + ); + + return { decorated, redis, reads }; +} + +async function seedRun( + decorated: TaskRunExecutionSnapshotStore, + env: SnapshotFixtureEnv +): Promise { + const runId = generateInternalId(); + await decorated.createRun({ + data: buildCreateRunData(runId, env), + snapshot: { + id: generateInternalId(), + engine: "V2", + executionStatus: "RUN_CREATED", + description: "Run was created", + runStatus: "PENDING", + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }, + }); + return runId; +} + +function snapshotInput(runId: string, env: SnapshotFixtureEnv, description: string) { + return { + run: { id: runId, status: "EXECUTING" as const, attemptNumber: 1 }, + snapshot: { executionStatus: "EXECUTING" as const, description }, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }; +} + +describe("snapshot reads", () => { + containerTest("serves the latest snapshot from Redis", async ({ prisma, redisOptions }) => { + const { decorated, redis, reads } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, env); + const created = await decorated.createExecutionSnapshot( + snapshotInput(runId, env, "Run started") + ); + + const latest = await decorated.findLatestExecutionSnapshot(runId); + + expect(latest).not.toBeNull(); + expect(latest!.id).toBe(created.id); + expect(latest!.executionStatus).toBe("EXECUTING"); + expect(latest!.description).toBe("Run started"); + expect(latest!.runId).toBe(runId); + expect(latest!.checkpoint).toBeNull(); + expect(latest!.completedWaitpoints).toEqual([]); + expect(reads).toContainEqual({ method: "findLatestExecutionSnapshot", source: "redis" }); + } finally { + await redis.quit(); + } + }); + + containerTest("returns the same payload Postgres would", async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + const postgresOnly = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, env); + await decorated.createExecutionSnapshot(snapshotInput(runId, env, "Run started")); + + const fromRedis = await decorated.findLatestExecutionSnapshot(runId); + const fromPostgres = await postgresOnly.findLatestExecutionSnapshot(runId); + + expect(fromRedis!.id).toBe(fromPostgres!.id); + expect(fromRedis!.executionStatus).toBe(fromPostgres!.executionStatus); + expect(fromRedis!.description).toBe(fromPostgres!.description); + expect(fromRedis!.runStatus).toBe(fromPostgres!.runStatus); + expect(fromRedis!.attemptNumber).toBe(fromPostgres!.attemptNumber); + expect(fromRedis!.isValid).toBe(fromPostgres!.isValid); + expect(fromRedis!.environmentId).toBe(fromPostgres!.environmentId); + expect(fromRedis!.createdAt.toISOString()).toBe(fromPostgres!.createdAt.toISOString()); + } finally { + await redis.quit(); + } + }); + + containerTest( + "returns the same field set Postgres does, key for key", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + const postgresOnly = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, env); + await decorated.createExecutionSnapshot(snapshotInput(runId, env, "Run started")); + + const fromRedis = await decorated.findLatestExecutionSnapshot(runId); + const fromPostgres = await postgresOnly.findLatestExecutionSnapshot(runId); + + // Not a value comparison: a column the hydrator forgets is absent rather than wrong, so it + // shows up as a missing KEY. lastHeartbeatAt was omitted this way and read back undefined + // where Postgres returns null, on every Redis-served read. + expect(Object.keys(fromRedis!).sort()).toEqual(Object.keys(fromPostgres!).sort()); + } finally { + await redis.quit(); + } + } + ); + + containerTest( + "reads a foreign environment as not found, so the caller's 404 still fires", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, env); + await decorated.createExecutionSnapshot(snapshotInput(runId, env, "Run started")); + + const foreign = await decorated.findLatestExecutionSnapshot(runId, undefined, "env_other"); + + expect(foreign).toBeNull(); + } finally { + await redis.quit(); + } + } + ); + + containerTest( + "falls back to Postgres for a run with no keyspace", + async ({ prisma, redisOptions }) => { + const { decorated, redis, reads } = build(prisma as never, redisOptions as never); + const postgresOnly = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + // A pre-cutover run: it exists in Postgres and Redis has never seen it. + await postgresOnly.createRun({ + data: buildCreateRunData(runId, env), + snapshot: { + id: generateInternalId(), + engine: "V2", + executionStatus: "RUN_CREATED", + description: "Run was created", + runStatus: "PENDING", + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }, + }); + + const latest = await decorated.findLatestExecutionSnapshot(runId); + + expect(latest).not.toBeNull(); + expect(latest!.executionStatus).toBe("RUN_CREATED"); + expect(reads).toContainEqual({ method: "findLatestExecutionSnapshot", source: "postgres" }); + } finally { + await redis.quit(); + } + } + ); + + containerTest("reads from Postgres at readPercent 0", async ({ prisma, redisOptions }) => { + const { decorated, redis, reads } = build(prisma as never, redisOptions as never, { + readPercent: 0, + }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, env); + + const latest = await decorated.findLatestExecutionSnapshot(runId); + + expect(latest).not.toBeNull(); + expect(reads).toEqual([]); + } finally { + await redis.quit(); + } + }); + + containerTest("reads from Postgres at mode dual-write", async ({ prisma, redisOptions }) => { + const { decorated, redis, reads } = build(prisma as never, redisOptions as never, { + mode: "dual-write", + }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, env); + + const latest = await decorated.findLatestExecutionSnapshot(runId); + + expect(latest).not.toBeNull(); + expect(reads).toEqual([]); + } finally { + await redis.quit(); + } + }); + + containerTest("serves the since-cursor lookup from Redis", async ({ prisma, redisOptions }) => { + const { decorated, redis, reads } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, env); + const created = await decorated.createExecutionSnapshot( + snapshotInput(runId, env, "Run started") + ); + + const cursor = await decorated.findExecutionSnapshot({ + where: { id: created.id, runId }, + select: { createdAt: true }, + }); + + expect(cursor).not.toBeNull(); + expect((cursor as { createdAt: Date }).createdAt.toISOString()).toBe( + created.createdAt.toISOString() + ); + expect(reads).toContainEqual({ method: "findExecutionSnapshot", source: "redis" }); + } finally { + await redis.quit(); + } + }); + + containerTest( + "delegates a snapshot lookup it does not recognise", + async ({ prisma, redisOptions }) => { + const { decorated, redis, reads } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, env); + const created = await decorated.createExecutionSnapshot( + snapshotInput(runId, env, "Run started") + ); + + // A different selection: Redis must not answer it approximately. + const row = await decorated.findExecutionSnapshot({ + where: { id: created.id }, + select: { description: true }, + }); + + expect(row).toEqual({ description: "Run started" }); + expect(reads.filter((r) => r.method === "findExecutionSnapshot")).toEqual([]); + } finally { + await redis.quit(); + } + } + ); + + containerTest("serves the since window from Redis", async ({ prisma, redisOptions }) => { + const { decorated, redis, reads } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, env); + const first = await decorated.createExecutionSnapshot(snapshotInput(runId, env, "First")); + await new Promise((resolve) => setTimeout(resolve, 5)); + const second = await decorated.createExecutionSnapshot(snapshotInput(runId, env, "Second")); + await new Promise((resolve) => setTimeout(resolve, 5)); + const third = await decorated.createExecutionSnapshot(snapshotInput(runId, env, "Third")); + + const window = await decorated.findManyExecutionSnapshots({ + where: { runId, isValid: true, createdAt: { gt: first.createdAt } }, + include: { checkpoint: true }, + orderBy: { createdAt: "desc" }, + take: 50, + }); + + // Descending, exactly as the engine asked; it reverses app-side. + expect(window.map((s) => s.id)).toEqual([third.id, second.id]); + expect(reads).toContainEqual({ method: "findManyExecutionSnapshots", source: "redis" }); + } finally { + await redis.quit(); + } + }); + + containerTest( + "delegates a window query it does not recognise", + async ({ prisma, redisOptions }) => { + const { decorated, redis, reads } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, env); + await decorated.createExecutionSnapshot(snapshotInput(runId, env, "First")); + + const rows = await decorated.findManyExecutionSnapshots({ + where: { runId }, + orderBy: { createdAt: "asc" }, + }); + + expect(rows.length).toBeGreaterThan(0); + expect(reads.filter((r) => r.method === "findManyExecutionSnapshots")).toEqual([]); + } finally { + await redis.quit(); + } + } + ); + + containerTest( + "serves the waitpoint id projections from Redis", + async ({ prisma, redisOptions }) => { + const { decorated, redis, reads } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, env); + const created = await decorated.createExecutionSnapshot( + snapshotInput(runId, env, "Run started") + ); + + const ids = await decorated.findSnapshotCompletedWaitpointIds(created.id, undefined, runId); + const withPresence = await decorated.findSnapshotCompletedWaitpointIdsWithPresence( + created.id, + undefined, + runId + ); + + expect(ids).toEqual([]); + // present distinguishes "no waitpoints" from "this reader cannot see the snapshot", which is + // what the engine's read-repair keys off. + expect(withPresence).toEqual({ present: true, ids: [] }); + expect(reads).toContainEqual({ + method: "findSnapshotCompletedWaitpointIds", + source: "redis", + }); + } finally { + await redis.quit(); + } + } + ); + + containerTest( + "delegates a waitpoint id projection with no run id", + async ({ prisma, redisOptions }) => { + const { decorated, redis, reads } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, env); + const created = await decorated.createExecutionSnapshot( + snapshotInput(runId, env, "Run started") + ); + + // Without a run id there is no keyspace to look in. + const ids = await decorated.findSnapshotCompletedWaitpointIds(created.id); + + expect(ids).toEqual([]); + expect(reads.filter((r) => r.method.startsWith("findSnapshot"))).toEqual([]); + } finally { + await redis.quit(); + } + } + ); + + containerTest("never touches Redis for reads at mode off", async ({ prisma, redisOptions }) => { + const { decorated, redis, reads } = build(prisma as never, redisOptions as never, { + mode: "off", + }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + await decorated.createRun({ + data: buildCreateRunData(runId, env), + snapshot: { + id: generateInternalId(), + engine: "V2", + executionStatus: "RUN_CREATED", + description: "Run was created", + runStatus: "PENDING", + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }, + }); + + const latest = await decorated.findLatestExecutionSnapshot(runId); + + expect(latest).not.toBeNull(); + expect(await redis.getLatest(runId)).toBeNull(); + expect(reads).toEqual([]); + } finally { + await redis.quit(); + } + }); +}); diff --git a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.redisOnly.test.ts b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.redisOnly.test.ts new file mode 100644 index 00000000000..b308cb20988 --- /dev/null +++ b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.redisOnly.test.ts @@ -0,0 +1,321 @@ +// `redis-only` is the terminal cutover, and it is the only dial position where Postgres stops being +// authoritative: it cannot be rolled back by turning the dial down, because the snapshots written +// while it was on exist nowhere else. It is also the only position that is a PAIR of settings, not +// one — the decorator's mode AND `snapshotWrites: false` on the store underneath it — and the two +// are set by different tickets. Every test here builds the pair, because testing the mode against a +// store that still writes snapshots would exercise a configuration that never ships. +import { describe, expect } from "vitest"; +import { containerTest } from "@internal/testcontainers"; +import { generateInternalId } from "@trigger.dev/core/v3/isomorphic"; +import { PostgresRunStore } from "./PostgresRunStore.js"; +import { RedisSnapshotStore } from "./redisSnapshotStore.js"; +import { TaskRunExecutionSnapshotStore } from "./taskRunExecutionSnapshotStore.js"; +import type { RunStore } from "./types.js"; +import { + buildCreateRunData, + seedSnapshotEnvironment, + seedSnapshotWaitpoints, + type SnapshotFixtureEnv, +} from "./testFixtures/snapshotIdFixture.js"; + +const COMPLETED_TTL_MS = 72 * 60 * 60 * 1000; + +/** The shipping pair: decorator at `redis-only` over a store that writes no snapshot rows. */ +function build(prisma: never, redisOptions: never) { + const redis = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + const reads: { method: string; source: string }[] = []; + + const decorated = new TaskRunExecutionSnapshotStore( + new PostgresRunStore({ + prisma, + readOnlyPrisma: prisma, + snapshotWrites: false, + }) as unknown as RunStore, + { + store: redis, + mode: "redis-only", + metrics: { + recordWrite: () => {}, + recordAppendFailed: () => {}, + recordRead: (method, source) => reads.push({ method, source }), + }, + } + ); + + return { decorated, redis, reads }; +} + +function birth(env: SnapshotFixtureEnv, id: string) { + return { + id, + engine: "V2" as const, + executionStatus: "RUN_CREATED" as const, + description: "Run was created", + runStatus: "PENDING" as const, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }; +} + +async function seedRun(decorated: TaskRunExecutionSnapshotStore, env: SnapshotFixtureEnv) { + const runId = generateInternalId(); + const snapshotId = generateInternalId(); + await decorated.createRun({ + data: buildCreateRunData(runId, env), + snapshot: birth(env, snapshotId), + }); + return { runId, snapshotId }; +} + +function transition(runId: string, env: SnapshotFixtureEnv, description: string) { + return { + run: { id: runId, status: "EXECUTING" as const, attemptNumber: 1 }, + snapshot: { executionStatus: "EXECUTING" as const, description }, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }; +} + +describe("redis-only: Postgres stops holding snapshots", () => { + containerTest("the run row lands but no snapshot row does", async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const { runId, snapshotId } = await seedRun(decorated, env); + + // The run itself is still Postgres-authoritative at this position. Only its snapshots move. + expect(await prisma.taskRun.count({ where: { id: runId } })).toBe(1); + expect(await prisma.taskRunExecutionSnapshot.count({ where: { runId } })).toBe(0); + + // And the snapshot is genuinely in Redis under the id the caller minted. + const head = await redis.getLatest(runId); + expect(head?.id).toBe(snapshotId); + } finally { + await redis.quit(); + } + }); + + containerTest("transitions write no snapshot row either", async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const { runId } = await seedRun(decorated, env); + + await decorated.createExecutionSnapshot(transition(runId, env, "Run started")); + await decorated.createExecutionSnapshot(transition(runId, env, "Run continued")); + + expect(await prisma.taskRunExecutionSnapshot.count({ where: { runId } })).toBe(0); + const since = await redis.getSinceCreatedAt(runId, new Date(Date.now() - 60_000), { + limit: 50, + }); + expect(since.kind).toBe("hit"); + expect(since.kind === "hit" ? since.entries.length : 0).toBeGreaterThanOrEqual(2); + } finally { + await redis.quit(); + } + }); + + containerTest( + "a completion still updates the run row while writing no snapshot", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const { runId } = await seedRun(decorated, env); + + await decorated.completeAttemptSuccess( + runId, + { + completedAt: new Date(), + outputType: "application/json", + usageDurationMs: 1, + costInCents: 0, + snapshot: { + id: generateInternalId(), + executionStatus: "FINISHED", + description: "Run completed", + runStatus: "COMPLETED_SUCCESSFULLY", + attemptNumber: 1, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }, + }, + { select: { id: true } } + ); + + // The mutation half of a nested write must still land, or the run never finishes. + const run = await prisma.taskRun.findFirstOrThrow({ where: { id: runId } }); + expect(run.status).toBe("COMPLETED_SUCCESSFULLY"); + expect(await prisma.taskRunExecutionSnapshot.count({ where: { runId } })).toBe(0); + } finally { + await redis.quit(); + } + } + ); + + containerTest( + "no completed-waitpoint join rows are written for a snapshot Postgres does not have", + async ({ prisma, redisOptions }) => { + // The join rows point at a snapshot row. With snapshot writes off there is no such row, so + // inserting them would leave links dangling at a snapshot only Redis holds. + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const { runId } = await seedRun(decorated, env); + const [wpA, wpB] = await seedSnapshotWaitpoints(prisma, env, 2); + + await decorated.createExecutionSnapshot({ + ...transition(runId, env, "Run resumed"), + completedWaitpoints: [ + { id: wpA, index: 0 }, + { id: wpB, index: 1 }, + ], + }); + + expect(await prisma.taskRunExecutionSnapshot.count({ where: { runId } })).toBe(0); + const joins = await prisma.$queryRawUnsafe<{ n: bigint }[]>( + `SELECT count(*) AS n FROM "_completedWaitpoints" WHERE "B" = ANY($1::text[])`, + [wpA, wpB] + ); + expect(Number(joins[0]!.n)).toBe(0); + } finally { + await redis.quit(); + } + } + ); +}); + +describe("redis-only: every read is served from Redis", () => { + containerTest( + "the hot read, the since window and the waitpoint lookups all come from Redis", + async ({ prisma, redisOptions }) => { + // At every earlier position a Redis miss falls back to Postgres and the caller never notices. + // Here Postgres holds nothing, so a read that fell back would answer empty rather than wrong, + // and a run would silently lose its state. Each read is asserted to be Redis-sourced. + const { decorated, redis, reads } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const { runId } = await seedRun(decorated, env); + const [wpA] = await seedSnapshotWaitpoints(prisma, env, 1); + const created = await decorated.createExecutionSnapshot({ + ...transition(runId, env, "Run resumed"), + completedWaitpoints: [{ id: wpA, index: 0 }], + }); + + const latest = await decorated.findLatestExecutionSnapshot(runId); + expect(latest!.id).toBe(created.id); + expect(latest!.completedWaitpointOrder).toEqual([wpA]); + + const window = await decorated.findManyExecutionSnapshots({ + where: { runId, isValid: true, createdAt: { gt: new Date(Date.now() - 60_000) } }, + include: { checkpoint: true }, + orderBy: { createdAt: "desc" }, + take: 50, + }); + expect(window.length).toBeGreaterThan(0); + + const withPresence = await decorated.findSnapshotCompletedWaitpointIdsWithPresence( + created.id, + undefined, + runId + ); + expect(withPresence.ids).toEqual([wpA]); + + expect(reads.length).toBeGreaterThan(0); + expect(reads.every((r) => r.source === "redis")).toBe(true); + } finally { + await redis.quit(); + } + } + ); + + containerTest( + "an unrecognised read shape falls through to a Postgres that holds nothing", + async ({ prisma, redisOptions }) => { + // CHARACTERISATION, NOT AN ENDORSEMENT. `findManyExecutionSnapshots` serves from Redis only + // for the since-window shape `matchSinceWindow` recognises; anything else delegates. At every + // dial position before this one that is harmless, because Postgres holds the same rows. Here + // it holds none, so the caller gets an EMPTY result rather than an error, and empty is a + // valid answer to this query. The same is true of the `miss` and `danglingCycle` fallbacks in + // that method: all three are safe everywhere except the one position that cannot fall back. + // + // Only the engine's own call shapes reach this method today, and it issues the since-window + // one, so nothing is broken. It is pinned here so the terminal-cutover ticket decides + // deliberately whether a fall-through at `redis-only` should throw instead of answering + // empty, rather than discovering this shape in production. + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const { runId } = await seedRun(decorated, env); + await decorated.createExecutionSnapshot(transition(runId, env, "Run started")); + + // No `createdAt` cursor, so the shape does not match and the read is delegated. + const unmatched = await decorated.findManyExecutionSnapshots({ + where: { runId, isValid: true }, + include: { checkpoint: true }, + orderBy: { createdAt: "desc" }, + take: 50, + }); + + expect(unmatched).toEqual([]); + + // The same run, asked the shape the engine actually issues, answers in full from Redis. + const matched = await decorated.findManyExecutionSnapshots({ + where: { runId, isValid: true, createdAt: { gt: new Date(Date.now() - 60_000) } }, + include: { checkpoint: true }, + orderBy: { createdAt: "desc" }, + take: 50, + }); + expect(matched.length).toBeGreaterThan(0); + } finally { + await redis.quit(); + } + } + ); + + containerTest( + "the read cohort dial cannot route a run away from Redis", + async ({ prisma, redisOptions }) => { + // readPercent is a ramp control for `redis-read`. At `redis-only` a run routed to Postgres + // would read a database that holds no snapshots at all, so the dial must be ignored here + // whatever it is set to. + const redis = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + const reads: string[] = []; + const decorated = new TaskRunExecutionSnapshotStore( + new PostgresRunStore({ + prisma: prisma as never, + readOnlyPrisma: prisma as never, + snapshotWrites: false, + }) as unknown as RunStore, + { + store: redis, + mode: "redis-only", + readPercent: 0, + metrics: { + recordWrite: () => {}, + recordAppendFailed: () => {}, + recordRead: (_m, source) => reads.push(source), + }, + } + ); + + try { + const env = await seedSnapshotEnvironment(prisma); + const { runId, snapshotId } = await seedRun(decorated, env); + + const latest = await decorated.findLatestExecutionSnapshot(runId); + + expect(latest!.id).toBe(snapshotId); + expect(reads).not.toContain("postgres"); + } finally { + await redis.quit(); + } + } + ); +}); diff --git a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.staging.test.ts b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.staging.test.ts new file mode 100644 index 00000000000..9952f569e2b --- /dev/null +++ b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.staging.test.ts @@ -0,0 +1,286 @@ +// Inside a transaction the Redis append cannot run until the Postgres side commits, or a rollback +// leaves Redis holding a transition that never happened. These tests observe the buffer from inside +// the callback, so the deferral is proved rather than assumed. +import { describe, expect } from "vitest"; +import { containerTest } from "@internal/testcontainers"; +import { generateInternalId } from "@trigger.dev/core/v3/isomorphic"; +import { PostgresRunStore } from "./PostgresRunStore.js"; +import { RedisSnapshotStore } from "./redisSnapshotStore.js"; +import { entryFromCreateRun } from "./snapshotEntry.js"; +import { TaskRunExecutionSnapshotStore } from "./taskRunExecutionSnapshotStore.js"; +import type { RunStore } from "./types.js"; +import { + buildCreateRunData, + seedSnapshotEnvironment, + type SnapshotFixtureEnv, +} from "./testFixtures/snapshotIdFixture.js"; + +const COMPLETED_TTL_MS = 72 * 60 * 60 * 1000; + +function build(prisma: never, redisOptions: never, mode: "off" | "dual-write" = "dual-write") { + const redis = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + const decorated = new TaskRunExecutionSnapshotStore( + new PostgresRunStore({ prisma, readOnlyPrisma: prisma }) as unknown as RunStore, + { store: redis, mode } + ); + return { decorated, redis }; +} + +async function seedBirth( + decorated: TaskRunExecutionSnapshotStore, + redis: RedisSnapshotStore, + runId: string, + env: SnapshotFixtureEnv +): Promise { + const snapshot = { + id: generateInternalId(), + engine: "V2" as const, + executionStatus: "RUN_CREATED" as const, + description: "Run was created", + runStatus: "PENDING" as const, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }; + + await redis.append({ + entry: entryFromCreateRun({ id: snapshot.id, runId, createdAt: new Date() }, snapshot), + kind: "birth", + isTerminal: false, + }); + await decorated.createRun({ data: buildCreateRunData(runId, env), snapshot }); +} + +function snapshotInput(runId: string, env: SnapshotFixtureEnv, id: string, description: string) { + return { + id, + run: { id: runId, status: "EXECUTING" as const, attemptNumber: 1 }, + snapshot: { executionStatus: "EXECUTING" as const, description }, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }; +} + +describe("the staging facade", () => { + containerTest("flushes the append after the commit", async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + await seedBirth(decorated, redis, runId, env); + const id = generateInternalId(); + + await decorated.runInTransaction(runId, async (store, tx) => { + await store.createExecutionSnapshot(snapshotInput(runId, env, id, "Run started"), tx); + + // Still inside the transaction: nothing has reached Redis yet. + expect(await redis.getById(runId, id)).toBeNull(); + }); + + expect(await redis.getById(runId, id)).not.toBeNull(); + expect(await prisma.taskRunExecutionSnapshot.count({ where: { id } })).toBe(1); + } finally { + await redis.quit(); + } + }); + + containerTest( + "writes nothing to Redis when the transaction rolls back", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + await seedBirth(decorated, redis, runId, env); + const id = generateInternalId(); + + await expect( + decorated.runInTransaction(runId, async (store, tx) => { + await store.createExecutionSnapshot(snapshotInput(runId, env, id, "Run started"), tx); + throw new Error("rolled back"); + }) + ).rejects.toThrow("rolled back"); + + // Both sides agree that the transition never happened. + expect(await prisma.taskRunExecutionSnapshot.count({ where: { id } })).toBe(0); + expect(await redis.getById(runId, id)).toBeNull(); + } finally { + await redis.quit(); + } + } + ); + + containerTest("flushes several staged appends in order", async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + await seedBirth(decorated, redis, runId, env); + const first = generateInternalId(); + const second = generateInternalId(); + + await decorated.runInTransaction(runId, async (store, tx) => { + await store.createExecutionSnapshot(snapshotInput(runId, env, first, "First"), tx); + await store.createExecutionSnapshot(snapshotInput(runId, env, second, "Second"), tx); + }); + + const firstRead = await redis.getById(runId, first); + const secondRead = await redis.getById(runId, second); + expect(firstRead).not.toBeNull(); + expect(secondRead).not.toBeNull(); + // Order matters: the log is append-only and its seq is what orders a read. + expect(firstRead!.seq).toBeLessThan(secondRead!.seq); + } finally { + await redis.quit(); + } + }); + + containerTest( + "keeps the fork guard on an append staged inside a transaction", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + await seedBirth(decorated, redis, runId, env); + const id = generateInternalId(); + + // A stale expectation: this names a head that was never current. Outside a transaction the + // append is rejected as forked and never written. Staging must not weaken that, or a write + // the store would have refused becomes the head purely because it ran inside a transaction. + await decorated.runInTransaction(runId, async (store, tx) => { + await store.createExecutionSnapshot( + { ...snapshotInput(runId, env, id, "stale"), previousSnapshotId: generateInternalId() }, + tx + ); + }); + + expect(await redis.getById(runId, id)).toBeNull(); + + const head = await redis.getLatest(runId); + expect(head?.id).not.toBe(id); + } finally { + await redis.quit(); + } + } + ); + + containerTest( + "honours a correct expectation on an append staged inside a transaction", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + await seedBirth(decorated, redis, runId, env); + const head = await redis.getLatest(runId); + const id = generateInternalId(); + + await decorated.runInTransaction(runId, async (store, tx) => { + await store.createExecutionSnapshot( + { ...snapshotInput(runId, env, id, "expected"), previousSnapshotId: head!.id }, + tx + ); + }); + + expect((await redis.getById(runId, id))?.entry.description).toBe("expected"); + } finally { + await redis.quit(); + } + } + ); + + containerTest( + "hands the transaction callback a decorated store", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + await seedBirth(decorated, redis, runId, env); + let seen: unknown; + + await decorated.runInTransaction(runId, async (store) => { + seen = store; + }); + + expect(seen).toBeInstanceOf(TaskRunExecutionSnapshotStore); + expect((seen as TaskRunExecutionSnapshotStore).mode).toBe("dual-write"); + } finally { + await redis.quit(); + } + } + ); + + containerTest( + "hands the transaction callback the plain delegate at mode off", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never, "off"); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + await decorated.createRun({ + data: buildCreateRunData(runId, env), + snapshot: { + id: generateInternalId(), + engine: "V2", + executionStatus: "RUN_CREATED", + description: "Run was created", + runStatus: "PENDING", + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }, + }); + let seen: unknown; + + await decorated.runInTransaction(runId, async (store) => { + seen = store; + }); + + expect(seen).not.toBeInstanceOf(TaskRunExecutionSnapshotStore); + expect(await redis.getLatest(runId)).toBeNull(); + } finally { + await redis.quit(); + } + } + ); + + containerTest( + "wraps the store handle from forWaitpointCompletion", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const handle = await decorated.forWaitpointCompletion(generateInternalId(), { + routeKind: "MANUAL", + } as never); + + // No snapshot write goes through this handle today. Wrapping it is what stops a future one + // from bypassing the decorator with no signal. + expect(handle).toBeInstanceOf(TaskRunExecutionSnapshotStore); + } finally { + await redis.quit(); + } + } + ); + + containerTest( + "returns the plain handle from forWaitpointCompletion at mode off", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never, "off"); + try { + const handle = await decorated.forWaitpointCompletion(generateInternalId(), { + routeKind: "MANUAL", + } as never); + + expect(handle).not.toBeInstanceOf(TaskRunExecutionSnapshotStore); + } finally { + await redis.quit(); + } + } + ); +}); diff --git a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.transitions.test.ts b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.transitions.test.ts new file mode 100644 index 00000000000..e6468c4f09f --- /dev/null +++ b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.transitions.test.ts @@ -0,0 +1,454 @@ +// A transition writes Postgres first and Redis second. The order is proved by observation, not by +// reading the code: with the Redis half made to fail, the Postgres row is still there and the caller +// sees no error, which is only possible if Postgres went first. +import { describe, expect } from "vitest"; +import { containerTest } from "@internal/testcontainers"; +import { generateInternalId } from "@trigger.dev/core/v3/isomorphic"; +import { PostgresRunStore } from "./PostgresRunStore.js"; +import { RedisSnapshotStore } from "./redisSnapshotStore.js"; +import { entryFromCreateRun } from "./snapshotEntry.js"; +import { InjectedSnapshotFault } from "./snapshotFaultInjection.js"; +import { + TaskRunExecutionSnapshotStore, + type SnapshotStoreMode, +} from "./taskRunExecutionSnapshotStore.js"; +import type { RunStore } from "./types.js"; +import { + buildCreateRunData, + seedSnapshotEnvironment, + seedSnapshotWorker, + setupSnapshotIdFixture, + type SnapshotFixtureEnv, +} from "./testFixtures/snapshotIdFixture.js"; + +const COMPLETED_TTL_MS = 72 * 60 * 60 * 1000; + +type Harness = { + decorated: TaskRunExecutionSnapshotStore; + redis: RedisSnapshotStore; + repairs: { runId: string; snapshotId: string; executionStatus: string }[]; + writes: { site: string; outcome: string }[]; +}; + +function harness( + prisma: never, + redisOptions: never, + opts?: { + mode?: SnapshotStoreMode; + faults?: ConstructorParameters[1]["faults"]; + } +): Harness { + const redis = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + const repairs: Harness["repairs"] = []; + const writes: Harness["writes"] = []; + + const decorated = new TaskRunExecutionSnapshotStore( + new PostgresRunStore({ prisma, readOnlyPrisma: prisma }) as unknown as RunStore, + { + store: redis, + mode: opts?.mode ?? "dual-write", + ...(opts?.faults && { faults: opts.faults }), + onAppendFailure: async (args) => { + repairs.push(args); + }, + metrics: { + recordWrite: (site, outcome) => writes.push({ site, outcome }), + recordAppendFailed: () => {}, + recordRead: () => {}, + }, + } + ); + + return { decorated, redis, repairs, writes }; +} + +/** + * Creates the run and its keyspace, so a following transition is not skippedNoKeyspace. + * + * The birth is appended through the raw store rather than the decorator, because the decorator's + * own birth path is a separate concern with its own suite. Keeping it out here means a failure in + * this file is a failure of the transition path and nothing else. + */ +async function seedBirth( + decorated: TaskRunExecutionSnapshotStore, + redis: RedisSnapshotStore, + runId: string, + env: SnapshotFixtureEnv +): Promise { + const snapshot = { + id: generateInternalId(), + engine: "V2" as const, + executionStatus: "RUN_CREATED" as const, + description: "Run was created", + runStatus: "PENDING" as const, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }; + + await redis.append({ + entry: entryFromCreateRun({ id: snapshot.id, runId, createdAt: new Date() }, snapshot), + kind: "birth", + isTerminal: false, + }); + + await decorated.createRun({ data: buildCreateRunData(runId, env), snapshot }); +} + +function completionInput(env: SnapshotFixtureEnv) { + return { + completedAt: new Date(), + outputType: "application/json", + usageDurationMs: 1, + costInCents: 0, + snapshot: { + executionStatus: "FINISHED" as const, + description: "Run completed", + runStatus: "COMPLETED_SUCCESSFULLY" as const, + attemptNumber: 1, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }, + }; +} + +function expireInput(env: SnapshotFixtureEnv) { + return { + error: { type: "STRING_ERROR" as const, raw: "expired" }, + completedAt: new Date(), + expiredAt: new Date(), + snapshot: { + engine: "V2" as const, + executionStatus: "FINISHED" as const, + description: "Run expired", + runStatus: "EXPIRED" as const, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }, + }; +} + +describe("transition write ordering", () => { + containerTest("writes Postgres then Redis", async ({ prisma, redisOptions }) => { + const { decorated, redis, writes } = harness(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + await seedBirth(decorated, redis, runId, env); + + await decorated.completeAttemptSuccess(runId, completionInput(env), { select: { id: true } }); + + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ + where: { runId, executionStatus: "FINISHED" }, + }); + const read = await redis.getById(runId, row.id); + + expect(read).not.toBeNull(); + expect(read!.entry.id).toBe(row.id); + expect(read!.entry.executionStatus).toBe("FINISHED"); + expect(writes).toContainEqual({ site: "completeAttemptSuccess", outcome: "written" }); + } finally { + await redis.quit(); + } + }); + + containerTest( + "keeps the Postgres write and enqueues one repair when the append fails", + async ({ prisma, redisOptions }) => { + const { decorated, redis, repairs } = harness(prisma as never, redisOptions as never, { + faults: (boundary) => { + if (boundary === "afterPgBeforeRedis") throw new InjectedSnapshotFault(boundary); + }, + }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + await seedBirth(decorated, redis, runId, env); + + // The caller must NOT see an error: the Postgres mutation already committed, and the stall + // watchdog is the designed compensator. + await decorated.completeAttemptSuccess(runId, completionInput(env), { + select: { id: true }, + }); + + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ + where: { runId, executionStatus: "FINISHED" }, + }); + + expect(await redis.getById(runId, row.id)).toBeNull(); + expect(repairs).toEqual([{ runId, snapshotId: row.id, executionStatus: "FINISHED" }]); + } finally { + await redis.quit(); + } + } + ); + + containerTest( + "treats a transition on a run with no keyspace as skipped, not failed", + async ({ prisma, redisOptions }) => { + const { decorated, redis, repairs, writes } = harness(prisma as never, redisOptions as never); + try { + // No birth: this is every pre-cutover run's first transition after the dial moves. + const { run, env } = await setupSnapshotIdFixture(prisma); + + await decorated.expireRun(run.id, expireInput(env), { select: { id: true } }); + + expect(await prisma.taskRunExecutionSnapshot.count({ where: { runId: run.id } })).toBe(1); + expect(await redis.getLatest(run.id)).toBeNull(); + expect(repairs).toEqual([]); + expect(writes).toEqual([{ site: "expireRun", outcome: "skippedNoKeyspace" }]); + } finally { + await redis.quit(); + } + } + ); + + containerTest("appends for expireRun", async ({ prisma, redisOptions }) => { + const { decorated, redis } = harness(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + await seedBirth(decorated, redis, runId, env); + + await decorated.expireRun(runId, expireInput(env), { select: { id: true } }); + + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ + where: { runId, executionStatus: "FINISHED" }, + }); + const read = await redis.getById(runId, row.id); + expect(read?.entry.description).toBe("Run expired"); + } finally { + await redis.quit(); + } + }); + + containerTest("appends for expireParkedRun", async ({ prisma, redisOptions }) => { + const { decorated, redis } = harness(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + await seedBirth(decorated, redis, runId, env); + await prisma.taskRun.update({ where: { id: runId }, data: { status: "PENDING_VERSION" } }); + + const result = await decorated.expireParkedRun(runId, { + ...expireInput(env), + statusReason: "VERSION_NEVER_ARRIVED", + snapshot: { ...expireInput(env).snapshot, description: "Parked run expired" }, + }); + + expect(result.count).toBe(1); + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ + where: { runId, executionStatus: "FINISHED" }, + }); + expect((await redis.getById(runId, row.id))?.entry.description).toBe("Parked run expired"); + } finally { + await redis.quit(); + } + }); + + containerTest( + "appends nothing when expireParkedRun matches no run", + async ({ prisma, redisOptions }) => { + const { decorated, redis, writes } = harness(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + await seedBirth(decorated, redis, runId, env); + // The run is PENDING, so the delegate's `status: PENDING_VERSION` guard matches nothing. + + const result = await decorated.expireParkedRun(runId, { + ...expireInput(env), + statusReason: "VERSION_NEVER_ARRIVED", + }); + + expect(result.count).toBe(0); + expect(writes.filter((w) => w.site === "expireParkedRun")).toEqual([]); + const latest = await redis.getLatest(runId); + expect(latest?.entry.executionStatus).toBe("RUN_CREATED"); + } finally { + await redis.quit(); + } + } + ); + + containerTest("appends for rescheduleRun", async ({ prisma, redisOptions }) => { + const { decorated, redis } = harness(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + await seedBirth(decorated, redis, runId, env); + + await decorated.rescheduleRun(runId, { + delayUntil: new Date(Date.now() + 60_000), + snapshot: { + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }, + }); + + const row = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ + where: { runId, executionStatus: "DELAYED" }, + }); + expect((await redis.getById(runId, row.id))?.entry.description).toBe( + "Delayed run was rescheduled to a future date" + ); + } finally { + await redis.quit(); + } + }); + + containerTest( + "appends nothing when rescheduleRun carries no snapshot", + async ({ prisma, redisOptions }) => { + const { decorated, redis, writes } = harness(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + await seedBirth(decorated, redis, runId, env); + + await decorated.rescheduleRun(runId, { delayUntil: new Date(Date.now() + 60_000) }); + + expect(writes.filter((w) => w.site === "rescheduleRun")).toEqual([]); + expect((await redis.getLatest(runId))?.entry.executionStatus).toBe("RUN_CREATED"); + } finally { + await redis.quit(); + } + } + ); + + containerTest("appends for lockRunToWorker under a CAS", async ({ prisma, redisOptions }) => { + const { decorated, redis, writes } = harness(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const { workerId, taskId } = await seedSnapshotWorker(prisma, env); + const runId = generateInternalId(); + await seedBirth(decorated, redis, runId, env); + + const head = await redis.getLatest(runId); + const snapshotId = generateInternalId(); + + await decorated.lockRunToWorker(runId, { + lockedAt: new Date(), + lockedById: taskId, + lockedToVersionId: workerId, + lockedQueueId: undefined, + startedAt: new Date(), + baseCostInCents: 0, + machinePreset: "small-1x", + taskVersion: "1.0.0", + snapshot: { + id: snapshotId, + previousSnapshotId: head!.id, + attemptNumber: 1, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + completedWaitpointIds: [], + completedWaitpointOrder: [], + }, + }); + + const read = await redis.getById(runId, snapshotId); + expect(read?.entry.executionStatus).toBe("PENDING_EXECUTING"); + expect(read?.entry.previousSnapshotId).toBe(head!.id); + expect(writes).toContainEqual({ site: "lockRunToWorker", outcome: "written" }); + } finally { + await redis.quit(); + } + }); + + containerTest( + "reports a forked append without enqueuing a repair", + async ({ prisma, redisOptions }) => { + const { decorated, redis, repairs, writes } = harness(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const { workerId, taskId } = await seedSnapshotWorker(prisma, env); + const runId = generateInternalId(); + await seedBirth(decorated, redis, runId, env); + + // A stale previousSnapshotId: another writer advanced the head. A repair cannot help, so the + // outcome is counted and dropped. + await decorated.lockRunToWorker(runId, { + lockedAt: new Date(), + lockedById: taskId, + lockedToVersionId: workerId, + lockedQueueId: undefined, + startedAt: new Date(), + baseCostInCents: 0, + machinePreset: "small-1x", + taskVersion: "1.0.0", + snapshot: { + id: generateInternalId(), + previousSnapshotId: generateInternalId(), + attemptNumber: 1, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + completedWaitpointIds: [], + completedWaitpointOrder: [], + }, + }); + + expect(writes).toContainEqual({ site: "lockRunToWorker", outcome: "forked" }); + expect(repairs).toEqual([]); + } finally { + await redis.quit(); + } + } + ); + + containerTest( + "appends for the standalone createExecutionSnapshot", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = harness(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + await seedBirth(decorated, redis, runId, env); + + const created = await decorated.createExecutionSnapshot({ + run: { id: runId, status: "EXECUTING", attemptNumber: 1 }, + snapshot: { executionStatus: "EXECUTING", description: "Run started" }, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }); + + const read = await redis.getById(runId, created.id); + expect(read).not.toBeNull(); + expect(read!.entry.executionStatus).toBe("EXECUTING"); + // The standalone path is the one whose delegate returns the row, so both stores agree exactly. + expect(read!.entry.createdAt).toBe(created.createdAt.toISOString()); + } finally { + await redis.quit(); + } + } + ); + + containerTest("writes nothing to Redis at mode off", async ({ prisma, redisOptions }) => { + const { decorated, redis } = harness(prisma as never, redisOptions as never, { mode: "off" }); + try { + const { run, env } = await setupSnapshotIdFixture(prisma); + + await decorated.completeAttemptSuccess(run.id, completionInput(env), { + select: { id: true }, + }); + + expect(await prisma.taskRunExecutionSnapshot.count({ where: { runId: run.id } })).toBe(1); + expect(await redis.getLatest(run.id)).toBeNull(); + } finally { + await redis.quit(); + } + }); +}); diff --git a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.ts b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.ts new file mode 100644 index 00000000000..c6bc145a91d --- /dev/null +++ b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.ts @@ -0,0 +1,980 @@ +// Decorates any RunStore so execution snapshots also land in Redis. It overrides only the methods +// that touch a snapshot and inherits the rest from the generated pass-through base. +// +// Write ORDER is the correctness property, and the two orders are deliberately different: +// +// transition Postgres first, Redis second. A crash in the gap leaves a run whose latest snapshot +// is stale, which is exactly the state the heartbeat stall watchdog already heals. +// birth Redis first, Postgres second. A crash in the gap leaves an unreachable key for a run +// that does not exist. Postgres-first would leave a run with no snapshot at all, and +// getLatestExecutionSnapshot treats that as a hard error. +// +// Each order is chosen so the crash state is the harmless one. A lost cross-store write is never +// recovered by a transaction or an outbox: recovery is always the existing stall-and-repair job. +import { Logger } from "@trigger.dev/core/logger"; +import { generateInternalId } from "@trigger.dev/core/v3/isomorphic"; +import { DelegatingRunStore } from "./delegatingRunStore.js"; +import type { + CompletedWaitpointRef, + RedisSnapshotStore, + SnapshotEntryInput, + SnapshotRead, +} from "./redisSnapshotStore.js"; +import { deriveDistinctIds, deriveOrder } from "./redisSnapshotStore.js"; +import { + entryFromCompletion, + entryFromCreateExecutionSnapshot, + entryFromCreateRun, + entryFromExpire, + entryFromLock, + entryFromReschedule, + isTerminalEntry, +} from "./snapshotEntry.js"; +import { isInjectedFault, type SnapshotFaultInjector } from "./snapshotFaultInjection.js"; +import type { + ReadClient, + CompletionSnapshotInput, + CreateCancelledRunInput, + CreateExecutionSnapshotInput, + CreateRunInput, + ExpireSnapshotInput, + LockRunData, + RescheduleSnapshotInput, + RunStore, + TaskRunWithWaitpoint, +} from "./types.js"; +import { matchSinceCursorLookup, matchSinceWindow } from "./snapshotReadShapes.js"; +import { boundedIn } from "@trigger.dev/database"; +import type { Prisma, PrismaClientOrTransaction, TaskRun } from "@trigger.dev/database"; + +/** One initial attempt plus three retries, per the write protocol. */ +const APPEND_ATTEMPTS = 4; + +/** + * Matches the engine's own chunked waitpoint fetch. A batch can complete a thousand waitpoints at + * once, and an unbounded `in:` makes each distinct list length its own prepared statement. + */ +const WAITPOINT_CHUNK_SIZE = 100; + +/** + * The rollout dial. Postgres stays fully written and authoritative in every position before + * `redis-only`, so every earlier position rolls back losslessly by turning the dial down. + * + * A `compare` position was named here before its behaviour existed, and it read from this type as a + * real dial position while behaving in every respect exactly like `dual-write`. A dial value that + * silently does something other than its name is worse than a missing one: turning it on would have + * looked like enabling divergence reporting and delivered plain dual-write. It is added back by the + * ticket that implements the sampled dual-read and diff, at which point the name will be true. + */ +export type SnapshotStoreMode = "off" | "dual-write" | "redis-read" | "redis-only"; + +/** + * Enqueues the existing `repairSnapshot` job for a run whose append was lost. The decorator lives in + * run-store and cannot reach the engine's worker, so the binding is injected. That binding must + * reuse the stall watchdog's job id for the run, or the watchdog and this path can start two + * concurrent repairs on one run. + */ +export type SnapshotRepairEnqueuer = (args: { + runId: string; + snapshotId: string; + executionStatus: string; +}) => Promise; + +export type DecoratorMetrics = { + recordWrite(site: string, outcome: string): void; + recordAppendFailed(site: string): void; + recordRead(method: string, source: "redis" | "postgres"): void; +}; + +export type TaskRunExecutionSnapshotStoreOptions = { + store: RedisSnapshotStore; + /** Defaults to `off`, which is a pure pass-through that never touches Redis. */ + mode?: SnapshotStoreMode; + /** Percentage of runs whose reads come from Redis at `redis-read` and `redis-only`. Defaults to 0. */ + readPercent?: number; + onAppendFailure?: SnapshotRepairEnqueuer; + faults?: SnapshotFaultInjector; + metrics?: DecoratorMetrics; + logger?: Logger; + /** + * Internal. Set only by the staging facade this class builds for `runInTransaction`. When present, + * an intercepted write does its Postgres half and pushes its entry here instead of appending, and + * the outer instance flushes the buffer after the transaction commits. + */ + staging?: StagedAppend[]; +}; + +/** One deferred append: the entry, plus the wait cycle it carries, if any. */ +export type StagedAppend = { + entry: SnapshotEntryInput; + /** + * The head this append expects, carried through staging so the compare-and-set survives the + * deferral. Dropping it would silently disable the fork guard for every snapshot written inside a + * transaction, and a stale append that should be rejected would instead become the head. + */ + expectedCur?: string; + completedWaitpoints?: CompletedWaitpointRef[]; +}; + +export class TaskRunExecutionSnapshotStore extends DelegatingRunStore { + readonly mode: SnapshotStoreMode; + protected readonly redis: RedisSnapshotStore; + protected readonly readPercent: number; + protected readonly onAppendFailure?: SnapshotRepairEnqueuer; + protected readonly faults?: SnapshotFaultInjector; + protected readonly metrics?: DecoratorMetrics; + protected readonly logger: Logger; + protected readonly staging?: StagedAppend[]; + + constructor(delegate: RunStore, options: TaskRunExecutionSnapshotStoreOptions) { + super(delegate); + this.redis = options.store; + this.mode = options.mode ?? "off"; + this.readPercent = options.readPercent ?? 0; + this.onAppendFailure = options.onAppendFailure; + this.faults = options.faults; + this.metrics = options.metrics; + this.logger = options.logger ?? new Logger("TaskRunExecutionSnapshotStore", "debug"); + this.staging = options.staging; + } + + /** True in every position that appends to Redis. */ + protected get writesRedis(): boolean { + return this.mode !== "off"; + } + + /** + * The staging facade. Two writes share one Postgres transaction here, and the Redis half of each + * cannot run until that transaction commits: a rollback would otherwise leave Redis holding a + * transition that never happened. + * + * The callback gets a second decorator over the transaction-bound store, carrying a staging + * buffer. An intercepted write does its Postgres half through that store and pushes its entry + * onto the buffer. After the transaction resolves, this instance flushes the buffer in order + * through the same retry-and-repair path a lone transition uses. If the callback throws, the + * delegate rejects, the flush never runs, and the buffer goes away with the stack — so the + * Postgres rollback and the Redis silence agree. + */ + override async runInTransaction( + runId: string | undefined, + fn: (store: RunStore, tx: PrismaClientOrTransaction) => Promise + ): Promise { + if (!this.writesRedis) { + // At `off` the callback must receive the delegate's own store, untouched, so a transaction + // behaves exactly as it does without the decorator in the chain. + return this.delegate.runInTransaction(runId, fn); + } + + const staged: StagedAppend[] = []; + + const result = await this.delegate.runInTransaction(runId, (store, tx) => + fn(this.#wrap(store, staged), tx) + ); + + // The transaction committed. Only now can a snapshot claim its partner is durable. + for (const item of staged) { + await this.#appendTransition( + "runInTransaction", + item.entry, + item.expectedCur, + item.completedWaitpoints + ); + } + + return result; + } + + /** + * `forWaitpointCompletion` hands the caller a store to apply a completion on. No snapshot write + * goes through that handle today, so wrapping it changes nothing now; leaving it unwrapped is the + * one hole that would let a future snapshot write bypass the decorator with no signal at all. + */ + override async forWaitpointCompletion( + waitpointId: string, + context: Parameters[1] + ): Promise { + const store = await this.delegate.forWaitpointCompletion(waitpointId, context); + + if (!this.writesRedis) { + return store; + } + + // Carry the staging buffer through. Without it, a handle taken inside a transaction appends + // immediately, which is the exact ordering the facade exists to prevent. + return this.#wrap(store, this.staging); + } + + /** + * A second decorator over another store, sharing this one's options. One class in both roles keeps + * the write-ordering logic in exactly one place. Passing no buffer gives a plain decorator that + * appends immediately; passing one makes it stage instead. + */ + #wrap(store: RunStore, staging?: StagedAppend[]): TaskRunExecutionSnapshotStore { + return new TaskRunExecutionSnapshotStore(store, { + store: this.redis, + mode: this.mode, + readPercent: this.readPercent, + logger: this.logger, + ...(this.onAppendFailure && { onAppendFailure: this.onAppendFailure }), + ...(this.faults && { faults: this.faults }), + ...(this.metrics && { metrics: this.metrics }), + ...(staging && { staging }), + }); + } + + // --------------------------------------------------------------------------------------------- + // Births: Redis first, Postgres second. + // --------------------------------------------------------------------------------------------- + + override async createRun( + params: CreateRunInput, + tx?: PrismaClientOrTransaction + ): Promise { + if (!this.writesRedis) { + return this.delegate.createRun(params, tx); + } + + const ctx = this.#context(params.data.id, params.snapshot.id); + const snapshot = { ...params.snapshot, id: ctx.id, createdAt: ctx.createdAt }; + + await this.#appendBirth("createRun", entryFromCreateRun(ctx, snapshot)); + + return this.delegate.createRun({ ...params, snapshot }, tx); + } + + override async createCancelledRun( + params: CreateCancelledRunInput, + tx?: PrismaClientOrTransaction + ): Promise { + if (!this.writesRedis) { + return this.delegate.createCancelledRun(params, tx); + } + + const ctx = this.#context(params.data.id, params.snapshot.id); + const snapshot = { ...params.snapshot, id: ctx.id, createdAt: ctx.createdAt }; + + await this.#appendBirth("createCancelledRun", entryFromCreateRun(ctx, snapshot)); + + return this.delegate.createCancelledRun({ ...params, snapshot }, tx); + } + + // --------------------------------------------------------------------------------------------- + // Transitions: Postgres first, Redis second. + // --------------------------------------------------------------------------------------------- + + override async completeAttemptSuccess( + runId: string, + data: { + completedAt: Date; + output?: string; + outputType: string; + usageDurationMs: number; + costInCents: number; + snapshot: CompletionSnapshotInput; + }, + args: { select: S }, + tx?: PrismaClientOrTransaction + ): Promise> { + if (!this.writesRedis) { + return this.delegate.completeAttemptSuccess(runId, data, args, tx); + } + + const ctx = this.#context(runId, data.snapshot.id); + const withId = { + ...data, + snapshot: { ...data.snapshot, id: ctx.id, createdAt: ctx.createdAt }, + }; + + const result = await this.delegate.completeAttemptSuccess(runId, withId, args, tx); + + await this.#appendTransition( + "completeAttemptSuccess", + entryFromCompletion(ctx, withId.snapshot) + ); + return result; + } + + override async expireRun( + runId: string, + data: { error: unknown; completedAt: Date; expiredAt: Date; snapshot: ExpireSnapshotInput }, + args: { select: S }, + tx?: PrismaClientOrTransaction + ): Promise> { + if (!this.writesRedis) { + return this.delegate.expireRun(runId, data as never, args, tx); + } + + const ctx = this.#context(runId, data.snapshot.id); + const withId = { + ...data, + snapshot: { ...data.snapshot, id: ctx.id, createdAt: ctx.createdAt }, + }; + + const result = await this.delegate.expireRun(runId, withId as never, args, tx); + + await this.#appendTransition("expireRun", entryFromExpire(ctx, withId.snapshot)); + return result; + } + + override async expireParkedRun( + runId: string, + data: { + error: unknown; + completedAt: Date; + expiredAt: Date; + statusReason: string; + snapshot: ExpireSnapshotInput; + }, + tx?: PrismaClientOrTransaction + ): Promise<{ count: number }> { + if (!this.writesRedis) { + return this.delegate.expireParkedRun(runId, data as never, tx); + } + + const ctx = this.#context(runId, data.snapshot.id); + const withId = { + ...data, + snapshot: { ...data.snapshot, id: ctx.id, createdAt: ctx.createdAt }, + }; + + const result = await this.delegate.expireParkedRun(runId, withId as never, tx); + + // The delegate writes nothing when the run is no longer PENDING_VERSION, so neither does Redis. + if (result.count > 0) { + await this.#appendTransition("expireParkedRun", entryFromExpire(ctx, withId.snapshot)); + } + return result; + } + + override async rescheduleRun( + runId: string, + data: { delayUntil: Date; queueTimestamp?: Date; snapshot?: RescheduleSnapshotInput }, + tx?: PrismaClientOrTransaction + ): Promise { + // The delegate writes a snapshot only when one is supplied, so an absent snapshot is a plain run + // update with nothing for Redis to mirror. + if (!this.writesRedis || !data.snapshot) { + return this.delegate.rescheduleRun(runId, data, tx); + } + + const ctx = this.#context(runId, data.snapshot.id); + const withId = { + ...data, + snapshot: { ...data.snapshot, id: ctx.id, createdAt: ctx.createdAt }, + }; + + const result = await this.delegate.rescheduleRun(runId, withId, tx); + + await this.#appendTransition("rescheduleRun", entryFromReschedule(ctx, withId.snapshot)); + return result; + } + + override async lockRunToWorker( + runId: string, + data: LockRunData, + tx?: PrismaClientOrTransaction + ): Promise>> { + if (!this.writesRedis) { + return this.delegate.lockRunToWorker(runId, data, tx); + } + + // This is the one transition whose input already carries both an id and the previous snapshot + // id, so it is also the one that can append under a compare-and-set on the current head. + const ctx = { id: data.snapshot.id, runId, createdAt: new Date() }; + const withStamp = { ...data, snapshot: { ...data.snapshot, createdAt: ctx.createdAt } }; + + const result = await this.delegate.lockRunToWorker(runId, withStamp, tx); + + await this.#appendTransition( + "lockRunToWorker", + entryFromLock(ctx, withStamp.snapshot), + withStamp.snapshot.previousSnapshotId, + // Built from the COMPLETE id set, which is what the delegate connects in Postgres, with the + // index taken from the ordered list where the id appears in it. Building from the ordered list + // instead would drop every id with no batch index, exactly the ids Postgres still records. + lockCycleRefs( + withStamp.snapshot.completedWaitpointIds, + withStamp.snapshot.completedWaitpointOrder + ) + ); + return result; + } + + override async createExecutionSnapshot( + input: CreateExecutionSnapshotInput, + tx?: PrismaClientOrTransaction + ): Promise> { + if (!this.writesRedis) { + return this.delegate.createExecutionSnapshot(input, tx); + } + + const ctx = this.#context(input.run.id, input.id); + const created = await this.delegate.createExecutionSnapshot( + { ...input, id: ctx.id, createdAt: ctx.createdAt }, + tx + ); + + // The standalone path is the only one whose delegate returns the row, so its entry can take the + // exact createdAt Postgres recorded rather than the decorator's own clock. + await this.#appendTransition( + "createExecutionSnapshot", + entryFromCreateExecutionSnapshot(ctx, input), + input.previousSnapshotId, + input.completedWaitpoints + ); + return created; + } + + // --------------------------------------------------------------------------------------------- + // The append protocol. + // --------------------------------------------------------------------------------------------- + + /** Mints the id when the caller did not, and stamps one clock for both stores. */ + #context(runId: string, suppliedId?: string) { + return { id: suppliedId ?? generateInternalId(), runId, createdAt: new Date() }; + } + + /** + * Births invert the order. Postgres-first would leave a run with no snapshot at all, and + * `getLatestExecutionSnapshot` treats that as a hard error, so the run would be stuck. Redis-first + * leaves an orphaned keyspace for a run that does not exist, which nothing can reach and the + * sweep's second rule reaps. + * + * Being first is also what lets this path refuse. Before `redis-only` a failed birth append is + * survivable, because Postgres is authoritative and holds the snapshot; at `redis-only` Postgres + * writes no snapshot, so a run created without its Redis birth would have no snapshot anywhere. + * Throwing here happens before the run row exists, so the caller retries a clean creation. + */ + async #appendBirth(site: string, entry: SnapshotEntryInput): Promise { + if (this.staging) { + // A birth inside a transaction cannot be staged: staging flushes after the commit, which is + // the opposite of what a birth needs. No caller does this today, so say so and append now. + this.logger.error("a run birth inside a transaction cannot be staged", { + runId: entry.runId, + site, + }); + } + + for (let attempt = 0; attempt < APPEND_ATTEMPTS; attempt++) { + try { + const result = await this.redis.append({ + entry, + kind: "birth", + isTerminal: isTerminalEntry(entry), + }); + this.#recordOutcome(site, entry, result); + + // Modelled AFTER the successful append: the crash this boundary represents is a process that + // died between the two stores, not an append that failed. + this.faults?.("afterRedisBirthBeforePg", { runId: entry.runId, snapshotId: entry.id }); + return; + } catch (error) { + if (isInjectedFault(error)) { + throw error; + } + + if (attempt === APPEND_ATTEMPTS - 1) { + this.metrics?.recordAppendFailed(site); + this.logger.error("snapshot birth append failed after retries", { + runId: entry.runId, + snapshotId: entry.id, + site, + mode: this.mode, + error, + }); + + if (this.mode === "redis-only") { + throw error; + } + return; + } + + await new Promise((resolve) => setTimeout(resolve, 10 * 2 ** attempt)); + } + } + } + + /** + * Postgres has already committed by the time this runs. A throw here would turn a gap the stall + * watchdog heals into a caller-visible failure, so it never rethrows: it retries, then hands the + * run to the repair job and returns. + */ + async #appendTransition( + site: string, + entry: SnapshotEntryInput, + expectedCur?: string, + completedWaitpoints?: CompletedWaitpointRef[] + ): Promise { + if (this.staging) { + // Inside a transaction the append cannot run until the Postgres side commits, or a rollback + // leaves Redis holding a transition that never happened. + this.staging.push({ + entry, + ...(expectedCur !== undefined && { expectedCur }), + ...(completedWaitpoints && { completedWaitpoints }), + }); + return; + } + + for (let attempt = 0; attempt < APPEND_ATTEMPTS; attempt++) { + try { + this.faults?.(attempt === 0 ? "afterPgBeforeRedis" : "midFlushRetry", { + runId: entry.runId, + snapshotId: entry.id, + }); + + const cycle = await this.#resolveCycle(entry.runId, completedWaitpoints); + + const result = await this.redis.append({ + entry, + kind: "transition", + isTerminal: isTerminalEntry(entry), + ...(expectedCur !== undefined && { expectedCur }), + ...(cycle && { cycle }), + }); + + this.#recordOutcome(site, entry, result); + return; + } catch (error) { + // An injected fault models a dead process, not a retryable append failure. + if (isInjectedFault(error)) { + this.metrics?.recordAppendFailed(site); + await this.#enqueueRepair(entry); + return; + } + + if (attempt === APPEND_ATTEMPTS - 1) { + this.metrics?.recordAppendFailed(site); + this.logger.error("snapshot append failed after retries", { + runId: entry.runId, + snapshotId: entry.id, + site, + error, + }); + await this.#enqueueRepair(entry); + return; + } + + await new Promise((resolve) => setTimeout(resolve, 10 * 2 ** attempt)); + } + } + } + + /** + * Decides whether this append mints a new wait cycle or points at the one already there. + * + * A resume append carries a newly-differing id set, so it mints a cycle and the record set is + * written once. Every copy-forward append that follows re-passes the SAME list, and re-minting on + * each would rewrite the record set once per entry in the resume chain — the write amplification + * the pointer model exists to remove. So an unchanged id set carries the previous cycleSeq + * forward and writes no key. + * + * The extra read only happens for an append that actually carries waitpoints, which is the resume + * path rather than the hot path. + * + * `records` is deliberately left unset. The record envelope belongs to the waitpoint lane and + * ships empty in this build, so dual-write never re-versions the entry when it arrives. + */ + async #resolveCycle( + runId: string, + completedWaitpoints?: CompletedWaitpointRef[] + ): Promise< + | { kind: "new"; completedWaitpoints: CompletedWaitpointRef[] } + | { kind: "carryForward"; cycleSeq: number; completedWaitpoints: CompletedWaitpointRef[] } + | undefined + > { + if (!completedWaitpoints || completedWaitpoints.length === 0) { + return undefined; + } + + const order = deriveOrder(completedWaitpoints); + const distinct = deriveDistinctIds(completedWaitpoints); + + try { + const head = await this.redis.getLatest(runId); + const previousIds = head?.completedWaitpointIds; + + // Both halves must match. Comparing the order alone is not enough: it holds only indexed ids, + // so two DIFFERENT single waits both present an empty order and would compare equal, and the + // second would inherit the first's waitpoint set instead of minting its own. + if ( + head?.cycle && + previousIds && + sameOrder(previousIds.order, order) && + sameSet(previousIds.distinctIds, distinct) + ) { + return { + kind: "carryForward", + cycleSeq: head.cycle.cycleSeq, + completedWaitpoints, + }; + } + } catch (error) { + // A failed probe must not lose the waitpoints. Minting a fresh cycle is the safe direction: + // it costs one duplicated record set, where a wrong carryForward would point at another + // cycle's ids. + this.logger.warn("snapshot cycle probe failed, minting a new cycle", { runId, error }); + } + + return { kind: "new", completedWaitpoints }; + } + + /** + * None of the four append outcomes is a failure, and none of them enqueues a repair. + * + * `skippedNoKeyspace` is every pre-cutover run's transitions. `forked` means another writer + * advanced the head, which a repair cannot help. `duplicate` is a retry that already landed. + * `cycleMismatch` means the store refused an untrustworthy waitpoint pointer on purpose. + */ + #recordOutcome( + site: string, + entry: SnapshotEntryInput, + result: Awaited> + ): void { + this.metrics?.recordWrite(site, result.outcome); + + if (result.outcome === "forked") { + this.logger.warn("snapshot append forked", { + runId: entry.runId, + snapshotId: entry.id, + site, + actualCur: result.actualCur, + }); + } + } + + // --------------------------------------------------------------------------------------------- + // Reads. + // + // Only three production call sites exist, all in the engine's executionSnapshotSystem, all with + // fixed argument shapes. The two generic Prisma-args methods therefore recognise exactly the + // shapes the engine sends and delegate everything else: an unrecognised shape must go to Postgres, + // never get an approximate answer from Redis. + // --------------------------------------------------------------------------------------------- + + /** + * Whether this run's reads come from Redis. Hashed on the run id so a run does not change store + * between two reads of the same poll, which would let a caller see the log go backwards. + */ + protected readsFromRedis(runId: string): boolean { + if (this.mode !== "redis-read" && this.mode !== "redis-only") return false; + + // At `redis-only` the cohort dial has no meaning. Postgres holds no snapshot rows at that + // position, so a run routed away from Redis reads nothing at all. Ignoring the percentage here + // makes that misconfiguration unreachable rather than merely documented. + if (this.mode === "redis-only") return true; + + if (this.readPercent >= 100) return true; + if (this.readPercent <= 0) return false; + + let hash = 0; + for (let i = 0; i < runId.length; i++) { + hash = (hash * 31 + runId.charCodeAt(i)) >>> 0; + } + return hash % 100 < this.readPercent; + } + + override async findLatestExecutionSnapshot( + runId: string, + client?: ReadClient, + environmentId?: string + ): Promise | null> { + if (!this.readsFromRedis(runId)) { + return this.delegate.findLatestExecutionSnapshot(runId, client, environmentId); + } + + const read = await this.redis.getLatest(runId, { ...(environmentId && { environmentId }) }); + if (!read) { + // A miss is the coexistence path: a pre-cutover run, or expired history. It is not an error. + this.metrics?.recordRead("findLatestExecutionSnapshot", "postgres"); + return this.delegate.findLatestExecutionSnapshot(runId, client, environmentId); + } + + if (read.danglingCycle) { + // The entry says it has waitpoints and the cycle key holding them is gone. Serving it would + // hand back an empty set that looks authoritative, and the run would resume with no waits. + // Postgres still has the join rows. + this.metrics?.recordRead("findLatestExecutionSnapshot", "postgres"); + return this.delegate.findLatestExecutionSnapshot(runId, client, environmentId); + } + + this.metrics?.recordRead("findLatestExecutionSnapshot", "redis"); + return this.#hydrate(read, runId, client, { hydrateWaitpointRows: true }); + } + + override async findExecutionSnapshot( + args: Prisma.SelectSubset, + client?: ReadClient + ): Promise | null> { + const shape = matchSinceCursorLookup(args); + if (!shape || !this.readsFromRedis(shape.runId)) { + return this.delegate.findExecutionSnapshot(args, client); + } + + const found = await this.redis.getById(shape.runId, shape.id, { + ...(shape.environmentId && { environmentId: shape.environmentId }), + }); + + if (!found) { + this.metrics?.recordRead("findExecutionSnapshot", "postgres"); + return this.delegate.findExecutionSnapshot(args, client); + } + + this.metrics?.recordRead("findExecutionSnapshot", "redis"); + // The engine selects createdAt only, so the answer is the cursor and nothing else. + return { + createdAt: new Date(found.entry.createdAt as string), + } as unknown as Prisma.TaskRunExecutionSnapshotGetPayload; + } + + override async findManyExecutionSnapshots( + args: Prisma.SelectSubset, + client?: ReadClient + ): Promise[]> { + const shape = matchSinceWindow(args); + if (!shape || !this.readsFromRedis(shape.runId)) { + return this.delegate.findManyExecutionSnapshots(args, client); + } + + const result = await this.redis.getSinceCreatedAt(shape.runId, shape.createdAt, { + limit: shape.take, + ...(shape.environmentId && { environmentId: shape.environmentId }), + }); + + if (result.kind === "miss") { + this.metrics?.recordRead("findManyExecutionSnapshots", "postgres"); + return this.delegate.findManyExecutionSnapshots(args, client); + } + + if (result.entries.some((entry) => entry.danglingCycle)) { + this.metrics?.recordRead("findManyExecutionSnapshots", "postgres"); + return this.delegate.findManyExecutionSnapshots(args, client); + } + + this.metrics?.recordRead("findManyExecutionSnapshots", "redis"); + + // The engine asks for createdAt DESC and reverses app-side; the store returns ascending. + const descending = [...result.entries].reverse(); + // Rows are hydrated for no entry here: the engine fetches the head's waitpoints itself, from + // the ids this call's head row reports. Each row still carries its own order. + const hydrated = await Promise.all( + descending.map((entry) => this.#hydrate(entry, shape.runId, client)) + ); + return hydrated as unknown as Prisma.TaskRunExecutionSnapshotGetPayload[]; + } + + override async findSnapshotCompletedWaitpointIds( + snapshotId: string, + client?: ReadClient, + runId?: string + ): Promise { + // Without a run id there is no keyspace to look in, so the router's fan-out is the only answer. + if (!runId || !this.readsFromRedis(runId)) { + return this.delegate.findSnapshotCompletedWaitpointIds(snapshotId, client, runId); + } + + const ids = await this.redis.getSnapshotWaitpointIds(runId, snapshotId); + if (!ids.present) { + this.metrics?.recordRead("findSnapshotCompletedWaitpointIds", "postgres"); + return this.delegate.findSnapshotCompletedWaitpointIds(snapshotId, client, runId); + } + + this.metrics?.recordRead("findSnapshotCompletedWaitpointIds", "redis"); + return ids.distinctIds; + } + + override async findSnapshotCompletedWaitpointIdsWithPresence( + snapshotId: string, + client?: ReadClient, + runId?: string + ): Promise<{ present: boolean; ids: string[] }> { + if (!runId || !this.readsFromRedis(runId)) { + return this.delegate.findSnapshotCompletedWaitpointIdsWithPresence(snapshotId, client, runId); + } + + const ids = await this.redis.getSnapshotWaitpointIds(runId, snapshotId); + if (!ids.present) { + // present=false means this reader cannot see the snapshot, so its empty list is not + // authoritative and the engine's read-repair needs the Postgres answer. + this.metrics?.recordRead("findSnapshotCompletedWaitpointIdsWithPresence", "postgres"); + return this.delegate.findSnapshotCompletedWaitpointIdsWithPresence(snapshotId, client, runId); + } + + this.metrics?.recordRead("findSnapshotCompletedWaitpointIdsWithPresence", "redis"); + return { present: true, ids: ids.distinctIds }; + } + + /** + * Turns a store entry into the Prisma payload the interface promises. + * + * The entry supplies every scalar column. `checkpoint` and the full waitpoint rows still live in + * Postgres, so they are read back through the delegate — but only when the entry says they exist, + * which keeps the common read (a running run with neither) free of any Postgres call at all. + */ + async #hydrate( + read: SnapshotRead, + runId: string, + client?: ReadClient, + opts?: { hydrateWaitpointRows?: boolean } + ): Promise< + Prisma.TaskRunExecutionSnapshotGetPayload<{ + include: { completedWaitpoints: true; checkpoint: true }; + }> + > { + const entry = read.entry as Record; + + const checkpoint = entry.checkpointId + ? await this.#hydrateCheckpoint(runId, read.id, client) + : null; + + // `completedWaitpointOrder` is a scalar column, NOT the join. The engine reads it off the head + // row as the index oracle that gives each completed waitpoint its position in a batch, so it + // must be populated even when the waitpoint ROWS are not fetched. Returning an empty order here + // resumes every batched triggerAndWait with `index: undefined`. + // Three cases, and only the last needs a second Redis call. The read already carries the ids + // when the store decoded them. An entry with no wait cycle has no waitpoints by construction, + // which is the common case and used to cost a round trip to rediscover. Anything else asks. + const ids = + read.completedWaitpointIds ?? + (read.cycle === undefined + ? { present: true, distinctIds: [], order: [] } + : await this.redis.getSnapshotWaitpointIds(runId, read.id)); + const completedWaitpointOrder = ids.order; + + // The rows themselves are head-only, mirroring the engine's own N x M avoidance. + const completedWaitpoints = opts?.hydrateWaitpointRows + ? await this.#fetchWaitpointsInChunks(ids.distinctIds, runId, client) + : []; + + return { + id: read.id, + engine: entry.engine ?? "V2", + executionStatus: entry.executionStatus, + description: entry.description, + previousSnapshotId: entry.previousSnapshotId ?? null, + runId: entry.runId, + runStatus: entry.runStatus, + attemptNumber: entry.attemptNumber ?? null, + batchId: entry.batchId ?? null, + environmentId: entry.environmentId, + environmentType: entry.environmentType, + projectId: entry.projectId, + organizationId: entry.organizationId, + checkpointId: entry.checkpointId ?? null, + workerId: entry.workerId ?? null, + runnerId: entry.runnerId ?? null, + metadata: entry.metadata ?? null, + // A column no code writes, so Postgres returns null for it on every row. The entry does not + // carry it, and omitting it here would hand back undefined where Postgres hands back null, + // on every single read served from Redis. + lastHeartbeatAt: null, + completedWaitpointOrder, + isValid: read.isValid, + error: entry.error ?? null, + createdAt: new Date(entry.createdAt as string), + // A snapshot row is write-once, so both columns hold the one instant the decorator minted. + updatedAt: new Date(entry.createdAt as string), + checkpoint, + completedWaitpoints, + } as unknown as Prisma.TaskRunExecutionSnapshotGetPayload<{ + include: { completedWaitpoints: true; checkpoint: true }; + }>; + } + + /** + * Chunked, and bounded within each chunk, mirroring the engine's own waitpoint fetch. The run id + * routes each chunk to the owning store rather than fanning every one across both databases. + */ + async #fetchWaitpointsInChunks( + waitpointIds: string[], + runId: string, + client?: ReadClient + ): Promise { + if (waitpointIds.length === 0) return []; + + const all: unknown[] = []; + for (let i = 0; i < waitpointIds.length; i += WAITPOINT_CHUNK_SIZE) { + const chunk = waitpointIds.slice(i, i + WAITPOINT_CHUNK_SIZE); + const rows = await this.delegate.findManyWaitpoints( + { where: { id: { in: boundedIn(chunk) } } }, + client, + runId + ); + all.push(...rows); + } + return all; + } + + /** + * Reads the checkpoint row through the snapshot the delegate still holds, so the read stays + * residency-aware: the run id in the where is what routes it to the owning database, and the + * decorator sits above the router and has no client of its own. + * + * At `redis-only` the Postgres snapshot row is gone, so this returns null. The checkpoint row + * itself stays in Postgres, but the interface has no residency-aware way to read one directly. + * Closing that needs a narrow lookup on the interface, which the plan freezes for this ticket. + */ + async #hydrateCheckpoint( + runId: string, + snapshotId: string, + client?: ReadClient + ): Promise { + const row = await this.delegate.findExecutionSnapshot( + { where: { id: snapshotId, runId }, include: { checkpoint: true } }, + client + ); + return (row as { checkpoint?: unknown } | null)?.checkpoint ?? null; + } + + async #enqueueRepair(entry: SnapshotEntryInput): Promise { + if (!this.onAppendFailure) { + return; + } + + try { + await this.onAppendFailure({ + runId: entry.runId, + snapshotId: entry.id, + executionStatus: entry.executionStatus, + }); + } catch (error) { + // The repair enqueue is itself best-effort. Failing it must not fail the caller's write. + this.logger.error("snapshot repair enqueue failed", { runId: entry.runId, error }); + } + } +} + +/** Position-sensitive: the same ids in a different order are a different wait cycle. */ +function sameOrder(a: string[], b: string[]): boolean { + return a.length === b.length && a.every((id, index) => id === b[index]); +} + +/** + * Turns the lock site's two lists into cycle refs. `completedWaitpointIds` is the complete set the + * delegate connects; `completedWaitpointOrder` gives a position only to the ids that have one, and a + * repeated id keeps each of its positions. + */ +function lockCycleRefs(ids: string[], order: string[]): { id: string; index?: number }[] { + const refs: { id: string; index?: number }[] = []; + const indexed = new Set(); + + order.forEach((id, index) => { + refs.push({ id, index }); + indexed.add(id); + }); + + for (const id of ids) { + if (!indexed.has(id)) refs.push({ id }); + } + + return refs; +} + +/** Membership only, for the id set, which has no meaningful order. */ +function sameSet(a: string[], b: string[]): boolean { + if (a.length !== b.length) return false; + const seen = new Set(a); + return b.every((id) => seen.has(id)); +} diff --git a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.waitpointCycles.test.ts b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.waitpointCycles.test.ts new file mode 100644 index 00000000000..4bc2d0dc271 --- /dev/null +++ b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.waitpointCycles.test.ts @@ -0,0 +1,706 @@ +// The completed-waitpoint path, which every other suite here was blind to. +// +// Two defects hid behind that blindness. The decorator passed no cycle to `append`, so no +// wp: key was ever written and the Redis waitpoint side was permanently empty. And the +// since-window hydration returned an empty `completedWaitpointOrder`, which is the index oracle the +// engine uses to give each completed waitpoint its position in a batch — an empty order resumes +// every batched triggerAndWait with `index: undefined`. +// +// So these tests all use a snapshot that ACTUALLY carries waitpoints. A test that does not cannot +// tell a working cycle from a missing one. +import { describe, expect } from "vitest"; +import { containerTest } from "@internal/testcontainers"; +import { createRedisClient } from "@internal/redis"; +import { generateInternalId } from "@trigger.dev/core/v3/isomorphic"; +import { PostgresRunStore } from "./PostgresRunStore.js"; +import { RedisSnapshotStore } from "./redisSnapshotStore.js"; +import { entryFromCreateRun } from "./snapshotEntry.js"; +import { TaskRunExecutionSnapshotStore } from "./taskRunExecutionSnapshotStore.js"; +import type { RunStore } from "./types.js"; +import { + buildCreateRunData, + seedSnapshotEnvironment, + seedSnapshotWaitpoints, + type SnapshotFixtureEnv, +} from "./testFixtures/snapshotIdFixture.js"; + +const COMPLETED_TTL_MS = 72 * 60 * 60 * 1000; + +function build( + prisma: never, + redisOptions: never, + mode: "dual-write" | "redis-read" = "redis-read" +) { + const redis = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + const writes: { site: string; outcome: string }[] = []; + const decorated = new TaskRunExecutionSnapshotStore( + new PostgresRunStore({ prisma, readOnlyPrisma: prisma }) as unknown as RunStore, + { + store: redis, + mode, + readPercent: 100, + metrics: { + recordWrite: (site, outcome) => writes.push({ site, outcome }), + recordAppendFailed: () => {}, + recordRead: () => {}, + }, + } + ); + return { decorated, redis, writes }; +} + +async function seedRun( + decorated: TaskRunExecutionSnapshotStore, + redis: RedisSnapshotStore, + env: SnapshotFixtureEnv +): Promise { + const runId = generateInternalId(); + const snapshot = { + id: generateInternalId(), + engine: "V2" as const, + executionStatus: "RUN_CREATED" as const, + description: "Run was created", + runStatus: "PENDING" as const, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }; + await redis.append({ + entry: entryFromCreateRun({ id: snapshot.id, runId, createdAt: new Date() }, snapshot), + kind: "birth", + isTerminal: false, + }); + await decorated.createRun({ data: buildCreateRunData(runId, env), snapshot }); + return runId; +} + +function resumeInput( + runId: string, + env: SnapshotFixtureEnv, + completedWaitpoints: { id: string; index?: number }[], + description = "Run resumed" +) { + return { + id: generateInternalId(), + run: { id: runId, status: "EXECUTING" as const, attemptNumber: 1 }, + snapshot: { executionStatus: "EXECUTING" as const, description }, + completedWaitpoints, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }; +} + +describe("completed-waitpoint cycles", () => { + containerTest("a resume append mints a cycle key", async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, redis, env); + const [wpA, wpB] = await seedSnapshotWaitpoints(prisma, env, 2); + + const created = await decorated.createExecutionSnapshot( + resumeInput(runId, env, [ + { id: wpA, index: 0 }, + { id: wpB, index: 1 }, + ]) + ); + + // The key exists at all — before the fix, none was ever written. + const cycleKeys = await probe.keys(`snap:{${runId}}:wp:*`); + expect(cycleKeys.length).toBe(1); + + const ids = await redis.getSnapshotWaitpointIds(runId, created.id); + expect(ids.present).toBe(true); + expect(ids.order).toEqual([wpA, wpB]); + expect(ids.distinctIds).toEqual([wpA, wpB]); + } finally { + await Promise.all([redis.quit(), probe.quit().catch(() => {})]); + } + }); + + containerTest( + "a copy-forward reuses the cycle and writes no second key", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, redis, env); + const [wpA, wpB] = await seedSnapshotWaitpoints(prisma, env, 2); + const waitpoints = [ + { id: wpA, index: 0 }, + { id: wpB, index: 1 }, + ]; + + await decorated.createExecutionSnapshot(resumeInput(runId, env, waitpoints, "resume")); + // The same id set again: this is the copy-forward every dequeue and checkpoint site does. + const second = await decorated.createExecutionSnapshot( + resumeInput(runId, env, waitpoints, "carry one") + ); + const third = await decorated.createExecutionSnapshot( + resumeInput(runId, env, waitpoints, "carry two") + ); + + // Still ONE key. Re-minting per entry is the write amplification the pointer model removes. + expect((await probe.keys(`snap:{${runId}}:wp:*`)).length).toBe(1); + + // And every entry still resolves the same order. + for (const id of [second.id, third.id]) { + expect((await redis.getSnapshotWaitpointIds(runId, id)).order).toEqual([wpA, wpB]); + } + } finally { + await Promise.all([redis.quit(), probe.quit().catch(() => {})]); + } + } + ); + + containerTest( + "a newly-differing id set mints a second cycle", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, redis, env); + const [wpA, wpB] = await seedSnapshotWaitpoints(prisma, env, 2); + + await decorated.createExecutionSnapshot( + resumeInput(runId, env, [{ id: wpA, index: 0 }], "first wait") + ); + const second = await decorated.createExecutionSnapshot( + resumeInput(runId, env, [{ id: wpB, index: 0 }], "second wait") + ); + + expect((await probe.keys(`snap:{${runId}}:wp:*`)).length).toBe(2); + expect((await redis.getSnapshotWaitpointIds(runId, second.id)).order).toEqual([wpB]); + } finally { + await Promise.all([redis.quit(), probe.quit().catch(() => {})]); + } + } + ); + + containerTest( + "the same ids in a different order are a new cycle", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, redis, env); + const [wpA, wpB] = await seedSnapshotWaitpoints(prisma, env, 2); + + await decorated.createExecutionSnapshot( + resumeInput(runId, env, [ + { id: wpA, index: 0 }, + { id: wpB, index: 1 }, + ]) + ); + // Order IS the index oracle, so a reordering is a different cycle, not a carry-forward. + const reordered = await decorated.createExecutionSnapshot( + resumeInput(runId, env, [ + { id: wpB, index: 0 }, + { id: wpA, index: 1 }, + ]) + ); + + expect((await probe.keys(`snap:{${runId}}:wp:*`)).length).toBe(2); + expect((await redis.getSnapshotWaitpointIds(runId, reordered.id)).order).toEqual([ + wpB, + wpA, + ]); + } finally { + await Promise.all([redis.quit(), probe.quit().catch(() => {})]); + } + } + ); + + containerTest("a repeated id keeps both of its positions", async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, redis, env); + const [wpX] = await seedSnapshotWaitpoints(prisma, env, 1); + + // One run batched twice under a single idempotency key: the id repeats, and each position + // must survive, because the runner matches results to positions. + const created = await decorated.createExecutionSnapshot( + resumeInput(runId, env, [ + { id: wpX, index: 0 }, + { id: wpX, index: 1 }, + ]) + ); + + const ids = await redis.getSnapshotWaitpointIds(runId, created.id); + expect(ids.order).toEqual([wpX, wpX]); + expect(ids.distinctIds).toEqual([wpX]); + } finally { + await redis.quit(); + } + }); + + containerTest( + "keeps a completed waitpoint that has no batch index", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, redis, env); + const [wpA] = await seedSnapshotWaitpoints(prisma, env, 1); + + // Every wait.for, every single triggerAndWait and every token resumes with no batch index: + // the engine passes `index: b.batchIndex ?? undefined`. Postgres records the id in the + // completed-waitpoint join regardless. The ordered list cannot hold it, because its + // positions ARE the indexes, so the complete set has to be stored separately or the wait's + // result vanishes on a Redis read. + const created = await decorated.createExecutionSnapshot( + resumeInput(runId, env, [{ id: wpA }], "single wait") + ); + + const ids = await redis.getSnapshotWaitpointIds(runId, created.id); + expect(ids.present).toBe(true); + expect(ids.distinctIds).toEqual([wpA]); + // No position, so it is absent from the oracle. That part is correct. + expect(ids.order).toEqual([]); + } finally { + await redis.quit(); + } + } + ); + + containerTest( + "matches the Postgres join for a mix of indexed and index-less waits", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, redis, env); + const [wpA, wpB, wpC] = await seedSnapshotWaitpoints(prisma, env, 3); + + const created = await decorated.createExecutionSnapshot( + resumeInput( + runId, + env, + [{ id: wpA, index: 0 }, { id: wpB }, { id: wpC, index: 1 }], + "mixed wait" + ) + ); + + // Parity with what Postgres holds is the actual requirement: the engine iterates the rows + // this set fetches, and uses the order only to assign each one its index. + const fromRedis = await redis.getSnapshotWaitpointIds(runId, created.id); + const fromPostgres = await new PostgresRunStore({ + prisma, + readOnlyPrisma: prisma, + }).findSnapshotCompletedWaitpointIds(created.id, undefined, runId); + + expect([...fromRedis.distinctIds].sort()).toEqual([...fromPostgres].sort()); + expect(fromRedis.order).toEqual([wpA, wpC]); + } finally { + await redis.quit(); + } + } + ); + + containerTest( + "two consecutive index-less waits do not share a cycle", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, redis, env); + const [wpA, wpB] = await seedSnapshotWaitpoints(prisma, env, 2); + + // Neither wait has a batch index, so both present an EMPTY order. Deciding carry-forward on + // the order alone makes them compare equal, and the second silently inherits the first's + // waitpoint set: its own result is never stored and a read returns the wrong id. + const first = await decorated.createExecutionSnapshot( + resumeInput(runId, env, [{ id: wpA }], "first single wait") + ); + const second = await decorated.createExecutionSnapshot( + resumeInput(runId, env, [{ id: wpB }], "second single wait") + ); + + expect((await redis.getSnapshotWaitpointIds(runId, first.id)).distinctIds).toEqual([wpA]); + expect((await redis.getSnapshotWaitpointIds(runId, second.id)).distinctIds).toEqual([wpB]); + expect((await probe.keys(`snap:{${runId}}:wp:*`)).length).toBe(2); + } finally { + await Promise.all([redis.quit(), probe.quit().catch(() => {})]); + } + } + ); + + containerTest( + "the same index-less wait repeated does still carry forward", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, redis, env); + const [wpA] = await seedSnapshotWaitpoints(prisma, env, 1); + + // The copy-forward case must survive the stricter comparison: the same id set, still one key. + await decorated.createExecutionSnapshot(resumeInput(runId, env, [{ id: wpA }], "wait")); + const carried = await decorated.createExecutionSnapshot( + resumeInput(runId, env, [{ id: wpA }], "carry") + ); + + expect((await probe.keys(`snap:{${runId}}:wp:*`)).length).toBe(1); + expect((await redis.getSnapshotWaitpointIds(runId, carried.id)).distinctIds).toEqual([wpA]); + } finally { + await Promise.all([redis.quit(), probe.quit().catch(() => {})]); + } + } + ); + + containerTest( + "the dequeue snapshot keeps an index-less waitpoint", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, redis, env); + const [wpA, wpB] = await seedSnapshotWaitpoints(prisma, env, 2); + const head = await redis.getLatest(runId); + const snapshotId = generateInternalId(); + + // Postgres connects completedWaitpointIds, the COMPLETE set. Building the Redis refs from + // completedWaitpointOrder instead drops every id that has no position in it. + await decorated.lockRunToWorker(runId, { + lockedAt: new Date(), + lockedById: undefined, + lockedToVersionId: undefined, + lockedQueueId: undefined, + startedAt: new Date(), + baseCostInCents: 0, + machinePreset: "small-1x", + taskVersion: "1.0.0", + snapshot: { + id: snapshotId, + previousSnapshotId: head!.id, + attemptNumber: 1, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + completedWaitpointIds: [wpA, wpB], + completedWaitpointOrder: [wpA], + }, + } as never); + + const ids = await redis.getSnapshotWaitpointIds(runId, snapshotId); + expect([...ids.distinctIds].sort()).toEqual([wpA, wpB].sort()); + expect(ids.order).toEqual([wpA]); + } finally { + await redis.quit(); + } + } + ); + + containerTest( + "findLatestExecutionSnapshot hydrates an index-less waitpoint row", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, redis, env); + const [wpA] = await seedSnapshotWaitpoints(prisma, env, 1); + + await decorated.createExecutionSnapshot(resumeInput(runId, env, [{ id: wpA }], "single")); + + // The hot read hydrates the rows from the id set, so an incomplete set means the resume + // gets no waitpoint at all. + const latest = await decorated.findLatestExecutionSnapshot(runId); + expect(latest!.completedWaitpoints.map((w) => w.id)).toEqual([wpA]); + } finally { + await redis.quit(); + } + } + ); + + containerTest( + "a refused carry mints a fresh cycle rather than writing a pointerless head", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, redis, env); + const [wpA] = await seedSnapshotWaitpoints(prisma, env, 1); + const snapshotId = generateInternalId(); + + // Driven at the store, because the decorator cannot reach this state deliberately: its + // probe reads the head first, sees the id set no longer matches, and mints a new cycle. + // The refusal is only reachable when the key vanishes BETWEEN that probe and the append, + // which is a race. Naming a cycle this incarnation never minted reproduces the same + // refusal deterministically. + const result = await redis.append({ + entry: { + id: snapshotId, + engine: "V2", + executionStatus: "EXECUTING", + description: "carry a cycle that was never minted", + runId, + runStatus: "EXECUTING", + createdAt: new Date().toISOString(), + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }, + kind: "transition", + isTerminal: false, + cycle: { + kind: "carryForward", + cycleSeq: 9999, + completedWaitpoints: [{ id: wpA, index: 0 }], + }, + }); + + expect(result.outcome).toBe("written"); + if (result.outcome !== "written") return; + + // Refusing the pointer is right. Writing the entry with NO pointer is not: it becomes the + // head, and a read of it answers present-with-nothing, which is the one answer that stops + // the engine's read-repair from looking. + expect(result.cycleMismatch).toBe(true); + expect(result.cycleSeq).toBeGreaterThan(0); + + const ids = await redis.getSnapshotWaitpointIds(runId, snapshotId); + expect(ids.present).toBe(true); + expect(ids.distinctIds).toEqual([wpA]); + } finally { + await redis.quit(); + } + } + ); + + containerTest( + "a dangling pointer reads as not present, not as an empty set", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, redis, env); + const [wpA] = await seedSnapshotWaitpoints(prisma, env, 1); + + const created = await decorated.createExecutionSnapshot( + resumeInput(runId, env, [{ id: wpA, index: 0 }], "resume") + ); + + // The entry keeps its pointer and the cycle key goes. Reachable by eviction, and by the + // completion TTL, which is set on every key for a run at once but expires them separately. + for (const key of await probe.keys(`snap:{${runId}}:wp:*`)) await probe.del(key); + + const ids = await redis.getSnapshotWaitpointIds(runId, created.id); + // present:false is what sends the caller to Postgres, which still holds the join rows. + // present:true with an empty set would suppress the engine's read-repair. + expect(ids.present).toBe(false); + expect(ids.distinctIds).toEqual([]); + + // And the projections the engine actually calls fall back rather than answering empty. + const withPresence = await decorated.findSnapshotCompletedWaitpointIdsWithPresence( + created.id, + undefined, + runId + ); + expect(withPresence.ids).toEqual([wpA]); + } finally { + await Promise.all([redis.quit(), probe.quit().catch(() => {})]); + } + } + ); + + containerTest( + "the hot read falls back to Postgres when the cycle key is gone", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, redis, env); + const [wpA] = await seedSnapshotWaitpoints(prisma, env, 1); + + await decorated.createExecutionSnapshot( + resumeInput(runId, env, [{ id: wpA, index: 0 }], "resume") + ); + for (const key of await probe.keys(`snap:{${runId}}:wp:*`)) await probe.del(key); + + const latest = await decorated.findLatestExecutionSnapshot(runId); + + // Served from Postgres, so the waitpoint is still there and the resume is not silently + // stripped of it. + expect(latest!.completedWaitpoints.map((w) => w.id)).toEqual([wpA]); + } finally { + await Promise.all([redis.quit(), probe.quit().catch(() => {})]); + } + } + ); + + containerTest( + "findLatestExecutionSnapshot returns the index oracle", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, redis, env); + const [wpA, wpB] = await seedSnapshotWaitpoints(prisma, env, 2); + await decorated.createExecutionSnapshot( + resumeInput(runId, env, [ + { id: wpA, index: 0 }, + { id: wpB, index: 1 }, + ]) + ); + + const latest = await decorated.findLatestExecutionSnapshot(runId); + + // completedWaitpointOrder is a scalar column, not the join. Empty here means every batched + // waitpoint resumes with index undefined. + expect(latest!.completedWaitpointOrder).toEqual([wpA, wpB]); + } finally { + await redis.quit(); + } + } + ); + + containerTest( + "the since-window head carries the index oracle", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, redis, env); + const [wpA, wpB] = await seedSnapshotWaitpoints(prisma, env, 2); + const first = await decorated.createExecutionSnapshot( + resumeInput(runId, env, [], "before the wait") + ); + await new Promise((resolve) => setTimeout(resolve, 5)); + await decorated.createExecutionSnapshot( + resumeInput(runId, env, [ + { id: wpA, index: 0 }, + { id: wpB, index: 1 }, + ]) + ); + + const window = await decorated.findManyExecutionSnapshots({ + where: { runId, isValid: true, createdAt: { gt: first.createdAt } }, + include: { checkpoint: true }, + orderBy: { createdAt: "desc" }, + take: 50, + }); + + // The head is first in a descending window. This is the row the engine reads the oracle off. + expect(window[0]!.completedWaitpointOrder).toEqual([wpA, wpB]); + } finally { + await redis.quit(); + } + } + ); + + containerTest( + "the since-window falls back to Postgres when the head cycle key is gone", + async ({ prisma, redisOptions }) => { + // The hot read has always handled this. The since-window did not: its Lua returned the head's + // order and distinct set but never its dangling flag, so the decorator's fallback guard was + // dead code and an expired cycle key came back as an EMPTY order. Empty means "no indexed + // waitpoints" to the engine, which is how a batched triggerAndWait resumes with every + // position lost rather than falling back to the store that still knows them. + const { decorated, redis } = build(prisma as never, redisOptions as never); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, redis, env); + const [wpA, wpB] = await seedSnapshotWaitpoints(prisma, env, 2); + const first = await decorated.createExecutionSnapshot( + resumeInput(runId, env, [], "before the wait") + ); + await new Promise((resolve) => setTimeout(resolve, 5)); + await decorated.createExecutionSnapshot( + resumeInput(runId, env, [ + { id: wpA, index: 0 }, + { id: wpB, index: 1 }, + ]) + ); + + // The entry hash survives; only the cycle key goes. The completion TTL is applied per key, + // so this is a state the keyspace reaches on its own. + for (const key of await probe.keys(`snap:{${runId}}:wp:*`)) await probe.del(key); + + const window = await decorated.findManyExecutionSnapshots({ + where: { runId, isValid: true, createdAt: { gt: first.createdAt } }, + include: { checkpoint: true }, + orderBy: { createdAt: "desc" }, + take: 50, + }); + + expect(window[0]!.completedWaitpointOrder).toEqual([wpA, wpB]); + } finally { + await Promise.all([redis.quit(), probe.quit().catch(() => {})]); + } + } + ); + + containerTest( + "lockRunToWorker carries its resolved order into the cycle", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, redis, env); + const [wpA, wpB] = await seedSnapshotWaitpoints(prisma, env, 2); + const head = await redis.getLatest(runId); + const snapshotId = generateInternalId(); + + await decorated.lockRunToWorker(runId, { + lockedAt: new Date(), + lockedById: undefined, + lockedToVersionId: undefined, + lockedQueueId: undefined, + startedAt: new Date(), + baseCostInCents: 0, + machinePreset: "small-1x", + taskVersion: "1.0.0", + snapshot: { + id: snapshotId, + previousSnapshotId: head!.id, + attemptNumber: 1, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + completedWaitpointIds: [wpA, wpB], + completedWaitpointOrder: [wpA, wpB], + }, + } as never); + + const ids = await redis.getSnapshotWaitpointIds(runId, snapshotId); + expect(ids.order).toEqual([wpA, wpB]); + } finally { + await redis.quit(); + } + } + ); + + containerTest( + "an append with no waitpoints writes no cycle key", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = build(prisma as never, redisOptions as never); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = await seedRun(decorated, redis, env); + + await decorated.createExecutionSnapshot(resumeInput(runId, env, [], "no waitpoints")); + + expect(await probe.keys(`snap:{${runId}}:wp:*`)).toEqual([]); + } finally { + await Promise.all([redis.quit(), probe.quit().catch(() => {})]); + } + } + ); +}); diff --git a/internal-packages/run-store/src/testFixtures/snapshotIdFixture.ts b/internal-packages/run-store/src/testFixtures/snapshotIdFixture.ts new file mode 100644 index 00000000000..e0b8a1a2107 --- /dev/null +++ b/internal-packages/run-store/src/testFixtures/snapshotIdFixture.ts @@ -0,0 +1,167 @@ +// Shared setup for the snapshot-id, snapshot-writes and entry-parity suites. Modelled on the +// seedEnvironment/buildCreateRunInput pair in PostgresRunStore.test.ts; the slugs are suffixed so +// several fixtures can coexist in one database. +import type { PrismaClient, TaskRunStatus } from "@trigger.dev/database"; +import { generateInternalId } from "@trigger.dev/core/v3/isomorphic"; +import type { CreateRunData } from "../types.js"; + +export type SnapshotFixtureEnv = { + id: string; + type: "DEVELOPMENT"; + projectId: string; + organizationId: string; +}; + +export type SnapshotIdFixture = { + run: { id: string }; + env: SnapshotFixtureEnv; +}; + +export async function seedSnapshotEnvironment(prisma: PrismaClient): Promise { + const suffix = generateInternalId().slice(-12); + + const organization = await prisma.organization.create({ + data: { title: `Snapshot Org ${suffix}`, slug: `snapshot-org-${suffix}` }, + }); + + const project = await prisma.project.create({ + data: { + name: `Snapshot Project ${suffix}`, + slug: `snapshot-project-${suffix}`, + externalRef: `proj_${suffix}`, + organizationId: organization.id, + }, + }); + + const environment = await prisma.runtimeEnvironment.create({ + data: { + type: "DEVELOPMENT", + slug: `dev-${suffix}`, + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_dev_${suffix}`, + pkApiKey: `pk_dev_${suffix}`, + shortcode: `short_${suffix}`, + }, + }); + + return { + id: environment.id, + type: "DEVELOPMENT", + projectId: project.id, + organizationId: organization.id, + }; +} + +export function buildCreateRunData(runId: string, env: SnapshotFixtureEnv): CreateRunData { + return { + id: runId, + engine: "V2", + status: "PENDING", + friendlyId: `run_${runId.slice(-16)}`, + runtimeEnvironmentId: env.id, + environmentType: env.type, + organizationId: env.organizationId, + projectId: env.projectId, + taskIdentifier: "my-task", + payload: "{}", + payloadType: "application/json", + traceContext: {}, + traceId: `trace_${runId.slice(-8)}`, + spanId: `span_${runId.slice(-8)}`, + queue: "task/my-task", + isTest: false, + taskEventStore: "taskEvent", + depth: 0, + }; +} + +export type SnapshotWorkerFixture = { workerId: string; taskId: string }; + +/** + * Seeds a BackgroundWorker and one of its tasks. The snapshot's `workerId` and the run's + * `lockedById` are both foreign keys, so a made-up id fails the constraint rather than the + * assertion, and the test reports a fixture fault as if it were a parity fault. + */ +export async function seedSnapshotWorker( + prisma: PrismaClient, + env: SnapshotFixtureEnv +): Promise { + const suffix = generateInternalId().slice(-12); + + const worker = await prisma.backgroundWorker.create({ + data: { + friendlyId: `worker_${suffix}`, + engine: "V2", + contentHash: `hash_${suffix}`, + projectId: env.projectId, + runtimeEnvironmentId: env.id, + version: "20260824.1", + metadata: {}, + }, + }); + + const task = await prisma.backgroundWorkerTask.create({ + data: { + slug: "my-task", + friendlyId: `task_${suffix}`, + filePath: "src/trigger/my-task.ts", + exportName: "myTask", + workerId: worker.id, + projectId: env.projectId, + runtimeEnvironmentId: env.id, + }, + }); + + return { workerId: worker.id, taskId: task.id }; +} + +/** + * Seeds real Waitpoint rows and returns their ids. The legacy completed-waitpoint join carries a + * real foreign key, so an invented id fails the constraint rather than the assertion — the test + * then reports a fixture fault as if it were a defect in the code under test. + */ +export async function seedSnapshotWaitpoints( + prisma: PrismaClient, + env: SnapshotFixtureEnv, + count: number +): Promise { + const ids: string[] = []; + + for (let i = 0; i < count; i++) { + const suffix = generateInternalId().slice(-12); + const waitpoint = await prisma.waitpoint.create({ + data: { + friendlyId: `waitpoint_${suffix}`, + type: "MANUAL", + status: "COMPLETED", + completedAt: new Date(), + idempotencyKey: `idem_${suffix}`, + userProvidedIdempotencyKey: false, + projectId: env.projectId, + environmentId: env.id, + }, + }); + ids.push(waitpoint.id); + } + + return ids; +} + +/** + * Seeds an environment plus one run in `status`, with no execution snapshot. The suites that use it + * assert on the snapshot rows a store method writes, so the run must start with none. + */ +export async function setupSnapshotIdFixture( + prisma: PrismaClient, + opts?: { status?: TaskRunStatus } +): Promise { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + + await prisma.taskRun.create({ + data: { ...buildCreateRunData(runId, env), status: opts?.status ?? "PENDING" }, + }); + + return { run: { id: runId }, env }; +} diff --git a/internal-packages/run-store/src/types.ts b/internal-packages/run-store/src/types.ts index 3ceb0a462dd..9ea39473e5b 100644 --- a/internal-packages/run-store/src/types.ts +++ b/internal-packages/run-store/src/types.ts @@ -31,6 +31,11 @@ export type IdempotencyKeyRunMatch = { }; export type CreateRunSnapshotInput = { + /** Caller-minted creation instant. Absent, Postgres applies its own default. The decorator sets + * it so a snapshot carries the SAME instant in Postgres and in the Redis store: the field is + * compared directly under dual-write, and the since-window cursor is resolved from one store and + * applied in the other, so two different instants misfilter that window. */ + createdAt?: Date; id?: string; engine: "V2"; executionStatus: TaskRunExecutionStatus; @@ -45,6 +50,14 @@ export type CreateRunSnapshotInput = { }; export type CompletionSnapshotInput = { + /** Caller-minted creation instant. Absent, Postgres applies its own default. The decorator sets + * it so a snapshot carries the SAME instant in Postgres and in the Redis store: the field is + * compared directly under dual-write, and the since-window cursor is resolved from one store and + * applied in the other, so two different instants misfilter that window. */ + createdAt?: Date; + /** Caller-minted snapshot id. Absent, Prisma's `@default(cuid())` supplies one. The decorator + * sets it so a snapshot carries the same id in Postgres and in the Redis store. */ + id?: string; executionStatus: "FINISHED"; description: string; runStatus: TaskRunStatus; @@ -66,6 +79,14 @@ export type PromotePendingVersionArgs = { }; export type ExpireSnapshotInput = { + /** Caller-minted creation instant. Absent, Postgres applies its own default. The decorator sets + * it so a snapshot carries the SAME instant in Postgres and in the Redis store: the field is + * compared directly under dual-write, and the since-window cursor is resolved from one store and + * applied in the other, so two different instants misfilter that window. */ + createdAt?: Date; + /** Caller-minted snapshot id. Absent, Prisma's `@default(cuid())` supplies one. The decorator + * sets it so a snapshot carries the same id in Postgres and in the Redis store. */ + id?: string; engine: "V2"; executionStatus: "FINISHED"; description: string; @@ -77,6 +98,14 @@ export type ExpireSnapshotInput = { }; export type RescheduleSnapshotInput = { + /** Caller-minted creation instant. Absent, Postgres applies its own default. The decorator sets + * it so a snapshot carries the SAME instant in Postgres and in the Redis store: the field is + * compared directly under dual-write, and the since-window cursor is resolved from one store and + * applied in the other, so two different instants misfilter that window. */ + createdAt?: Date; + /** Caller-minted snapshot id. Absent, Prisma's `@default(cuid())` supplies one. The decorator + * sets it so a snapshot carries the same id in Postgres and in the Redis store. */ + id?: string; environmentId: string; environmentType: RuntimeEnvironmentType; projectId: string; @@ -87,6 +116,11 @@ export type RescheduleSnapshotInput = { }; export type LockSnapshotInput = { + /** Caller-minted creation instant. Absent, Postgres applies its own default. The decorator sets + * it so a snapshot carries the SAME instant in Postgres and in the Redis store: the field is + * compared directly under dual-write, and the since-window cursor is resolved from one store and + * applied in the other, so two different instants misfilter that window. */ + createdAt?: Date; id: string; previousSnapshotId: string; attemptNumber?: number; @@ -294,6 +328,14 @@ export type TaskRunWithWaitpoint = TaskRun & { associatedWaitpoint: Waitpoint | * input — callers pass the high-level shape, not a raw Prisma `data`/`include`. */ export type CreateExecutionSnapshotInput = { + /** Caller-minted creation instant. Absent, Postgres applies its own default. The decorator sets + * it so a snapshot carries the SAME instant in Postgres and in the Redis store: the field is + * compared directly under dual-write, and the since-window cursor is resolved from one store and + * applied in the other, so two different instants misfilter that window. */ + createdAt?: Date; + /** Caller-minted snapshot id. Absent, Prisma's `@default(cuid())` supplies one. The decorator + * sets it so a snapshot carries the same id in Postgres and in the Redis store. */ + id?: string; run: { id: string; status: TaskRunStatus; attemptNumber?: number | null }; snapshot: { executionStatus: TaskRunExecutionStatus;