From fa0a0b39263aa615bebeb9b597c396a7967fc260 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Sat, 29 Aug 2026 10:36:57 +0100 Subject: [PATCH 01/30] feat(run-engine): per-concurrency-key limit override storage and methods Sparse ckLimits HASH at the base queue whose fields are the exact ck-variant queue names, plus engine methods to set (atomic cardinality cap, default 1000 per queue), remove, and read the overrides. The admit-path gate wiring follows. --- .../run-engine/src/run-queue/index.ts | 95 +++++++++++++++++++ .../run-engine/src/run-queue/keyProducer.ts | 15 +++ .../run-engine/src/run-queue/types.ts | 2 + 3 files changed, 112 insertions(+) diff --git a/internal-packages/run-engine/src/run-queue/index.ts b/internal-packages/run-engine/src/run-queue/index.ts index f8e28d55731..848e56a3a63 100644 --- a/internal-packages/run-engine/src/run-queue/index.ts +++ b/internal-packages/run-engine/src/run-queue/index.ts @@ -255,6 +255,13 @@ export interface RunQueueMetricsEmitter { emitGauge(shardKey: string, fields: Record): void; } +export class RunQueueConcurrencyKeyLimitExceededError extends Error { + constructor(message: string) { + super(message); + this.name = "RunQueueConcurrencyKeyLimitExceededError"; + } +} + export type RunQueueOptions = { name: string; tracer: Tracer; @@ -314,6 +321,8 @@ export type RunQueueOptions = { * the total cap covering releases from builds without the mirror. */ gatesEnabled?: boolean; + /** Cap on per-concurrency-key limit overrides stored per queue. Default 1000. */ + maxConcurrencyKeyOverridesPerQueue?: number; workerOptions?: { pollIntervalMs?: number; immediatePollIntervalMs?: number; @@ -427,6 +436,7 @@ export class RunQueue { private queueSelectionStrategy: RunQueueSelectionStrategy; private shardCount: number; private counterTtlSeconds: number; + private maxConcurrencyKeyOverridesPerQueue: number; private abortController: AbortController; private worker: Worker; private workerQueueResolver: WorkerQueueResolver; @@ -437,6 +447,7 @@ export class RunQueue { constructor(public readonly options: RunQueueOptions) { this.shardCount = options.shardCount ?? 2; this.counterTtlSeconds = options.counterTtlSeconds ?? 86400; + this.maxConcurrencyKeyOverridesPerQueue = options.maxConcurrencyKeyOverridesPerQueue ?? 1000; this.retryOptions = options.retryOptions ?? defaultRetrySettings; this.redis = createRedisClient(options.redis, { onError: (error) => { @@ -631,6 +642,62 @@ export class RunQueue { return this.redis.scard(this.keys.queueGroupConcurrencyKey(env, queue)); } + /** + * Sets a per-concurrency-key limit override for a queue. The stored value is the + * raw requested limit; admit paths clamp to the environment limit at read time. + * Throws RunQueueConcurrencyKeyLimitExceededError when a NEW key would push the + * queue past maxConcurrencyKeyOverridesPerQueue (updates to existing keys always + * succeed). + */ + public async updateQueueConcurrencyKeyLimit( + env: MinimalAuthenticatedEnvironment, + queue: string, + concurrencyKey: string, + limit: number + ) { + const result = await this.redis.setQueueConcurrencyKeyLimit( + this.keys.queueCkLimitsKey(env, queue), + this.keys.queueKey(env, queue, concurrencyKey), + String(limit), + String(this.maxConcurrencyKeyOverridesPerQueue) + ); + + if (result === 0) { + throw new RunQueueConcurrencyKeyLimitExceededError( + `Cannot add a concurrency key override to queue ${queue}: the queue already has ${this.maxConcurrencyKeyOverridesPerQueue} overrides` + ); + } + } + + public async removeQueueConcurrencyKeyLimit( + env: MinimalAuthenticatedEnvironment, + queue: string, + concurrencyKey: string + ) { + return this.redis.hdel( + this.keys.queueCkLimitsKey(env, queue), + this.keys.queueKey(env, queue, concurrencyKey) + ); + } + + /** Returns the raw per-concurrency-key limit overrides for a queue, keyed by concurrency key value. */ + public async getQueueConcurrencyKeyLimits( + env: MinimalAuthenticatedEnvironment, + queue: string + ): Promise> { + const raw = await this.redis.hgetall(this.keys.queueCkLimitsKey(env, queue)); + + const limits: Record = {}; + for (const [variantName, value] of Object.entries(raw)) { + const ckIndex = variantName.indexOf(":ck:"); + if (ckIndex === -1) { + continue; + } + limits[variantName.slice(ckIndex + 4)] = Number(value); + } + return limits; + } + public async updateEnvConcurrencyLimits(env: MinimalAuthenticatedEnvironment) { await this.#callUpdateEnvironmentConcurrencyLimits({ envConcurrencyLimitKey: this.keys.envConcurrencyLimitKey(env), @@ -5893,6 +5960,26 @@ __gatesRelease(keyPrefix, redis.call('GET', messageKey), messageId) `, }); + this.redis.defineCommand("setQueueConcurrencyKeyLimit", { + numberOfKeys: 1, + lua: ` +local ckLimitsKey = KEYS[1] + +local fieldName = ARGV[1] +local limit = ARGV[2] +local maxFields = tonumber(ARGV[3]) + +if redis.call('HEXISTS', ckLimitsKey, fieldName) == 0 then + if redis.call('HLEN', ckLimitsKey) >= maxFields then + return 0 + end +end + +redis.call('HSET', ckLimitsKey, fieldName, limit) +return 1 +`, + }); + this.redis.defineCommand("updateEnvironmentConcurrencyLimits", { numberOfKeys: 2, lua: ` @@ -6254,6 +6341,14 @@ declare module "@internal/redis" { callback?: Callback ): Result; + setQueueConcurrencyKeyLimit( + ckLimitsKey: string, + fieldName: string, + limit: string, + maxFields: string, + callback?: Callback + ): Result; + updateEnvironmentConcurrencyLimits( // keys envConcurrencyLimitKey: string, diff --git a/internal-packages/run-engine/src/run-queue/keyProducer.ts b/internal-packages/run-engine/src/run-queue/keyProducer.ts index 98028f5af7b..7b997043244 100644 --- a/internal-packages/run-engine/src/run-queue/keyProducer.ts +++ b/internal-packages/run-engine/src/run-queue/keyProducer.ts @@ -26,6 +26,7 @@ const constants = { RUNNING_COUNTER_PART: "runningCounter", GROUP_CONCURRENCY_PART: "groupConcurrency", TOTAL_CONCURRENCY_LIMIT_PART: "totalConcurrency", + CK_LIMITS_PART: "ckLimits", } as const; export class RunQueueFullKeyProducer implements RunQueueKeyProducer { @@ -366,6 +367,20 @@ export class RunQueueFullKeyProducer implements RunQueueKeyProducer { return `${this.baseQueueKeyFromQueue(queue)}:${constants.TOTAL_CONCURRENCY_LIMIT_PART}`; } + /** + * HASH of per-concurrency-key limit overrides for a queue. Lives at the base + * queue; each field is the EXACT full ck-variant queue name (the ckIndex ZSET + * member), so reads need no parsing, and values are the raw requested limits + * (readers clamp to the environment limit). + */ + queueCkLimitsKey(env: RunQueueKeyProducerEnvironment, queue: string): string { + return `${this.queueKey(env, queue)}:${constants.CK_LIMITS_PART}`; + } + + queueCkLimitsKeyFromQueue(queue: string): string { + return `${this.baseQueueKeyFromQueue(queue)}:${constants.CK_LIMITS_PART}`; + } + isCkWildcard(queue: string): boolean { return queue.endsWith(":ck:*"); } diff --git a/internal-packages/run-engine/src/run-queue/types.ts b/internal-packages/run-engine/src/run-queue/types.ts index 75651a1f847..639d206be5c 100644 --- a/internal-packages/run-engine/src/run-queue/types.ts +++ b/internal-packages/run-engine/src/run-queue/types.ts @@ -111,6 +111,8 @@ export interface RunQueueKeyProducer { queueGroupConcurrencyKeyFromQueue(queue: string): string; queueTotalConcurrencyLimitKey(env: RunQueueKeyProducerEnvironment, queue: string): string; queueTotalConcurrencyLimitKeyFromQueue(queue: string): string; + queueCkLimitsKey(env: RunQueueKeyProducerEnvironment, queue: string): string; + queueCkLimitsKeyFromQueue(queue: string): string; //env oncurrency envCurrentConcurrencyKey(env: EnvDescriptor): string; From 43b478d0ce840854cbd8319357f57ac388fab75e Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Sat, 29 Aug 2026 10:46:04 +0100 Subject: [PATCH 02/30] feat(run-engine): enforce per-key limit overrides at admit time The ck dequeue admit and both enqueue fast paths read the queue's ckLimits HASH for the variant being admitted and use the env-clamped override in place of the queue's per-key limit, behind the totalConcurrencyEnabled flag. Covered by tests for lowered and raised keys, removal, the cardinality cap, and flag-off behavior. --- .../run-engine/src/run-queue/index.ts | 38 ++- .../tests/concurrencyKeyOverrides.test.ts | 270 ++++++++++++++++++ 2 files changed, 304 insertions(+), 4 deletions(-) create mode 100644 internal-packages/run-engine/src/run-queue/tests/concurrencyKeyOverrides.test.ts diff --git a/internal-packages/run-engine/src/run-queue/index.ts b/internal-packages/run-engine/src/run-queue/index.ts index 848e56a3a63..1c983c3ccb6 100644 --- a/internal-packages/run-engine/src/run-queue/index.ts +++ b/internal-packages/run-engine/src/run-queue/index.ts @@ -2447,6 +2447,7 @@ export class RunQueue { const totalConcurrencyLimitKey = this.keys.queueTotalConcurrencyLimitKeyFromQueue( message.queue ); + const ckLimitsKey = this.keys.queueCkLimitsKeyFromQueue(message.queue); const totalConcurrencyEnabledArg = this.options.totalConcurrencyEnabled ? "1" : "0"; if (ttlInfo) { @@ -2470,6 +2471,7 @@ export class RunQueue { baseQueueKey, groupConcurrencyKey, totalConcurrencyLimitKey, + ckLimitsKey, // args queueName, messageId, @@ -2509,6 +2511,7 @@ export class RunQueue { baseQueueKey, groupConcurrencyKey, totalConcurrencyLimitKey, + ckLimitsKey, // args queueName, messageId, @@ -2796,6 +2799,7 @@ export class RunQueue { runningCounterKey, this.keys.queueGroupConcurrencyKeyFromQueue(ckWildcardQueue), this.keys.queueTotalConcurrencyLimitKeyFromQueue(ckWildcardQueue), + this.keys.queueCkLimitsKeyFromQueue(ckWildcardQueue), //args ckWildcardQueue, String(Date.now()), @@ -4084,7 +4088,7 @@ return __qmret(0) // *Tracked variants of dequeueMessageFromKey and the ack/nack/dlq/release/clear // scripts. this.redis.defineCommand("enqueueMessageCkTracked", { - numberOfKeys: 17, + numberOfKeys: 18, lua: ` local masterQueueKey = KEYS[1] local queueKey = KEYS[2] @@ -4106,6 +4110,7 @@ local baseQueueKey = KEYS[15] -- Total-cap keys (KEYS 16-17) local groupConcurrencyKey = KEYS[16] local totalConcurrencyLimitKey = KEYS[17] +local ckLimitsKey = KEYS[18] local queueName = ARGV[1] local messageId = ARGV[2] @@ -4143,6 +4148,12 @@ if enableFastPath == '1' then tonumber(redis.call('GET', queueConcurrencyLimitKey) or '1000000'), envLimit ) + if totalConcurrencyEnabled then + local perKeyOverride = redis.call('HGET', ckLimitsKey, queueName) + if perKeyOverride then + queueLimit = math.min(tonumber(perKeyOverride), envLimit) + end + end if queueCurrent < queueLimit then -- Total-cap gate: a fast-path admit consumes a group slot, so it must @@ -4254,7 +4265,7 @@ return __qmret(0) }); this.redis.defineCommand("enqueueMessageWithTtlCkTracked", { - numberOfKeys: 18, + numberOfKeys: 19, lua: ` local masterQueueKey = KEYS[1] local queueKey = KEYS[2] @@ -4277,6 +4288,7 @@ local baseQueueKey = KEYS[16] -- Total-cap keys (KEYS 17-18) local groupConcurrencyKey = KEYS[17] local totalConcurrencyLimitKey = KEYS[18] +local ckLimitsKey = KEYS[19] local queueName = ARGV[1] local messageId = ARGV[2] @@ -4316,6 +4328,12 @@ if enableFastPath == '1' then tonumber(redis.call('GET', queueConcurrencyLimitKey) or '1000000'), envLimit ) + if totalConcurrencyEnabled then + local perKeyOverride = redis.call('HGET', ckLimitsKey, queueName) + if perKeyOverride then + queueLimit = math.min(tonumber(perKeyOverride), envLimit) + end + end if queueCurrent < queueLimit then -- Total-cap gate: see enqueueMessageCkTracked. @@ -4940,7 +4958,7 @@ return results // (normal dequeue, TTL-expired, or stale-orphan path — all of which were // counted at enqueue time). this.redis.defineCommand("dequeueMessagesFromCkQueueTracked", { - numberOfKeys: 13, + numberOfKeys: 14, lua: ` local ckIndexKey = KEYS[1] local queueConcurrencyLimitKey = KEYS[2] @@ -4955,6 +4973,7 @@ local lengthCounterKey = KEYS[10] local runningCounterKey = KEYS[11] local groupConcurrencyKey = KEYS[12] local totalConcurrencyLimitKey = KEYS[13] +local ckLimitsKey = KEYS[14] local ckWildcardName = ARGV[1] local currentTime = tonumber(ARGV[2]) @@ -5044,7 +5063,15 @@ for _, ckQueueName in ipairs(ckQueues) do local ckConcurrencyKey = fullQueueKey .. ':currentConcurrency' local ckCurrentConcurrency = tonumber(redis.call('SCARD', ckConcurrencyKey) or '0') - if ckCurrentConcurrency < queueConcurrencyLimit then + local perKeyLimit = queueConcurrencyLimit + if totalConcurrencyEnabled then + local perKeyOverride = redis.call('HGET', ckLimitsKey, ckQueueName) + if perKeyOverride then + perKeyLimit = math.min(tonumber(perKeyOverride), envConcurrencyLimit) + end + end + + if ckCurrentConcurrency < perKeyLimit then local messages = redis.call('ZRANGEBYSCORE', fullQueueKey, '-inf', tostring(currentTime), 'WITHSCORES', 'LIMIT', 0, 1) if #messages >= 2 then @@ -6535,6 +6562,7 @@ declare module "@internal/redis" { baseQueueKey: string, groupConcurrencyKey: string, totalConcurrencyLimitKey: string, + ckLimitsKey: string, queueName: string, messageId: string, messageData: string, @@ -6572,6 +6600,7 @@ declare module "@internal/redis" { baseQueueKey: string, groupConcurrencyKey: string, totalConcurrencyLimitKey: string, + ckLimitsKey: string, queueName: string, messageId: string, messageData: string, @@ -6606,6 +6635,7 @@ declare module "@internal/redis" { runningCounterKey: string, groupConcurrencyKey: string, totalConcurrencyLimitKey: string, + ckLimitsKey: string, ckWildcardName: string, currentTime: string, defaultEnvConcurrencyLimit: string, diff --git a/internal-packages/run-engine/src/run-queue/tests/concurrencyKeyOverrides.test.ts b/internal-packages/run-engine/src/run-queue/tests/concurrencyKeyOverrides.test.ts new file mode 100644 index 00000000000..7995369bc3c --- /dev/null +++ b/internal-packages/run-engine/src/run-queue/tests/concurrencyKeyOverrides.test.ts @@ -0,0 +1,270 @@ +import { redisTest } from "@internal/testcontainers"; +import { trace } from "@internal/tracing"; +import { setTimeout } from "node:timers/promises"; +import { describe } from "vitest"; +import { FairQueueSelectionStrategy } from "../fairQueueSelectionStrategy.js"; +import { RunQueue, RunQueueConcurrencyKeyLimitExceededError } from "../index.js"; +import { RunQueueFullKeyProducer } from "../keyProducer.js"; +import type { InputPayload } from "../types.js"; +import { Decimal } from "@trigger.dev/database"; + +const testOptions = { + name: "rq", + tracer: trace.getTracer("rq"), + workers: 1, + defaultEnvConcurrency: 25, + retryOptions: { + maxAttempts: 5, + factor: 1.1, + minTimeoutInMs: 100, + maxTimeoutInMs: 1_000, + randomize: true, + }, + keys: new RunQueueFullKeyProducer(), +}; + +const authenticatedEnvDev = { + id: "e1234", + type: "DEVELOPMENT" as const, + maximumConcurrencyLimit: 10, + concurrencyLimitBurstFactor: new Decimal(2.0), + project: { id: "p1234" }, + organization: { id: "o1234" }, +}; + +function createQueue(redisContainer: any, totalConcurrencyEnabled: boolean, maxOverrides?: number) { + return new RunQueue({ + ...testOptions, + totalConcurrencyEnabled, + maxConcurrencyKeyOverridesPerQueue: maxOverrides, + queueSelectionStrategy: new FairQueueSelectionStrategy({ + redis: { + keyPrefix: "runqueue:test:", + host: redisContainer.getHost(), + port: redisContainer.getPort(), + }, + keys: testOptions.keys, + }), + redis: { + keyPrefix: "runqueue:test:", + host: redisContainer.getHost(), + port: redisContainer.getPort(), + }, + }); +} + +function makeMessage(overrides: Partial = {}): InputPayload { + return { + runId: "r1", + taskIdentifier: "task/my-task", + orgId: "o1234", + projectId: "p1234", + environmentId: "e1234", + environmentType: "DEVELOPMENT", + queue: "task/my-task", + timestamp: Date.now(), + attempt: 0, + ...overrides, + }; +} + +async function waitFor(condition: () => Promise, timeoutMs = 20_000): Promise { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + if (await condition()) { + return true; + } + await setTimeout(250); + } + return condition(); +} + +vi.setConfig({ testTimeout: 60_000 }); + +describe("RunQueue per-concurrency-key limit overrides", () => { + redisTest( + "a lowered key is capped while other keys keep the queue limit", + async ({ redisContainer }) => { + const queue = createQueue(redisContainer, true); + try { + await queue.updateQueueConcurrencyLimits(authenticatedEnvDev, "task/my-task", 2); + await queue.updateQueueConcurrencyKeyLimit(authenticatedEnvDev, "task/my-task", "ck-a", 1); + + const now = Date.now(); + const messages = [ + ["ck-a", "a0"], + ["ck-a", "a1"], + ["ck-b", "b0"], + ["ck-b", "b1"], + ] as const; + for (const [i, [ck, id]] of messages.entries()) { + await queue.enqueueMessage({ + env: authenticatedEnvDev, + message: makeMessage({ runId: id, concurrencyKey: ck, timestamp: now - 1000 + i }), + workerQueue: "main", + }); + } + + const settled = await waitFor(async () => { + const a = await queue.currentConcurrencyOfQueue( + authenticatedEnvDev, + "task/my-task", + "ck-a" + ); + const b = await queue.currentConcurrencyOfQueue( + authenticatedEnvDev, + "task/my-task", + "ck-b" + ); + return a === 1 && b === 2; + }); + expect(settled).toBe(true); + + await setTimeout(2000); + expect( + await queue.currentConcurrencyOfQueue(authenticatedEnvDev, "task/my-task", "ck-a") + ).toBe(1); + expect(await queue.lengthOfQueue(authenticatedEnvDev, "task/my-task")).toBe(1); + } finally { + await queue.quit(); + } + } + ); + + redisTest("a raised key admits past the queue limit", async ({ redisContainer }) => { + const queue = createQueue(redisContainer, true); + try { + await queue.updateQueueConcurrencyLimits(authenticatedEnvDev, "task/my-task", 1); + await queue.updateQueueConcurrencyKeyLimit(authenticatedEnvDev, "task/my-task", "ck-a", 3); + + const now = Date.now(); + for (const i of [0, 1, 2]) { + await queue.enqueueMessage({ + env: authenticatedEnvDev, + message: makeMessage({ + runId: `a${i}`, + concurrencyKey: "ck-a", + timestamp: now - 1000 + i, + }), + workerQueue: "main", + }); + } + await queue.enqueueMessage({ + env: authenticatedEnvDev, + message: makeMessage({ runId: "b0", concurrencyKey: "ck-b", timestamp: now - 500 }), + workerQueue: "main", + }); + await queue.enqueueMessage({ + env: authenticatedEnvDev, + message: makeMessage({ runId: "b1", concurrencyKey: "ck-b", timestamp: now - 499 }), + workerQueue: "main", + }); + + const settled = await waitFor(async () => { + const a = await queue.currentConcurrencyOfQueue( + authenticatedEnvDev, + "task/my-task", + "ck-a" + ); + const b = await queue.currentConcurrencyOfQueue( + authenticatedEnvDev, + "task/my-task", + "ck-b" + ); + return a === 3 && b === 1; + }); + expect(settled).toBe(true); + + await setTimeout(2000); + expect(await queue.lengthOfQueue(authenticatedEnvDev, "task/my-task")).toBe(1); + } finally { + await queue.quit(); + } + }); + + redisTest("removing an override restores the queue limit", async ({ redisContainer }) => { + const queue = createQueue(redisContainer, true); + try { + await queue.updateQueueConcurrencyLimits(authenticatedEnvDev, "task/my-task", 2); + await queue.updateQueueConcurrencyKeyLimit(authenticatedEnvDev, "task/my-task", "ck-a", 1); + expect(await queue.getQueueConcurrencyKeyLimits(authenticatedEnvDev, "task/my-task")).toEqual( + { "ck-a": 1 } + ); + + await queue.removeQueueConcurrencyKeyLimit(authenticatedEnvDev, "task/my-task", "ck-a"); + expect(await queue.getQueueConcurrencyKeyLimits(authenticatedEnvDev, "task/my-task")).toEqual( + {} + ); + + const now = Date.now(); + for (const i of [0, 1]) { + await queue.enqueueMessage({ + env: authenticatedEnvDev, + message: makeMessage({ + runId: `a${i}`, + concurrencyKey: "ck-a", + timestamp: now - 1000 + i, + }), + workerQueue: "main", + }); + } + + const settled = await waitFor( + async () => + (await queue.currentConcurrencyOfQueue(authenticatedEnvDev, "task/my-task", "ck-a")) === 2 + ); + expect(settled).toBe(true); + } finally { + await queue.quit(); + } + }); + + redisTest("the per-queue override count is capped", async ({ redisContainer }) => { + const queue = createQueue(redisContainer, true, 2); + try { + await queue.updateQueueConcurrencyKeyLimit(authenticatedEnvDev, "task/my-task", "ck-a", 1); + await queue.updateQueueConcurrencyKeyLimit(authenticatedEnvDev, "task/my-task", "ck-b", 1); + + await expect( + queue.updateQueueConcurrencyKeyLimit(authenticatedEnvDev, "task/my-task", "ck-c", 1) + ).rejects.toThrow(RunQueueConcurrencyKeyLimitExceededError); + + /** Updates to existing keys always succeed at the cap. */ + await queue.updateQueueConcurrencyKeyLimit(authenticatedEnvDev, "task/my-task", "ck-a", 4); + expect(await queue.getQueueConcurrencyKeyLimits(authenticatedEnvDev, "task/my-task")).toEqual( + { "ck-a": 4, "ck-b": 1 } + ); + } finally { + await queue.quit(); + } + }); + + redisTest("overrides are ignored when disabled", async ({ redisContainer }) => { + const queue = createQueue(redisContainer, false); + try { + await queue.updateQueueConcurrencyLimits(authenticatedEnvDev, "task/my-task", 2); + await queue.updateQueueConcurrencyKeyLimit(authenticatedEnvDev, "task/my-task", "ck-a", 1); + + const now = Date.now(); + for (const i of [0, 1]) { + await queue.enqueueMessage({ + env: authenticatedEnvDev, + message: makeMessage({ + runId: `a${i}`, + concurrencyKey: "ck-a", + timestamp: now - 1000 + i, + }), + workerQueue: "main", + }); + } + + const settled = await waitFor( + async () => + (await queue.currentConcurrencyOfQueue(authenticatedEnvDev, "task/my-task", "ck-a")) === 2 + ); + expect(settled).toBe(true); + } finally { + await queue.quit(); + } + }); +}); From a3cdd3b431b17c468f61a6511f7499117dbf3dc8 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Sat, 29 Aug 2026 10:54:53 +0100 Subject: [PATCH 03/30] feat(database): total-override bookkeeping and per-key override table Three nullable TaskQueue columns record when, by whom, and from what declared base the total concurrency limit was overridden, and a new TaskQueueConcurrencyKeyOverride child table stores per-key limit overrides, unique per queue and key and cascading with the queue. --- .../migration.sql | 24 +++++++++++++++ .../database/prisma/schema.prisma | 30 ++++++++++++++++++- 2 files changed, 53 insertions(+), 1 deletion(-) create mode 100644 internal-packages/database/prisma/migrations/20260829150000_add_concurrency_overrides/migration.sql diff --git a/internal-packages/database/prisma/migrations/20260829150000_add_concurrency_overrides/migration.sql b/internal-packages/database/prisma/migrations/20260829150000_add_concurrency_overrides/migration.sql new file mode 100644 index 00000000000..d95778f276d --- /dev/null +++ b/internal-packages/database/prisma/migrations/20260829150000_add_concurrency_overrides/migration.sql @@ -0,0 +1,24 @@ +-- AlterTable +ALTER TABLE "TaskQueue" ADD COLUMN "totalConcurrencyLimitOverriddenAt" TIMESTAMP(3); +ALTER TABLE "TaskQueue" ADD COLUMN "totalConcurrencyLimitOverriddenBy" TEXT; +ALTER TABLE "TaskQueue" ADD COLUMN "totalConcurrencyLimitBase" INTEGER; + +-- CreateTable +CREATE TABLE "TaskQueueConcurrencyKeyOverride" ( + "id" TEXT NOT NULL, + "taskQueueId" TEXT NOT NULL, + "concurrencyKey" TEXT NOT NULL, + "concurrencyLimit" INTEGER NOT NULL, + "overriddenAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "overriddenBy" TEXT, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + + CONSTRAINT "TaskQueueConcurrencyKeyOverride_pkey" PRIMARY KEY ("id") +); + +-- CreateIndex +CREATE UNIQUE INDEX "TaskQueueConcurrencyKeyOverride_taskQueueId_concurrencyKey_key" ON "TaskQueueConcurrencyKeyOverride"("taskQueueId", "concurrencyKey"); + +-- AddForeignKey +ALTER TABLE "TaskQueueConcurrencyKeyOverride" ADD CONSTRAINT "TaskQueueConcurrencyKeyOverride_taskQueueId_fkey" FOREIGN KEY ("taskQueueId") REFERENCES "TaskQueue"("id") ON DELETE CASCADE ON UPDATE CASCADE; diff --git a/internal-packages/database/prisma/schema.prisma b/internal-packages/database/prisma/schema.prisma index c88ba5b5887..65a84ff1645 100644 --- a/internal-packages/database/prisma/schema.prisma +++ b/internal-packages/database/prisma/schema.prisma @@ -1983,7 +1983,13 @@ model TaskQueue { concurrencyLimitOverridePercent Decimal? @db.Decimal(5, 2) /// Caps total concurrent runs across ALL concurrencyKey values of this queue /// (concurrencyLimit applies per key value). Null = no total cap. - totalConcurrencyLimit Int? + totalConcurrencyLimit Int? + /// When the total concurrency limit was overridden + totalConcurrencyLimitOverriddenAt DateTime? + /// Who overrode the total concurrency limit (null when overridden via the API) + totalConcurrencyLimitOverriddenBy String? + /// If totalConcurrencyLimit is overridden, the declared value it reverts to on reset + totalConcurrencyLimitBase Int? rateLimit Json? paused Boolean @default(false) @@ -1995,9 +2001,31 @@ model TaskQueue { tasks BackgroundWorkerTask[] workers BackgroundWorker[] + concurrencyKeyOverrides TaskQueueConcurrencyKeyOverride[] + @@unique([runtimeEnvironmentId, name]) } +/// A per-concurrency-key limit override for a queue: the named key value gets this +/// limit instead of the queue's concurrencyLimit. Deleting the row resets the key. +model TaskQueueConcurrencyKeyOverride { + id String @id @default(cuid()) + + taskQueue TaskQueue @relation(fields: [taskQueueId], references: [id], onDelete: Cascade, onUpdate: Cascade) + taskQueueId String + + concurrencyKey String + concurrencyLimit Int + + overriddenAt DateTime @default(now()) + overriddenBy String? + + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + @@unique([taskQueueId, concurrencyKey]) +} + enum TaskQueueType { VIRTUAL NAMED From eeb3e44f349af763bfb890e7ccd7e93e9b781928 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Sat, 29 Aug 2026 11:02:54 +0100 Subject: [PATCH 04/30] feat(sdk,core,webapp): runtime overrides for total and per-key limits queues.overrideConcurrencyLimit accepts a named concurrencyKey to adjust one key's limit independently, and new overrideTotalConcurrencyLimit and resetTotalConcurrencyLimit calls adjust the cap across all keys. Four API routes back them; the concurrency system validates against the environment limit, captures the declared base on first override, persists per-key overrides in the child table alongside the engine hash, and deploys keep an overridden total instead of clobbering it from the manifest. --- .changeset/queue-concurrency-overrides.md | 18 ++ ...es.$queueParam.concurrency.key.override.ts | 103 ++++++++ ...ueues.$queueParam.concurrency.key.reset.ts | 95 +++++++ ....$queueParam.concurrency.total.override.ts | 96 +++++++ ...ues.$queueParam.concurrency.total.reset.ts | 97 +++++++ .../v3/services/concurrencySystem.server.ts | 244 +++++++++++++++++- .../services/createBackgroundWorker.server.ts | 4 +- internal-packages/run-engine/src/index.ts | 1 + packages/core/src/v3/apiClient/index.ts | 97 +++++++ packages/trigger-sdk/src/v3/queues.ts | 98 ++++++- 10 files changed, 847 insertions(+), 6 deletions(-) create mode 100644 .changeset/queue-concurrency-overrides.md create mode 100644 apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.key.override.ts create mode 100644 apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.key.reset.ts create mode 100644 apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.total.override.ts create mode 100644 apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.total.reset.ts diff --git a/.changeset/queue-concurrency-overrides.md b/.changeset/queue-concurrency-overrides.md new file mode 100644 index 00000000000..65579b0d81a --- /dev/null +++ b/.changeset/queue-concurrency-overrides.md @@ -0,0 +1,18 @@ +--- +"@trigger.dev/sdk": patch +"@trigger.dev/core": patch +--- + +Adjust queue concurrency at runtime, per key and in total. `queues.overrideConcurrencyLimit` accepts a `concurrencyKey` to raise or lower one key's limit without touching the rest of the queue, and the new `queues.overrideTotalConcurrencyLimit` and `queues.resetTotalConcurrencyLimit` adjust the cap across all keys. + +```ts +import { queues } from "@trigger.dev/sdk"; + +await queues.overrideConcurrencyLimit("my-queue", 20, { concurrencyKey: "tenant-123" }); +await queues.resetConcurrencyLimit("my-queue", { concurrencyKey: "tenant-123" }); + +await queues.overrideTotalConcurrencyLimit("my-queue", 100); +await queues.resetTotalConcurrencyLimit("my-queue"); +``` + +Overrides survive deploys and reset back to the declared configuration. Enforcement happens server-side on servers with total concurrency limits enabled. diff --git a/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.key.override.ts b/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.key.override.ts new file mode 100644 index 00000000000..9a72155d4fc --- /dev/null +++ b/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.key.override.ts @@ -0,0 +1,103 @@ +import { json } from "@remix-run/server-runtime"; +import { type RetrieveQueueParam, RetrieveQueueType } from "@trigger.dev/core/v3"; +import { z } from "zod"; +import { toQueueItem } from "~/presenters/v3/QueueRetrievePresenter.server"; +import { createActionApiRoute } from "~/services/routeBuilders/apiBuilder.server"; +import { concurrencySystem } from "~/v3/services/concurrencySystemInstance.server"; + +const BodySchema = z.object({ + type: RetrieveQueueType.default("id"), + concurrencyKey: z.string().min(1).max(128), + concurrencyLimit: z.number().int().min(0).max(100000), +}); + +const route = createActionApiRoute( + { + body: BodySchema, + params: z.object({ + queueParam: z.string().transform((val) => val.replace(/%2F/g, "/")), + }), + authorization: { + action: "write", + resource: () => ({ type: "queues" }), + }, + }, + async ({ params, body, authentication }) => { + const input: RetrieveQueueParam = + body.type === "id" + ? params.queueParam + : { + type: body.type, + name: decodeURIComponent(params.queueParam).replace(/%2F/g, "/"), + }; + + return concurrencySystem.queues + .overrideConcurrencyKeyLimit( + authentication.environment, + input, + body.concurrencyKey, + body.concurrencyLimit + ) + .match( + (queue) => { + return json( + toQueueItem({ + friendlyId: queue.friendlyId, + name: queue.name, + type: queue.type, + running: queue.running, + queued: queue.queued, + concurrencyLimit: queue.concurrencyLimit, + concurrencyLimitBase: queue.concurrencyLimitBase, + concurrencyLimitOverriddenAt: queue.concurrencyLimitOverriddenAt, + concurrencyLimitOverriddenBy: null, + paused: queue.paused, + }), + { status: 200 } + ); + }, + (error) => { + switch (error.type) { + case "queue_not_found": { + return json({ error: "Queue not found" }, { status: 404 }); + } + case "invalid_override": + case "concurrency_limit_exceeds_maximum": + case "too_many_key_overrides": { + return json({ error: error.message }, { status: 400 }); + } + case "queue_update_failed": { + return json( + { error: "Failed to update queue concurrency key limit" }, + { status: 500 } + ); + } + case "sync_queue_concurrency_to_engine_failed": { + return json({ error: "Failed to sync the concurrency key limit" }, { status: 500 }); + } + case "get_queue_stats_failed": { + return json({ error: "Failed to read queue stats" }, { status: 500 }); + } + case "other": { + return json( + { error: "Failed to update queue concurrency key limit" }, + { + status: 500, + } + ); + } + default: { + return json( + { error: "Failed to update queue concurrency key limit" }, + { + status: 500, + } + ); + } + } + } + ); + } +); + +export const { action } = route; diff --git a/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.key.reset.ts b/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.key.reset.ts new file mode 100644 index 00000000000..3024ac84b72 --- /dev/null +++ b/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.key.reset.ts @@ -0,0 +1,95 @@ +import { json } from "@remix-run/server-runtime"; +import { type RetrieveQueueParam, RetrieveQueueType } from "@trigger.dev/core/v3"; +import { z } from "zod"; +import { toQueueItem } from "~/presenters/v3/QueueRetrievePresenter.server"; +import { createActionApiRoute } from "~/services/routeBuilders/apiBuilder.server"; +import { concurrencySystem } from "~/v3/services/concurrencySystemInstance.server"; + +const BodySchema = z.object({ + type: RetrieveQueueType.default("id"), + concurrencyKey: z.string().min(1).max(128), +}); + +const route = createActionApiRoute( + { + body: BodySchema, + params: z.object({ + queueParam: z.string().transform((val) => val.replace(/%2F/g, "/")), + }), + authorization: { + action: "write", + resource: () => ({ type: "queues" }), + }, + }, + async ({ params, body, authentication }) => { + const input: RetrieveQueueParam = + body.type === "id" + ? params.queueParam + : { + type: body.type, + name: decodeURIComponent(params.queueParam).replace(/%2F/g, "/"), + }; + + return concurrencySystem.queues + .resetConcurrencyKeyLimit(authentication.environment, input, body.concurrencyKey) + .match( + (queue) => { + return json( + toQueueItem({ + friendlyId: queue.friendlyId, + name: queue.name, + type: queue.type, + running: queue.running, + queued: queue.queued, + concurrencyLimit: queue.concurrencyLimit, + concurrencyLimitBase: queue.concurrencyLimitBase, + concurrencyLimitOverriddenAt: queue.concurrencyLimitOverriddenAt, + concurrencyLimitOverriddenBy: null, + paused: queue.paused, + }), + { status: 200 } + ); + }, + (error) => { + switch (error.type) { + case "queue_not_found": { + return json({ error: "Queue not found" }, { status: 404 }); + } + case "queue_not_overridden": { + return json( + { error: "This concurrency key does not have an override" }, + { status: 400 } + ); + } + case "queue_update_failed": { + return json({ error: "Failed to reset the concurrency key limit" }, { status: 500 }); + } + case "sync_queue_concurrency_to_engine_failed": { + return json({ error: "Failed to sync the concurrency key limit" }, { status: 500 }); + } + case "get_queue_stats_failed": { + return json({ error: "Failed to read queue stats" }, { status: 500 }); + } + case "other": { + return json( + { error: "Failed to reset the concurrency key limit" }, + { + status: 500, + } + ); + } + default: { + return json( + { error: "Failed to reset the concurrency key limit" }, + { + status: 500, + } + ); + } + } + } + ); + } +); + +export const { action } = route; diff --git a/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.total.override.ts b/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.total.override.ts new file mode 100644 index 00000000000..b0c34ae1379 --- /dev/null +++ b/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.total.override.ts @@ -0,0 +1,96 @@ +import { json } from "@remix-run/server-runtime"; +import { type RetrieveQueueParam, RetrieveQueueType } from "@trigger.dev/core/v3"; +import { z } from "zod"; +import { toQueueItem } from "~/presenters/v3/QueueRetrievePresenter.server"; +import { createActionApiRoute } from "~/services/routeBuilders/apiBuilder.server"; +import { concurrencySystem } from "~/v3/services/concurrencySystemInstance.server"; + +const BodySchema = z.object({ + type: RetrieveQueueType.default("id"), + concurrencyLimit: z.number().int().min(0).max(100000), +}); + +const route = createActionApiRoute( + { + body: BodySchema, + params: z.object({ + queueParam: z.string().transform((val) => val.replace(/%2F/g, "/")), + }), + authorization: { + action: "write", + resource: () => ({ type: "queues" }), + }, + }, + async ({ params, body, authentication }) => { + const input: RetrieveQueueParam = + body.type === "id" + ? params.queueParam + : { + type: body.type, + name: decodeURIComponent(params.queueParam).replace(/%2F/g, "/"), + }; + + return concurrencySystem.queues + .overrideTotalConcurrencyLimit(authentication.environment, input, body.concurrencyLimit) + .match( + (queue) => { + return json( + toQueueItem({ + friendlyId: queue.friendlyId, + name: queue.name, + type: queue.type, + running: queue.running, + queued: queue.queued, + concurrencyLimit: queue.concurrencyLimit, + concurrencyLimitBase: queue.concurrencyLimitBase, + concurrencyLimitOverriddenAt: queue.concurrencyLimitOverriddenAt, + concurrencyLimitOverriddenBy: null, + paused: queue.paused, + }), + { status: 200 } + ); + }, + (error) => { + switch (error.type) { + case "queue_not_found": { + return json({ error: "Queue not found" }, { status: 404 }); + } + case "invalid_override": + case "concurrency_limit_exceeds_maximum": { + return json({ error: error.message }, { status: 400 }); + } + case "queue_update_failed": { + return json( + { error: "Failed to update queue total concurrency limit" }, + { status: 500 } + ); + } + case "sync_queue_concurrency_to_engine_failed": { + return json({ error: "Failed to sync the total concurrency limit" }, { status: 500 }); + } + case "get_queue_stats_failed": { + return json({ error: "Failed to read queue stats" }, { status: 500 }); + } + case "other": { + return json( + { error: "Failed to update queue total concurrency limit" }, + { + status: 500, + } + ); + } + default: { + return json( + { error: "Failed to update queue total concurrency limit" }, + { + status: 500, + } + ); + } + } + } + ); + } +); + +export const { action } = route; diff --git a/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.total.reset.ts b/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.total.reset.ts new file mode 100644 index 00000000000..2eacbb3ed6a --- /dev/null +++ b/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.total.reset.ts @@ -0,0 +1,97 @@ +import { json } from "@remix-run/server-runtime"; +import { type RetrieveQueueParam, RetrieveQueueType } from "@trigger.dev/core/v3"; +import { z } from "zod"; +import { toQueueItem } from "~/presenters/v3/QueueRetrievePresenter.server"; +import { createActionApiRoute } from "~/services/routeBuilders/apiBuilder.server"; +import { concurrencySystem } from "~/v3/services/concurrencySystemInstance.server"; + +const BodySchema = z.object({ + type: RetrieveQueueType.default("id"), +}); + +const route = createActionApiRoute( + { + body: BodySchema, + params: z.object({ + queueParam: z.string().transform((val) => val.replace(/%2F/g, "/")), + }), + authorization: { + action: "write", + resource: () => ({ type: "queues" }), + }, + }, + async ({ params, body, authentication }) => { + const input: RetrieveQueueParam = + body.type === "id" + ? params.queueParam + : { + type: body.type, + name: decodeURIComponent(params.queueParam).replace(/%2F/g, "/"), + }; + + return concurrencySystem.queues + .resetTotalConcurrencyLimit(authentication.environment, input) + .match( + (queue) => { + return json( + toQueueItem({ + friendlyId: queue.friendlyId, + name: queue.name, + type: queue.type, + running: queue.running, + queued: queue.queued, + concurrencyLimit: queue.concurrencyLimit, + concurrencyLimitBase: queue.concurrencyLimitBase, + concurrencyLimitOverriddenAt: queue.concurrencyLimitOverriddenAt, + concurrencyLimitOverriddenBy: null, + paused: queue.paused, + }), + { status: 200 } + ); + }, + (error) => { + switch (error.type) { + case "queue_not_found": { + return json({ error: "Queue not found" }, { status: 404 }); + } + case "queue_not_overridden": { + return json( + { error: "The queue total concurrency limit is not overridden" }, + { status: 400 } + ); + } + case "queue_update_failed": { + return json( + { error: "Failed to reset the queue total concurrency limit" }, + { status: 500 } + ); + } + case "sync_queue_concurrency_to_engine_failed": { + return json({ error: "Failed to sync the total concurrency limit" }, { status: 500 }); + } + case "get_queue_stats_failed": { + return json({ error: "Failed to read queue stats" }, { status: 500 }); + } + case "other": { + return json( + { error: "Failed to reset the queue total concurrency limit" }, + { + status: 500, + } + ); + } + default: { + return json( + { error: "Failed to reset the queue total concurrency limit" }, + { + status: 500, + } + ); + } + } + } + ); + } +); + +export const { action } = route; diff --git a/apps/webapp/app/v3/services/concurrencySystem.server.ts b/apps/webapp/app/v3/services/concurrencySystem.server.ts index 51c51674234..3945c087053 100644 --- a/apps/webapp/app/v3/services/concurrencySystem.server.ts +++ b/apps/webapp/app/v3/services/concurrencySystem.server.ts @@ -3,7 +3,13 @@ import { errAsync, fromPromise, okAsync } from "neverthrow"; import type { PrismaClientOrTransaction } from "~/db.server"; import type { AuthenticatedEnvironment } from "~/services/apiAuth.server"; import { logger } from "~/services/logger.server"; -import { removeQueueConcurrencyLimits, updateQueueConcurrencyLimits } from "../runQueue.server"; +import { + removeQueueConcurrencyLimits, + removeQueueTotalConcurrencyLimits, + updateQueueConcurrencyLimits, + updateQueueTotalConcurrencyLimits, +} from "../runQueue.server"; +import { RunQueueConcurrencyKeyLimitExceededError } from "@internal/run-engine"; import { engine } from "../runEngine.server"; export type ConcurrencySystemOptions = { @@ -77,6 +83,60 @@ export class ConcurrencySystem { .andThen((queue) => syncQueueConcurrencyToEngine(environment, queue)) .andThen((queue) => getQueueStats(environment, queue)); }, + overrideTotalConcurrencyLimit: ( + environment: AuthenticatedEnvironment, + queue: QueueInput, + totalConcurrencyLimit: number, + overriddenBy?: User + ) => { + return findQueueFromInput(this.db, environment, queue) + .andThen((queue) => + overrideQueueTotalConcurrencyLimit( + this.db, + environment, + queue, + totalConcurrencyLimit, + overriddenBy + ) + ) + .andThen((queue) => syncQueueTotalConcurrencyToEngine(environment, queue)) + .andThen((queue) => getQueueStats(environment, queue)); + }, + resetTotalConcurrencyLimit: (environment: AuthenticatedEnvironment, queue: QueueInput) => { + return findQueueFromInput(this.db, environment, queue) + .andThen((queue) => resetQueueTotalConcurrencyLimit(this.db, queue)) + .andThen((queue) => syncQueueTotalConcurrencyToEngine(environment, queue)) + .andThen((queue) => getQueueStats(environment, queue)); + }, + overrideConcurrencyKeyLimit: ( + environment: AuthenticatedEnvironment, + queue: QueueInput, + concurrencyKey: string, + concurrencyLimit: number, + overriddenBy?: User + ) => { + return findQueueFromInput(this.db, environment, queue) + .andThen((queue) => + overrideQueueConcurrencyKeyLimit( + this.db, + environment, + queue, + concurrencyKey, + concurrencyLimit, + overriddenBy + ) + ) + .andThen((queue) => getQueueStats(environment, queue)); + }, + resetConcurrencyKeyLimit: ( + environment: AuthenticatedEnvironment, + queue: QueueInput, + concurrencyKey: string + ) => { + return findQueueFromInput(this.db, environment, queue) + .andThen((queue) => resetQueueConcurrencyKeyLimit(this.db, environment, queue, concurrencyKey)) + .andThen((queue) => getQueueStats(environment, queue)); + }, /** * Recalculates the materialized limit of every percent-based override in the environment * against its CURRENT maximumConcurrencyLimit and syncs changed queues to the run engine. @@ -316,6 +376,188 @@ function syncQueueConcurrencyToEngine(environment: AuthenticatedEnvironment, que } } +function overrideQueueTotalConcurrencyLimit( + db: PrismaClientOrTransaction, + environment: AuthenticatedEnvironment, + queue: TaskQueue, + totalConcurrencyLimit: number, + overriddenBy?: User +) { + const maximum = environment.maximumConcurrencyLimit; + + if (!Number.isFinite(totalConcurrencyLimit) || totalConcurrencyLimit < 0) { + return errAsync({ + type: "invalid_override" as const, + message: "Total concurrency limit must be a non-negative number", + }); + } + + if (totalConcurrencyLimit > maximum) { + return errAsync({ + type: "concurrency_limit_exceeds_maximum" as const, + message: `Total concurrency limit (${totalConcurrencyLimit}) cannot exceed the environment limit (${maximum})`, + }); + } + + const totalConcurrencyLimitBase = queue.totalConcurrencyLimitOverriddenAt + ? queue.totalConcurrencyLimitBase + : queue.totalConcurrencyLimit; + + return fromPromise( + db.taskQueue.update({ + where: { id: queue.id }, + data: { + totalConcurrencyLimit, + totalConcurrencyLimitBase: totalConcurrencyLimitBase ?? null, + totalConcurrencyLimitOverriddenAt: new Date(), + totalConcurrencyLimitOverriddenBy: overriddenBy?.id ?? null, + }, + }), + (error) => ({ + type: "queue_update_failed" as const, + cause: error, + }) + ); +} + +function resetQueueTotalConcurrencyLimit(db: PrismaClientOrTransaction, queue: TaskQueue) { + if (queue.totalConcurrencyLimitOverriddenAt === null) { + return errAsync({ type: "queue_not_overridden" as const }); + } + + return fromPromise( + db.taskQueue.update({ + where: { id: queue.id }, + data: { + totalConcurrencyLimit: queue.totalConcurrencyLimitBase, + totalConcurrencyLimitBase: null, + totalConcurrencyLimitOverriddenAt: null, + totalConcurrencyLimitOverriddenBy: null, + }, + }), + (error) => ({ + type: "queue_update_failed" as const, + cause: error, + }) + ); +} + +/** + * The total limit key is separate from the per-queue limit key that pause zeroes, + * so it syncs regardless of the paused state. + */ +function syncQueueTotalConcurrencyToEngine( + environment: AuthenticatedEnvironment, + queue: TaskQueue +) { + if (typeof queue.totalConcurrencyLimit === "number") { + return fromPromise( + updateQueueTotalConcurrencyLimits(environment, queue.name, queue.totalConcurrencyLimit), + (error) => ({ + type: "sync_queue_concurrency_to_engine_failed" as const, + cause: error, + }) + ).andThen(() => okAsync(queue)); + } + + return fromPromise(removeQueueTotalConcurrencyLimits(environment, queue.name), (error) => ({ + type: "sync_queue_concurrency_to_engine_failed" as const, + cause: error, + })).andThen(() => okAsync(queue)); +} + +function overrideQueueConcurrencyKeyLimit( + db: PrismaClientOrTransaction, + environment: AuthenticatedEnvironment, + queue: TaskQueue, + concurrencyKey: string, + concurrencyLimit: number, + overriddenBy?: User +) { + const maximum = environment.maximumConcurrencyLimit; + + if (!Number.isFinite(concurrencyLimit) || concurrencyLimit < 0) { + return errAsync({ + type: "invalid_override" as const, + message: "Concurrency limit must be a non-negative number", + }); + } + + if (concurrencyLimit > maximum) { + return errAsync({ + type: "concurrency_limit_exceeds_maximum" as const, + message: `Concurrency limit (${concurrencyLimit}) cannot exceed the environment limit (${maximum})`, + }); + } + + return fromPromise( + engine.runQueue.updateQueueConcurrencyKeyLimit( + environment, + queue.name, + concurrencyKey, + concurrencyLimit + ), + (error) => { + if (error instanceof RunQueueConcurrencyKeyLimitExceededError) { + return { type: "too_many_key_overrides" as const, message: error.message }; + } + return { type: "sync_queue_concurrency_to_engine_failed" as const, cause: error }; + } + ) + .andThen(() => + fromPromise( + db.taskQueueConcurrencyKeyOverride.upsert({ + where: { taskQueueId_concurrencyKey: { taskQueueId: queue.id, concurrencyKey } }, + create: { + taskQueueId: queue.id, + concurrencyKey, + concurrencyLimit, + overriddenBy: overriddenBy?.id ?? null, + }, + update: { + concurrencyLimit, + overriddenAt: new Date(), + overriddenBy: overriddenBy?.id ?? null, + }, + }), + (error) => ({ + type: "queue_update_failed" as const, + cause: error, + }) + ) + ) + .andThen(() => okAsync(queue)); +} + +function resetQueueConcurrencyKeyLimit( + db: PrismaClientOrTransaction, + environment: AuthenticatedEnvironment, + queue: TaskQueue, + concurrencyKey: string +) { + return fromPromise( + db.taskQueueConcurrencyKeyOverride.deleteMany({ + where: { taskQueueId: queue.id, concurrencyKey }, + }), + (error) => ({ + type: "queue_update_failed" as const, + cause: error, + }) + ).andThen((deleted) => { + if (deleted.count === 0) { + return errAsync({ type: "queue_not_overridden" as const }); + } + + return fromPromise( + engine.runQueue.removeQueueConcurrencyKeyLimit(environment, queue.name, concurrencyKey), + (error) => ({ + type: "sync_queue_concurrency_to_engine_failed" as const, + cause: error, + }) + ).andThen(() => okAsync(queue)); + }); +} + function getQueueStats(environment: AuthenticatedEnvironment, queue: TaskQueue) { return fromPromise( Promise.all([ diff --git a/apps/webapp/app/v3/services/createBackgroundWorker.server.ts b/apps/webapp/app/v3/services/createBackgroundWorker.server.ts index 386f8f038d6..147248bd376 100644 --- a/apps/webapp/app/v3/services/createBackgroundWorker.server.ts +++ b/apps/webapp/app/v3/services/createBackgroundWorker.server.ts @@ -660,6 +660,7 @@ async function upsertWorkerQueueRecord( }); } else { const hasOverride = taskQueue.concurrencyLimitOverriddenAt !== null; + const hasTotalOverride = taskQueue.totalConcurrencyLimitOverriddenAt !== null; taskQueue = await prisma.taskQueue.update({ where: { @@ -672,7 +673,8 @@ async function upsertWorkerQueueRecord( // If overridden, keep current limit and update base; otherwise update limit normally concurrencyLimit: hasOverride ? undefined : concurrencyLimit, concurrencyLimitBase: hasOverride ? concurrencyLimit : undefined, - totalConcurrencyLimit, + totalConcurrencyLimit: hasTotalOverride ? undefined : totalConcurrencyLimit, + totalConcurrencyLimitBase: hasTotalOverride ? totalConcurrencyLimit : undefined, }, }); } diff --git a/internal-packages/run-engine/src/index.ts b/internal-packages/run-engine/src/index.ts index f30e271d5a4..77758c6db11 100644 --- a/internal-packages/run-engine/src/index.ts +++ b/internal-packages/run-engine/src/index.ts @@ -1,4 +1,5 @@ export { RunEngine } from "./engine/index.js"; +export { RunQueueConcurrencyKeyLimitExceededError } from "./run-queue/index.js"; export { RunDuplicateIdempotencyKeyError, RunOneTimeUseTokenError, diff --git a/packages/core/src/v3/apiClient/index.ts b/packages/core/src/v3/apiClient/index.ts index 461091cc918..6a4d24ace35 100644 --- a/packages/core/src/v3/apiClient/index.ts +++ b/packages/core/src/v3/apiClient/index.ts @@ -1765,6 +1765,103 @@ export class ApiClient { ); } + overrideQueueTotalConcurrencyLimit( + queue: RetrieveQueueParam, + concurrencyLimit: number, + requestOptions?: ZodFetchOptions + ) { + const type = typeof queue === "string" ? "id" : queue.type; + const value = typeof queue === "string" ? queue : queue.name; + + const encodedValue = encodeURIComponent(value.replace(/\//g, "%2F")); + + return zodfetch( + QueueItem, + `${this.baseUrl}/api/v1/queues/${encodedValue}/concurrency/total/override`, + { + method: "POST", + headers: this.#getHeaders(false), + body: JSON.stringify({ + type, + concurrencyLimit, + }), + }, + mergeRequestOptions(this.defaultRequestOptions, requestOptions) + ); + } + + resetQueueTotalConcurrencyLimit(queue: RetrieveQueueParam, requestOptions?: ZodFetchOptions) { + const type = typeof queue === "string" ? "id" : queue.type; + const value = typeof queue === "string" ? queue : queue.name; + + const encodedValue = encodeURIComponent(value.replace(/\//g, "%2F")); + + return zodfetch( + QueueItem, + `${this.baseUrl}/api/v1/queues/${encodedValue}/concurrency/total/reset`, + { + method: "POST", + headers: this.#getHeaders(false), + body: JSON.stringify({ + type, + }), + }, + mergeRequestOptions(this.defaultRequestOptions, requestOptions) + ); + } + + overrideQueueConcurrencyKeyLimit( + queue: RetrieveQueueParam, + concurrencyKey: string, + concurrencyLimit: number, + requestOptions?: ZodFetchOptions + ) { + const type = typeof queue === "string" ? "id" : queue.type; + const value = typeof queue === "string" ? queue : queue.name; + + const encodedValue = encodeURIComponent(value.replace(/\//g, "%2F")); + + return zodfetch( + QueueItem, + `${this.baseUrl}/api/v1/queues/${encodedValue}/concurrency/key/override`, + { + method: "POST", + headers: this.#getHeaders(false), + body: JSON.stringify({ + type, + concurrencyKey, + concurrencyLimit, + }), + }, + mergeRequestOptions(this.defaultRequestOptions, requestOptions) + ); + } + + resetQueueConcurrencyKeyLimit( + queue: RetrieveQueueParam, + concurrencyKey: string, + requestOptions?: ZodFetchOptions + ) { + const type = typeof queue === "string" ? "id" : queue.type; + const value = typeof queue === "string" ? queue : queue.name; + + const encodedValue = encodeURIComponent(value.replace(/\//g, "%2F")); + + return zodfetch( + QueueItem, + `${this.baseUrl}/api/v1/queues/${encodedValue}/concurrency/key/reset`, + { + method: "POST", + headers: this.#getHeaders(false), + body: JSON.stringify({ + type, + concurrencyKey, + }), + }, + mergeRequestOptions(this.defaultRequestOptions, requestOptions) + ); + } + subscribeToRun( runId: string, options?: { diff --git a/packages/trigger-sdk/src/v3/queues.ts b/packages/trigger-sdk/src/v3/queues.ts index 7e76c5f940b..e4f3a9b6641 100644 --- a/packages/trigger-sdk/src/v3/queues.ts +++ b/packages/trigger-sdk/src/v3/queues.ts @@ -144,9 +144,10 @@ export function pause( export function overrideConcurrencyLimit( queue: RetrieveQueueParam, concurrencyLimit: number, - requestOptions?: ApiRequestOptions + options?: ApiRequestOptions & { concurrencyKey?: string } ): ApiPromise { const apiClient = apiClientManager.clientOrThrow(); + const { concurrencyKey, ...requestOptions } = options ?? {}; const $requestOptions = mergeRequestOptions( { @@ -154,7 +155,7 @@ export function overrideConcurrencyLimit( name: "queues.overrideConcurrencyLimit()", icon: "queue", attributes: { - ...flattenAttributes({ queue }), + ...flattenAttributes({ queue, concurrencyKey }), ...accessoryAttributes({ items: [ { @@ -169,9 +170,93 @@ export function overrideConcurrencyLimit( requestOptions ); + if (concurrencyKey) { + return apiClient.overrideQueueConcurrencyKeyLimit( + queue, + concurrencyKey, + concurrencyLimit, + $requestOptions + ); + } + return apiClient.overrideQueueConcurrencyLimit(queue, concurrencyLimit, $requestOptions); } +/** + * Overrides the total concurrency limit of a queue: the cap on concurrent runs across + * all of its `concurrencyKey` values. + * + * @param queue - The ID of the queue, or the type and name + * @param concurrencyLimit - The total concurrency limit to apply + * @returns The updated queue state + */ +export function overrideTotalConcurrencyLimit( + queue: RetrieveQueueParam, + concurrencyLimit: number, + requestOptions?: ApiRequestOptions +): ApiPromise { + const apiClient = apiClientManager.clientOrThrow(); + + const $requestOptions = mergeRequestOptions( + { + tracer, + name: "queues.overrideTotalConcurrencyLimit()", + icon: "queue", + attributes: { + ...flattenAttributes({ queue }), + ...accessoryAttributes({ + items: [ + { + text: typeof queue === "string" ? queue : queue.name, + variant: "normal", + }, + ], + style: "codepath", + }), + }, + }, + requestOptions + ); + + return apiClient.overrideQueueTotalConcurrencyLimit(queue, concurrencyLimit, $requestOptions); +} + +/** + * Resets the total concurrency limit of a queue back to its declared value. + * + * @param queue - The ID of the queue, or the type and name + * @returns The updated queue state + */ +export function resetTotalConcurrencyLimit( + queue: RetrieveQueueParam, + requestOptions?: ApiRequestOptions +): ApiPromise { + const apiClient = apiClientManager.clientOrThrow(); + + const $requestOptions = mergeRequestOptions( + { + tracer, + name: "queues.resetTotalConcurrencyLimit()", + icon: "queue", + attributes: { + ...flattenAttributes({ queue }), + ...accessoryAttributes({ + items: [ + { + text: typeof queue === "string" ? queue : queue.name, + variant: "normal", + }, + ], + style: "codepath", + }), + }, + }, + requestOptions + ); + + return apiClient.resetQueueTotalConcurrencyLimit(queue, $requestOptions); +} + /** * Resets the concurrency limit of a queue to the base value. * @@ -180,9 +265,10 @@ export function overrideConcurrencyLimit( */ export function resetConcurrencyLimit( queue: RetrieveQueueParam, - requestOptions?: ApiRequestOptions + options?: ApiRequestOptions & { concurrencyKey?: string } ): ApiPromise { const apiClient = apiClientManager.clientOrThrow(); + const { concurrencyKey, ...requestOptions } = options ?? {}; const $requestOptions = mergeRequestOptions( { @@ -190,7 +276,7 @@ export function resetConcurrencyLimit( name: "queues.resetConcurrencyLimit()", icon: "queue", attributes: { - ...flattenAttributes({ queue }), + ...flattenAttributes({ queue, concurrencyKey }), ...accessoryAttributes({ items: [ { @@ -205,6 +291,10 @@ export function resetConcurrencyLimit( requestOptions ); + if (concurrencyKey) { + return apiClient.resetQueueConcurrencyKeyLimit(queue, concurrencyKey, $requestOptions); + } + return apiClient.resetQueueConcurrencyLimit(queue, $requestOptions); } From 7927d1df5e5cc8e595bfe616fe40788e4e42326c Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Sat, 29 Aug 2026 11:15:06 +0100 Subject: [PATCH 05/30] fix(run-engine,webapp,sdk): converge override failures and unpin blocked keys A variant blocked at its per-key limit or by a gate now backs off in the candidate index instead of pinning the bounded window, so zero-limit keys can never starve runnable keys behind them; acks and nacks rebalance the score back the moment capacity frees. Override writes persist before enforcing and resets enforce before clearing, so a failure on either side leaves a state a retry converges from, with the cap-rejected row compensated away. Deploys restore per-key overrides from their durable rows into the engine, and an empty concurrencyKey in the SDK no longer falls through to the queue-wide endpoint. --- .../v3/services/concurrencySystem.server.ts | 119 ++++++++++++++---- .../services/createBackgroundWorker.server.ts | 25 ++++ .../run-engine/src/run-queue/index.ts | 28 ++++- .../tests/concurrencyKeyOverrides.test.ts | 63 +++++++++- packages/trigger-sdk/src/v3/queues.ts | 4 +- 5 files changed, 209 insertions(+), 30 deletions(-) diff --git a/apps/webapp/app/v3/services/concurrencySystem.server.ts b/apps/webapp/app/v3/services/concurrencySystem.server.ts index 3945c087053..28fb32b868d 100644 --- a/apps/webapp/app/v3/services/concurrencySystem.server.ts +++ b/apps/webapp/app/v3/services/concurrencySystem.server.ts @@ -104,6 +104,7 @@ export class ConcurrencySystem { }, resetTotalConcurrencyLimit: (environment: AuthenticatedEnvironment, queue: QueueInput) => { return findQueueFromInput(this.db, environment, queue) + .andThen((queue) => syncQueueTotalConcurrencyResetToEngine(environment, queue)) .andThen((queue) => resetQueueTotalConcurrencyLimit(this.db, queue)) .andThen((queue) => syncQueueTotalConcurrencyToEngine(environment, queue)) .andThen((queue) => getQueueStats(environment, queue)); @@ -134,7 +135,9 @@ export class ConcurrencySystem { concurrencyKey: string ) => { return findQueueFromInput(this.db, environment, queue) - .andThen((queue) => resetQueueConcurrencyKeyLimit(this.db, environment, queue, concurrencyKey)) + .andThen((queue) => + resetQueueConcurrencyKeyLimit(this.db, environment, queue, concurrencyKey) + ) .andThen((queue) => getQueueStats(environment, queue)); }, /** @@ -420,6 +423,35 @@ function overrideQueueTotalConcurrencyLimit( ); } +/** + * Enforce first, then persist: syncs the engine to the declared base BEFORE clearing + * the override marker, so an engine failure leaves the marker set and a retry + * converges instead of being rejected while the overridden limit stays enforced. + */ +function syncQueueTotalConcurrencyResetToEngine( + environment: AuthenticatedEnvironment, + queue: TaskQueue +) { + if (queue.totalConcurrencyLimitOverriddenAt === null) { + return errAsync({ type: "queue_not_overridden" as const }); + } + + if (typeof queue.totalConcurrencyLimitBase === "number") { + return fromPromise( + updateQueueTotalConcurrencyLimits(environment, queue.name, queue.totalConcurrencyLimitBase), + (error) => ({ + type: "sync_queue_concurrency_to_engine_failed" as const, + cause: error, + }) + ).andThen(() => okAsync(queue)); + } + + return fromPromise(removeQueueTotalConcurrencyLimits(environment, queue.name), (error) => ({ + type: "sync_queue_concurrency_to_engine_failed" as const, + cause: error, + })).andThen(() => okAsync(queue)); +} + function resetQueueTotalConcurrencyLimit(db: PrismaClientOrTransaction, queue: TaskQueue) { if (queue.totalConcurrencyLimitOverriddenAt === null) { return errAsync({ type: "queue_not_overridden" as const }); @@ -466,6 +498,13 @@ function syncQueueTotalConcurrencyToEngine( })).andThen(() => okAsync(queue)); } +/** + * Persist first, then enforce: a database failure leaves nothing enforced and the + * request errors cleanly, while an engine failure after persistence leaves a durable + * record and a retry converges (the upsert is idempotent). When the engine rejects a + * NEW key for exceeding the per-queue override cap, the just-created row is removed + * again so the record never claims an override the engine refused. + */ function overrideQueueConcurrencyKeyLimit( db: PrismaClientOrTransaction, environment: AuthenticatedEnvironment, @@ -491,20 +530,13 @@ function overrideQueueConcurrencyKeyLimit( } return fromPromise( - engine.runQueue.updateQueueConcurrencyKeyLimit( - environment, - queue.name, - concurrencyKey, - concurrencyLimit - ), - (error) => { - if (error instanceof RunQueueConcurrencyKeyLimitExceededError) { - return { type: "too_many_key_overrides" as const, message: error.message }; - } - return { type: "sync_queue_concurrency_to_engine_failed" as const, cause: error }; - } + db.taskQueueConcurrencyKeyOverride.findFirst({ + where: { taskQueueId: queue.id, concurrencyKey }, + select: { id: true }, + }), + (error) => ({ type: "other" as const, cause: error }) ) - .andThen(() => + .andThen((existing) => fromPromise( db.taskQueueConcurrencyKeyOverride.upsert({ where: { taskQueueId_concurrencyKey: { taskQueueId: queue.id, concurrencyKey } }, @@ -524,11 +556,42 @@ function overrideQueueConcurrencyKeyLimit( type: "queue_update_failed" as const, cause: error, }) - ) + ).map(() => existing) + ) + .andThen((existing) => + fromPromise( + engine.runQueue.updateQueueConcurrencyKeyLimit( + environment, + queue.name, + concurrencyKey, + concurrencyLimit + ), + (error) => { + if (error instanceof RunQueueConcurrencyKeyLimitExceededError) { + return { type: "too_many_key_overrides" as const, message: error.message }; + } + return { type: "sync_queue_concurrency_to_engine_failed" as const, cause: error }; + } + ).orElse((error) => { + if (!existing && error.type === "too_many_key_overrides") { + return fromPromise( + db.taskQueueConcurrencyKeyOverride.deleteMany({ + where: { taskQueueId: queue.id, concurrencyKey }, + }), + () => error + ).andThen(() => errAsync(error)); + } + return errAsync(error); + }) ) .andThen(() => okAsync(queue)); } +/** + * Enforce first, then persist: removing the engine limit is idempotent, so an engine + * failure leaves the override row in place and a retry converges instead of being + * rejected as not overridden while the old limit is still enforced. + */ function resetQueueConcurrencyKeyLimit( db: PrismaClientOrTransaction, environment: AuthenticatedEnvironment, @@ -536,15 +599,13 @@ function resetQueueConcurrencyKeyLimit( concurrencyKey: string ) { return fromPromise( - db.taskQueueConcurrencyKeyOverride.deleteMany({ + db.taskQueueConcurrencyKeyOverride.findFirst({ where: { taskQueueId: queue.id, concurrencyKey }, + select: { id: true }, }), - (error) => ({ - type: "queue_update_failed" as const, - cause: error, - }) - ).andThen((deleted) => { - if (deleted.count === 0) { + (error) => ({ type: "other" as const, cause: error }) + ).andThen((existing) => { + if (!existing) { return errAsync({ type: "queue_not_overridden" as const }); } @@ -554,7 +615,19 @@ function resetQueueConcurrencyKeyLimit( type: "sync_queue_concurrency_to_engine_failed" as const, cause: error, }) - ).andThen(() => okAsync(queue)); + ) + .andThen(() => + fromPromise( + db.taskQueueConcurrencyKeyOverride.deleteMany({ + where: { taskQueueId: queue.id, concurrencyKey }, + }), + (error) => ({ + type: "queue_update_failed" as const, + cause: error, + }) + ) + ) + .andThen(() => okAsync(queue)); }); } diff --git a/apps/webapp/app/v3/services/createBackgroundWorker.server.ts b/apps/webapp/app/v3/services/createBackgroundWorker.server.ts index 147248bd376..6799c925af8 100644 --- a/apps/webapp/app/v3/services/createBackgroundWorker.server.ts +++ b/apps/webapp/app/v3/services/createBackgroundWorker.server.ts @@ -582,6 +582,31 @@ async function createWorkerQueue( await removeQueueTotalConcurrencyLimits(environment, taskQueue.name); } + /** + * Restore per-key limit overrides into the engine so a fresh or flushed Redis + * converges back to the durable records on the next deploy. Row-level failures + * are logged rather than failing the deploy; the next deploy retries them. + */ + const keyOverrides = await prisma.taskQueueConcurrencyKeyOverride.findMany({ + where: { taskQueueId: taskQueue.id }, + }); + for (const keyOverride of keyOverrides) { + try { + await engine.runQueue.updateQueueConcurrencyKeyLimit( + environment, + taskQueue.name, + keyOverride.concurrencyKey, + keyOverride.concurrencyLimit + ); + } catch (error) { + logger.error("createWorkerQueue: failed to restore concurrency key override", { + taskQueueId: taskQueue.id, + concurrencyKey: keyOverride.concurrencyKey, + error, + }); + } + } + if (!taskQueue.paused) { if (typeof newConcurrencyLimit === "number") { logger.debug("createWorkerQueue: updating concurrency limit", { diff --git a/internal-packages/run-engine/src/run-queue/index.ts b/internal-packages/run-engine/src/run-queue/index.ts index 1c983c3ccb6..17c95592ef7 100644 --- a/internal-packages/run-engine/src/run-queue/index.ts +++ b/internal-packages/run-engine/src/run-queue/index.ts @@ -310,6 +310,10 @@ export type RunQueueOptions = { * that dead-lettered or suspended through a mirror-less path. Enabling only after * every instance runs this build avoids the noise but is no longer load-bearing * for correctness. + * + * Per-concurrency-key limit overrides are part of the same concurrency-limits + * feature and are deliberately enforced behind this flag too: writes are always + * accepted and durable, and enforcement of both arrives together. */ totalConcurrencyEnabled?: boolean; /** @@ -5059,6 +5063,7 @@ for _, ckQueueName in ipairs(ckQueues) do end local fullQueueKey = keyPrefix .. ckQueueName + local blockedByGates = false local ckConcurrencyKey = fullQueueKey .. ':currentConcurrency' local ckCurrentConcurrency = tonumber(redis.call('SCARD', ckConcurrencyKey) or '0') @@ -5071,6 +5076,14 @@ for _, ckQueueName in ipairs(ckQueues) do end end + if ckCurrentConcurrency >= perKeyLimit then + -- Back a blocked variant off so it cannot pin the bounded candidate window + -- and starve later keys (acute with a zero per-key override, which never + -- self-clears). Acks and nacks rebalance the score back to the oldest + -- message, so the key is eligible again the moment capacity frees. + redis.call('ZADD', ckIndexKey, currentTime + 1000, ckQueueName) + end + if ckCurrentConcurrency < perKeyLimit then local messages = redis.call('ZRANGEBYSCORE', fullQueueKey, '-inf', tostring(currentTime), 'WITHSCORES', 'LIMIT', 0, 1) @@ -5098,6 +5111,9 @@ for _, ckQueueName in ipairs(ckQueues) do if gatesEnabled then gatesAllow = __gatesHaveCapacity(keyPrefix, messageData, messageId, envConcurrencyLimit, messageKeyPrefix) end + if not gatesAllow then + blockedByGates = true + end local alreadyInGroup = false local totalAllows = true @@ -5140,11 +5156,15 @@ for _, ckQueueName in ipairs(ckQueues) do decrLengthCounter() end - local earliest = redis.call('ZRANGE', fullQueueKey, 0, 0, 'WITHSCORES') - if #earliest == 0 then - redis.call('ZREM', ckIndexKey, ckQueueName) + if blockedByGates then + redis.call('ZADD', ckIndexKey, currentTime + 1000, ckQueueName) else - redis.call('ZADD', ckIndexKey, earliest[2], ckQueueName) + local earliest = redis.call('ZRANGE', fullQueueKey, 0, 0, 'WITHSCORES') + if #earliest == 0 then + redis.call('ZREM', ckIndexKey, ckQueueName) + else + redis.call('ZADD', ckIndexKey, earliest[2], ckQueueName) + end end else local any = redis.call('ZRANGE', fullQueueKey, 0, 0, 'WITHSCORES') diff --git a/internal-packages/run-engine/src/run-queue/tests/concurrencyKeyOverrides.test.ts b/internal-packages/run-engine/src/run-queue/tests/concurrencyKeyOverrides.test.ts index 7995369bc3c..22a52a46625 100644 --- a/internal-packages/run-engine/src/run-queue/tests/concurrencyKeyOverrides.test.ts +++ b/internal-packages/run-engine/src/run-queue/tests/concurrencyKeyOverrides.test.ts @@ -32,11 +32,17 @@ const authenticatedEnvDev = { organization: { id: "o1234" }, }; -function createQueue(redisContainer: any, totalConcurrencyEnabled: boolean, maxOverrides?: number) { +function createQueue( + redisContainer: any, + totalConcurrencyEnabled: boolean, + maxOverrides?: number, + dequeueCount?: number +) { return new RunQueue({ ...testOptions, totalConcurrencyEnabled, maxConcurrencyKeyOverridesPerQueue: maxOverrides, + masterQueueConsumerDequeueCount: dequeueCount, queueSelectionStrategy: new FairQueueSelectionStrategy({ redis: { keyPrefix: "runqueue:test:", @@ -239,6 +245,61 @@ describe("RunQueue per-concurrency-key limit overrides", () => { } }); + redisTest( + "blocked keys cannot pin the candidate window and starve later keys", + async ({ redisContainer }) => { + /** dequeueCount 2 makes the candidate window 6 variants wide. */ + const queue = createQueue(redisContainer, true, undefined, 2); + try { + await queue.updateQueueConcurrencyLimits(authenticatedEnvDev, "task/my-task", 5); + + /** + * Ten zero-limit keys with OLDER messages fill the window many times over; + * without the blocked-key backoff the runnable key behind them would never + * be examined. + */ + const now = Date.now(); + for (let i = 0; i < 10; i++) { + const ck = `blocked-${i}`; + await queue.updateQueueConcurrencyKeyLimit(authenticatedEnvDev, "task/my-task", ck, 0); + await queue.enqueueMessage({ + env: authenticatedEnvDev, + message: makeMessage({ + runId: `b${i}`, + concurrencyKey: ck, + timestamp: now - 10_000 + i, + }), + workerQueue: "main", + }); + } + + await queue.enqueueMessage({ + env: authenticatedEnvDev, + message: makeMessage({ + runId: "good-0", + concurrencyKey: "ck-good", + timestamp: now - 500, + }), + workerQueue: "main", + }); + + const goodAdmitted = await waitFor( + async () => + (await queue.currentConcurrencyOfQueue( + authenticatedEnvDev, + "task/my-task", + "ck-good" + )) === 1, + 30_000 + ); + expect(goodAdmitted).toBe(true); + expect(await queue.lengthOfQueue(authenticatedEnvDev, "task/my-task")).toBe(10); + } finally { + await queue.quit(); + } + } + ); + redisTest("overrides are ignored when disabled", async ({ redisContainer }) => { const queue = createQueue(redisContainer, false); try { diff --git a/packages/trigger-sdk/src/v3/queues.ts b/packages/trigger-sdk/src/v3/queues.ts index e4f3a9b6641..7f16d96e58b 100644 --- a/packages/trigger-sdk/src/v3/queues.ts +++ b/packages/trigger-sdk/src/v3/queues.ts @@ -170,7 +170,7 @@ export function overrideConcurrencyLimit( requestOptions ); - if (concurrencyKey) { + if (concurrencyKey !== undefined) { return apiClient.overrideQueueConcurrencyKeyLimit( queue, concurrencyKey, @@ -291,7 +291,7 @@ export function resetConcurrencyLimit( requestOptions ); - if (concurrencyKey) { + if (concurrencyKey !== undefined) { return apiClient.resetQueueConcurrencyKeyLimit(queue, concurrencyKey, $requestOptions); } From a3c8c6a13c05debdc8038d8b5c6e785aca54d9c2 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Sat, 29 Aug 2026 11:54:58 +0100 Subject: [PATCH 06/30] fix(run-engine,webapp): gate admission honors per-key overrides; harden races Gate capacity now reads the gate queue's ckLimits hash, so an override on a key applies whether runs meet it as their own queue or as a gate. A reset deletes only the exact row generation it read, so a concurrent override's newer record survives, and a cap rejection deletes its row unconditionally since the cap can only reject keys absent from the engine hash. --- .../v3/services/concurrencySystem.server.ts | 21 +++++++--- .../run-engine/src/run-queue/index.ts | 6 +++ .../tests/concurrencyKeyOverrides.test.ts | 40 ++++++++++++++++++- 3 files changed, 60 insertions(+), 7 deletions(-) diff --git a/apps/webapp/app/v3/services/concurrencySystem.server.ts b/apps/webapp/app/v3/services/concurrencySystem.server.ts index 28fb32b868d..94417abf7df 100644 --- a/apps/webapp/app/v3/services/concurrencySystem.server.ts +++ b/apps/webapp/app/v3/services/concurrencySystem.server.ts @@ -501,9 +501,10 @@ function syncQueueTotalConcurrencyToEngine( /** * Persist first, then enforce: a database failure leaves nothing enforced and the * request errors cleanly, while an engine failure after persistence leaves a durable - * record and a retry converges (the upsert is idempotent). When the engine rejects a - * NEW key for exceeding the per-queue override cap, the just-created row is removed - * again so the record never claims an override the engine refused. + * record and a retry converges (the upsert is idempotent). A cap rejection deletes + * the row unconditionally: the cap only rejects keys absent from the engine hash + * (updates to present keys always succeed), so a cap-rejected row is never backing + * an enforced limit and must not survive as an authoritative override. */ function overrideQueueConcurrencyKeyLimit( db: PrismaClientOrTransaction, @@ -573,7 +574,7 @@ function overrideQueueConcurrencyKeyLimit( return { type: "sync_queue_concurrency_to_engine_failed" as const, cause: error }; } ).orElse((error) => { - if (!existing && error.type === "too_many_key_overrides") { + if (error.type === "too_many_key_overrides") { return fromPromise( db.taskQueueConcurrencyKeyOverride.deleteMany({ where: { taskQueueId: queue.id, concurrencyKey }, @@ -601,7 +602,7 @@ function resetQueueConcurrencyKeyLimit( return fromPromise( db.taskQueueConcurrencyKeyOverride.findFirst({ where: { taskQueueId: queue.id, concurrencyKey }, - select: { id: true }, + select: { id: true, overriddenAt: true }, }), (error) => ({ type: "other" as const, cause: error }) ).andThen((existing) => { @@ -618,8 +619,16 @@ function resetQueueConcurrencyKeyLimit( ) .andThen(() => fromPromise( + /** + * Deletes only the exact row generation this reset read, so a concurrent + * override that re-wrote the row after the reset began keeps its record + * (its next write, or the deploy-time restore, re-syncs the engine). + */ db.taskQueueConcurrencyKeyOverride.deleteMany({ - where: { taskQueueId: queue.id, concurrencyKey }, + where: { + id: existing.id, + overriddenAt: existing.overriddenAt, + }, }), (error) => ({ type: "queue_update_failed" as const, diff --git a/internal-packages/run-engine/src/run-queue/index.ts b/internal-packages/run-engine/src/run-queue/index.ts index 17c95592ef7..f00060520dc 100644 --- a/internal-packages/run-engine/src/run-queue/index.ts +++ b/internal-packages/run-engine/src/run-queue/index.ts @@ -122,6 +122,12 @@ local function __gatesHaveCapacity(gatesKeyPrefix, msg, messageId, envLimit, msg local base, variant, gateKey = __gateKeys(gatesKeyPrefix, msg, gate) local occupancy = tonumber(redis.call('SCARD', variant .. ':currentConcurrency') or '0') local perKeyLimit = math.min(tonumber(redis.call('GET', base .. ':concurrency') or '1000000'), envLimit) + if gateKey and gateKey ~= '' then + local gateOverride = redis.call('HGET', base .. ':ckLimits', string.sub(variant, #gatesKeyPrefix + 1)) + if gateOverride then + perKeyLimit = math.min(tonumber(gateOverride), envLimit) + end + end if occupancy >= perKeyLimit and redis.call('SISMEMBER', variant .. ':currentConcurrency', messageId) == 0 then __gateReconcile(variant .. ':currentConcurrency', msgKeyPrefix, gatesKeyPrefix) return false diff --git a/internal-packages/run-engine/src/run-queue/tests/concurrencyKeyOverrides.test.ts b/internal-packages/run-engine/src/run-queue/tests/concurrencyKeyOverrides.test.ts index 22a52a46625..8c10a8ac2bf 100644 --- a/internal-packages/run-engine/src/run-queue/tests/concurrencyKeyOverrides.test.ts +++ b/internal-packages/run-engine/src/run-queue/tests/concurrencyKeyOverrides.test.ts @@ -36,11 +36,13 @@ function createQueue( redisContainer: any, totalConcurrencyEnabled: boolean, maxOverrides?: number, - dequeueCount?: number + dequeueCount?: number, + gatesEnabled?: boolean ) { return new RunQueue({ ...testOptions, totalConcurrencyEnabled, + gatesEnabled, maxConcurrencyKeyOverridesPerQueue: maxOverrides, masterQueueConsumerDequeueCount: dequeueCount, queueSelectionStrategy: new FairQueueSelectionStrategy({ @@ -300,6 +302,42 @@ describe("RunQueue per-concurrency-key limit overrides", () => { } ); + redisTest("gate admission honors the gate queue per-key override", async ({ redisContainer }) => { + const queue = createQueue(redisContainer, true, undefined, undefined, true); + try { + await queue.updateQueueConcurrencyLimits(authenticatedEnvDev, "task/my-task", 5); + await queue.updateQueueConcurrencyLimits(authenticatedEnvDev, "tenant", 1); + await queue.updateQueueConcurrencyKeyLimit(authenticatedEnvDev, "tenant", "acme", 2); + + const now = Date.now(); + for (const [i, ck] of ["ck-a", "ck-b", "ck-c"].entries()) { + await queue.enqueueMessage({ + env: authenticatedEnvDev, + message: makeMessage({ + runId: `r${i}`, + concurrencyKey: ck, + timestamp: now - 1000 + i, + gates: [{ queue: "tenant", concurrencyKey: "acme" }], + }), + workerQueue: "main", + }); + } + + /** The declared gate limit is 1; the override raises acme to 2. */ + const twoAdmitted = await waitFor( + async () => + (await queue.currentConcurrencyOfQueue(authenticatedEnvDev, "tenant", "acme")) === 2 + ); + expect(twoAdmitted).toBe(true); + + await setTimeout(2000); + expect(await queue.currentConcurrencyOfQueue(authenticatedEnvDev, "tenant", "acme")).toBe(2); + expect(await queue.lengthOfQueue(authenticatedEnvDev, "task/my-task")).toBe(1); + } finally { + await queue.quit(); + } + }); + redisTest("overrides are ignored when disabled", async ({ redisContainer }) => { const queue = createQueue(redisContainer, false); try { From 7b2f3664ea92ab907875f07b7ab33cd877a22342 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Sat, 29 Aug 2026 12:03:40 +0100 Subject: [PATCH 07/30] fix(run-engine,webapp): flag-consistent gate overrides; generation-safe cap cleanup Gate admission reads a per-key override only when concurrency limit enforcement is enabled, matching the primary admit paths (the flag now threads through the unkeyed enqueue and dequeue scripts too). The cap-rejection cleanup deletes only the exact row generation the rejected request wrote, so a concurrent request that succeeded after capacity freed keeps its durable record. --- .../v3/services/concurrencySystem.server.ts | 13 +++++++--- .../run-engine/src/run-queue/index.ts | 25 +++++++++++++------ 2 files changed, 26 insertions(+), 12 deletions(-) diff --git a/apps/webapp/app/v3/services/concurrencySystem.server.ts b/apps/webapp/app/v3/services/concurrencySystem.server.ts index 94417abf7df..571ed5daf64 100644 --- a/apps/webapp/app/v3/services/concurrencySystem.server.ts +++ b/apps/webapp/app/v3/services/concurrencySystem.server.ts @@ -537,7 +537,7 @@ function overrideQueueConcurrencyKeyLimit( }), (error) => ({ type: "other" as const, cause: error }) ) - .andThen((existing) => + .andThen(() => fromPromise( db.taskQueueConcurrencyKeyOverride.upsert({ where: { taskQueueId_concurrencyKey: { taskQueueId: queue.id, concurrencyKey } }, @@ -557,9 +557,9 @@ function overrideQueueConcurrencyKeyLimit( type: "queue_update_failed" as const, cause: error, }) - ).map(() => existing) + ) ) - .andThen((existing) => + .andThen((written) => fromPromise( engine.runQueue.updateQueueConcurrencyKeyLimit( environment, @@ -576,8 +576,13 @@ function overrideQueueConcurrencyKeyLimit( ).orElse((error) => { if (error.type === "too_many_key_overrides") { return fromPromise( + /** + * Deletes only the exact row generation this request wrote, so a + * concurrent request that succeeded after capacity freed keeps its + * durable record. + */ db.taskQueueConcurrencyKeyOverride.deleteMany({ - where: { taskQueueId: queue.id, concurrencyKey }, + where: { id: written.id, overriddenAt: written.overriddenAt }, }), () => error ).andThen(() => errAsync(error)); diff --git a/internal-packages/run-engine/src/run-queue/index.ts b/internal-packages/run-engine/src/run-queue/index.ts index f00060520dc..1dcbcb33a88 100644 --- a/internal-packages/run-engine/src/run-queue/index.ts +++ b/internal-packages/run-engine/src/run-queue/index.ts @@ -116,13 +116,13 @@ local function __gateReconcile(setKey, msgKeyPrefix, reconcileKeyPrefix) end end -local function __gatesHaveCapacity(gatesKeyPrefix, msg, messageId, envLimit, msgKeyPrefix) +local function __gatesHaveCapacity(gatesKeyPrefix, msg, messageId, envLimit, msgKeyPrefix, ckOverridesEnabled) if not msg.gates then return true end for _, gate in ipairs(msg.gates) do local base, variant, gateKey = __gateKeys(gatesKeyPrefix, msg, gate) local occupancy = tonumber(redis.call('SCARD', variant .. ':currentConcurrency') or '0') local perKeyLimit = math.min(tonumber(redis.call('GET', base .. ':concurrency') or '1000000'), envLimit) - if gateKey and gateKey ~= '' then + if ckOverridesEnabled and gateKey and gateKey ~= '' then local gateOverride = redis.call('HGET', base .. ':ckLimits', string.sub(variant, #gatesKeyPrefix + 1)) if gateOverride then perKeyLimit = math.min(tonumber(gateOverride), envLimit) @@ -2571,6 +2571,7 @@ export class RunQueue { enableFastPathArg, this.options.redis.keyPrefix ?? "", this.options.gatesEnabled ? "1" : "0", + this.options.totalConcurrencyEnabled ? "1" : "0", metricsGaugeArg ); } else { @@ -2600,6 +2601,7 @@ export class RunQueue { enableFastPathArg, this.options.redis.keyPrefix ?? "", this.options.gatesEnabled ? "1" : "0", + this.options.totalConcurrencyEnabled ? "1" : "0", metricsGaugeArg ); } @@ -2681,6 +2683,7 @@ export class RunQueue { this.options.redis.keyPrefix ?? "", String(maxCount), this.options.gatesEnabled ? "1" : "0", + this.options.totalConcurrencyEnabled ? "1" : "0", metricsGaugeArg ); @@ -3678,6 +3681,7 @@ local currentTime = ARGV[8] local enableFastPath = ARGV[9] local keyPrefix = ARGV[10] local gatesEnabled = ARGV[11] == '1' +local totalConcurrencyEnabled = ARGV[12] == '1' ${QUEUE_METRICS_GAUGE_PRELUDE} ${QUEUE_GATES_LUA_HELPERS} @@ -3705,7 +3709,7 @@ if enableFastPath == '1' then local okDecode, decoded = pcall(cjson.decode, messageData) if okDecode and type(decoded) == 'table' and decoded.gates then gateMsg = decoded - gatesAllowFastPath = __gatesHaveCapacity(keyPrefix, decoded, messageId, envLimit, nil) + gatesAllowFastPath = __gatesHaveCapacity(keyPrefix, decoded, messageId, envLimit, nil, totalConcurrencyEnabled) end end @@ -3792,6 +3796,7 @@ local currentTime = ARGV[10] local enableFastPath = ARGV[11] local keyPrefix = ARGV[12] local gatesEnabled = ARGV[13] == '1' +local totalConcurrencyEnabled = ARGV[14] == '1' ${QUEUE_METRICS_GAUGE_PRELUDE} ${QUEUE_GATES_LUA_HELPERS} @@ -3819,7 +3824,7 @@ if enableFastPath == '1' then local okDecode, decoded = pcall(cjson.decode, messageData) if okDecode and type(decoded) == 'table' and decoded.gates then gateMsg = decoded - gatesAllowFastPath = __gatesHaveCapacity(keyPrefix, decoded, messageId, envLimit, nil) + gatesAllowFastPath = __gatesHaveCapacity(keyPrefix, decoded, messageId, envLimit, nil, totalConcurrencyEnabled) end end @@ -4186,7 +4191,7 @@ if enableFastPath == '1' then local okDecode, decoded = pcall(cjson.decode, messageData) if okDecode and type(decoded) == 'table' and decoded.gates then gateMsg = decoded - gatesAllowFastPath = __gatesHaveCapacity(keyPrefix, decoded, messageId, envLimit, nil) + gatesAllowFastPath = __gatesHaveCapacity(keyPrefix, decoded, messageId, envLimit, nil, totalConcurrencyEnabled) end end @@ -4364,7 +4369,7 @@ if enableFastPath == '1' then local okDecode, decoded = pcall(cjson.decode, messageData) if okDecode and type(decoded) == 'table' and decoded.gates then gateMsg = decoded - gatesAllowFastPath = __gatesHaveCapacity(keyPrefix, decoded, messageId, envLimit, nil) + gatesAllowFastPath = __gatesHaveCapacity(keyPrefix, decoded, messageId, envLimit, nil, totalConcurrencyEnabled) end end @@ -4694,6 +4699,7 @@ local defaultEnvConcurrencyBurstFactor = ARGV[4] local keyPrefix = ARGV[5] local maxCount = tonumber(ARGV[6] or '1') local gatesEnabled = ARGV[7] == '1' +local totalConcurrencyEnabled = ARGV[8] == '1' ${QUEUE_METRICS_GAUGE_PRELUDE} ${QUEUE_GATES_LUA_HELPERS} ${QUEUE_METRICS_GAUGE_LUA} @@ -4766,7 +4772,7 @@ for i = 1, #messages, 2 do else local gatesAllow = true if gatesEnabled then - gatesAllow = __gatesHaveCapacity(keyPrefix, messageData, messageId, envConcurrencyLimit, messageKeyPrefix) + gatesAllow = __gatesHaveCapacity(keyPrefix, messageData, messageId, envConcurrencyLimit, messageKeyPrefix, totalConcurrencyEnabled) end if gatesAllow then @@ -5115,7 +5121,7 @@ for _, ckQueueName in ipairs(ckQueues) do else local gatesAllow = true if gatesEnabled then - gatesAllow = __gatesHaveCapacity(keyPrefix, messageData, messageId, envConcurrencyLimit, messageKeyPrefix) + gatesAllow = __gatesHaveCapacity(keyPrefix, messageData, messageId, envConcurrencyLimit, messageKeyPrefix, totalConcurrencyEnabled) end if not gatesAllow then blockedByGates = true @@ -6223,6 +6229,7 @@ declare module "@internal/redis" { enableFastPath: string, keyPrefix: string, gatesEnabled: string, + totalConcurrencyEnabled: string, metricsEnabled: string, callback?: Callback<[number, number[] | null]> ): Result<[number, number[] | null], Context>; @@ -6256,6 +6263,7 @@ declare module "@internal/redis" { enableFastPath: string, keyPrefix: string, gatesEnabled: string, + totalConcurrencyEnabled: string, metricsEnabled: string, callback?: Callback<[number, number[] | null]> ): Result<[number, number[] | null], Context>; @@ -6294,6 +6302,7 @@ declare module "@internal/redis" { keyPrefix: string, maxCount: string, gatesEnabled: string, + totalConcurrencyEnabled: string, metricsEnabled: string, callback?: Callback<[string[] | null, number[] | null]> ): Result<[string[] | null, number[] | null], Context>; From 0e29d90e2e910d41762ee9ccf38bdd8d3416c094 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Sat, 29 Aug 2026 16:52:12 +0100 Subject: [PATCH 08/30] fix(webapp): export queue concurrency route handlers by property access The client build's export analyzer cannot statically resolve destructured route exports, so it treats the module's exports as depending on server-only code and the build fails. Also export the builder's loader so non-POST methods get a 405, matching the other concurrency routes. --- .../api.v1.queues.$queueParam.concurrency.key.override.ts | 4 +++- .../routes/api.v1.queues.$queueParam.concurrency.key.reset.ts | 4 +++- .../api.v1.queues.$queueParam.concurrency.total.override.ts | 4 +++- .../api.v1.queues.$queueParam.concurrency.total.reset.ts | 4 +++- 4 files changed, 12 insertions(+), 4 deletions(-) diff --git a/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.key.override.ts b/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.key.override.ts index 9a72155d4fc..5a37b4526ec 100644 --- a/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.key.override.ts +++ b/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.key.override.ts @@ -100,4 +100,6 @@ const route = createActionApiRoute( } ); -export const { action } = route; +export const action = route.action; +/** The builder's loader answers non-POST methods with a 405. */ +export const loader = route.loader; diff --git a/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.key.reset.ts b/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.key.reset.ts index 3024ac84b72..51d14642e2c 100644 --- a/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.key.reset.ts +++ b/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.key.reset.ts @@ -92,4 +92,6 @@ const route = createActionApiRoute( } ); -export const { action } = route; +export const action = route.action; +/** The builder's loader answers non-POST methods with a 405. */ +export const loader = route.loader; diff --git a/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.total.override.ts b/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.total.override.ts index b0c34ae1379..c643b77965a 100644 --- a/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.total.override.ts +++ b/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.total.override.ts @@ -93,4 +93,6 @@ const route = createActionApiRoute( } ); -export const { action } = route; +export const action = route.action; +/** The builder's loader answers non-POST methods with a 405. */ +export const loader = route.loader; diff --git a/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.total.reset.ts b/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.total.reset.ts index 2eacbb3ed6a..b2841f1efe6 100644 --- a/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.total.reset.ts +++ b/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.total.reset.ts @@ -94,4 +94,6 @@ const route = createActionApiRoute( } ); -export const { action } = route; +export const action = route.action; +/** The builder's loader answers non-POST methods with a 405. */ +export const loader = route.loader; From 109540f4ea3380dae4bd7553814ff07084544398 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Sat, 29 Aug 2026 19:10:38 +0100 Subject: [PATCH 09/30] refactor(sdk,core,webapp): combined concurrency override API names The override and reset endpoints, client methods and SDK functions now say combinedConcurrencyLimit, matching the queue option. --- .changeset/queue-concurrency-overrides.md | 8 ++++---- ...ueueParam.concurrency.combined.override.ts} | 0 ....$queueParam.concurrency.combined.reset.ts} | 0 packages/core/src/v3/apiClient/index.ts | 8 ++++---- packages/trigger-sdk/src/v3/queues.ts | 18 +++++++++--------- 5 files changed, 17 insertions(+), 17 deletions(-) rename apps/webapp/app/routes/{api.v1.queues.$queueParam.concurrency.total.override.ts => api.v1.queues.$queueParam.concurrency.combined.override.ts} (100%) rename apps/webapp/app/routes/{api.v1.queues.$queueParam.concurrency.total.reset.ts => api.v1.queues.$queueParam.concurrency.combined.reset.ts} (100%) diff --git a/.changeset/queue-concurrency-overrides.md b/.changeset/queue-concurrency-overrides.md index 65579b0d81a..6bbe8f909ee 100644 --- a/.changeset/queue-concurrency-overrides.md +++ b/.changeset/queue-concurrency-overrides.md @@ -3,7 +3,7 @@ "@trigger.dev/core": patch --- -Adjust queue concurrency at runtime, per key and in total. `queues.overrideConcurrencyLimit` accepts a `concurrencyKey` to raise or lower one key's limit without touching the rest of the queue, and the new `queues.overrideTotalConcurrencyLimit` and `queues.resetTotalConcurrencyLimit` adjust the cap across all keys. +Adjust queue concurrency at runtime, per key and combined. `queues.overrideConcurrencyLimit` accepts a `concurrencyKey` to raise or lower one key's limit without touching the rest of the queue, and the new `queues.overrideCombinedConcurrencyLimit` and `queues.resetCombinedConcurrencyLimit` adjust the cap across all keys. ```ts import { queues } from "@trigger.dev/sdk"; @@ -11,8 +11,8 @@ import { queues } from "@trigger.dev/sdk"; await queues.overrideConcurrencyLimit("my-queue", 20, { concurrencyKey: "tenant-123" }); await queues.resetConcurrencyLimit("my-queue", { concurrencyKey: "tenant-123" }); -await queues.overrideTotalConcurrencyLimit("my-queue", 100); -await queues.resetTotalConcurrencyLimit("my-queue"); +await queues.overrideCombinedConcurrencyLimit("my-queue", 100); +await queues.resetCombinedConcurrencyLimit("my-queue"); ``` -Overrides survive deploys and reset back to the declared configuration. Enforcement happens server-side on servers with total concurrency limits enabled. +Overrides survive deploys and reset back to the declared configuration. Enforcement happens server-side on servers with combined concurrency limits enabled. diff --git a/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.total.override.ts b/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.combined.override.ts similarity index 100% rename from apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.total.override.ts rename to apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.combined.override.ts diff --git a/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.total.reset.ts b/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.combined.reset.ts similarity index 100% rename from apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.total.reset.ts rename to apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.combined.reset.ts diff --git a/packages/core/src/v3/apiClient/index.ts b/packages/core/src/v3/apiClient/index.ts index 6a4d24ace35..d67661ebd13 100644 --- a/packages/core/src/v3/apiClient/index.ts +++ b/packages/core/src/v3/apiClient/index.ts @@ -1765,7 +1765,7 @@ export class ApiClient { ); } - overrideQueueTotalConcurrencyLimit( + overrideQueueCombinedConcurrencyLimit( queue: RetrieveQueueParam, concurrencyLimit: number, requestOptions?: ZodFetchOptions @@ -1777,7 +1777,7 @@ export class ApiClient { return zodfetch( QueueItem, - `${this.baseUrl}/api/v1/queues/${encodedValue}/concurrency/total/override`, + `${this.baseUrl}/api/v1/queues/${encodedValue}/concurrency/combined/override`, { method: "POST", headers: this.#getHeaders(false), @@ -1790,7 +1790,7 @@ export class ApiClient { ); } - resetQueueTotalConcurrencyLimit(queue: RetrieveQueueParam, requestOptions?: ZodFetchOptions) { + resetQueueCombinedConcurrencyLimit(queue: RetrieveQueueParam, requestOptions?: ZodFetchOptions) { const type = typeof queue === "string" ? "id" : queue.type; const value = typeof queue === "string" ? queue : queue.name; @@ -1798,7 +1798,7 @@ export class ApiClient { return zodfetch( QueueItem, - `${this.baseUrl}/api/v1/queues/${encodedValue}/concurrency/total/reset`, + `${this.baseUrl}/api/v1/queues/${encodedValue}/concurrency/combined/reset`, { method: "POST", headers: this.#getHeaders(false), diff --git a/packages/trigger-sdk/src/v3/queues.ts b/packages/trigger-sdk/src/v3/queues.ts index 7f16d96e58b..f2ec3dd6339 100644 --- a/packages/trigger-sdk/src/v3/queues.ts +++ b/packages/trigger-sdk/src/v3/queues.ts @@ -183,14 +183,14 @@ export function overrideConcurrencyLimit( } /** - * Overrides the total concurrency limit of a queue: the cap on concurrent runs across + * Overrides the combined concurrency limit of a queue: the cap on concurrent runs across * all of its `concurrencyKey` values. * * @param queue - The ID of the queue, or the type and name - * @param concurrencyLimit - The total concurrency limit to apply + * @param concurrencyLimit - The combined concurrency limit to apply * @returns The updated queue state */ -export function overrideTotalConcurrencyLimit( +export function overrideCombinedConcurrencyLimit( queue: RetrieveQueueParam, concurrencyLimit: number, requestOptions?: ApiRequestOptions @@ -200,7 +200,7 @@ export function overrideTotalConcurrencyLimit( const $requestOptions = mergeRequestOptions( { tracer, - name: "queues.overrideTotalConcurrencyLimit()", + name: "queues.overrideCombinedConcurrencyLimit()", icon: "queue", attributes: { ...flattenAttributes({ queue }), @@ -218,16 +218,16 @@ export function overrideTotalConcurrencyLimit( requestOptions ); - return apiClient.overrideQueueTotalConcurrencyLimit(queue, concurrencyLimit, $requestOptions); + return apiClient.overrideQueueCombinedConcurrencyLimit(queue, concurrencyLimit, $requestOptions); } /** - * Resets the total concurrency limit of a queue back to its declared value. + * Resets the combined concurrency limit of a queue back to its declared value. * * @param queue - The ID of the queue, or the type and name * @returns The updated queue state */ -export function resetTotalConcurrencyLimit( +export function resetCombinedConcurrencyLimit( queue: RetrieveQueueParam, requestOptions?: ApiRequestOptions ): ApiPromise { @@ -236,7 +236,7 @@ export function resetTotalConcurrencyLimit( const $requestOptions = mergeRequestOptions( { tracer, - name: "queues.resetTotalConcurrencyLimit()", + name: "queues.resetCombinedConcurrencyLimit()", icon: "queue", attributes: { ...flattenAttributes({ queue }), @@ -254,7 +254,7 @@ export function resetTotalConcurrencyLimit( requestOptions ); - return apiClient.resetQueueTotalConcurrencyLimit(queue, $requestOptions); + return apiClient.resetQueueCombinedConcurrencyLimit(queue, $requestOptions); } /** From c7927deee9c016a3eee9ead56371b14be5ace6ba Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Mon, 31 Aug 2026 12:57:07 +0100 Subject: [PATCH 10/30] refactor(run-engine,webapp,sdk,database): remove per-key concurrency limit overrides Per-key runtime overrides are deferred: composing a per-key hash with pause semantics and keeping it convergent with durable rows across deploys and resets needs its own design. Combined-limit overrides stay. Declared per-key behavior is unchanged (concurrencyLimit applies per key as before). --- .changeset/queue-concurrency-overrides.md | 7 +- .../v3/services/concurrencySystem.server.ts | 179 --------- .../services/createBackgroundWorker.server.ts | 25 -- .../migration.sql | 20 - .../database/prisma/schema.prisma | 20 - internal-packages/run-engine/src/index.ts | 1 - .../run-engine/src/run-queue/keyProducer.ts | 15 - .../tests/concurrencyKeyOverrides.test.ts | 369 ------------------ .../run-engine/src/run-queue/types.ts | 2 - packages/core/src/v3/apiClient/index.ts | 52 --- packages/trigger-sdk/src/v3/queues.ts | 23 +- 11 files changed, 6 insertions(+), 707 deletions(-) delete mode 100644 internal-packages/run-engine/src/run-queue/tests/concurrencyKeyOverrides.test.ts diff --git a/.changeset/queue-concurrency-overrides.md b/.changeset/queue-concurrency-overrides.md index 6bbe8f909ee..22f79c5e72c 100644 --- a/.changeset/queue-concurrency-overrides.md +++ b/.changeset/queue-concurrency-overrides.md @@ -3,16 +3,13 @@ "@trigger.dev/core": patch --- -Adjust queue concurrency at runtime, per key and combined. `queues.overrideConcurrencyLimit` accepts a `concurrencyKey` to raise or lower one key's limit without touching the rest of the queue, and the new `queues.overrideCombinedConcurrencyLimit` and `queues.resetCombinedConcurrencyLimit` adjust the cap across all keys. +Adjust a queue's combined concurrency limit at runtime. `queues.overrideCombinedConcurrencyLimit` raises or lowers the cap on concurrent runs across all of a queue's `concurrencyKey` values, and `queues.resetCombinedConcurrencyLimit` reverts to the declared configuration. ```ts import { queues } from "@trigger.dev/sdk"; -await queues.overrideConcurrencyLimit("my-queue", 20, { concurrencyKey: "tenant-123" }); -await queues.resetConcurrencyLimit("my-queue", { concurrencyKey: "tenant-123" }); - await queues.overrideCombinedConcurrencyLimit("my-queue", 100); await queues.resetCombinedConcurrencyLimit("my-queue"); ``` -Overrides survive deploys and reset back to the declared configuration. Enforcement happens server-side on servers with combined concurrency limits enabled. +Overrides survive deploys. Enforcement happens server-side on servers with combined concurrency limits enabled. diff --git a/apps/webapp/app/v3/services/concurrencySystem.server.ts b/apps/webapp/app/v3/services/concurrencySystem.server.ts index 571ed5daf64..e1b285136fa 100644 --- a/apps/webapp/app/v3/services/concurrencySystem.server.ts +++ b/apps/webapp/app/v3/services/concurrencySystem.server.ts @@ -9,7 +9,6 @@ import { updateQueueConcurrencyLimits, updateQueueTotalConcurrencyLimits, } from "../runQueue.server"; -import { RunQueueConcurrencyKeyLimitExceededError } from "@internal/run-engine"; import { engine } from "../runEngine.server"; export type ConcurrencySystemOptions = { @@ -109,37 +108,6 @@ export class ConcurrencySystem { .andThen((queue) => syncQueueTotalConcurrencyToEngine(environment, queue)) .andThen((queue) => getQueueStats(environment, queue)); }, - overrideConcurrencyKeyLimit: ( - environment: AuthenticatedEnvironment, - queue: QueueInput, - concurrencyKey: string, - concurrencyLimit: number, - overriddenBy?: User - ) => { - return findQueueFromInput(this.db, environment, queue) - .andThen((queue) => - overrideQueueConcurrencyKeyLimit( - this.db, - environment, - queue, - concurrencyKey, - concurrencyLimit, - overriddenBy - ) - ) - .andThen((queue) => getQueueStats(environment, queue)); - }, - resetConcurrencyKeyLimit: ( - environment: AuthenticatedEnvironment, - queue: QueueInput, - concurrencyKey: string - ) => { - return findQueueFromInput(this.db, environment, queue) - .andThen((queue) => - resetQueueConcurrencyKeyLimit(this.db, environment, queue, concurrencyKey) - ) - .andThen((queue) => getQueueStats(environment, queue)); - }, /** * Recalculates the materialized limit of every percent-based override in the environment * against its CURRENT maximumConcurrencyLimit and syncs changed queues to the run engine. @@ -498,153 +466,6 @@ function syncQueueTotalConcurrencyToEngine( })).andThen(() => okAsync(queue)); } -/** - * Persist first, then enforce: a database failure leaves nothing enforced and the - * request errors cleanly, while an engine failure after persistence leaves a durable - * record and a retry converges (the upsert is idempotent). A cap rejection deletes - * the row unconditionally: the cap only rejects keys absent from the engine hash - * (updates to present keys always succeed), so a cap-rejected row is never backing - * an enforced limit and must not survive as an authoritative override. - */ -function overrideQueueConcurrencyKeyLimit( - db: PrismaClientOrTransaction, - environment: AuthenticatedEnvironment, - queue: TaskQueue, - concurrencyKey: string, - concurrencyLimit: number, - overriddenBy?: User -) { - const maximum = environment.maximumConcurrencyLimit; - - if (!Number.isFinite(concurrencyLimit) || concurrencyLimit < 0) { - return errAsync({ - type: "invalid_override" as const, - message: "Concurrency limit must be a non-negative number", - }); - } - - if (concurrencyLimit > maximum) { - return errAsync({ - type: "concurrency_limit_exceeds_maximum" as const, - message: `Concurrency limit (${concurrencyLimit}) cannot exceed the environment limit (${maximum})`, - }); - } - - return fromPromise( - db.taskQueueConcurrencyKeyOverride.findFirst({ - where: { taskQueueId: queue.id, concurrencyKey }, - select: { id: true }, - }), - (error) => ({ type: "other" as const, cause: error }) - ) - .andThen(() => - fromPromise( - db.taskQueueConcurrencyKeyOverride.upsert({ - where: { taskQueueId_concurrencyKey: { taskQueueId: queue.id, concurrencyKey } }, - create: { - taskQueueId: queue.id, - concurrencyKey, - concurrencyLimit, - overriddenBy: overriddenBy?.id ?? null, - }, - update: { - concurrencyLimit, - overriddenAt: new Date(), - overriddenBy: overriddenBy?.id ?? null, - }, - }), - (error) => ({ - type: "queue_update_failed" as const, - cause: error, - }) - ) - ) - .andThen((written) => - fromPromise( - engine.runQueue.updateQueueConcurrencyKeyLimit( - environment, - queue.name, - concurrencyKey, - concurrencyLimit - ), - (error) => { - if (error instanceof RunQueueConcurrencyKeyLimitExceededError) { - return { type: "too_many_key_overrides" as const, message: error.message }; - } - return { type: "sync_queue_concurrency_to_engine_failed" as const, cause: error }; - } - ).orElse((error) => { - if (error.type === "too_many_key_overrides") { - return fromPromise( - /** - * Deletes only the exact row generation this request wrote, so a - * concurrent request that succeeded after capacity freed keeps its - * durable record. - */ - db.taskQueueConcurrencyKeyOverride.deleteMany({ - where: { id: written.id, overriddenAt: written.overriddenAt }, - }), - () => error - ).andThen(() => errAsync(error)); - } - return errAsync(error); - }) - ) - .andThen(() => okAsync(queue)); -} - -/** - * Enforce first, then persist: removing the engine limit is idempotent, so an engine - * failure leaves the override row in place and a retry converges instead of being - * rejected as not overridden while the old limit is still enforced. - */ -function resetQueueConcurrencyKeyLimit( - db: PrismaClientOrTransaction, - environment: AuthenticatedEnvironment, - queue: TaskQueue, - concurrencyKey: string -) { - return fromPromise( - db.taskQueueConcurrencyKeyOverride.findFirst({ - where: { taskQueueId: queue.id, concurrencyKey }, - select: { id: true, overriddenAt: true }, - }), - (error) => ({ type: "other" as const, cause: error }) - ).andThen((existing) => { - if (!existing) { - return errAsync({ type: "queue_not_overridden" as const }); - } - - return fromPromise( - engine.runQueue.removeQueueConcurrencyKeyLimit(environment, queue.name, concurrencyKey), - (error) => ({ - type: "sync_queue_concurrency_to_engine_failed" as const, - cause: error, - }) - ) - .andThen(() => - fromPromise( - /** - * Deletes only the exact row generation this reset read, so a concurrent - * override that re-wrote the row after the reset began keeps its record - * (its next write, or the deploy-time restore, re-syncs the engine). - */ - db.taskQueueConcurrencyKeyOverride.deleteMany({ - where: { - id: existing.id, - overriddenAt: existing.overriddenAt, - }, - }), - (error) => ({ - type: "queue_update_failed" as const, - cause: error, - }) - ) - ) - .andThen(() => okAsync(queue)); - }); -} - function getQueueStats(environment: AuthenticatedEnvironment, queue: TaskQueue) { return fromPromise( Promise.all([ diff --git a/apps/webapp/app/v3/services/createBackgroundWorker.server.ts b/apps/webapp/app/v3/services/createBackgroundWorker.server.ts index 6799c925af8..147248bd376 100644 --- a/apps/webapp/app/v3/services/createBackgroundWorker.server.ts +++ b/apps/webapp/app/v3/services/createBackgroundWorker.server.ts @@ -582,31 +582,6 @@ async function createWorkerQueue( await removeQueueTotalConcurrencyLimits(environment, taskQueue.name); } - /** - * Restore per-key limit overrides into the engine so a fresh or flushed Redis - * converges back to the durable records on the next deploy. Row-level failures - * are logged rather than failing the deploy; the next deploy retries them. - */ - const keyOverrides = await prisma.taskQueueConcurrencyKeyOverride.findMany({ - where: { taskQueueId: taskQueue.id }, - }); - for (const keyOverride of keyOverrides) { - try { - await engine.runQueue.updateQueueConcurrencyKeyLimit( - environment, - taskQueue.name, - keyOverride.concurrencyKey, - keyOverride.concurrencyLimit - ); - } catch (error) { - logger.error("createWorkerQueue: failed to restore concurrency key override", { - taskQueueId: taskQueue.id, - concurrencyKey: keyOverride.concurrencyKey, - error, - }); - } - } - if (!taskQueue.paused) { if (typeof newConcurrencyLimit === "number") { logger.debug("createWorkerQueue: updating concurrency limit", { diff --git a/internal-packages/database/prisma/migrations/20260829150000_add_concurrency_overrides/migration.sql b/internal-packages/database/prisma/migrations/20260829150000_add_concurrency_overrides/migration.sql index d95778f276d..4b8b6ec6077 100644 --- a/internal-packages/database/prisma/migrations/20260829150000_add_concurrency_overrides/migration.sql +++ b/internal-packages/database/prisma/migrations/20260829150000_add_concurrency_overrides/migration.sql @@ -2,23 +2,3 @@ ALTER TABLE "TaskQueue" ADD COLUMN "totalConcurrencyLimitOverriddenAt" TIMESTAMP(3); ALTER TABLE "TaskQueue" ADD COLUMN "totalConcurrencyLimitOverriddenBy" TEXT; ALTER TABLE "TaskQueue" ADD COLUMN "totalConcurrencyLimitBase" INTEGER; - --- CreateTable -CREATE TABLE "TaskQueueConcurrencyKeyOverride" ( - "id" TEXT NOT NULL, - "taskQueueId" TEXT NOT NULL, - "concurrencyKey" TEXT NOT NULL, - "concurrencyLimit" INTEGER NOT NULL, - "overriddenAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, - "overriddenBy" TEXT, - "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, - "updatedAt" TIMESTAMP(3) NOT NULL, - - CONSTRAINT "TaskQueueConcurrencyKeyOverride_pkey" PRIMARY KEY ("id") -); - --- CreateIndex -CREATE UNIQUE INDEX "TaskQueueConcurrencyKeyOverride_taskQueueId_concurrencyKey_key" ON "TaskQueueConcurrencyKeyOverride"("taskQueueId", "concurrencyKey"); - --- AddForeignKey -ALTER TABLE "TaskQueueConcurrencyKeyOverride" ADD CONSTRAINT "TaskQueueConcurrencyKeyOverride_taskQueueId_fkey" FOREIGN KEY ("taskQueueId") REFERENCES "TaskQueue"("id") ON DELETE CASCADE ON UPDATE CASCADE; diff --git a/internal-packages/database/prisma/schema.prisma b/internal-packages/database/prisma/schema.prisma index 65a84ff1645..406caec57e5 100644 --- a/internal-packages/database/prisma/schema.prisma +++ b/internal-packages/database/prisma/schema.prisma @@ -2001,30 +2001,10 @@ model TaskQueue { tasks BackgroundWorkerTask[] workers BackgroundWorker[] - concurrencyKeyOverrides TaskQueueConcurrencyKeyOverride[] @@unique([runtimeEnvironmentId, name]) } -/// A per-concurrency-key limit override for a queue: the named key value gets this -/// limit instead of the queue's concurrencyLimit. Deleting the row resets the key. -model TaskQueueConcurrencyKeyOverride { - id String @id @default(cuid()) - - taskQueue TaskQueue @relation(fields: [taskQueueId], references: [id], onDelete: Cascade, onUpdate: Cascade) - taskQueueId String - - concurrencyKey String - concurrencyLimit Int - - overriddenAt DateTime @default(now()) - overriddenBy String? - - createdAt DateTime @default(now()) - updatedAt DateTime @updatedAt - - @@unique([taskQueueId, concurrencyKey]) -} enum TaskQueueType { VIRTUAL diff --git a/internal-packages/run-engine/src/index.ts b/internal-packages/run-engine/src/index.ts index 77758c6db11..f30e271d5a4 100644 --- a/internal-packages/run-engine/src/index.ts +++ b/internal-packages/run-engine/src/index.ts @@ -1,5 +1,4 @@ export { RunEngine } from "./engine/index.js"; -export { RunQueueConcurrencyKeyLimitExceededError } from "./run-queue/index.js"; export { RunDuplicateIdempotencyKeyError, RunOneTimeUseTokenError, diff --git a/internal-packages/run-engine/src/run-queue/keyProducer.ts b/internal-packages/run-engine/src/run-queue/keyProducer.ts index 7b997043244..98028f5af7b 100644 --- a/internal-packages/run-engine/src/run-queue/keyProducer.ts +++ b/internal-packages/run-engine/src/run-queue/keyProducer.ts @@ -26,7 +26,6 @@ const constants = { RUNNING_COUNTER_PART: "runningCounter", GROUP_CONCURRENCY_PART: "groupConcurrency", TOTAL_CONCURRENCY_LIMIT_PART: "totalConcurrency", - CK_LIMITS_PART: "ckLimits", } as const; export class RunQueueFullKeyProducer implements RunQueueKeyProducer { @@ -367,20 +366,6 @@ export class RunQueueFullKeyProducer implements RunQueueKeyProducer { return `${this.baseQueueKeyFromQueue(queue)}:${constants.TOTAL_CONCURRENCY_LIMIT_PART}`; } - /** - * HASH of per-concurrency-key limit overrides for a queue. Lives at the base - * queue; each field is the EXACT full ck-variant queue name (the ckIndex ZSET - * member), so reads need no parsing, and values are the raw requested limits - * (readers clamp to the environment limit). - */ - queueCkLimitsKey(env: RunQueueKeyProducerEnvironment, queue: string): string { - return `${this.queueKey(env, queue)}:${constants.CK_LIMITS_PART}`; - } - - queueCkLimitsKeyFromQueue(queue: string): string { - return `${this.baseQueueKeyFromQueue(queue)}:${constants.CK_LIMITS_PART}`; - } - isCkWildcard(queue: string): boolean { return queue.endsWith(":ck:*"); } diff --git a/internal-packages/run-engine/src/run-queue/tests/concurrencyKeyOverrides.test.ts b/internal-packages/run-engine/src/run-queue/tests/concurrencyKeyOverrides.test.ts deleted file mode 100644 index 8c10a8ac2bf..00000000000 --- a/internal-packages/run-engine/src/run-queue/tests/concurrencyKeyOverrides.test.ts +++ /dev/null @@ -1,369 +0,0 @@ -import { redisTest } from "@internal/testcontainers"; -import { trace } from "@internal/tracing"; -import { setTimeout } from "node:timers/promises"; -import { describe } from "vitest"; -import { FairQueueSelectionStrategy } from "../fairQueueSelectionStrategy.js"; -import { RunQueue, RunQueueConcurrencyKeyLimitExceededError } from "../index.js"; -import { RunQueueFullKeyProducer } from "../keyProducer.js"; -import type { InputPayload } from "../types.js"; -import { Decimal } from "@trigger.dev/database"; - -const testOptions = { - name: "rq", - tracer: trace.getTracer("rq"), - workers: 1, - defaultEnvConcurrency: 25, - retryOptions: { - maxAttempts: 5, - factor: 1.1, - minTimeoutInMs: 100, - maxTimeoutInMs: 1_000, - randomize: true, - }, - keys: new RunQueueFullKeyProducer(), -}; - -const authenticatedEnvDev = { - id: "e1234", - type: "DEVELOPMENT" as const, - maximumConcurrencyLimit: 10, - concurrencyLimitBurstFactor: new Decimal(2.0), - project: { id: "p1234" }, - organization: { id: "o1234" }, -}; - -function createQueue( - redisContainer: any, - totalConcurrencyEnabled: boolean, - maxOverrides?: number, - dequeueCount?: number, - gatesEnabled?: boolean -) { - return new RunQueue({ - ...testOptions, - totalConcurrencyEnabled, - gatesEnabled, - maxConcurrencyKeyOverridesPerQueue: maxOverrides, - masterQueueConsumerDequeueCount: dequeueCount, - queueSelectionStrategy: new FairQueueSelectionStrategy({ - redis: { - keyPrefix: "runqueue:test:", - host: redisContainer.getHost(), - port: redisContainer.getPort(), - }, - keys: testOptions.keys, - }), - redis: { - keyPrefix: "runqueue:test:", - host: redisContainer.getHost(), - port: redisContainer.getPort(), - }, - }); -} - -function makeMessage(overrides: Partial = {}): InputPayload { - return { - runId: "r1", - taskIdentifier: "task/my-task", - orgId: "o1234", - projectId: "p1234", - environmentId: "e1234", - environmentType: "DEVELOPMENT", - queue: "task/my-task", - timestamp: Date.now(), - attempt: 0, - ...overrides, - }; -} - -async function waitFor(condition: () => Promise, timeoutMs = 20_000): Promise { - const deadline = Date.now() + timeoutMs; - while (Date.now() < deadline) { - if (await condition()) { - return true; - } - await setTimeout(250); - } - return condition(); -} - -vi.setConfig({ testTimeout: 60_000 }); - -describe("RunQueue per-concurrency-key limit overrides", () => { - redisTest( - "a lowered key is capped while other keys keep the queue limit", - async ({ redisContainer }) => { - const queue = createQueue(redisContainer, true); - try { - await queue.updateQueueConcurrencyLimits(authenticatedEnvDev, "task/my-task", 2); - await queue.updateQueueConcurrencyKeyLimit(authenticatedEnvDev, "task/my-task", "ck-a", 1); - - const now = Date.now(); - const messages = [ - ["ck-a", "a0"], - ["ck-a", "a1"], - ["ck-b", "b0"], - ["ck-b", "b1"], - ] as const; - for (const [i, [ck, id]] of messages.entries()) { - await queue.enqueueMessage({ - env: authenticatedEnvDev, - message: makeMessage({ runId: id, concurrencyKey: ck, timestamp: now - 1000 + i }), - workerQueue: "main", - }); - } - - const settled = await waitFor(async () => { - const a = await queue.currentConcurrencyOfQueue( - authenticatedEnvDev, - "task/my-task", - "ck-a" - ); - const b = await queue.currentConcurrencyOfQueue( - authenticatedEnvDev, - "task/my-task", - "ck-b" - ); - return a === 1 && b === 2; - }); - expect(settled).toBe(true); - - await setTimeout(2000); - expect( - await queue.currentConcurrencyOfQueue(authenticatedEnvDev, "task/my-task", "ck-a") - ).toBe(1); - expect(await queue.lengthOfQueue(authenticatedEnvDev, "task/my-task")).toBe(1); - } finally { - await queue.quit(); - } - } - ); - - redisTest("a raised key admits past the queue limit", async ({ redisContainer }) => { - const queue = createQueue(redisContainer, true); - try { - await queue.updateQueueConcurrencyLimits(authenticatedEnvDev, "task/my-task", 1); - await queue.updateQueueConcurrencyKeyLimit(authenticatedEnvDev, "task/my-task", "ck-a", 3); - - const now = Date.now(); - for (const i of [0, 1, 2]) { - await queue.enqueueMessage({ - env: authenticatedEnvDev, - message: makeMessage({ - runId: `a${i}`, - concurrencyKey: "ck-a", - timestamp: now - 1000 + i, - }), - workerQueue: "main", - }); - } - await queue.enqueueMessage({ - env: authenticatedEnvDev, - message: makeMessage({ runId: "b0", concurrencyKey: "ck-b", timestamp: now - 500 }), - workerQueue: "main", - }); - await queue.enqueueMessage({ - env: authenticatedEnvDev, - message: makeMessage({ runId: "b1", concurrencyKey: "ck-b", timestamp: now - 499 }), - workerQueue: "main", - }); - - const settled = await waitFor(async () => { - const a = await queue.currentConcurrencyOfQueue( - authenticatedEnvDev, - "task/my-task", - "ck-a" - ); - const b = await queue.currentConcurrencyOfQueue( - authenticatedEnvDev, - "task/my-task", - "ck-b" - ); - return a === 3 && b === 1; - }); - expect(settled).toBe(true); - - await setTimeout(2000); - expect(await queue.lengthOfQueue(authenticatedEnvDev, "task/my-task")).toBe(1); - } finally { - await queue.quit(); - } - }); - - redisTest("removing an override restores the queue limit", async ({ redisContainer }) => { - const queue = createQueue(redisContainer, true); - try { - await queue.updateQueueConcurrencyLimits(authenticatedEnvDev, "task/my-task", 2); - await queue.updateQueueConcurrencyKeyLimit(authenticatedEnvDev, "task/my-task", "ck-a", 1); - expect(await queue.getQueueConcurrencyKeyLimits(authenticatedEnvDev, "task/my-task")).toEqual( - { "ck-a": 1 } - ); - - await queue.removeQueueConcurrencyKeyLimit(authenticatedEnvDev, "task/my-task", "ck-a"); - expect(await queue.getQueueConcurrencyKeyLimits(authenticatedEnvDev, "task/my-task")).toEqual( - {} - ); - - const now = Date.now(); - for (const i of [0, 1]) { - await queue.enqueueMessage({ - env: authenticatedEnvDev, - message: makeMessage({ - runId: `a${i}`, - concurrencyKey: "ck-a", - timestamp: now - 1000 + i, - }), - workerQueue: "main", - }); - } - - const settled = await waitFor( - async () => - (await queue.currentConcurrencyOfQueue(authenticatedEnvDev, "task/my-task", "ck-a")) === 2 - ); - expect(settled).toBe(true); - } finally { - await queue.quit(); - } - }); - - redisTest("the per-queue override count is capped", async ({ redisContainer }) => { - const queue = createQueue(redisContainer, true, 2); - try { - await queue.updateQueueConcurrencyKeyLimit(authenticatedEnvDev, "task/my-task", "ck-a", 1); - await queue.updateQueueConcurrencyKeyLimit(authenticatedEnvDev, "task/my-task", "ck-b", 1); - - await expect( - queue.updateQueueConcurrencyKeyLimit(authenticatedEnvDev, "task/my-task", "ck-c", 1) - ).rejects.toThrow(RunQueueConcurrencyKeyLimitExceededError); - - /** Updates to existing keys always succeed at the cap. */ - await queue.updateQueueConcurrencyKeyLimit(authenticatedEnvDev, "task/my-task", "ck-a", 4); - expect(await queue.getQueueConcurrencyKeyLimits(authenticatedEnvDev, "task/my-task")).toEqual( - { "ck-a": 4, "ck-b": 1 } - ); - } finally { - await queue.quit(); - } - }); - - redisTest( - "blocked keys cannot pin the candidate window and starve later keys", - async ({ redisContainer }) => { - /** dequeueCount 2 makes the candidate window 6 variants wide. */ - const queue = createQueue(redisContainer, true, undefined, 2); - try { - await queue.updateQueueConcurrencyLimits(authenticatedEnvDev, "task/my-task", 5); - - /** - * Ten zero-limit keys with OLDER messages fill the window many times over; - * without the blocked-key backoff the runnable key behind them would never - * be examined. - */ - const now = Date.now(); - for (let i = 0; i < 10; i++) { - const ck = `blocked-${i}`; - await queue.updateQueueConcurrencyKeyLimit(authenticatedEnvDev, "task/my-task", ck, 0); - await queue.enqueueMessage({ - env: authenticatedEnvDev, - message: makeMessage({ - runId: `b${i}`, - concurrencyKey: ck, - timestamp: now - 10_000 + i, - }), - workerQueue: "main", - }); - } - - await queue.enqueueMessage({ - env: authenticatedEnvDev, - message: makeMessage({ - runId: "good-0", - concurrencyKey: "ck-good", - timestamp: now - 500, - }), - workerQueue: "main", - }); - - const goodAdmitted = await waitFor( - async () => - (await queue.currentConcurrencyOfQueue( - authenticatedEnvDev, - "task/my-task", - "ck-good" - )) === 1, - 30_000 - ); - expect(goodAdmitted).toBe(true); - expect(await queue.lengthOfQueue(authenticatedEnvDev, "task/my-task")).toBe(10); - } finally { - await queue.quit(); - } - } - ); - - redisTest("gate admission honors the gate queue per-key override", async ({ redisContainer }) => { - const queue = createQueue(redisContainer, true, undefined, undefined, true); - try { - await queue.updateQueueConcurrencyLimits(authenticatedEnvDev, "task/my-task", 5); - await queue.updateQueueConcurrencyLimits(authenticatedEnvDev, "tenant", 1); - await queue.updateQueueConcurrencyKeyLimit(authenticatedEnvDev, "tenant", "acme", 2); - - const now = Date.now(); - for (const [i, ck] of ["ck-a", "ck-b", "ck-c"].entries()) { - await queue.enqueueMessage({ - env: authenticatedEnvDev, - message: makeMessage({ - runId: `r${i}`, - concurrencyKey: ck, - timestamp: now - 1000 + i, - gates: [{ queue: "tenant", concurrencyKey: "acme" }], - }), - workerQueue: "main", - }); - } - - /** The declared gate limit is 1; the override raises acme to 2. */ - const twoAdmitted = await waitFor( - async () => - (await queue.currentConcurrencyOfQueue(authenticatedEnvDev, "tenant", "acme")) === 2 - ); - expect(twoAdmitted).toBe(true); - - await setTimeout(2000); - expect(await queue.currentConcurrencyOfQueue(authenticatedEnvDev, "tenant", "acme")).toBe(2); - expect(await queue.lengthOfQueue(authenticatedEnvDev, "task/my-task")).toBe(1); - } finally { - await queue.quit(); - } - }); - - redisTest("overrides are ignored when disabled", async ({ redisContainer }) => { - const queue = createQueue(redisContainer, false); - try { - await queue.updateQueueConcurrencyLimits(authenticatedEnvDev, "task/my-task", 2); - await queue.updateQueueConcurrencyKeyLimit(authenticatedEnvDev, "task/my-task", "ck-a", 1); - - const now = Date.now(); - for (const i of [0, 1]) { - await queue.enqueueMessage({ - env: authenticatedEnvDev, - message: makeMessage({ - runId: `a${i}`, - concurrencyKey: "ck-a", - timestamp: now - 1000 + i, - }), - workerQueue: "main", - }); - } - - const settled = await waitFor( - async () => - (await queue.currentConcurrencyOfQueue(authenticatedEnvDev, "task/my-task", "ck-a")) === 2 - ); - expect(settled).toBe(true); - } finally { - await queue.quit(); - } - }); -}); diff --git a/internal-packages/run-engine/src/run-queue/types.ts b/internal-packages/run-engine/src/run-queue/types.ts index 639d206be5c..75651a1f847 100644 --- a/internal-packages/run-engine/src/run-queue/types.ts +++ b/internal-packages/run-engine/src/run-queue/types.ts @@ -111,8 +111,6 @@ export interface RunQueueKeyProducer { queueGroupConcurrencyKeyFromQueue(queue: string): string; queueTotalConcurrencyLimitKey(env: RunQueueKeyProducerEnvironment, queue: string): string; queueTotalConcurrencyLimitKeyFromQueue(queue: string): string; - queueCkLimitsKey(env: RunQueueKeyProducerEnvironment, queue: string): string; - queueCkLimitsKeyFromQueue(queue: string): string; //env oncurrency envCurrentConcurrencyKey(env: EnvDescriptor): string; diff --git a/packages/core/src/v3/apiClient/index.ts b/packages/core/src/v3/apiClient/index.ts index d67661ebd13..c31f99b32de 100644 --- a/packages/core/src/v3/apiClient/index.ts +++ b/packages/core/src/v3/apiClient/index.ts @@ -1810,58 +1810,6 @@ export class ApiClient { ); } - overrideQueueConcurrencyKeyLimit( - queue: RetrieveQueueParam, - concurrencyKey: string, - concurrencyLimit: number, - requestOptions?: ZodFetchOptions - ) { - const type = typeof queue === "string" ? "id" : queue.type; - const value = typeof queue === "string" ? queue : queue.name; - - const encodedValue = encodeURIComponent(value.replace(/\//g, "%2F")); - - return zodfetch( - QueueItem, - `${this.baseUrl}/api/v1/queues/${encodedValue}/concurrency/key/override`, - { - method: "POST", - headers: this.#getHeaders(false), - body: JSON.stringify({ - type, - concurrencyKey, - concurrencyLimit, - }), - }, - mergeRequestOptions(this.defaultRequestOptions, requestOptions) - ); - } - - resetQueueConcurrencyKeyLimit( - queue: RetrieveQueueParam, - concurrencyKey: string, - requestOptions?: ZodFetchOptions - ) { - const type = typeof queue === "string" ? "id" : queue.type; - const value = typeof queue === "string" ? queue : queue.name; - - const encodedValue = encodeURIComponent(value.replace(/\//g, "%2F")); - - return zodfetch( - QueueItem, - `${this.baseUrl}/api/v1/queues/${encodedValue}/concurrency/key/reset`, - { - method: "POST", - headers: this.#getHeaders(false), - body: JSON.stringify({ - type, - concurrencyKey, - }), - }, - mergeRequestOptions(this.defaultRequestOptions, requestOptions) - ); - } - subscribeToRun( runId: string, options?: { diff --git a/packages/trigger-sdk/src/v3/queues.ts b/packages/trigger-sdk/src/v3/queues.ts index f2ec3dd6339..1292cde1a3c 100644 --- a/packages/trigger-sdk/src/v3/queues.ts +++ b/packages/trigger-sdk/src/v3/queues.ts @@ -144,10 +144,9 @@ export function pause( export function overrideConcurrencyLimit( queue: RetrieveQueueParam, concurrencyLimit: number, - options?: ApiRequestOptions & { concurrencyKey?: string } + requestOptions?: ApiRequestOptions ): ApiPromise { const apiClient = apiClientManager.clientOrThrow(); - const { concurrencyKey, ...requestOptions } = options ?? {}; const $requestOptions = mergeRequestOptions( { @@ -155,7 +154,7 @@ export function overrideConcurrencyLimit( name: "queues.overrideConcurrencyLimit()", icon: "queue", attributes: { - ...flattenAttributes({ queue, concurrencyKey }), + ...flattenAttributes({ queue }), ...accessoryAttributes({ items: [ { @@ -170,15 +169,6 @@ export function overrideConcurrencyLimit( requestOptions ); - if (concurrencyKey !== undefined) { - return apiClient.overrideQueueConcurrencyKeyLimit( - queue, - concurrencyKey, - concurrencyLimit, - $requestOptions - ); - } - return apiClient.overrideQueueConcurrencyLimit(queue, concurrencyLimit, $requestOptions); } @@ -265,10 +255,9 @@ export function resetCombinedConcurrencyLimit( */ export function resetConcurrencyLimit( queue: RetrieveQueueParam, - options?: ApiRequestOptions & { concurrencyKey?: string } + requestOptions?: ApiRequestOptions ): ApiPromise { const apiClient = apiClientManager.clientOrThrow(); - const { concurrencyKey, ...requestOptions } = options ?? {}; const $requestOptions = mergeRequestOptions( { @@ -276,7 +265,7 @@ export function resetConcurrencyLimit( name: "queues.resetConcurrencyLimit()", icon: "queue", attributes: { - ...flattenAttributes({ queue, concurrencyKey }), + ...flattenAttributes({ queue }), ...accessoryAttributes({ items: [ { @@ -291,10 +280,6 @@ export function resetConcurrencyLimit( requestOptions ); - if (concurrencyKey !== undefined) { - return apiClient.resetQueueConcurrencyKeyLimit(queue, concurrencyKey, $requestOptions); - } - return apiClient.resetQueueConcurrencyLimit(queue, $requestOptions); } From 119a9624eb02d71c2c7c7087dd2e6ab9837a9dfe Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Mon, 31 Aug 2026 12:58:37 +0100 Subject: [PATCH 11/30] refactor(run-engine,webapp): drop per-key override reads and endpoints Removes the per-key override endpoints, the override-aware admit and gauge reads, and the per-key limit column, following the removal of runtime per-key overrides from this stack. --- ...es.$queueParam.concurrency.key.override.ts | 105 ------------------ ...ueues.$queueParam.concurrency.key.reset.ts | 97 ---------------- 2 files changed, 202 deletions(-) delete mode 100644 apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.key.override.ts delete mode 100644 apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.key.reset.ts diff --git a/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.key.override.ts b/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.key.override.ts deleted file mode 100644 index 5a37b4526ec..00000000000 --- a/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.key.override.ts +++ /dev/null @@ -1,105 +0,0 @@ -import { json } from "@remix-run/server-runtime"; -import { type RetrieveQueueParam, RetrieveQueueType } from "@trigger.dev/core/v3"; -import { z } from "zod"; -import { toQueueItem } from "~/presenters/v3/QueueRetrievePresenter.server"; -import { createActionApiRoute } from "~/services/routeBuilders/apiBuilder.server"; -import { concurrencySystem } from "~/v3/services/concurrencySystemInstance.server"; - -const BodySchema = z.object({ - type: RetrieveQueueType.default("id"), - concurrencyKey: z.string().min(1).max(128), - concurrencyLimit: z.number().int().min(0).max(100000), -}); - -const route = createActionApiRoute( - { - body: BodySchema, - params: z.object({ - queueParam: z.string().transform((val) => val.replace(/%2F/g, "/")), - }), - authorization: { - action: "write", - resource: () => ({ type: "queues" }), - }, - }, - async ({ params, body, authentication }) => { - const input: RetrieveQueueParam = - body.type === "id" - ? params.queueParam - : { - type: body.type, - name: decodeURIComponent(params.queueParam).replace(/%2F/g, "/"), - }; - - return concurrencySystem.queues - .overrideConcurrencyKeyLimit( - authentication.environment, - input, - body.concurrencyKey, - body.concurrencyLimit - ) - .match( - (queue) => { - return json( - toQueueItem({ - friendlyId: queue.friendlyId, - name: queue.name, - type: queue.type, - running: queue.running, - queued: queue.queued, - concurrencyLimit: queue.concurrencyLimit, - concurrencyLimitBase: queue.concurrencyLimitBase, - concurrencyLimitOverriddenAt: queue.concurrencyLimitOverriddenAt, - concurrencyLimitOverriddenBy: null, - paused: queue.paused, - }), - { status: 200 } - ); - }, - (error) => { - switch (error.type) { - case "queue_not_found": { - return json({ error: "Queue not found" }, { status: 404 }); - } - case "invalid_override": - case "concurrency_limit_exceeds_maximum": - case "too_many_key_overrides": { - return json({ error: error.message }, { status: 400 }); - } - case "queue_update_failed": { - return json( - { error: "Failed to update queue concurrency key limit" }, - { status: 500 } - ); - } - case "sync_queue_concurrency_to_engine_failed": { - return json({ error: "Failed to sync the concurrency key limit" }, { status: 500 }); - } - case "get_queue_stats_failed": { - return json({ error: "Failed to read queue stats" }, { status: 500 }); - } - case "other": { - return json( - { error: "Failed to update queue concurrency key limit" }, - { - status: 500, - } - ); - } - default: { - return json( - { error: "Failed to update queue concurrency key limit" }, - { - status: 500, - } - ); - } - } - } - ); - } -); - -export const action = route.action; -/** The builder's loader answers non-POST methods with a 405. */ -export const loader = route.loader; diff --git a/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.key.reset.ts b/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.key.reset.ts deleted file mode 100644 index 51d14642e2c..00000000000 --- a/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.key.reset.ts +++ /dev/null @@ -1,97 +0,0 @@ -import { json } from "@remix-run/server-runtime"; -import { type RetrieveQueueParam, RetrieveQueueType } from "@trigger.dev/core/v3"; -import { z } from "zod"; -import { toQueueItem } from "~/presenters/v3/QueueRetrievePresenter.server"; -import { createActionApiRoute } from "~/services/routeBuilders/apiBuilder.server"; -import { concurrencySystem } from "~/v3/services/concurrencySystemInstance.server"; - -const BodySchema = z.object({ - type: RetrieveQueueType.default("id"), - concurrencyKey: z.string().min(1).max(128), -}); - -const route = createActionApiRoute( - { - body: BodySchema, - params: z.object({ - queueParam: z.string().transform((val) => val.replace(/%2F/g, "/")), - }), - authorization: { - action: "write", - resource: () => ({ type: "queues" }), - }, - }, - async ({ params, body, authentication }) => { - const input: RetrieveQueueParam = - body.type === "id" - ? params.queueParam - : { - type: body.type, - name: decodeURIComponent(params.queueParam).replace(/%2F/g, "/"), - }; - - return concurrencySystem.queues - .resetConcurrencyKeyLimit(authentication.environment, input, body.concurrencyKey) - .match( - (queue) => { - return json( - toQueueItem({ - friendlyId: queue.friendlyId, - name: queue.name, - type: queue.type, - running: queue.running, - queued: queue.queued, - concurrencyLimit: queue.concurrencyLimit, - concurrencyLimitBase: queue.concurrencyLimitBase, - concurrencyLimitOverriddenAt: queue.concurrencyLimitOverriddenAt, - concurrencyLimitOverriddenBy: null, - paused: queue.paused, - }), - { status: 200 } - ); - }, - (error) => { - switch (error.type) { - case "queue_not_found": { - return json({ error: "Queue not found" }, { status: 404 }); - } - case "queue_not_overridden": { - return json( - { error: "This concurrency key does not have an override" }, - { status: 400 } - ); - } - case "queue_update_failed": { - return json({ error: "Failed to reset the concurrency key limit" }, { status: 500 }); - } - case "sync_queue_concurrency_to_engine_failed": { - return json({ error: "Failed to sync the concurrency key limit" }, { status: 500 }); - } - case "get_queue_stats_failed": { - return json({ error: "Failed to read queue stats" }, { status: 500 }); - } - case "other": { - return json( - { error: "Failed to reset the concurrency key limit" }, - { - status: 500, - } - ); - } - default: { - return json( - { error: "Failed to reset the concurrency key limit" }, - { - status: 500, - } - ); - } - } - } - ); - } -); - -export const action = route.action; -/** The builder's loader answers non-POST methods with a 405. */ -export const loader = route.loader; From 84a961988e615fe02eac0049dea96e1f26cce340 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Mon, 31 Aug 2026 13:10:59 +0100 Subject: [PATCH 12/30] fix(run-engine): keep the ck-limits key builders while the Lua reads remain The per-key admit reads are removed in the PR above alongside the rest of the engine-internal plumbing; the key builders they reference stay until then so every level of the stack compiles. --- internal-packages/run-engine/src/run-queue/keyProducer.ts | 8 ++++++++ internal-packages/run-engine/src/run-queue/types.ts | 3 +++ 2 files changed, 11 insertions(+) diff --git a/internal-packages/run-engine/src/run-queue/keyProducer.ts b/internal-packages/run-engine/src/run-queue/keyProducer.ts index 98028f5af7b..120e04f8c38 100644 --- a/internal-packages/run-engine/src/run-queue/keyProducer.ts +++ b/internal-packages/run-engine/src/run-queue/keyProducer.ts @@ -366,6 +366,14 @@ export class RunQueueFullKeyProducer implements RunQueueKeyProducer { return `${this.baseQueueKeyFromQueue(queue)}:${constants.TOTAL_CONCURRENCY_LIMIT_PART}`; } + queueCkLimitsKey(env: RunQueueKeyProducerEnvironment, queue: string): string { + return `${this.queueKey(env, queue)}:ckLimits`; + } + + queueCkLimitsKeyFromQueue(queue: string): string { + return `${this.baseQueueKeyFromQueue(queue)}:ckLimits`; + } + isCkWildcard(queue: string): boolean { return queue.endsWith(":ck:*"); } diff --git a/internal-packages/run-engine/src/run-queue/types.ts b/internal-packages/run-engine/src/run-queue/types.ts index 75651a1f847..ad358a04cbb 100644 --- a/internal-packages/run-engine/src/run-queue/types.ts +++ b/internal-packages/run-engine/src/run-queue/types.ts @@ -112,6 +112,9 @@ export interface RunQueueKeyProducer { queueTotalConcurrencyLimitKey(env: RunQueueKeyProducerEnvironment, queue: string): string; queueTotalConcurrencyLimitKeyFromQueue(queue: string): string; + queueCkLimitsKey(env: RunQueueKeyProducerEnvironment, queue: string): string; + queueCkLimitsKeyFromQueue(queue: string): string; + //env oncurrency envCurrentConcurrencyKey(env: EnvDescriptor): string; envCurrentConcurrencyKey(env: RunQueueKeyProducerEnvironment): string; From f69ba93d2936e03fa1b86cd95045c84b32c74174 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Mon, 31 Aug 2026 14:02:52 +0100 Subject: [PATCH 13/30] fix(webapp): combined override error messages use the public name --- apps/webapp/app/v3/services/concurrencySystem.server.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/webapp/app/v3/services/concurrencySystem.server.ts b/apps/webapp/app/v3/services/concurrencySystem.server.ts index e1b285136fa..09cc5f9ce11 100644 --- a/apps/webapp/app/v3/services/concurrencySystem.server.ts +++ b/apps/webapp/app/v3/services/concurrencySystem.server.ts @@ -359,14 +359,14 @@ function overrideQueueTotalConcurrencyLimit( if (!Number.isFinite(totalConcurrencyLimit) || totalConcurrencyLimit < 0) { return errAsync({ type: "invalid_override" as const, - message: "Total concurrency limit must be a non-negative number", + message: "Combined concurrency limit must be a non-negative number", }); } if (totalConcurrencyLimit > maximum) { return errAsync({ type: "concurrency_limit_exceeds_maximum" as const, - message: `Total concurrency limit (${totalConcurrencyLimit}) cannot exceed the environment limit (${maximum})`, + message: `Combined concurrency limit (${totalConcurrencyLimit}) cannot exceed the environment limit (${maximum})`, }); } From 2a38daa9a0281d0fed8533e82873969ba413d79e Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Sun, 6 Sep 2026 13:44:56 +0100 Subject: [PATCH 14/30] feat(database): TaskQueue concurrencyVersion and role columns concurrencyVersion records how a queue's concurrency was declared (V1 = legacy contextual concurrencyLimit, V2 = the explicit perKey/total shape) so display never has to guess what a stored number meant. role distinguishes real queues from the rows that back named concurrency limits, which every queue-facing surface filters out. Defaults cover all existing rows with no backfill. --- .../migration.sql | 9 +++++++++ internal-packages/database/prisma/schema.prisma | 17 +++++++++++++++++ 2 files changed, 26 insertions(+) create mode 100644 internal-packages/database/prisma/migrations/20260906140000_add_task_queue_concurrency_version_and_role/migration.sql diff --git a/internal-packages/database/prisma/migrations/20260906140000_add_task_queue_concurrency_version_and_role/migration.sql b/internal-packages/database/prisma/migrations/20260906140000_add_task_queue_concurrency_version_and_role/migration.sql new file mode 100644 index 00000000000..faff8327eca --- /dev/null +++ b/internal-packages/database/prisma/migrations/20260906140000_add_task_queue_concurrency_version_and_role/migration.sql @@ -0,0 +1,9 @@ +-- CreateEnum +CREATE TYPE "TaskQueueConcurrencyVersion" AS ENUM ('V1', 'V2'); + +-- CreateEnum +CREATE TYPE "TaskQueueRole" AS ENUM ('QUEUE', 'LIMIT'); + +-- AlterTable +ALTER TABLE "TaskQueue" ADD COLUMN "concurrencyVersion" "TaskQueueConcurrencyVersion" NOT NULL DEFAULT 'V1', +ADD COLUMN "role" "TaskQueueRole" NOT NULL DEFAULT 'QUEUE'; diff --git a/internal-packages/database/prisma/schema.prisma b/internal-packages/database/prisma/schema.prisma index 406caec57e5..b0621525231 100644 --- a/internal-packages/database/prisma/schema.prisma +++ b/internal-packages/database/prisma/schema.prisma @@ -1963,6 +1963,13 @@ model TaskQueue { version TaskQueueVersion @default(V1) orderableName String? + /// How the queue's concurrency was declared: V1 = legacy concurrencyLimit semantics + /// (per key when keyed, whole queue otherwise), V2 = the explicit perKey/total shape. + concurrencyVersion TaskQueueConcurrencyVersion @default(V1) + /// QUEUE rows are real queues; LIMIT rows back named concurrency limits and are + /// excluded from every queue-facing surface. + role TaskQueueRole @default(QUEUE) + project Project @relation(fields: [projectId], references: [id], onDelete: Cascade, onUpdate: Cascade) projectId String @@ -2016,6 +2023,16 @@ enum TaskQueueVersion { V2 } +enum TaskQueueConcurrencyVersion { + V1 + V2 +} + +enum TaskQueueRole { + QUEUE + LIMIT +} + model BatchTaskRun { id String @id @default(cuid()) friendlyId String @unique From 905d6884a04c13b1d3dcec1d2ac07fc3aa734161 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Sun, 6 Sep 2026 14:22:58 +0100 Subject: [PATCH 15/30] feat(sdk,core): drop the combined concurrency override client methods The combined override and reset never ship; runtime management of the new named limits arrives with its own management API. --- packages/core/src/v3/apiClient/index.ts | 45 --------------- packages/trigger-sdk/src/v3/queues.ts | 75 ------------------------- 2 files changed, 120 deletions(-) diff --git a/packages/core/src/v3/apiClient/index.ts b/packages/core/src/v3/apiClient/index.ts index c31f99b32de..461091cc918 100644 --- a/packages/core/src/v3/apiClient/index.ts +++ b/packages/core/src/v3/apiClient/index.ts @@ -1765,51 +1765,6 @@ export class ApiClient { ); } - overrideQueueCombinedConcurrencyLimit( - queue: RetrieveQueueParam, - concurrencyLimit: number, - requestOptions?: ZodFetchOptions - ) { - const type = typeof queue === "string" ? "id" : queue.type; - const value = typeof queue === "string" ? queue : queue.name; - - const encodedValue = encodeURIComponent(value.replace(/\//g, "%2F")); - - return zodfetch( - QueueItem, - `${this.baseUrl}/api/v1/queues/${encodedValue}/concurrency/combined/override`, - { - method: "POST", - headers: this.#getHeaders(false), - body: JSON.stringify({ - type, - concurrencyLimit, - }), - }, - mergeRequestOptions(this.defaultRequestOptions, requestOptions) - ); - } - - resetQueueCombinedConcurrencyLimit(queue: RetrieveQueueParam, requestOptions?: ZodFetchOptions) { - const type = typeof queue === "string" ? "id" : queue.type; - const value = typeof queue === "string" ? queue : queue.name; - - const encodedValue = encodeURIComponent(value.replace(/\//g, "%2F")); - - return zodfetch( - QueueItem, - `${this.baseUrl}/api/v1/queues/${encodedValue}/concurrency/combined/reset`, - { - method: "POST", - headers: this.#getHeaders(false), - body: JSON.stringify({ - type, - }), - }, - mergeRequestOptions(this.defaultRequestOptions, requestOptions) - ); - } - subscribeToRun( runId: string, options?: { diff --git a/packages/trigger-sdk/src/v3/queues.ts b/packages/trigger-sdk/src/v3/queues.ts index 1292cde1a3c..7e76c5f940b 100644 --- a/packages/trigger-sdk/src/v3/queues.ts +++ b/packages/trigger-sdk/src/v3/queues.ts @@ -172,81 +172,6 @@ export function overrideConcurrencyLimit( return apiClient.overrideQueueConcurrencyLimit(queue, concurrencyLimit, $requestOptions); } -/** - * Overrides the combined concurrency limit of a queue: the cap on concurrent runs across - * all of its `concurrencyKey` values. - * - * @param queue - The ID of the queue, or the type and name - * @param concurrencyLimit - The combined concurrency limit to apply - * @returns The updated queue state - */ -export function overrideCombinedConcurrencyLimit( - queue: RetrieveQueueParam, - concurrencyLimit: number, - requestOptions?: ApiRequestOptions -): ApiPromise { - const apiClient = apiClientManager.clientOrThrow(); - - const $requestOptions = mergeRequestOptions( - { - tracer, - name: "queues.overrideCombinedConcurrencyLimit()", - icon: "queue", - attributes: { - ...flattenAttributes({ queue }), - ...accessoryAttributes({ - items: [ - { - text: typeof queue === "string" ? queue : queue.name, - variant: "normal", - }, - ], - style: "codepath", - }), - }, - }, - requestOptions - ); - - return apiClient.overrideQueueCombinedConcurrencyLimit(queue, concurrencyLimit, $requestOptions); -} - -/** - * Resets the combined concurrency limit of a queue back to its declared value. - * - * @param queue - The ID of the queue, or the type and name - * @returns The updated queue state - */ -export function resetCombinedConcurrencyLimit( - queue: RetrieveQueueParam, - requestOptions?: ApiRequestOptions -): ApiPromise { - const apiClient = apiClientManager.clientOrThrow(); - - const $requestOptions = mergeRequestOptions( - { - tracer, - name: "queues.resetCombinedConcurrencyLimit()", - icon: "queue", - attributes: { - ...flattenAttributes({ queue }), - ...accessoryAttributes({ - items: [ - { - text: typeof queue === "string" ? queue : queue.name, - variant: "normal", - }, - ], - style: "codepath", - }), - }, - }, - requestOptions - ); - - return apiClient.resetQueueCombinedConcurrencyLimit(queue, $requestOptions); -} - /** * Resets the concurrency limit of a queue to the base value. * From 6c3b4171d88720583ae120f23e3a00e38c4499ed Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Sun, 6 Sep 2026 15:16:01 +0100 Subject: [PATCH 16/30] feat(webapp): compile task concurrency declarations at deploy Named limits (declared or merely referenced) materialize as LIMIT-role queue rows under the reserved limit/ prefix, with a total-only limit storing its total as the per-key limit too so it truly caps keyless runs. A task's inline limit maps onto its own default queue (stamped concurrencyVersion V2) or, on a shared queue, onto an anonymous limit/task row holding one of the two gate slots. Declaring both the legacy queue concurrencyLimit and concurrency is a deploy error. Trigger-time concurrency names resolve to limit gates, replacing the task's declared set, and every queue read filters to QUEUE-role rows. Drops the release note for the combined override methods that no longer ship. --- .changeset/queue-concurrency-overrides.md | 15 -- .../v3/QueueListPresenter.server.ts | 1 + .../v3/QueueRetrievePresenter.server.ts | 2 + .../app/runEngine/concerns/queues.server.ts | 10 +- .../services/createBackgroundWorker.server.ts | 136 ++++++++++++++++-- 5 files changed, 137 insertions(+), 27 deletions(-) delete mode 100644 .changeset/queue-concurrency-overrides.md diff --git a/.changeset/queue-concurrency-overrides.md b/.changeset/queue-concurrency-overrides.md deleted file mode 100644 index 22f79c5e72c..00000000000 --- a/.changeset/queue-concurrency-overrides.md +++ /dev/null @@ -1,15 +0,0 @@ ---- -"@trigger.dev/sdk": patch -"@trigger.dev/core": patch ---- - -Adjust a queue's combined concurrency limit at runtime. `queues.overrideCombinedConcurrencyLimit` raises or lowers the cap on concurrent runs across all of a queue's `concurrencyKey` values, and `queues.resetCombinedConcurrencyLimit` reverts to the declared configuration. - -```ts -import { queues } from "@trigger.dev/sdk"; - -await queues.overrideCombinedConcurrencyLimit("my-queue", 100); -await queues.resetCombinedConcurrencyLimit("my-queue"); -``` - -Overrides survive deploys. Enforcement happens server-side on servers with combined concurrency limits enabled. diff --git a/apps/webapp/app/presenters/v3/QueueListPresenter.server.ts b/apps/webapp/app/presenters/v3/QueueListPresenter.server.ts index 0dc3daa9856..7db2a6d2e39 100644 --- a/apps/webapp/app/presenters/v3/QueueListPresenter.server.ts +++ b/apps/webapp/app/presenters/v3/QueueListPresenter.server.ts @@ -70,6 +70,7 @@ function buildQueueListWhere( return { runtimeEnvironmentId: environmentId, + role: "QUEUE" as const, version: "V2", name: trimmedQuery ? { diff --git a/apps/webapp/app/presenters/v3/QueueRetrievePresenter.server.ts b/apps/webapp/app/presenters/v3/QueueRetrievePresenter.server.ts index f6918394e5c..6385b388d1f 100644 --- a/apps/webapp/app/presenters/v3/QueueRetrievePresenter.server.ts +++ b/apps/webapp/app/presenters/v3/QueueRetrievePresenter.server.ts @@ -31,6 +31,7 @@ export async function getQueue( where: { friendlyId: queue, runtimeEnvironmentId: environment.id, + role: "QUEUE", }, }) ); @@ -44,6 +45,7 @@ export async function getQueue( where: { name: queueName, runtimeEnvironmentId: environment.id, + role: "QUEUE", }, }) ); diff --git a/apps/webapp/app/runEngine/concerns/queues.server.ts b/apps/webapp/app/runEngine/concerns/queues.server.ts index 9214e91b9b3..6d09e7bc3f1 100644 --- a/apps/webapp/app/runEngine/concerns/queues.server.ts +++ b/apps/webapp/app/runEngine/concerns/queues.server.ts @@ -220,7 +220,15 @@ export class DefaultQueueManager implements QueueManager { queueName = sanitizedQueueName; } - const requestedGates = request.body.options?.gates ?? taskGates ?? undefined; + const triggerLimits = request.body.options?.concurrency; + const concurrencyGates = triggerLimits?.map( + (name): { queue: string; concurrencyKey?: string } => ({ + queue: `limit/${sanitizeQueueName(name)}`, + }) + ); + + const requestedGates = + concurrencyGates ?? request.body.options?.gates ?? taskGates ?? undefined; const gates = requestedGates ?.flatMap((gate) => { const sanitized = sanitizeQueueName(gate.queue); diff --git a/apps/webapp/app/v3/services/createBackgroundWorker.server.ts b/apps/webapp/app/v3/services/createBackgroundWorker.server.ts index 147248bd376..22e8832e5a9 100644 --- a/apps/webapp/app/v3/services/createBackgroundWorker.server.ts +++ b/apps/webapp/app/v3/services/createBackgroundWorker.server.ts @@ -16,7 +16,13 @@ import { stringifyDuration, } from "@trigger.dev/core/v3/isomorphic"; import { randomBytes } from "node:crypto"; -import type { BackgroundWorker, TaskQueue, TaskQueueType } from "@trigger.dev/database"; +import type { + BackgroundWorker, + TaskQueue, + TaskQueueConcurrencyVersion, + TaskQueueRole, + TaskQueueType, +} from "@trigger.dev/database"; import cronstrue from "cronstrue"; import type { PrismaClientOrTransaction, WebhookDatabase } from "~/db.server"; import { $transaction, Prisma, boundedIn, webhookPrisma } from "~/db.server"; @@ -337,6 +343,7 @@ export async function createWorkerResources( // Create the queues const queues = await createWorkerQueues(metadata, worker, environment, prisma); + await createWorkerConcurrencyLimits(metadata, worker, environment, prisma); // Create the tasks const taskEntries = await createWorkerTasks( @@ -391,10 +398,55 @@ async function createWorkerTask( ): Promise { // Hoisted so the P2002 catch branch can return the same entry shape. let queue: TaskQueue | undefined; + let compiledGates: Array<{ queue: string; concurrencyKey?: string }> = []; let resolvedTriggerSource: "SCHEDULED" | "AGENT" | "WEBHOOK" | "STANDARD" | undefined; let resolvedTtl: string | null | undefined; try { + const concurrency = task.concurrency; + + if (concurrency && typeof task.queue?.concurrencyLimit === "number") { + throw new ServiceValidationError( + `Task "${task.id}" declares both a queue concurrencyLimit and the concurrency option; use concurrency.` + ); + } + + compiledGates = (concurrency?.limits ?? []).map((name) => ({ + queue: concurrencyLimitQueueName(name), + })); + + let queueConcurrencyLimit = task.queue?.concurrencyLimit; + let queueTotalConcurrencyLimit = task.queue?.combinedConcurrencyLimit; + + if (concurrency?.inline) { + if (!task.queue?.name) { + queueConcurrencyLimit = concurrency.inline.perKey ?? concurrency.inline.total; + queueTotalConcurrencyLimit = concurrency.inline.total; + } else { + if (compiledGates.length > 1) { + throw new ServiceValidationError( + `Task "${task.id}": an inline limit on a shared queue uses a gate slot, so at most one named limit can be combined with it.` + ); + } + const anonymousName = `task/${task.id}`; + await createWorkerQueue( + { + name: concurrencyLimitQueueName(anonymousName), + concurrencyLimit: concurrency.inline.perKey ?? concurrency.inline.total ?? null, + combinedConcurrencyLimit: concurrency.inline.total ?? null, + }, + anonymousName, + "NAMED", + worker, + environment, + prisma, + "LIMIT", + "V2" + ); + compiledGates = [{ queue: concurrencyLimitQueueName(anonymousName) }, ...compiledGates]; + } + } + queue = queues.find((queue) => queue.name === task.queue?.name); if (!queue) { @@ -402,14 +454,16 @@ async function createWorkerTask( queue = await createWorkerQueue( { name: task.queue?.name ?? `task/${task.id}`, - concurrencyLimit: task.queue?.concurrencyLimit, - combinedConcurrencyLimit: task.queue?.combinedConcurrencyLimit, + concurrencyLimit: queueConcurrencyLimit, + combinedConcurrencyLimit: queueTotalConcurrencyLimit, }, task.queue?.name ?? task.id, task.queue?.name ? "NAMED" : "VIRTUAL", worker, environment, - prisma + prisma, + "QUEUE", + concurrency ? "V2" : "V1" ); } @@ -437,7 +491,7 @@ async function createWorkerTask( exportName: task.exportName, retryConfig: task.retry, queueConfig: task.queue, - gates: task.gates, + gates: compiledGates.length > 0 ? compiledGates : task.gates, machineConfig: task.machine, triggerSource: resolvedTriggerSource, config: task.agentConfig ? (task.agentConfig as any) : undefined, @@ -455,7 +509,7 @@ async function createWorkerTask( triggerSource: resolvedTriggerSource, queueId: queue.id, queueName: queue.name, - gates: task.gates ?? null, + gates: compiledGates.length > 0 ? compiledGates : (task.gates ?? null), }; } catch (error) { if (error instanceof Prisma.PrismaClientKnownRequestError) { @@ -479,7 +533,7 @@ async function createWorkerTask( triggerSource: resolvedTriggerSource, queueId: queue.id, queueName: queue.name, - gates: task.gates ?? null, + gates: compiledGates.length > 0 ? compiledGates : (task.gates ?? null), }; } } else { @@ -540,13 +594,63 @@ async function createWorkerQueues( return allQueues; } +/** Queue rows that back named concurrency limits live under this reserved prefix so + * they can never collide with a user's queue names. */ +export const CONCURRENCY_LIMIT_QUEUE_PREFIX = "limit/"; + +export function concurrencyLimitQueueName(limitName: string): string { + return `${CONCURRENCY_LIMIT_QUEUE_PREFIX}${sanitizeQueueName(limitName)}`; +} + +/** + * Materializes the worker's declared named concurrency limits (plus any names tasks + * reference without declaring, created uncapped) as LIMIT-role TaskQueue rows. A + * total-only limit stores the total as its per-key limit too, so it truly caps + * keyless runs as well as the keyed group. + */ +async function createWorkerConcurrencyLimits( + metadata: BackgroundWorkerMetadata, + worker: BackgroundWorker, + environment: AuthenticatedEnvironment, + prisma: PrismaClientOrTransaction +) { + const declared = new Map((metadata.concurrencyLimits ?? []).map((l) => [l.name, l])); + + for (const task of metadata.tasks) { + for (const name of task.concurrency?.limits ?? []) { + if (!declared.has(name)) { + declared.set(name, { name }); + } + } + } + + for (const limit of declared.values()) { + await createWorkerQueue( + { + name: concurrencyLimitQueueName(limit.name), + concurrencyLimit: limit.perKey ?? limit.total ?? null, + combinedConcurrencyLimit: limit.total ?? null, + }, + limit.name, + "NAMED", + worker, + environment, + prisma, + "LIMIT", + "V2" + ); + } +} + async function createWorkerQueue( queue: QueueManifest, orderableName: string, queueType: TaskQueueType, worker: BackgroundWorker, environment: AuthenticatedEnvironment, - prisma: PrismaClientOrTransaction + prisma: PrismaClientOrTransaction, + role: TaskQueueRole = "QUEUE", + concurrencyVersion: TaskQueueConcurrencyVersion = "V1" ) { let queueName = sanitizeQueueName(queue.name); @@ -562,7 +666,10 @@ async function createWorkerQueue( orderableName, queueType, worker, - prisma + prisma, + 0, + role, + concurrencyVersion ); const newConcurrencyLimit = taskQueue.concurrencyLimit; @@ -625,7 +732,9 @@ async function upsertWorkerQueueRecord( queueType: TaskQueueType, worker: BackgroundWorker, prisma: PrismaClientOrTransaction, - attempt: number = 0 + attempt: number = 0, + role: TaskQueueRole = "QUEUE", + concurrencyVersion: TaskQueueConcurrencyVersion = "V1" ): Promise { if (attempt > 3) { throw new Error("Failed to insert queue record"); @@ -644,6 +753,8 @@ async function upsertWorkerQueueRecord( data: { friendlyId: generateFriendlyId("queue"), version: "V2", + role, + concurrencyVersion, name: queueName, orderableName, concurrencyLimit, @@ -669,6 +780,7 @@ async function upsertWorkerQueueRecord( data: { workers: { connect: { id: worker.id } }, version: "V2", + concurrencyVersion, orderableName, // If overridden, keep current limit and update base; otherwise update limit normally concurrencyLimit: hasOverride ? undefined : concurrencyLimit, @@ -691,7 +803,9 @@ async function upsertWorkerQueueRecord( queueType, worker, prisma, - attempt + 1 + attempt + 1, + role, + concurrencyVersion ); } throw error; From e314856f4356fa85e70e8758f78bf0c2a985609c Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Sun, 6 Sep 2026 16:12:17 +0100 Subject: [PATCH 17/30] fix(webapp): deploy-time guards for the limit namespace Validation errors now fail the deploy instead of being swallowed into a silently incomplete worker, user queue names may not claim the reserved limit/ prefix, and a limit whose prefixed row name would exceed the 128-character queue maximum is rejected rather than truncated into the wrong row. --- .../services/createBackgroundWorker.server.ts | 26 ++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/apps/webapp/app/v3/services/createBackgroundWorker.server.ts b/apps/webapp/app/v3/services/createBackgroundWorker.server.ts index 22e8832e5a9..10f751a5af2 100644 --- a/apps/webapp/app/v3/services/createBackgroundWorker.server.ts +++ b/apps/webapp/app/v3/services/createBackgroundWorker.server.ts @@ -405,6 +405,10 @@ async function createWorkerTask( try { const concurrency = task.concurrency; + if (task.queue?.name) { + assertNotReservedQueueName(task.queue.name, `Task "${task.id}"`); + } + if (concurrency && typeof task.queue?.concurrencyLimit === "number") { throw new ServiceValidationError( `Task "${task.id}" declares both a queue concurrencyLimit and the concurrency option; use concurrency.` @@ -512,6 +516,9 @@ async function createWorkerTask( gates: compiledGates.length > 0 ? compiledGates : (task.gates ?? null), }; } catch (error) { + if (error instanceof ServiceValidationError) { + throw error; + } if (error instanceof Prisma.PrismaClientKnownRequestError) { // The error code for unique constraint violation in Prisma is P2002 if (error.code === "P2002") { @@ -585,6 +592,7 @@ async function createWorkerQueues( const chunk = metadata.queues.slice(i, i + CHUNK_SIZE); const queueChunk = await Promise.all( chunk.map(async (queue) => { + assertNotReservedQueueName(queue.name, `Queue "${queue.name}"`); return createWorkerQueue(queue, queue.name, "NAMED", worker, environment, prisma); }) ); @@ -599,7 +607,23 @@ async function createWorkerQueues( export const CONCURRENCY_LIMIT_QUEUE_PREFIX = "limit/"; export function concurrencyLimitQueueName(limitName: string): string { - return `${CONCURRENCY_LIMIT_QUEUE_PREFIX}${sanitizeQueueName(limitName)}`; + const sanitized = sanitizeQueueName(limitName); + const name = `${CONCURRENCY_LIMIT_QUEUE_PREFIX}${sanitized}`; + if (sanitized.length === 0 || name.length > 128) { + throw new ServiceValidationError( + `Concurrency limit name "${limitName}" must sanitize to between 1 and ${128 - CONCURRENCY_LIMIT_QUEUE_PREFIX.length} characters.` + ); + } + return name; +} + +/** User queue names may not claim the reserved limit/ namespace. */ +function assertNotReservedQueueName(name: string, context: string): void { + if (sanitizeQueueName(name).startsWith(CONCURRENCY_LIMIT_QUEUE_PREFIX)) { + throw new ServiceValidationError( + `${context}: queue names starting with "${CONCURRENCY_LIMIT_QUEUE_PREFIX}" are reserved for concurrency limits.` + ); + } } /** From 61b32fbca2176c0de445710c9ca8a0383d1591cd Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Sun, 6 Sep 2026 16:17:10 +0100 Subject: [PATCH 18/30] chore(webapp): the limit-name helpers are module-local --- apps/webapp/app/v3/services/createBackgroundWorker.server.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/webapp/app/v3/services/createBackgroundWorker.server.ts b/apps/webapp/app/v3/services/createBackgroundWorker.server.ts index 10f751a5af2..c337bfca847 100644 --- a/apps/webapp/app/v3/services/createBackgroundWorker.server.ts +++ b/apps/webapp/app/v3/services/createBackgroundWorker.server.ts @@ -604,9 +604,9 @@ async function createWorkerQueues( /** Queue rows that back named concurrency limits live under this reserved prefix so * they can never collide with a user's queue names. */ -export const CONCURRENCY_LIMIT_QUEUE_PREFIX = "limit/"; +const CONCURRENCY_LIMIT_QUEUE_PREFIX = "limit/"; -export function concurrencyLimitQueueName(limitName: string): string { +function concurrencyLimitQueueName(limitName: string): string { const sanitized = sanitizeQueueName(limitName); const name = `${CONCURRENCY_LIMIT_QUEUE_PREFIX}${sanitized}`; if (sanitized.length === 0 || name.length > 128) { From 550985c8e244f25cc37fe35124f6301d719b9d54 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Sun, 6 Sep 2026 16:20:50 +0100 Subject: [PATCH 19/30] fix(webapp): validate concurrency declarations before any worker rows exist A rejected deploy previously left a partial worker behind, which a same-content retry then returned as if the deploy had succeeded. The whole manifest's concurrency declarations are now validated up front. --- .../services/createBackgroundWorker.server.ts | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/apps/webapp/app/v3/services/createBackgroundWorker.server.ts b/apps/webapp/app/v3/services/createBackgroundWorker.server.ts index c337bfca847..c0d2b61c64f 100644 --- a/apps/webapp/app/v3/services/createBackgroundWorker.server.ts +++ b/apps/webapp/app/v3/services/createBackgroundWorker.server.ts @@ -97,6 +97,8 @@ export class CreateBackgroundWorkerService extends BaseService { }, }); + validateWorkerConcurrencyDeclarations(body.metadata); + const latestBackgroundWorker = project.backgroundWorkers[0]; if (latestBackgroundWorker?.contentHash === body.metadata.contentHash) { @@ -602,6 +604,50 @@ async function createWorkerQueues( return allQueues; } +/** + * Rejects invalid concurrency declarations before any worker rows are written, so a + * failed deploy leaves nothing behind for a same-content retry to return. + */ +function validateWorkerConcurrencyDeclarations(metadata: BackgroundWorkerMetadata): void { + for (const queue of metadata.queues ?? []) { + assertNotReservedQueueName(queue.name, `Queue "${queue.name}"`); + } + + for (const limit of metadata.concurrencyLimits ?? []) { + concurrencyLimitQueueName(limit.name); + } + + for (const task of metadata.tasks) { + if (task.queue?.name) { + assertNotReservedQueueName(task.queue.name, `Task "${task.id}"`); + } + + const concurrency = task.concurrency; + if (!concurrency) { + continue; + } + + if (typeof task.queue?.concurrencyLimit === "number") { + throw new ServiceValidationError( + `Task "${task.id}" declares both a queue concurrencyLimit and the concurrency option; use concurrency.` + ); + } + + for (const name of concurrency.limits ?? []) { + concurrencyLimitQueueName(name); + } + + if (concurrency.inline && task.queue?.name) { + concurrencyLimitQueueName(`task/${task.id}`); + if ((concurrency.limits ?? []).length > 1) { + throw new ServiceValidationError( + `Task "${task.id}": an inline limit on a shared queue uses a gate slot, so at most one named limit can be combined with it.` + ); + } + } + } +} + /** Queue rows that back named concurrency limits live under this reserved prefix so * they can never collide with a user's queue names. */ const CONCURRENCY_LIMIT_QUEUE_PREFIX = "limit/"; From 056b4f31e82926a19fd7ffc3fb7c246c6856614c Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Sun, 6 Sep 2026 16:40:13 +0100 Subject: [PATCH 20/30] fix(webapp): strict limit names at deploy and collision-proof anonymous limit rows Deploy validation now enforces the SDK's limit-name charset, so sanitization can never merge two declared limits onto one row. A task's anonymous inline limit row hashes the raw task id whenever sanitizing it would be lossy or overflow the queue-name length, so distinct task ids always keep distinct rows. --- .../services/createBackgroundWorker.server.ts | 53 +++++++++++++------ 1 file changed, 38 insertions(+), 15 deletions(-) diff --git a/apps/webapp/app/v3/services/createBackgroundWorker.server.ts b/apps/webapp/app/v3/services/createBackgroundWorker.server.ts index c0d2b61c64f..8857213e152 100644 --- a/apps/webapp/app/v3/services/createBackgroundWorker.server.ts +++ b/apps/webapp/app/v3/services/createBackgroundWorker.server.ts @@ -434,14 +434,14 @@ async function createWorkerTask( `Task "${task.id}": an inline limit on a shared queue uses a gate slot, so at most one named limit can be combined with it.` ); } - const anonymousName = `task/${task.id}`; + const anonymousQueueName = anonymousConcurrencyLimitQueueName(task.id); await createWorkerQueue( { - name: concurrencyLimitQueueName(anonymousName), + name: anonymousQueueName, concurrencyLimit: concurrency.inline.perKey ?? concurrency.inline.total ?? null, combinedConcurrencyLimit: concurrency.inline.total ?? null, }, - anonymousName, + `task/${task.id}`, "NAMED", worker, environment, @@ -449,7 +449,7 @@ async function createWorkerTask( "LIMIT", "V2" ); - compiledGates = [{ queue: concurrencyLimitQueueName(anonymousName) }, ...compiledGates]; + compiledGates = [{ queue: anonymousQueueName }, ...compiledGates]; } } @@ -614,7 +614,7 @@ function validateWorkerConcurrencyDeclarations(metadata: BackgroundWorkerMetadat } for (const limit of metadata.concurrencyLimits ?? []) { - concurrencyLimitQueueName(limit.name); + assertValidConcurrencyLimitName(limit.name); } for (const task of metadata.tasks) { @@ -634,11 +634,10 @@ function validateWorkerConcurrencyDeclarations(metadata: BackgroundWorkerMetadat } for (const name of concurrency.limits ?? []) { - concurrencyLimitQueueName(name); + assertValidConcurrencyLimitName(name); } if (concurrency.inline && task.queue?.name) { - concurrencyLimitQueueName(`task/${task.id}`); if ((concurrency.limits ?? []).length > 1) { throw new ServiceValidationError( `Task "${task.id}": an inline limit on a shared queue uses a gate slot, so at most one named limit can be combined with it.` @@ -652,15 +651,39 @@ function validateWorkerConcurrencyDeclarations(metadata: BackgroundWorkerMetadat * they can never collide with a user's queue names. */ const CONCURRENCY_LIMIT_QUEUE_PREFIX = "limit/"; -function concurrencyLimitQueueName(limitName: string): string { - const sanitized = sanitizeQueueName(limitName); - const name = `${CONCURRENCY_LIMIT_QUEUE_PREFIX}${sanitized}`; - if (sanitized.length === 0 || name.length > 128) { +const CONCURRENCY_LIMIT_NAME_MAX_LENGTH = 128 - CONCURRENCY_LIMIT_QUEUE_PREFIX.length; + +/** + * Named limits require a strict charset so distinct declared names can never merge + * onto one row after queue-name sanitization (which strips disallowed characters). + */ +function assertValidConcurrencyLimitName(name: string): void { + if (!new RegExp(`^[a-zA-Z0-9_-]{1,${CONCURRENCY_LIMIT_NAME_MAX_LENGTH}}$`).test(name)) { throw new ServiceValidationError( - `Concurrency limit name "${limitName}" must sanitize to between 1 and ${128 - CONCURRENCY_LIMIT_QUEUE_PREFIX.length} characters.` + `Concurrency limit name "${name}" must be 1-${CONCURRENCY_LIMIT_NAME_MAX_LENGTH} characters using only letters, numbers, underscores and hyphens.` ); } - return name; +} + +function concurrencyLimitQueueName(limitName: string): string { + assertValidConcurrencyLimitName(limitName); + return `${CONCURRENCY_LIMIT_QUEUE_PREFIX}${limitName}`; +} + +/** + * Row name for a task's anonymous inline limit. Task ids are not charset-restricted, + * so when sanitization would be lossy (or the name would overflow the 128-char queue + * name limit) a hash of the raw id keeps distinct task ids on distinct rows. + */ +function anonymousConcurrencyLimitQueueName(taskId: string): string { + const sanitized = sanitizeQueueName(taskId); + const name = `${CONCURRENCY_LIMIT_QUEUE_PREFIX}task/${sanitized}`; + if (sanitized === taskId && name.length <= 128) { + return name; + } + const hash = createHash("sha256").update(taskId).digest("hex").slice(0, 8); + const budget = 128 - `${CONCURRENCY_LIMIT_QUEUE_PREFIX}task/`.length - hash.length - 1; + return `${CONCURRENCY_LIMIT_QUEUE_PREFIX}task/${sanitized.slice(0, budget)}-${hash}`; } /** User queue names may not claim the reserved limit/ namespace. */ @@ -675,8 +698,8 @@ function assertNotReservedQueueName(name: string, context: string): void { /** * Materializes the worker's declared named concurrency limits (plus any names tasks * reference without declaring, created uncapped) as LIMIT-role TaskQueue rows. A - * total-only limit stores the total as its per-key limit too, so it truly caps - * keyless runs as well as the keyed group. + * total-only limit stores the total as its per-key limit too, so no single key (or + * the keyless pool) can exceed it even before the group check applies. */ async function createWorkerConcurrencyLimits( metadata: BackgroundWorkerMetadata, From 87ee30a91858440dc18617c6f19c21ae867fac7b Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Sun, 6 Sep 2026 16:40:23 +0100 Subject: [PATCH 21/30] fix(webapp): queue override and reset APIs resolve queue rows only The queue concurrency override/reset endpoints no longer resolve the TaskQueue rows that back named concurrency limits, matching the list and retrieve surfaces. Limits get their own management API instead of being mutable through a surface that cannot read them back. --- apps/webapp/app/v3/services/concurrencySystem.server.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/apps/webapp/app/v3/services/concurrencySystem.server.ts b/apps/webapp/app/v3/services/concurrencySystem.server.ts index 09cc5f9ce11..099f22bdd69 100644 --- a/apps/webapp/app/v3/services/concurrencySystem.server.ts +++ b/apps/webapp/app/v3/services/concurrencySystem.server.ts @@ -190,6 +190,7 @@ function findQueueByFriendlyId( where: { runtimeEnvironmentId: environment.id, friendlyId, + role: "QUEUE", }, }), (error) => ({ @@ -214,6 +215,7 @@ function findQueueByName( where: { runtimeEnvironmentId: environment.id, name: queue, + role: "QUEUE", }, }), (error) => ({ From 1a43646e7fc908b73273920b5c271621fc11a239 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Sun, 6 Sep 2026 16:41:37 +0100 Subject: [PATCH 22/30] fix(webapp): trigger-time concurrency keeps the task's inline limit gate Passing concurrency at trigger time replaces only the task's declared named limits. The anonymous limit/task/ gate that carries an inline limit on a shared queue is preserved, and an empty array clears named limits without dropping it. Trigger-time limit names are also validated against the queue-safe charset so lossy sanitization can never remap them onto another limit's row. --- .../app/runEngine/concerns/queues.server.ts | 31 ++++++++++++++++--- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/apps/webapp/app/runEngine/concerns/queues.server.ts b/apps/webapp/app/runEngine/concerns/queues.server.ts index 6d09e7bc3f1..3924739b93e 100644 --- a/apps/webapp/app/runEngine/concerns/queues.server.ts +++ b/apps/webapp/app/runEngine/concerns/queues.server.ts @@ -221,11 +221,32 @@ export class DefaultQueueManager implements QueueManager { } const triggerLimits = request.body.options?.concurrency; - const concurrencyGates = triggerLimits?.map( - (name): { queue: string; concurrencyKey?: string } => ({ - queue: `limit/${sanitizeQueueName(name)}`, - }) + + for (const name of triggerLimits ?? []) { + if (!/^[a-zA-Z0-9_-]{1,122}$/.test(name)) { + throw new ServiceValidationError( + `Invalid concurrency limit name "${name}": names are 1-122 characters using only letters, numbers, underscores and hyphens.` + ); + } + } + + /** + * Trigger-time names replace the task's declared NAMED limits only. The task's + * inline limit rides in its stored gates as an anonymous "limit/task/" gate and + * always applies, so it is carried over into the replacement (an empty array + * clears the named limits but keeps the inline one). + */ + const inlineTaskGates = (taskGates ?? []).filter((gate) => + gate.queue.startsWith("limit/task/") ); + const concurrencyGates = triggerLimits + ? [ + ...inlineTaskGates, + ...triggerLimits.map((name): { queue: string; concurrencyKey?: string } => ({ + queue: `limit/${name}`, + })), + ] + : undefined; const requestedGates = concurrencyGates ?? request.body.options?.gates ?? taskGates ?? undefined; @@ -234,7 +255,7 @@ export class DefaultQueueManager implements QueueManager { const sanitized = sanitizeQueueName(gate.queue); return sanitized ? [{ queue: sanitized, concurrencyKey: gate.concurrencyKey }] : []; }) - .slice(0, 2); + .slice(0, 3); return { queueName, From 8336d5cd04daadb2a1586cd4b1fde96829cd8127 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Sun, 6 Sep 2026 16:46:59 +0100 Subject: [PATCH 23/30] fix(webapp): deploy upserts never clobber concurrent overrides, V2 only for inline default queues The queue upsert's update carries the override markers in its where clause, so a concurrent override or reset makes the update miss and retry against a fresh read instead of silently overwriting what the operator just set. A queue row is marked concurrencyVersion V2 only when an inline limit compiles into the task's own default queue; shared queues stay V1 whichever code path materializes them. --- .../services/createBackgroundWorker.server.ts | 26 ++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/apps/webapp/app/v3/services/createBackgroundWorker.server.ts b/apps/webapp/app/v3/services/createBackgroundWorker.server.ts index 8857213e152..c5f238951dd 100644 --- a/apps/webapp/app/v3/services/createBackgroundWorker.server.ts +++ b/apps/webapp/app/v3/services/createBackgroundWorker.server.ts @@ -469,7 +469,13 @@ async function createWorkerTask( environment, prisma, "QUEUE", - concurrency ? "V2" : "V1" + /** + * V2 marks rows whose limit fields hold the new perKey/total vocabulary. That + * only happens when an inline limit compiles into the task's own default queue; + * a shared queue keeps V1 regardless of task concurrency (which lives in gates + * and LIMIT rows), matching rows materialized from queue() declarations. + */ + concurrency?.inline && !task.queue?.name ? "V2" : "V1" ); } @@ -866,9 +872,17 @@ async function upsertWorkerQueueRecord( const hasOverride = taskQueue.concurrencyLimitOverriddenAt !== null; const hasTotalOverride = taskQueue.totalConcurrencyLimitOverriddenAt !== null; + /** + * The override markers in the where clause make this an optimistic-concurrency + * update: a concurrent override/reset between the read above and this write + * changes a marker, the update misses (P2025) and the catch below retries with + * a fresh read, so a deploy can never clobber an operator's override. + */ taskQueue = await prisma.taskQueue.update({ where: { id: taskQueue.id, + concurrencyLimitOverriddenAt: taskQueue.concurrencyLimitOverriddenAt, + totalConcurrencyLimitOverriddenAt: taskQueue.totalConcurrencyLimitOverriddenAt, }, data: { workers: { connect: { id: worker.id } }, @@ -886,8 +900,14 @@ async function upsertWorkerQueueRecord( return taskQueue; } catch (error) { - // If the queue already exists, let's try again - if (error instanceof Prisma.PrismaClientKnownRequestError && error.code === "P2002") { + /** + * P2002: the queue was created concurrently. P2025: an override/reset moved a + * marker under the optimistic update. Both re-read and retry. + */ + if ( + error instanceof Prisma.PrismaClientKnownRequestError && + (error.code === "P2002" || error.code === "P2025") + ) { return await upsertWorkerQueueRecord( queueName, concurrencyLimit, From 010742b5aa5f6e72c30aabddfdb2f45f1ddb777a Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Sun, 6 Sep 2026 17:03:49 +0100 Subject: [PATCH 24/30] fix(webapp): re-sync engine limits when an override lands during a deploy The optimistic markers only guard the Postgres write; an override or reset landing between that write and the engine sync would leave the engine holding the deploy's stale values. The deploy now re-reads the markers after syncing and re-syncs once from the fresh row when they moved, so the last engine write always carries the freshest limits. --- .../services/createBackgroundWorker.server.ts | 92 ++++++++++++------- 1 file changed, 58 insertions(+), 34 deletions(-) diff --git a/apps/webapp/app/v3/services/createBackgroundWorker.server.ts b/apps/webapp/app/v3/services/createBackgroundWorker.server.ts index c5f238951dd..0922fa6f0b8 100644 --- a/apps/webapp/app/v3/services/createBackgroundWorker.server.ts +++ b/apps/webapp/app/v3/services/createBackgroundWorker.server.ts @@ -771,53 +771,77 @@ async function createWorkerQueue( concurrencyVersion ); - const newConcurrencyLimit = taskQueue.concurrencyLimit; - - /** - * The total limit key is separate from the per-queue limit key that pause zeroes, - * so it is safe to sync it regardless of the paused state. The engine clamps it - * to the environment limit at read time, so the raw declared value is stored. - */ - if (typeof taskQueue.totalConcurrencyLimit === "number") { - await updateQueueTotalConcurrencyLimits( - environment, - taskQueue.name, - taskQueue.totalConcurrencyLimit - ); - } else { - await removeQueueTotalConcurrencyLimits(environment, taskQueue.name); - } + const syncQueueLimitsToEngine = async (row: { + name: string; + paused: boolean; + concurrencyLimit: number | null; + totalConcurrencyLimit: number | null; + }) => { + /** + * The total limit key is separate from the per-queue limit key that pause zeroes, + * so it is safe to sync it regardless of the paused state. The engine clamps it + * to the environment limit at read time, so the raw declared value is stored. + */ + if (typeof row.totalConcurrencyLimit === "number") { + await updateQueueTotalConcurrencyLimits(environment, row.name, row.totalConcurrencyLimit); + } else { + await removeQueueTotalConcurrencyLimits(environment, row.name); + } - if (!taskQueue.paused) { - if (typeof newConcurrencyLimit === "number") { - logger.debug("createWorkerQueue: updating concurrency limit", { + if (!row.paused) { + logger.debug("createWorkerQueue: syncing concurrency limit", { workerId: worker.id, - taskQueue, + taskQueue: row, orgId: environment.organizationId, projectId: environment.projectId, environmentId: environment.id, - concurrencyLimit: newConcurrencyLimit, + concurrencyLimit: row.concurrencyLimit, }); - await updateQueueConcurrencyLimits(environment, taskQueue.name, newConcurrencyLimit); + if (typeof row.concurrencyLimit === "number") { + await updateQueueConcurrencyLimits(environment, row.name, row.concurrencyLimit); + } else { + await removeQueueConcurrencyLimits(environment, row.name); + } } else { - logger.debug("createWorkerQueue: removing concurrency limit", { + logger.debug("createWorkerQueue: queue is paused, not updating concurrency limit", { workerId: worker.id, - taskQueue, + taskQueue: row, orgId: environment.organizationId, projectId: environment.projectId, environmentId: environment.id, - concurrencyLimit: newConcurrencyLimit, }); - await removeQueueConcurrencyLimits(environment, taskQueue.name); } - } else { - logger.debug("createWorkerQueue: queue is paused, not updating concurrency limit", { - workerId: worker.id, - taskQueue, - orgId: environment.organizationId, - projectId: environment.projectId, - environmentId: environment.id, - }); + }; + + await syncQueueLimitsToEngine(taskQueue); + + /** + * The optimistic markers only guard the Postgres write; an override or reset can + * still land between that write and the engine sync above, which would leave the + * engine holding this deploy's stale values. Re-read the markers and re-sync once + * from the fresh row when they moved: every actor writes Postgres before its own + * engine sync, so whoever syncs last is syncing the freshest row. + */ + const freshQueue = await prisma.taskQueue.findFirst({ + where: { id: taskQueue.id }, + select: { + name: true, + paused: true, + concurrencyLimit: true, + totalConcurrencyLimit: true, + concurrencyLimitOverriddenAt: true, + totalConcurrencyLimitOverriddenAt: true, + }, + }); + + if ( + freshQueue && + (freshQueue.concurrencyLimitOverriddenAt?.getTime() !== + taskQueue.concurrencyLimitOverriddenAt?.getTime() || + freshQueue.totalConcurrencyLimitOverriddenAt?.getTime() !== + taskQueue.totalConcurrencyLimitOverriddenAt?.getTime()) + ) { + await syncQueueLimitsToEngine(freshQueue); } return taskQueue; From 5722cfe7fb227cc47aae8005b650c627e49be9e3 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Sun, 6 Sep 2026 17:08:08 +0100 Subject: [PATCH 25/30] fix(webapp): converge the post-deploy engine re-sync when markers keep moving The single re-read left a window where a second override during the same deploy could be overwritten by a stale re-sync. The deploy now re-reads and re-syncs until the markers stop moving (bounded), and any marker moving after the final read is healed by that actor's own engine sync. --- .../services/createBackgroundWorker.server.ts | 55 ++++++++++++------- 1 file changed, 34 insertions(+), 21 deletions(-) diff --git a/apps/webapp/app/v3/services/createBackgroundWorker.server.ts b/apps/webapp/app/v3/services/createBackgroundWorker.server.ts index 0922fa6f0b8..a9812690773 100644 --- a/apps/webapp/app/v3/services/createBackgroundWorker.server.ts +++ b/apps/webapp/app/v3/services/createBackgroundWorker.server.ts @@ -818,30 +818,43 @@ async function createWorkerQueue( /** * The optimistic markers only guard the Postgres write; an override or reset can * still land between that write and the engine sync above, which would leave the - * engine holding this deploy's stale values. Re-read the markers and re-sync once - * from the fresh row when they moved: every actor writes Postgres before its own - * engine sync, so whoever syncs last is syncing the freshest row. + * engine holding this deploy's stale values. Re-read the markers and re-sync from + * the fresh row until they stop moving (bounded): every actor writes Postgres + * before its own engine sync, so re-syncing whatever is freshest converges. A + * marker moving after the final read is healed by that actor's own engine sync + * or the next deploy. */ - const freshQueue = await prisma.taskQueue.findFirst({ - where: { id: taskQueue.id }, - select: { - name: true, - paused: true, - concurrencyLimit: true, - totalConcurrencyLimit: true, - concurrencyLimitOverriddenAt: true, - totalConcurrencyLimitOverriddenAt: true, - }, - }); + let syncedMarkers = { + concurrency: taskQueue.concurrencyLimitOverriddenAt?.getTime(), + total: taskQueue.totalConcurrencyLimitOverriddenAt?.getTime(), + }; + + for (let i = 0; i < 3; i++) { + const freshQueue = await prisma.taskQueue.findFirst({ + where: { id: taskQueue.id }, + select: { + name: true, + paused: true, + concurrencyLimit: true, + totalConcurrencyLimit: true, + concurrencyLimitOverriddenAt: true, + totalConcurrencyLimitOverriddenAt: true, + }, + }); + + if ( + !freshQueue || + (freshQueue.concurrencyLimitOverriddenAt?.getTime() === syncedMarkers.concurrency && + freshQueue.totalConcurrencyLimitOverriddenAt?.getTime() === syncedMarkers.total) + ) { + break; + } - if ( - freshQueue && - (freshQueue.concurrencyLimitOverriddenAt?.getTime() !== - taskQueue.concurrencyLimitOverriddenAt?.getTime() || - freshQueue.totalConcurrencyLimitOverriddenAt?.getTime() !== - taskQueue.totalConcurrencyLimitOverriddenAt?.getTime()) - ) { await syncQueueLimitsToEngine(freshQueue); + syncedMarkers = { + concurrency: freshQueue.concurrencyLimitOverriddenAt?.getTime(), + total: freshQueue.totalConcurrencyLimitOverriddenAt?.getTime(), + }; } return taskQueue; From 6eb28d404be08ef2a3ff8c621f5d38dc36f95974 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Sun, 6 Sep 2026 17:56:35 +0100 Subject: [PATCH 26/30] fix(webapp): raw gate replacement keeps the inline gate, V4 deploys validate first The low-level gates option now carries the task's anonymous inline limit gate over the same way the concurrency option does (a replay resending stored gates collapses back through dedupe). The V4 deploy path rejects invalid concurrency declarations before any worker rows or engine keys are written, so a failed deploy can no longer leave the running version's limits mutated. --- .../app/runEngine/concerns/queues.server.ts | 23 +++++++++++++++++-- .../services/createBackgroundWorker.server.ts | 2 +- ...eateDeploymentBackgroundWorkerV4.server.ts | 18 +++++++++++++++ 3 files changed, 40 insertions(+), 3 deletions(-) diff --git a/apps/webapp/app/runEngine/concerns/queues.server.ts b/apps/webapp/app/runEngine/concerns/queues.server.ts index 3924739b93e..3d93d36399f 100644 --- a/apps/webapp/app/runEngine/concerns/queues.server.ts +++ b/apps/webapp/app/runEngine/concerns/queues.server.ts @@ -248,12 +248,31 @@ export class DefaultQueueManager implements QueueManager { ] : undefined; + /** + * The raw gates option replaces stored gates the same way concurrency does, so + * it also carries the inline gate over; a replay resending the stored gates + * collapses back to the original set through the dedupe below. + */ + const rawGates = request.body.options?.gates; const requestedGates = - concurrencyGates ?? request.body.options?.gates ?? taskGates ?? undefined; + concurrencyGates ?? + (rawGates ? [...inlineTaskGates, ...rawGates] : undefined) ?? + taskGates ?? + undefined; + + const seenGates = new Set(); const gates = requestedGates ?.flatMap((gate) => { const sanitized = sanitizeQueueName(gate.queue); - return sanitized ? [{ queue: sanitized, concurrencyKey: gate.concurrencyKey }] : []; + if (!sanitized) { + return []; + } + const dedupeKey = `${sanitized}${gate.concurrencyKey ?? ""}`; + if (seenGates.has(dedupeKey)) { + return []; + } + seenGates.add(dedupeKey); + return [{ queue: sanitized, concurrencyKey: gate.concurrencyKey }]; }) .slice(0, 3); diff --git a/apps/webapp/app/v3/services/createBackgroundWorker.server.ts b/apps/webapp/app/v3/services/createBackgroundWorker.server.ts index a9812690773..1ea03d35e40 100644 --- a/apps/webapp/app/v3/services/createBackgroundWorker.server.ts +++ b/apps/webapp/app/v3/services/createBackgroundWorker.server.ts @@ -614,7 +614,7 @@ async function createWorkerQueues( * Rejects invalid concurrency declarations before any worker rows are written, so a * failed deploy leaves nothing behind for a same-content retry to return. */ -function validateWorkerConcurrencyDeclarations(metadata: BackgroundWorkerMetadata): void { +export function validateWorkerConcurrencyDeclarations(metadata: BackgroundWorkerMetadata): void { for (const queue of metadata.queues ?? []) { assertNotReservedQueueName(queue.name, `Queue "${queue.name}"`); } diff --git a/apps/webapp/app/v3/services/createDeploymentBackgroundWorkerV4.server.ts b/apps/webapp/app/v3/services/createDeploymentBackgroundWorkerV4.server.ts index bf2c95745f3..f6100a00b7c 100644 --- a/apps/webapp/app/v3/services/createDeploymentBackgroundWorkerV4.server.ts +++ b/apps/webapp/app/v3/services/createDeploymentBackgroundWorkerV4.server.ts @@ -15,6 +15,7 @@ import { createWorkerResources, syncDeclarativeSchedules, syncDeclarativeWebhooks, + validateWorkerConcurrencyDeclarations, } from "./createBackgroundWorker.server"; import { findOrCreateBackgroundWorker } from "./createDeploymentBackgroundWorkerV4/findOrCreateBackgroundWorker.server"; import { TimeoutDeploymentService } from "./timeoutDeployment.server"; @@ -67,6 +68,23 @@ export class CreateDeploymentBackgroundWorkerServiceV4 extends BaseService { return; } + /** + * Reject invalid concurrency declarations before any worker rows exist. Queue + * rows and their engine limit keys are per-environment, so a mid-creation + * failure would leave the running version's limits already mutated. + */ + try { + validateWorkerConcurrencyDeclarations(body.metadata); + } catch (concurrencyError) { + if (concurrencyError instanceof ServiceValidationError) { + logger.warn("Invalid worker concurrency declarations", { + error: concurrencyError.message, + }); + await this.#failBackgroundWorkerDeployment(deployment, concurrencyError, environment); + } + throw concurrencyError; + } + // Handle multi-platform builds const deploymentPlatforms = deployment.imagePlatform?.split(",") ?? []; if (deploymentPlatforms.length > 1) { From 979b99c35e75cd2e883e383003c113df128ba8a6 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Sun, 6 Sep 2026 18:23:17 +0100 Subject: [PATCH 27/30] fix(webapp): reject gate requests that exceed the three-gate capacity Prepending the task's inline gate to a full raw gates array could push the set past three; the resolver now throws a clear error instead of silently dropping the last gate. Replays are unaffected: resent stored gates collapse through the dedupe before the check. --- .../app/runEngine/concerns/queues.server.ts | 32 +++++++++++-------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/apps/webapp/app/runEngine/concerns/queues.server.ts b/apps/webapp/app/runEngine/concerns/queues.server.ts index 3d93d36399f..b42f6c83d4e 100644 --- a/apps/webapp/app/runEngine/concerns/queues.server.ts +++ b/apps/webapp/app/runEngine/concerns/queues.server.ts @@ -261,20 +261,24 @@ export class DefaultQueueManager implements QueueManager { undefined; const seenGates = new Set(); - const gates = requestedGates - ?.flatMap((gate) => { - const sanitized = sanitizeQueueName(gate.queue); - if (!sanitized) { - return []; - } - const dedupeKey = `${sanitized}${gate.concurrencyKey ?? ""}`; - if (seenGates.has(dedupeKey)) { - return []; - } - seenGates.add(dedupeKey); - return [{ queue: sanitized, concurrencyKey: gate.concurrencyKey }]; - }) - .slice(0, 3); + const gates = requestedGates?.flatMap((gate) => { + const sanitized = sanitizeQueueName(gate.queue); + if (!sanitized) { + return []; + } + const dedupeKey = `${sanitized}${gate.concurrencyKey ?? ""}`; + if (seenGates.has(dedupeKey)) { + return []; + } + seenGates.add(dedupeKey); + return [{ queue: sanitized, concurrencyKey: gate.concurrencyKey }]; + }); + + if (gates && gates.length > 3) { + throw new ServiceValidationError( + `A run can hold at most three gates (the task's inline limit plus two named limits); this request resolves to ${gates.length}.` + ); + } return { queueName, From 2b2a798f96dfc5ca638843acc02938b29f110699 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Sun, 6 Sep 2026 20:10:43 +0100 Subject: [PATCH 28/30] fix(webapp): deploys re-assert pause, reserve limit/ in task gates, four-gate backstop A deploy syncing a paused queue writes the paused zero limit instead of skipping, and the post-deploy convergence loop also re-syncs when the paused flag moved, so a pause or resume landing mid-deploy can no longer leave the engine enforcing the wrong limit. Client-supplied task gates may not claim the reserved limit/ namespace. The gate overflow backstop moves to four to match the engine capacity. --- .../app/runEngine/concerns/queues.server.ts | 10 ++++++++-- .../services/createBackgroundWorker.server.ts | 18 ++++++++++++++++-- 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/apps/webapp/app/runEngine/concerns/queues.server.ts b/apps/webapp/app/runEngine/concerns/queues.server.ts index b42f6c83d4e..36229db3503 100644 --- a/apps/webapp/app/runEngine/concerns/queues.server.ts +++ b/apps/webapp/app/runEngine/concerns/queues.server.ts @@ -274,9 +274,15 @@ export class DefaultQueueManager implements QueueManager { return [{ queue: sanitized, concurrencyKey: gate.concurrencyKey }]; }); - if (gates && gates.length > 3) { + /** + * Unreachable through the public schemas (three requested gates plus one inline + * gate is the ceiling), kept as a backstop so an overflowing set can never be + * silently truncated downstream. Replays of three-gate runs against a task that + * later gained an inline limit resolve to four and stay valid. + */ + if (gates && gates.length > 4) { throw new ServiceValidationError( - `A run can hold at most three gates (the task's inline limit plus two named limits); this request resolves to ${gates.length}.` + `A run can hold at most four gates; this request resolves to ${gates.length}.` ); } diff --git a/apps/webapp/app/v3/services/createBackgroundWorker.server.ts b/apps/webapp/app/v3/services/createBackgroundWorker.server.ts index 1ea03d35e40..9712d8d06f1 100644 --- a/apps/webapp/app/v3/services/createBackgroundWorker.server.ts +++ b/apps/webapp/app/v3/services/createBackgroundWorker.server.ts @@ -628,6 +628,10 @@ export function validateWorkerConcurrencyDeclarations(metadata: BackgroundWorker assertNotReservedQueueName(task.queue.name, `Task "${task.id}"`); } + for (const gate of task.gates ?? []) { + assertNotReservedQueueName(gate.queue, `Task "${task.id}" gate "${gate.queue}"`); + } + const concurrency = task.concurrency; if (!concurrency) { continue; @@ -803,13 +807,20 @@ async function createWorkerQueue( await removeQueueConcurrencyLimits(environment, row.name); } } else { - logger.debug("createWorkerQueue: queue is paused, not updating concurrency limit", { + /** + * A paused queue's engine limit is 0 (what pause wrote). Re-asserting it here + * heals the race where a pause lands between this deploy's row read and its + * engine sync, which would otherwise overwrite the 0 with the declared limit + * and leave a queue the dashboard shows as paused still dequeuing. + */ + logger.debug("createWorkerQueue: queue is paused, re-asserting the paused limit", { workerId: worker.id, taskQueue: row, orgId: environment.organizationId, projectId: environment.projectId, environmentId: environment.id, }); + await updateQueueConcurrencyLimits(environment, row.name, 0); } }; @@ -827,6 +838,7 @@ async function createWorkerQueue( let syncedMarkers = { concurrency: taskQueue.concurrencyLimitOverriddenAt?.getTime(), total: taskQueue.totalConcurrencyLimitOverriddenAt?.getTime(), + paused: taskQueue.paused, }; for (let i = 0; i < 3; i++) { @@ -845,7 +857,8 @@ async function createWorkerQueue( if ( !freshQueue || (freshQueue.concurrencyLimitOverriddenAt?.getTime() === syncedMarkers.concurrency && - freshQueue.totalConcurrencyLimitOverriddenAt?.getTime() === syncedMarkers.total) + freshQueue.totalConcurrencyLimitOverriddenAt?.getTime() === syncedMarkers.total && + freshQueue.paused === syncedMarkers.paused) ) { break; } @@ -854,6 +867,7 @@ async function createWorkerQueue( syncedMarkers = { concurrency: freshQueue.concurrencyLimitOverriddenAt?.getTime(), total: freshQueue.totalConcurrencyLimitOverriddenAt?.getTime(), + paused: freshQueue.paused, }; } From bcee429cb88c6a505d342ea364802261d6041cb4 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Mon, 7 Sep 2026 01:44:34 +0100 Subject: [PATCH 29/30] fix(webapp): resume syncs the fresh row's limit and honors a zero limit Resume writes the engine limit from the row the update returned, so a concurrent deploy's change is not overwritten with a stale snapshot, and a declared limit of zero is written as zero instead of removing the limit and leaving the queue uncapped. --- apps/webapp/app/v3/services/pauseQueue.server.ts | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/apps/webapp/app/v3/services/pauseQueue.server.ts b/apps/webapp/app/v3/services/pauseQueue.server.ts index aa3e21f9727..87d2d339864 100644 --- a/apps/webapp/app/v3/services/pauseQueue.server.ts +++ b/apps/webapp/app/v3/services/pauseQueue.server.ts @@ -57,11 +57,20 @@ export class PauseQueueService extends BaseService { }, }); + /** + * Resume syncs from the row the update returned, never the pre-update + * snapshot, so a limit changed by a concurrent deploy is not resurrected. + * A declared limit of zero is a real limit and must be written, not removed. + */ if (action === "paused") { await updateQueueConcurrencyLimits(environment, queue.name, 0); } else { - if (queue.concurrencyLimit) { - await updateQueueConcurrencyLimits(environment, queue.name, queue.concurrencyLimit); + if (typeof updatedQueue.concurrencyLimit === "number") { + await updateQueueConcurrencyLimits( + environment, + queue.name, + updatedQueue.concurrencyLimit + ); } else { await removeQueueConcurrencyLimits(environment, queue.name); } From 235e628049f01f681be32c1c4c8c0af2d3b63718 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Mon, 7 Sep 2026 18:25:30 +0100 Subject: [PATCH 30/30] chore: drop the combined concurrency changeset, the surface never ships --- .changeset/queue-combined-concurrency-stats.md | 5 ----- 1 file changed, 5 deletions(-) delete mode 100644 .changeset/queue-combined-concurrency-stats.md diff --git a/.changeset/queue-combined-concurrency-stats.md b/.changeset/queue-combined-concurrency-stats.md deleted file mode 100644 index 533aab34a08..00000000000 --- a/.changeset/queue-combined-concurrency-stats.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -"@trigger.dev/core": patch ---- - -Queue retrieve and list API responses now report combined concurrency usage. When a queue has a `combinedConcurrencyLimit`, `concurrency.combined` includes the effective cap, the declared base, any active override, and how many runs are in flight across all concurrency keys.