From 365bf200743c6df92a0c4f7fc83e244455f9fdac Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Sat, 29 Aug 2026 13:27:01 +0100 Subject: [PATCH 01/28] feat(webapp,run-engine,core,clickhouse): surface total concurrency in metrics and dashboard Queues with a totalConcurrencyLimit now report how they use it. The gauge pipeline emits total running and the stored cap, ClickHouse aggregates them into the queue metrics tiers, the queues list gets a Total column, the queue detail page charts total running against the cap, and the per-key table shows each key's effective limit including per-key overrides. Queue retrieve and list API responses include the same totals. --- .changeset/queue-total-concurrency-stats.md | 5 + .../v3/QueueListPresenter.server.ts | 25 +++- .../v3/QueueRetrievePresenter.server.ts | 20 ++++ .../route.tsx | 32 ++++- .../route.tsx | 69 ++++++++++- ...ueueParam.concurrency.combined.override.ts | 3 + ....$queueParam.concurrency.combined.reset.ts | 3 + ...queues.$queueParam.concurrency.override.ts | 3 + ...v1.queues.$queueParam.concurrency.reset.ts | 3 + .../resources.queues.concurrency-keys.ts | 10 +- apps/webapp/app/v3/querySchemas.ts | 24 ++++ apps/webapp/app/v3/queueMetricsMapping.ts | 2 + ...42_add_queue_metrics_total_concurrency.sql | 109 ++++++++++++++++++ .../clickhouse/src/queueMetrics.ts | 2 + internal-packages/metrics-pipeline/src/lua.ts | 18 ++- .../run-engine/src/engine/index.ts | 14 +++ .../run-engine/src/run-queue/index.ts | 49 +++++++- packages/core/src/v3/schemas/queues.ts | 15 +++ 18 files changed, 394 insertions(+), 12 deletions(-) create mode 100644 .changeset/queue-total-concurrency-stats.md create mode 100644 internal-packages/clickhouse/schema/042_add_queue_metrics_total_concurrency.sql diff --git a/.changeset/queue-total-concurrency-stats.md b/.changeset/queue-total-concurrency-stats.md new file mode 100644 index 00000000000..a70da24d1fb --- /dev/null +++ b/.changeset/queue-total-concurrency-stats.md @@ -0,0 +1,5 @@ +--- +"@trigger.dev/core": patch +--- + +Queue retrieve and list API responses now report total concurrency usage. When a queue has a `totalConcurrencyLimit`, `concurrency.total` includes the effective cap, the declared base, any active override, and how many runs are in flight across all concurrency keys. diff --git a/apps/webapp/app/presenters/v3/QueueListPresenter.server.ts b/apps/webapp/app/presenters/v3/QueueListPresenter.server.ts index 0dc3daa9856..d78e98ade4c 100644 --- a/apps/webapp/app/presenters/v3/QueueListPresenter.server.ts +++ b/apps/webapp/app/presenters/v3/QueueListPresenter.server.ts @@ -9,7 +9,10 @@ import { engine } from "~/v3/runEngine.server"; import { BasePresenter } from "./basePresenter.server"; import { toQueueItem } from "./QueueRetrievePresenter.server"; -type QueueListEngine = Pick; +type QueueListEngine = Pick< + RunEngine, + "lengthOfQueues" | "currentConcurrencyOfQueues" | "totalConcurrencyOfQueues" +>; export const QUEUE_LIST_DEFAULT_ITEMS_PER_PAGE = 25; const MAX_ITEMS_PER_PAGE = 100; @@ -34,6 +37,9 @@ const queueListSelect = { concurrencyLimitOverriddenAt: true, concurrencyLimitOverriddenBy: true, concurrencyLimitOverridePercent: true, + totalConcurrencyLimit: true, + totalConcurrencyLimitBase: true, + totalConcurrencyLimitOverriddenAt: true, type: true, paused: true, } satisfies Prisma.TaskQueueSelect; @@ -333,11 +339,15 @@ export class QueueListPresenter extends BasePresenter { concurrencyLimitOverriddenAt: Date | null; concurrencyLimitOverriddenBy: string | null; concurrencyLimitOverridePercent: Prisma.Decimal | null; + totalConcurrencyLimit: number | null; + totalConcurrencyLimitBase: number | null; + totalConcurrencyLimitOverriddenAt: Date | null; type: TaskQueueType; paused: boolean; }[] ): Promise { - const [queuedByQueue, runningByQueue] = await Promise.all([ + const queuesWithTotalCap = queues.filter((q) => q.totalConcurrencyLimit !== null); + const [queuedByQueue, runningByQueue, totalRunningByQueue] = await Promise.all([ this.engineClient.lengthOfQueues( environment, queues.map((q) => q.name) @@ -346,6 +356,12 @@ export class QueueListPresenter extends BasePresenter { environment, queues.map((q) => q.name) ), + queuesWithTotalCap.length > 0 + ? this.engineClient.totalConcurrencyOfQueues( + environment, + queuesWithTotalCap.map((q) => q.name) + ) + : Promise.resolve({} as Record), ]); // Manually "join" the overridden users because there is no way to implement the relationship @@ -373,6 +389,11 @@ export class QueueListPresenter extends BasePresenter { ? (overriddenByMap.get(queue.concurrencyLimitOverriddenBy) ?? null) : null, paused: queue.paused, + totalConcurrencyLimit: queue.totalConcurrencyLimit, + totalConcurrencyLimitBase: queue.totalConcurrencyLimitBase, + totalConcurrencyLimitOverriddenAt: queue.totalConcurrencyLimitOverriddenAt, + totalRunning: + queue.totalConcurrencyLimit !== null ? (totalRunningByQueue[queue.name] ?? 0) : null, }), // Prisma returns Decimal; the client only needs a plain number (null for absolute overrides). concurrencyLimitOverridePercent: diff --git a/apps/webapp/app/presenters/v3/QueueRetrievePresenter.server.ts b/apps/webapp/app/presenters/v3/QueueRetrievePresenter.server.ts index f6918394e5c..e4777ceb13e 100644 --- a/apps/webapp/app/presenters/v3/QueueRetrievePresenter.server.ts +++ b/apps/webapp/app/presenters/v3/QueueRetrievePresenter.server.ts @@ -90,6 +90,7 @@ export class QueueRetrievePresenter extends BasePresenter { const results = await Promise.all([ engine.lengthOfQueues(environment, [queue.name]), engine.currentConcurrencyOfQueues(environment, [queue.name]), + engine.totalConcurrencyOfQueues(environment, [queue.name]), ]); // Transform queues to include running and queued counts @@ -107,6 +108,11 @@ export class QueueRetrievePresenter extends BasePresenter { concurrencyLimitOverriddenAt: queue.concurrencyLimitOverriddenAt ?? null, concurrencyLimitOverriddenBy: queue.concurrencyLimitOverriddenBy ?? null, paused: queue.paused, + totalConcurrencyLimit: queue.totalConcurrencyLimit ?? null, + totalConcurrencyLimitBase: queue.totalConcurrencyLimitBase ?? null, + totalConcurrencyLimitOverriddenAt: queue.totalConcurrencyLimitOverriddenAt ?? null, + totalRunning: + queue.totalConcurrencyLimit != null ? (results[2]?.[queue.name] ?? 0) : null, }), // The percent source-of-truth for percent-based overrides isn't part of the shared // `QueueItem` schema (that's a public contract), so we surface it as an extra field on @@ -148,6 +154,10 @@ export function toQueueItem(data: { concurrencyLimitOverriddenAt: Date | null; concurrencyLimitOverriddenBy: User | null; paused: boolean; + totalConcurrencyLimit?: number | null; + totalConcurrencyLimitBase?: number | null; + totalConcurrencyLimitOverriddenAt?: Date | null; + totalRunning?: number | null; }): QueueItem & { releaseConcurrencyOnWaitpoint: boolean } { return { id: data.friendlyId, @@ -164,6 +174,16 @@ export function toQueueItem(data: { override: data.concurrencyLimitOverriddenAt ? data.concurrencyLimit : null, overriddenBy: toQueueConcurrencyOverriddenBy(data.concurrencyLimitOverriddenBy), overriddenAt: data.concurrencyLimitOverriddenAt, + total: + data.totalConcurrencyLimit !== undefined + ? { + current: data.totalConcurrencyLimit, + base: data.totalConcurrencyLimitBase ?? null, + override: data.totalConcurrencyLimitOverriddenAt ? data.totalConcurrencyLimit : null, + overriddenAt: data.totalConcurrencyLimitOverriddenAt ?? null, + running: data.totalRunning ?? null, + } + : undefined, }, // TODO: This needs to be removed but keeping this here for now to avoid breaking existing clients releaseConcurrencyOnWaitpoint: true, diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx index 05ac0e7b47d..a7c97b80718 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx @@ -707,6 +707,13 @@ function QueuesWithMetricsView() { Queued Running Limit + + Total + + = + Math.min( + queue.concurrency.total.current, + environment.concurrencyLimit + ) && + "text-warning" + )} + > + {queue.concurrency?.total?.current != null + ? `${queue.concurrency.total.running ?? 0}/${Math.min( + queue.concurrency.total.current, + environment.concurrencyLimit + )}` + : "–"} + - +
{hasFilters ? "No queues found matching your filters" : "No queues found"} diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues_.$queueParam/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues_.$queueParam/route.tsx index 8f8a91dca4f..22279ffcaba 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues_.$queueParam/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues_.$queueParam/route.tsx @@ -392,7 +392,12 @@ export default function Page() { ) ) : ( - + )} @@ -402,7 +407,13 @@ export default function Page() { {view === "keys" && hasKeys ? ( <> - + {selectedKey ? ( @@ -436,10 +447,12 @@ function OverviewCharts({ ids, timeRange, queueName, + hasTotalLimit, }: { ids: Ids; timeRange: TimeRangeParams; queueName: string; + hasTotalLimit: boolean; }) { const zoomToTimeFilter = useZoomToTimeFilter(); return ( @@ -479,6 +492,37 @@ function OverviewCharts({ // leading zeros so the reference line doesn't start with a false 0→limit step. carryBackfill={["limit"]} /> + {hasTotalLimit ? ( + + Runs in flight across ALL concurrency keys ( + ) versus the queue's total limit ( + + ). + + } + showLegend + className="aspect-[2/1]" + query={`SELECT timeBucket() AS t, max(max_total_running) AS running, least(max(max_total_limit), max(max_env_limit)) AS cap\nFROM queue_metrics\nGROUP BY t\nORDER BY t`} + fillGaps + minBucketSeconds={SYNCED_CHART_MIN_BUCKET_SECONDS} + ids={ids} + timeRange={timeRange} + queueName={queueName} + series={[ + { key: "cap", label: "Total limit", color: COLORS.limit }, + { key: "running", label: "Running", color: COLORS.running }, + ]} + thresholdStroke={{ + series: "running", + valueFromSeries: "cap", + aboveColor: "var(--color-warning)", + }} + carryBackfill={["cap"]} + /> + ) : null} Key Queued now Running now + + Limit + Oldest wait Started Peak backlog @@ -976,11 +1031,11 @@ function KeyStatsTable({ {showLoading ? ( - + Loading… ) : rows.length === 0 ? ( - + {search ? `No keys match “${search}”` : "No concurrency keys"} ) : ( @@ -994,6 +1049,12 @@ function KeyStatsTable({ {row.key} {row.queued.toLocaleString()} {row.running.toLocaleString()} + + {Math.min(row.limitOverride ?? defaultKeyLimit, envLimit).toLocaleString()} + {row.oldestWaitMs === null ? "–" : formatWaitMs(row.oldestWaitMs)} diff --git a/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.combined.override.ts b/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.combined.override.ts index c643b77965a..77688a9fcc5 100644 --- a/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.combined.override.ts +++ b/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.combined.override.ts @@ -46,6 +46,9 @@ const route = createActionApiRoute( concurrencyLimitOverriddenAt: queue.concurrencyLimitOverriddenAt, concurrencyLimitOverriddenBy: null, paused: queue.paused, + totalConcurrencyLimit: queue.totalConcurrencyLimit, + totalConcurrencyLimitBase: queue.totalConcurrencyLimitBase, + totalConcurrencyLimitOverriddenAt: queue.totalConcurrencyLimitOverriddenAt, }), { status: 200 } ); diff --git a/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.combined.reset.ts b/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.combined.reset.ts index b2841f1efe6..0e588716658 100644 --- a/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.combined.reset.ts +++ b/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.combined.reset.ts @@ -45,6 +45,9 @@ const route = createActionApiRoute( concurrencyLimitOverriddenAt: queue.concurrencyLimitOverriddenAt, concurrencyLimitOverriddenBy: null, paused: queue.paused, + totalConcurrencyLimit: queue.totalConcurrencyLimit, + totalConcurrencyLimitBase: queue.totalConcurrencyLimitBase, + totalConcurrencyLimitOverriddenAt: queue.totalConcurrencyLimitOverriddenAt, }), { status: 200 } ); diff --git a/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.override.ts b/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.override.ts index 90f5772c5d3..42bb2008682 100644 --- a/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.override.ts +++ b/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.override.ts @@ -61,6 +61,9 @@ const route = createActionApiRoute( concurrencyLimitOverriddenAt: queue.concurrencyLimitOverriddenAt, concurrencyLimitOverriddenBy: null, paused: queue.paused, + totalConcurrencyLimit: queue.totalConcurrencyLimit, + totalConcurrencyLimitBase: queue.totalConcurrencyLimitBase, + totalConcurrencyLimitOverriddenAt: queue.totalConcurrencyLimitOverriddenAt, }), { status: 200 } ); diff --git a/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.reset.ts b/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.reset.ts index 503d875e471..3f36e629f09 100644 --- a/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.reset.ts +++ b/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.reset.ts @@ -43,6 +43,9 @@ const route = createActionApiRoute( concurrencyLimitOverriddenAt: queue.concurrencyLimitOverriddenAt, concurrencyLimitOverriddenBy: null, paused: queue.paused, + totalConcurrencyLimit: queue.totalConcurrencyLimit, + totalConcurrencyLimitBase: queue.totalConcurrencyLimitBase, + totalConcurrencyLimitOverriddenAt: queue.totalConcurrencyLimitOverriddenAt, }), { status: 200 } ); diff --git a/apps/webapp/app/routes/resources.queues.concurrency-keys.ts b/apps/webapp/app/routes/resources.queues.concurrency-keys.ts index 67c2b9f500a..662013694ef 100644 --- a/apps/webapp/app/routes/resources.queues.concurrency-keys.ts +++ b/apps/webapp/app/routes/resources.queues.concurrency-keys.ts @@ -43,6 +43,8 @@ export type ConcurrencyKeyRow = { peakBacklog: number; peakRunning: number; meanWaitMs: number; + /** Per-key concurrency limit override, when one is set for this key (null = inherits the queue limit). */ + limitOverride: number | null; }; export type ConcurrencyKeysResponse = @@ -151,8 +153,11 @@ export const action = async ({ request }: ActionFunctionArgs) => { const total = rankingRows?.[0]?.ranked_total ?? 0; const keys = (rankingRows ?? []).map((r) => r.concurrency_key); - // Enrich just this page's keys with live "now" counts from Redis. - const live = await engine.concurrencyKeyLiveStats(environment, queueName, keys); + // Enrich just this page's keys with live "now" counts and any per-key limit overrides from Redis. + const [live, keyLimitOverrides] = await Promise.all([ + engine.concurrencyKeyLiveStats(environment, queueName, keys), + engine.runQueue.getQueueConcurrencyKeyLimits(environment, queueName), + ]); const loadedAt = Date.now(); const rows: ConcurrencyKeyRow[] = (rankingRows ?? []).map((r) => { @@ -168,6 +173,7 @@ export const action = async ({ request }: ActionFunctionArgs) => { peakBacklog: r.peak_backlog, peakRunning: r.peak_running, meanWaitMs: r.mean_wait_ms, + limitOverride: keyLimitOverrides[r.concurrency_key] ?? null, }; }); diff --git a/apps/webapp/app/v3/querySchemas.ts b/apps/webapp/app/v3/querySchemas.ts index 690bbaf5396..8267a0f8020 100644 --- a/apps/webapp/app/v3/querySchemas.ts +++ b/apps/webapp/app/v3/querySchemas.ts @@ -770,6 +770,22 @@ const queueMetricsSchema: TableSchema = { fillMode: "carry", }), }, + max_total_running: { + name: "max_total_running", + ...column("UInt32", { + description: + "Peak in-flight runs across ALL concurrency keys of the queue in the bucket (only emitted for keyed queues). Aggregate with max().", + fillMode: "carry", + }), + }, + max_total_limit: { + name: "max_total_limit", + ...column("UInt32", { + description: + "The queue's total concurrency limit across all keys, as stored (0 = no cap; clamp against max_env_limit). Aggregate with max().", + fillMode: "carry", + }), + }, max_ck_backlogged: { name: "max_ck_backlogged", ...column("UInt32", { @@ -1406,6 +1422,14 @@ const queueMetricsByKeySchema: TableSchema = { fillMode: "carry", }), }, + max_limit: { + name: "max_limit", + ...column("UInt32", { + description: + "The effective concurrency limit for this key (the queue limit, or its per-key override). Aggregate with max().", + fillMode: "carry", + }), + }, wait_ms_sum: { name: "wait_ms_sum", ...column("UInt64", { diff --git a/apps/webapp/app/v3/queueMetricsMapping.ts b/apps/webapp/app/v3/queueMetricsMapping.ts index 9433b361a88..d341f4a63cd 100644 --- a/apps/webapp/app/v3/queueMetricsMapping.ts +++ b/apps/webapp/app/v3/queueMetricsMapping.ts @@ -131,6 +131,8 @@ export function mapEntryToRows( throttled: num(f.thr), ck_backlogged: num(f.ckq), ck_max_wait_ms: num(f.ckw), + total_running: num(f.tcc), + total_limit: num(f.tlim), }, ]; } diff --git a/internal-packages/clickhouse/schema/042_add_queue_metrics_total_concurrency.sql b/internal-packages/clickhouse/schema/042_add_queue_metrics_total_concurrency.sql new file mode 100644 index 00000000000..7711effa6e3 --- /dev/null +++ b/internal-packages/clickhouse/schema/042_add_queue_metrics_total_concurrency.sql @@ -0,0 +1,109 @@ +-- +goose Up + +-- Total-concurrency gauges: total_running is the in-flight count across ALL +-- concurrency-key variants of a queue (the groupConcurrency set), total_limit the +-- RAW stored total cap (0 = none; readers clamp against max_env_limit). Emitted on +-- base-queue gauge rows only. Per-key gauge rows now carry the EFFECTIVE per-key +-- limit in queue_limit (override-aware), surfaced in the ck tier as max_limit. + +ALTER TABLE trigger_dev.queue_metrics_raw_v1 + ADD COLUMN IF NOT EXISTS total_running UInt32 DEFAULT 0, + ADD COLUMN IF NOT EXISTS total_limit UInt32 DEFAULT 0; + +ALTER TABLE trigger_dev.queue_metrics_v1 + ADD COLUMN IF NOT EXISTS max_total_running SimpleAggregateFunction(max, UInt32), + ADD COLUMN IF NOT EXISTS max_total_limit SimpleAggregateFunction(max, UInt32); + +ALTER TABLE trigger_dev.queue_metrics_5m_v1 + ADD COLUMN IF NOT EXISTS max_total_running SimpleAggregateFunction(max, UInt32), + ADD COLUMN IF NOT EXISTS max_total_limit SimpleAggregateFunction(max, UInt32); + +ALTER TABLE trigger_dev.queue_metrics_ck_v1 + ADD COLUMN IF NOT EXISTS max_limit SimpleAggregateFunction(max, UInt32); + +-- Materialized views cannot be altered: recreate them with the new columns. The 5m +-- MV MUST keep reading raw, never cascade off queue_metrics_v1 (out-of-time-order +-- deltaSumTimestamp merges double-count bridging spans). + +DROP VIEW IF EXISTS trigger_dev.queue_metrics_mv_v1; +CREATE MATERIALIZED VIEW IF NOT EXISTS trigger_dev.queue_metrics_mv_v1 +TO trigger_dev.queue_metrics_v1 AS +SELECT + organization_id, project_id, environment_id, queue_name, + toStartOfInterval(event_time, INTERVAL 10 SECOND) AS bucket_start, + deltaSumTimestampStateIf(cumulative, order_key, op = 'enqueue' AND concurrency_key = '') AS enqueue_delta, + deltaSumTimestampStateIf(cumulative, order_key, op = 'started' AND concurrency_key = '') AS started_delta, + deltaSumTimestampStateIf(cumulative, order_key, op = 'ack' AND concurrency_key = '') AS ack_delta, + deltaSumTimestampStateIf(cumulative, order_key, op = 'nack' AND concurrency_key = '') AS nack_delta, + deltaSumTimestampStateIf(cumulative, order_key, op = 'dlq' AND concurrency_key = '') AS dlq_delta, + sum(throttled) AS throttled_count, + max(queued) AS max_queued, + max(running) AS max_running, + max(queue_limit) AS max_limit, + max(env_queued) AS max_env_queued, + max(env_running) AS max_env_running, + max(env_limit) AS max_env_limit, + max(ck_backlogged) AS max_ck_backlogged, + max(ck_max_wait_ms) AS max_ck_wait_ms, + max(total_running) AS max_total_running, + max(total_limit) AS max_total_limit, + sumIf(wait_ms, op = 'started' AND concurrency_key = '') AS wait_ms_sum, + countIf(op = 'started' AND wait_ms > 0 AND concurrency_key = '') AS wait_ms_count, + quantilesStateIf(0.5, 0.9, 0.95, 0.99)(wait_ms, op = 'started' AND wait_ms > 0 AND concurrency_key = '') AS wait_quantiles +FROM trigger_dev.queue_metrics_raw_v1 +GROUP BY organization_id, project_id, environment_id, queue_name, bucket_start; + +DROP VIEW IF EXISTS trigger_dev.queue_metrics_5m_mv_v1; +CREATE MATERIALIZED VIEW IF NOT EXISTS trigger_dev.queue_metrics_5m_mv_v1 +TO trigger_dev.queue_metrics_5m_v1 AS +SELECT + organization_id, project_id, environment_id, queue_name, + toStartOfInterval(event_time, INTERVAL 5 MINUTE) AS bucket_start, + deltaSumTimestampStateIf(cumulative, order_key, op = 'enqueue' AND concurrency_key = '') AS enqueue_delta, + deltaSumTimestampStateIf(cumulative, order_key, op = 'started' AND concurrency_key = '') AS started_delta, + deltaSumTimestampStateIf(cumulative, order_key, op = 'ack' AND concurrency_key = '') AS ack_delta, + deltaSumTimestampStateIf(cumulative, order_key, op = 'nack' AND concurrency_key = '') AS nack_delta, + deltaSumTimestampStateIf(cumulative, order_key, op = 'dlq' AND concurrency_key = '') AS dlq_delta, + sum(throttled) AS throttled_count, + max(queued) AS max_queued, + max(running) AS max_running, + max(queue_limit) AS max_limit, + max(env_queued) AS max_env_queued, + max(env_running) AS max_env_running, + max(env_limit) AS max_env_limit, + max(ck_backlogged) AS max_ck_backlogged, + max(ck_max_wait_ms) AS max_ck_wait_ms, + max(total_running) AS max_total_running, + max(total_limit) AS max_total_limit, + sumIf(wait_ms, op = 'started' AND concurrency_key = '') AS wait_ms_sum, + countIf(op = 'started' AND wait_ms > 0 AND concurrency_key = '') AS wait_ms_count, + quantilesStateIf(0.5, 0.9, 0.95, 0.99)(wait_ms, op = 'started' AND wait_ms > 0 AND concurrency_key = '') AS wait_quantiles +FROM trigger_dev.queue_metrics_raw_v1 +GROUP BY organization_id, project_id, environment_id, queue_name, bucket_start; + +DROP VIEW IF EXISTS trigger_dev.queue_metrics_ck_mv_v1; +CREATE MATERIALIZED VIEW IF NOT EXISTS trigger_dev.queue_metrics_ck_mv_v1 +TO trigger_dev.queue_metrics_ck_v1 AS +SELECT + organization_id, project_id, environment_id, queue_name, concurrency_key, + toStartOfInterval(event_time, INTERVAL 10 SECOND) AS bucket_start, + deltaSumTimestampStateIf(cumulative, order_key, op = 'enqueue') AS enqueue_delta, + deltaSumTimestampStateIf(cumulative, order_key, op = 'started') AS started_delta, + deltaSumTimestampStateIf(cumulative, order_key, op = 'ack') AS ack_delta, + maxIf(queued, op = 'gauge') AS max_queued, + maxIf(running, op = 'gauge') AS max_running, + maxIf(queue_limit, op = 'gauge') AS max_limit, + sumIf(wait_ms, op = 'started') AS wait_ms_sum, + countIf(op = 'started' AND wait_ms > 0) AS wait_ms_count +FROM trigger_dev.queue_metrics_raw_v1 +WHERE concurrency_key != '' +GROUP BY organization_id, project_id, environment_id, queue_name, concurrency_key, bucket_start; + +-- +goose Down +DROP VIEW IF EXISTS trigger_dev.queue_metrics_ck_mv_v1; +DROP VIEW IF EXISTS trigger_dev.queue_metrics_5m_mv_v1; +DROP VIEW IF EXISTS trigger_dev.queue_metrics_mv_v1; +ALTER TABLE trigger_dev.queue_metrics_ck_v1 DROP COLUMN IF EXISTS max_limit; +ALTER TABLE trigger_dev.queue_metrics_5m_v1 DROP COLUMN IF EXISTS max_total_running, DROP COLUMN IF EXISTS max_total_limit; +ALTER TABLE trigger_dev.queue_metrics_v1 DROP COLUMN IF EXISTS max_total_running, DROP COLUMN IF EXISTS max_total_limit; +ALTER TABLE trigger_dev.queue_metrics_raw_v1 DROP COLUMN IF EXISTS total_running, DROP COLUMN IF EXISTS total_limit; diff --git a/internal-packages/clickhouse/src/queueMetrics.ts b/internal-packages/clickhouse/src/queueMetrics.ts index 39576b4a0a3..aa3cf5296d2 100644 --- a/internal-packages/clickhouse/src/queueMetrics.ts +++ b/internal-packages/clickhouse/src/queueMetrics.ts @@ -21,6 +21,8 @@ export const QueueMetricsRawV1Input = z.object({ throttled: z.number().optional(), ck_backlogged: z.number().optional(), ck_max_wait_ms: z.number().optional(), + total_running: z.number().optional(), + total_limit: z.number().optional(), wait_ms: z.number().optional(), cumulative: z.number().optional(), }); diff --git a/internal-packages/metrics-pipeline/src/lua.ts b/internal-packages/metrics-pipeline/src/lua.ts index 64f3b896c0d..701f608308a 100644 --- a/internal-packages/metrics-pipeline/src/lua.ts +++ b/internal-packages/metrics-pipeline/src/lua.ts @@ -17,6 +17,10 @@ export type GaugeComputeLuaParams = { // CK-health extras (both or neither): appended as an optional gauge tail, gauge[8]/gauge[9]. ckBacklogged?: string; ckMaxWaitMs?: string; + // Total-concurrency extras (both or neither, and only with the CK extras): appended as + // gauge[10]/gauge[11]. totalLimit is the RAW stored limit (0 = none); readers clamp. + totalRunning?: string; + totalLimit?: string; }; // Computes an op=gauge snapshot into the enclosing script's `__qm_g` local (a flat @@ -26,11 +30,21 @@ export type GaugeComputeLuaParams = { export function createMetricsGaugeComputeLua(params: GaugeComputeLuaParams): string { const throttled = params.throttledExpr ?? "__cc >= __lim and __ql > 0"; const hasCk = params.ckBacklogged != null && params.ckMaxWaitMs != null; - const gauge = hasCk + const hasTotal = params.totalRunning != null && params.totalLimit != null; + if (hasTotal && !hasCk) { + throw new Error("gauge totalRunning/totalLimit extras require the CK extras"); + } + const gauge = hasTotal ? ` local __ckq = tonumber(${params.ckBacklogged}) or 0 local __ckw = tonumber(${params.ckMaxWaitMs}) or 0 + local __tcc = tonumber(${params.totalRunning}) or 0 + local __tlim = tonumber(${params.totalLimit}) or 0 + __qm_g = {__ql, __cc, __lim, __eql, __ec, __elim, __thr, __ckq, __ckw, __tcc, __tlim}` + : hasCk + ? ` local __ckq = tonumber(${params.ckBacklogged}) or 0 + local __ckw = tonumber(${params.ckMaxWaitMs}) or 0 __qm_g = {__ql, __cc, __lim, __eql, __ec, __elim, __thr, __ckq, __ckw}` - : ` __qm_g = {__ql, __cc, __lim, __eql, __ec, __elim, __thr}`; + : ` __qm_g = {__ql, __cc, __lim, __eql, __ec, __elim, __thr}`; return ` if ${params.enabledArg} then diff --git a/internal-packages/run-engine/src/engine/index.ts b/internal-packages/run-engine/src/engine/index.ts index 9d7bb8ff947..504673bf052 100644 --- a/internal-packages/run-engine/src/engine/index.ts +++ b/internal-packages/run-engine/src/engine/index.ts @@ -1740,6 +1740,20 @@ export class RunEngine { return this.runQueue.currentConcurrencyOfQueues(environment, queues); } + async totalConcurrencyOfQueues( + environment: MinimalAuthenticatedEnvironment, + queues: string[] + ): Promise> { + return this.runQueue.totalConcurrencyOfQueues(environment, queues); + } + + async totalConcurrencyLimitsOfQueues( + environment: MinimalAuthenticatedEnvironment, + queues: string[] + ): Promise> { + return this.runQueue.totalConcurrencyLimitsOfQueues(environment, queues); + } + async concurrencyKeyBreakdown( environment: MinimalAuthenticatedEnvironment, queue: string, diff --git a/internal-packages/run-engine/src/run-queue/index.ts b/internal-packages/run-engine/src/run-queue/index.ts index 7917f3a48be..5349de09bce 100644 --- a/internal-packages/run-engine/src/run-queue/index.ts +++ b/internal-packages/run-engine/src/run-queue/index.ts @@ -219,7 +219,8 @@ const QUEUE_METRICS_CK_ENQUEUE_GAUGE_LUA = createMetricsGaugeComputeLua({ enabledArg: "ARGV[#ARGV] == '1'", queued: "redis.call('ZCARD', queueKey)", running: "redis.call('SCARD', queueCurrentConcurrencyKey)", - queueLimit: "redis.call('GET', queueConcurrencyLimitKey) or '1000000'", + queueLimit: + "redis.call('HGET', ckLimitsKey, queueName) or redis.call('GET', queueConcurrencyLimitKey) or '1000000'", envQueued: "redis.call('ZCARD', envQueueKey)", envRunning: "redis.call('SCARD', envCurrentConcurrencyKey)", envLimit: "redis.call('GET', envConcurrencyLimitKey) or defaultEnvConcurrencyLimit", @@ -250,6 +251,8 @@ const QUEUE_METRICS_CK_DEQUEUE_GAUGE_LUA = createMetricsGaugeComputeLua({ envLimit: "redis.call('GET', envConcurrencyLimitKey) or defaultEnvConcurrencyLimit", throttledExpr: "false", ...QUEUE_METRICS_CK_GAUGE_EXTRAS, + totalRunning: "redis.call('SCARD', groupConcurrencyKey)", + totalLimit: "redis.call('GET', totalConcurrencyLimitKey) or '0'", }); /** Injected queue-metrics stream emitter; all calls are no-ops when metrics are disabled. */ @@ -710,6 +713,46 @@ export class RunQueue { return limits; } + /** Batch variant of totalConcurrencyOfQueue: one pipeline of group SCARDs. */ + public async totalConcurrencyOfQueues( + env: MinimalAuthenticatedEnvironment, + queues: string[] + ): Promise> { + const pipeline = this.redis.pipeline(); + queues.forEach((queue) => { + pipeline.scard(this.keys.queueGroupConcurrencyKey(env, queue)); + }); + + const results = await pipeline.exec(); + + return queues.reduce( + (acc, queue, index) => { + const value = results?.[index]?.[1]; + acc[queue] = typeof value === "number" ? value : 0; + return acc; + }, + {} as Record + ); + } + + /** Batch read of the RAW stored total concurrency limits (undefined = no cap). */ + public async totalConcurrencyLimitsOfQueues( + env: MinimalAuthenticatedEnvironment, + queues: string[] + ): Promise> { + const keys = queues.map((queue) => this.keys.queueTotalConcurrencyLimitKey(env, queue)); + const values = keys.length > 0 ? await this.redis.mget(...keys) : []; + + return queues.reduce( + (acc, queue, index) => { + const value = values[index]; + acc[queue] = value != null ? Number(value) : undefined; + return acc; + }, + {} as Record + ); + } + public async updateEnvConcurrencyLimits(env: MinimalAuthenticatedEnvironment) { await this.#callUpdateEnvironmentConcurrencyLimits({ envConcurrencyLimitKey: this.keys.envConcurrencyLimitKey(env), @@ -2341,6 +2384,10 @@ export class RunQueue { fields.ckq = ckq; fields.ckw = ckw; } + if (gauge.length >= 11) { + fields.tcc = gauge[9]; + fields.tlim = gauge[10]; + } this.options.queueMetrics?.emitGauge(queue, fields); } diff --git a/packages/core/src/v3/schemas/queues.ts b/packages/core/src/v3/schemas/queues.ts index 34a47b34e3e..f3039f8075a 100644 --- a/packages/core/src/v3/schemas/queues.ts +++ b/packages/core/src/v3/schemas/queues.ts @@ -45,6 +45,21 @@ export const QueueItem = z.object({ overriddenAt: z.coerce.date().nullable(), /** Who overrode the concurrency limit (will be null if overridden via the API) */ overriddenBy: z.string().nullable(), + /** The total concurrency cap across all concurrencyKey values of the queue */ + total: z + .object({ + /** The effective/current total concurrency limit (null = no cap) */ + current: z.number().nullable(), + /** The declared total limit an override reverts to on reset */ + base: z.number().nullable(), + /** The overridden total limit, when an override is active */ + override: z.number().nullable(), + /** When the total override was applied */ + overriddenAt: z.coerce.date().nullable(), + /** Runs currently in flight across all concurrencyKey values */ + running: z.number().nullable(), + }) + .optional(), }) .optional(), }); From cc6ed7f135baaa148bc4be4b24f30c229dce7d63 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Sat, 29 Aug 2026 13:38:22 +0100 Subject: [PATCH 02/28] fix(run-engine,clickhouse,webapp): total gauges on enqueue paths; restore views on rollback The CK enqueue gauges (fast path and queued path) now sample total running and the stored cap, so metric buckets fed only by enqueues no longer record zero totals. The migration's down section recreates the pre-existing materialized view definitions so ingestion keeps flowing after a rollback. The per-key table reads only the page's overrides with one HMGET instead of loading the queue's whole override hash. --- .../resources.queues.concurrency-keys.ts | 2 +- ...42_add_queue_metrics_total_concurrency.sql | 68 +++++++++++++++++++ .../run-engine/src/run-queue/index.ts | 36 +++++++++- 3 files changed, 103 insertions(+), 3 deletions(-) diff --git a/apps/webapp/app/routes/resources.queues.concurrency-keys.ts b/apps/webapp/app/routes/resources.queues.concurrency-keys.ts index 662013694ef..8c590554e51 100644 --- a/apps/webapp/app/routes/resources.queues.concurrency-keys.ts +++ b/apps/webapp/app/routes/resources.queues.concurrency-keys.ts @@ -156,7 +156,7 @@ export const action = async ({ request }: ActionFunctionArgs) => { // Enrich just this page's keys with live "now" counts and any per-key limit overrides from Redis. const [live, keyLimitOverrides] = await Promise.all([ engine.concurrencyKeyLiveStats(environment, queueName, keys), - engine.runQueue.getQueueConcurrencyKeyLimits(environment, queueName), + engine.runQueue.getQueueConcurrencyKeyLimitsForKeys(environment, queueName, keys), ]); const loadedAt = Date.now(); diff --git a/internal-packages/clickhouse/schema/042_add_queue_metrics_total_concurrency.sql b/internal-packages/clickhouse/schema/042_add_queue_metrics_total_concurrency.sql index 7711effa6e3..8bb19faeef1 100644 --- a/internal-packages/clickhouse/schema/042_add_queue_metrics_total_concurrency.sql +++ b/internal-packages/clickhouse/schema/042_add_queue_metrics_total_concurrency.sql @@ -107,3 +107,71 @@ ALTER TABLE trigger_dev.queue_metrics_ck_v1 DROP COLUMN IF EXISTS max_limit; ALTER TABLE trigger_dev.queue_metrics_5m_v1 DROP COLUMN IF EXISTS max_total_running, DROP COLUMN IF EXISTS max_total_limit; ALTER TABLE trigger_dev.queue_metrics_v1 DROP COLUMN IF EXISTS max_total_running, DROP COLUMN IF EXISTS max_total_limit; ALTER TABLE trigger_dev.queue_metrics_raw_v1 DROP COLUMN IF EXISTS total_running, DROP COLUMN IF EXISTS total_limit; + +-- Recreate the pre-042 materialized views (the definitions from 036) so ingestion keeps +-- feeding every aggregate table after a rollback. +CREATE MATERIALIZED VIEW IF NOT EXISTS trigger_dev.queue_metrics_mv_v1 +TO trigger_dev.queue_metrics_v1 AS +SELECT + organization_id, project_id, environment_id, queue_name, + toStartOfInterval(event_time, INTERVAL 10 SECOND) AS bucket_start, + deltaSumTimestampStateIf(cumulative, order_key, op = 'enqueue' AND concurrency_key = '') AS enqueue_delta, + deltaSumTimestampStateIf(cumulative, order_key, op = 'started' AND concurrency_key = '') AS started_delta, + deltaSumTimestampStateIf(cumulative, order_key, op = 'ack' AND concurrency_key = '') AS ack_delta, + deltaSumTimestampStateIf(cumulative, order_key, op = 'nack' AND concurrency_key = '') AS nack_delta, + deltaSumTimestampStateIf(cumulative, order_key, op = 'dlq' AND concurrency_key = '') AS dlq_delta, + sum(throttled) AS throttled_count, + max(queued) AS max_queued, + max(running) AS max_running, + max(queue_limit) AS max_limit, + max(env_queued) AS max_env_queued, + max(env_running) AS max_env_running, + max(env_limit) AS max_env_limit, + max(ck_backlogged) AS max_ck_backlogged, + max(ck_max_wait_ms) AS max_ck_wait_ms, + sumIf(wait_ms, op = 'started' AND concurrency_key = '') AS wait_ms_sum, + countIf(op = 'started' AND wait_ms > 0 AND concurrency_key = '') AS wait_ms_count, + quantilesStateIf(0.5, 0.9, 0.95, 0.99)(wait_ms, op = 'started' AND wait_ms > 0 AND concurrency_key = '') AS wait_quantiles +FROM trigger_dev.queue_metrics_raw_v1 +GROUP BY organization_id, project_id, environment_id, queue_name, bucket_start; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trigger_dev.queue_metrics_5m_mv_v1 +TO trigger_dev.queue_metrics_5m_v1 AS +SELECT + organization_id, project_id, environment_id, queue_name, + toStartOfInterval(event_time, INTERVAL 5 MINUTE) AS bucket_start, + deltaSumTimestampStateIf(cumulative, order_key, op = 'enqueue' AND concurrency_key = '') AS enqueue_delta, + deltaSumTimestampStateIf(cumulative, order_key, op = 'started' AND concurrency_key = '') AS started_delta, + deltaSumTimestampStateIf(cumulative, order_key, op = 'ack' AND concurrency_key = '') AS ack_delta, + deltaSumTimestampStateIf(cumulative, order_key, op = 'nack' AND concurrency_key = '') AS nack_delta, + deltaSumTimestampStateIf(cumulative, order_key, op = 'dlq' AND concurrency_key = '') AS dlq_delta, + sum(throttled) AS throttled_count, + max(queued) AS max_queued, + max(running) AS max_running, + max(queue_limit) AS max_limit, + max(env_queued) AS max_env_queued, + max(env_running) AS max_env_running, + max(env_limit) AS max_env_limit, + max(ck_backlogged) AS max_ck_backlogged, + max(ck_max_wait_ms) AS max_ck_wait_ms, + sumIf(wait_ms, op = 'started' AND concurrency_key = '') AS wait_ms_sum, + countIf(op = 'started' AND wait_ms > 0 AND concurrency_key = '') AS wait_ms_count, + quantilesStateIf(0.5, 0.9, 0.95, 0.99)(wait_ms, op = 'started' AND wait_ms > 0 AND concurrency_key = '') AS wait_quantiles +FROM trigger_dev.queue_metrics_raw_v1 +GROUP BY organization_id, project_id, environment_id, queue_name, bucket_start; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trigger_dev.queue_metrics_ck_mv_v1 +TO trigger_dev.queue_metrics_ck_v1 AS +SELECT + organization_id, project_id, environment_id, queue_name, concurrency_key, + toStartOfInterval(event_time, INTERVAL 10 SECOND) AS bucket_start, + deltaSumTimestampStateIf(cumulative, order_key, op = 'enqueue') AS enqueue_delta, + deltaSumTimestampStateIf(cumulative, order_key, op = 'started') AS started_delta, + deltaSumTimestampStateIf(cumulative, order_key, op = 'ack') AS ack_delta, + maxIf(queued, op = 'gauge') AS max_queued, + maxIf(running, op = 'gauge') AS max_running, + sumIf(wait_ms, op = 'started') AS wait_ms_sum, + countIf(op = 'started' AND wait_ms > 0) AS wait_ms_count +FROM trigger_dev.queue_metrics_raw_v1 +WHERE concurrency_key != '' +GROUP BY organization_id, project_id, environment_id, queue_name, concurrency_key, bucket_start; diff --git a/internal-packages/run-engine/src/run-queue/index.ts b/internal-packages/run-engine/src/run-queue/index.ts index 5349de09bce..e4dc84e1ddc 100644 --- a/internal-packages/run-engine/src/run-queue/index.ts +++ b/internal-packages/run-engine/src/run-queue/index.ts @@ -214,6 +214,14 @@ const QUEUE_METRICS_CK_GAUGE_EXTRAS = { ckMaxWaitMs: "__ckwait", }; +// Total-concurrency tail (gauge[10]/gauge[11]): live group cardinality + raw stored cap. +// Requires groupConcurrencyKey/totalConcurrencyLimitKey locals; the CK scripts that actually +// run (the Tracked variants and the CK dequeue) all declare them for the total-cap gate. +const QUEUE_METRICS_TOTAL_GAUGE_EXTRAS = { + totalRunning: "redis.call('SCARD', groupConcurrencyKey)", + totalLimit: "redis.call('GET', totalConcurrencyLimitKey) or '0'", +}; + // CK enqueue variants of the two gauges above, extended with the CK-health tail. const QUEUE_METRICS_CK_ENQUEUE_GAUGE_LUA = createMetricsGaugeComputeLua({ enabledArg: "ARGV[#ARGV] == '1'", @@ -225,6 +233,7 @@ const QUEUE_METRICS_CK_ENQUEUE_GAUGE_LUA = createMetricsGaugeComputeLua({ envRunning: "redis.call('SCARD', envCurrentConcurrencyKey)", envLimit: "redis.call('GET', envConcurrencyLimitKey) or defaultEnvConcurrencyLimit", ...QUEUE_METRICS_CK_GAUGE_EXTRAS, + ...QUEUE_METRICS_TOTAL_GAUGE_EXTRAS, }); const QUEUE_METRICS_CK_ENQUEUE_FASTPATH_GAUGE_LUA = createMetricsGaugeComputeLua({ @@ -236,6 +245,7 @@ const QUEUE_METRICS_CK_ENQUEUE_FASTPATH_GAUGE_LUA = createMetricsGaugeComputeLua envRunning: "envCurrent", envLimit: "envLimit", ...QUEUE_METRICS_CK_GAUGE_EXTRAS, + ...QUEUE_METRICS_TOTAL_GAUGE_EXTRAS, }); // CK dequeue: depth/running from the per-base-queue aggregate counters the run-queue already @@ -251,8 +261,7 @@ const QUEUE_METRICS_CK_DEQUEUE_GAUGE_LUA = createMetricsGaugeComputeLua({ envLimit: "redis.call('GET', envConcurrencyLimitKey) or defaultEnvConcurrencyLimit", throttledExpr: "false", ...QUEUE_METRICS_CK_GAUGE_EXTRAS, - totalRunning: "redis.call('SCARD', groupConcurrencyKey)", - totalLimit: "redis.call('GET', totalConcurrencyLimitKey) or '0'", + ...QUEUE_METRICS_TOTAL_GAUGE_EXTRAS, }); /** Injected queue-metrics stream emitter; all calls are no-ops when metrics are disabled. */ @@ -713,6 +722,29 @@ export class RunQueue { return limits; } + /** Per-key limit overrides for just the given keys: one HMGET, O(keys) not O(overrides). */ + public async getQueueConcurrencyKeyLimitsForKeys( + env: MinimalAuthenticatedEnvironment, + queue: string, + concurrencyKeys: string[] + ): Promise> { + if (concurrencyKeys.length === 0) { + return {}; + } + + const fields = concurrencyKeys.map((key) => this.keys.queueKey(env, queue, key)); + const values = await this.redis.hmget(this.keys.queueCkLimitsKey(env, queue), ...fields); + + const limits: Record = {}; + concurrencyKeys.forEach((key, index) => { + const value = values[index]; + if (value != null) { + limits[key] = Number(value); + } + }); + return limits; + } + /** Batch variant of totalConcurrencyOfQueue: one pipeline of group SCARDs. */ public async totalConcurrencyOfQueues( env: MinimalAuthenticatedEnvironment, From 47314a985de6f890551fdc9c1880ca4046711f2d Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Sat, 29 Aug 2026 16:48:06 +0100 Subject: [PATCH 03/28] fix(clickhouse): keep migration comments semicolon-free The test harness splits a migration's up section on semicolons, so a semicolon inside a comment yields a comment-only statement that ClickHouse rejects as an empty query and every container-backed suite fails at setup. --- .../schema/042_add_queue_metrics_total_concurrency.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal-packages/clickhouse/schema/042_add_queue_metrics_total_concurrency.sql b/internal-packages/clickhouse/schema/042_add_queue_metrics_total_concurrency.sql index 8bb19faeef1..f45e6421ae3 100644 --- a/internal-packages/clickhouse/schema/042_add_queue_metrics_total_concurrency.sql +++ b/internal-packages/clickhouse/schema/042_add_queue_metrics_total_concurrency.sql @@ -2,7 +2,7 @@ -- Total-concurrency gauges: total_running is the in-flight count across ALL -- concurrency-key variants of a queue (the groupConcurrency set), total_limit the --- RAW stored total cap (0 = none; readers clamp against max_env_limit). Emitted on +-- RAW stored total cap (0 = none, readers clamp against max_env_limit). Emitted on -- base-queue gauge rows only. Per-key gauge rows now carry the EFFECTIVE per-key -- limit in queue_limit (override-aware), surfaced in the ck tier as max_limit. From 4ad7e5c77357f18fcd9c5473664e76ce3122ccc3 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Sat, 29 Aug 2026 17:01:59 +0100 Subject: [PATCH 04/28] fix(webapp): skip the total concurrency read when the queue has no cap Queue retrieve only asks the engine for total running when a total limit is set, matching the list presenter and avoiding a pointless read for the common uncapped case. --- .../webapp/app/presenters/v3/QueueRetrievePresenter.server.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/apps/webapp/app/presenters/v3/QueueRetrievePresenter.server.ts b/apps/webapp/app/presenters/v3/QueueRetrievePresenter.server.ts index e4777ceb13e..26811a8800e 100644 --- a/apps/webapp/app/presenters/v3/QueueRetrievePresenter.server.ts +++ b/apps/webapp/app/presenters/v3/QueueRetrievePresenter.server.ts @@ -90,7 +90,9 @@ export class QueueRetrievePresenter extends BasePresenter { const results = await Promise.all([ engine.lengthOfQueues(environment, [queue.name]), engine.currentConcurrencyOfQueues(environment, [queue.name]), - engine.totalConcurrencyOfQueues(environment, [queue.name]), + queue.totalConcurrencyLimit != null + ? engine.totalConcurrencyOfQueues(environment, [queue.name]) + : undefined, ]); // Transform queues to include running and queued counts From d90a9cfa933f64dd740af07a2c39948c93283012 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Sat, 29 Aug 2026 18:21:57 +0100 Subject: [PATCH 05/28] feat(webapp): show the Total column in the non-metrics queues table too The total concurrency numbers come from live Redis, not the metrics pipeline, so the column belongs in both tables rather than only behind the queue metrics UI gate. --- .../route.tsx | 29 ++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx index a7c97b80718..c06e47682aa 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx @@ -1815,6 +1815,12 @@ function ClassicQueuesView() { Queued Running Limit + + Total + {limit} + = + Math.min( + queue.concurrency.total.current, + environment.concurrencyLimit + ) && + "text-warning" + )} + > + {queue.concurrency?.total?.current != null + ? `${queue.concurrency.total.running ?? 0}/${Math.min( + queue.concurrency.total.current, + environment.concurrencyLimit + )}` + : "–"} + - +
{hasFilters ? "No queues found matching your filters" : "No queues found"} From c4866bbad9f3f992002248705f8a3ac04019579f Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Sat, 29 Aug 2026 18:33:02 +0100 Subject: [PATCH 06/28] feat(webapp): fold the total cap into the Limit column A separate Total column implied every queue should have one, and its dash read as a missing limit on queues that never use concurrency keys. Only queues that declare a totalConcurrencyLimit now change: their Limit cell reads as per-key plus total (e.g. 1 /key, 3 total) and Running turns warning-colored when the total cap is saturated. Plain queues are unchanged. --- .../route.tsx | 88 ++++++++----------- 1 file changed, 38 insertions(+), 50 deletions(-) diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx index c06e47682aa..b5249c91452 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx @@ -706,13 +706,12 @@ function QueuesWithMetricsView() { Name Queued Running - Limit - Total + Limit 0 && "text-text-bright" + queue.concurrency?.total?.current != null && + queue.running >= + Math.min( + queue.concurrency.total.current, + environment.concurrencyLimit + ) + ? "text-warning" + : queue.running > 0 && "text-text-bright" )} > {queue.running} @@ -881,29 +887,16 @@ function QueuesWithMetricsView() { ) : ( limit )} - - = - Math.min( - queue.concurrency.total.current, - environment.concurrencyLimit - ) && - "text-warning" - )} - > - {queue.concurrency?.total?.current != null - ? `${queue.concurrency.total.running ?? 0}/${Math.min( + {queue.concurrency?.total?.current != null ? ( + + /key ·{" "} + {Math.min( queue.concurrency.total.current, environment.concurrencyLimit - )}` - : "–"} + )}{" "} + total + + ) : null} - +
{hasFilters ? "No queues found matching your filters" : "No queues found"} @@ -1814,12 +1807,11 @@ function ClassicQueuesView() { Name Queued Running - Limit - Total + Limit 0 && "text-text-bright", + queue.concurrency?.total?.current != null && + queue.running >= + Math.min( + queue.concurrency.total.current, + environment.concurrencyLimit + ) + ? "text-warning" + : queue.running > 0 && "text-text-bright", isAtConcurrencyLimit && "text-warning" )} > @@ -1942,27 +1941,16 @@ function ClassicQueuesView() { )} > {limit} - - = - Math.min( - queue.concurrency.total.current, - environment.concurrencyLimit - ) && - "text-warning" - )} - > - {queue.concurrency?.total?.current != null - ? `${queue.concurrency.total.running ?? 0}/${Math.min( + {queue.concurrency?.total?.current != null ? ( + + /key ·{" "} + {Math.min( queue.concurrency.total.current, environment.concurrencyLimit - )}` - : "–"} + )}{" "} + total + + ) : null} - +
{hasFilters ? "No queues found matching your filters" : "No queues found"} From 518f3d6920e1fee510d557e25c70ed7d63550da0 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Sat, 29 Aug 2026 18:39:14 +0100 Subject: [PATCH 07/28] fix(webapp): saturate the total-cap warning on keyed runs only The total cap gates keyed admissions, so the Running cell now warns off the group count rather than the aggregate that also includes unkeyed runs. --- .../route.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx index b5249c91452..e60e6276607 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx @@ -856,7 +856,7 @@ function QueuesWithMetricsView() { "w-[1%]", queue.paused ? "opacity-50" : undefined, queue.concurrency?.total?.current != null && - queue.running >= + (queue.concurrency.total.running ?? 0) >= Math.min( queue.concurrency.total.current, environment.concurrencyLimit @@ -1920,7 +1920,7 @@ function ClassicQueuesView() { "w-[1%] pl-16 tabular-nums", queue.paused ? "opacity-50" : undefined, queue.concurrency?.total?.current != null && - queue.running >= + (queue.concurrency.total.running ?? 0) >= Math.min( queue.concurrency.total.current, environment.concurrencyLimit From badaadbc0333ba8c84fc43b84e81f22689e6007e Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Sat, 29 Aug 2026 19:11:06 +0100 Subject: [PATCH 08/28] refactor(webapp,core,clickhouse): combined concurrency in responses, dashboard and metrics Queue API responses expose concurrency.combined, the dashboard says combined, and the new metrics columns are named combined_running and combined_limit. --- .../v3/QueueRetrievePresenter.server.ts | 2 +- .../route.tsx | 28 ++++++++--------- .../route.tsx | 10 +++---- apps/webapp/app/v3/querySchemas.ts | 10 +++---- apps/webapp/app/v3/queueMetricsMapping.ts | 4 +-- ...dd_queue_metrics_combined_concurrency.sql} | 30 +++++++++---------- .../clickhouse/src/queueMetrics.ts | 4 +-- packages/core/src/v3/schemas/queues.ts | 12 ++++---- 8 files changed, 50 insertions(+), 50 deletions(-) rename internal-packages/clickhouse/schema/{042_add_queue_metrics_total_concurrency.sql => 042_add_queue_metrics_combined_concurrency.sql} (91%) diff --git a/apps/webapp/app/presenters/v3/QueueRetrievePresenter.server.ts b/apps/webapp/app/presenters/v3/QueueRetrievePresenter.server.ts index 26811a8800e..1ce08e4c628 100644 --- a/apps/webapp/app/presenters/v3/QueueRetrievePresenter.server.ts +++ b/apps/webapp/app/presenters/v3/QueueRetrievePresenter.server.ts @@ -176,7 +176,7 @@ export function toQueueItem(data: { override: data.concurrencyLimitOverriddenAt ? data.concurrencyLimit : null, overriddenBy: toQueueConcurrencyOverriddenBy(data.concurrencyLimitOverriddenBy), overriddenAt: data.concurrencyLimitOverriddenAt, - total: + combined: data.totalConcurrencyLimit !== undefined ? { current: data.totalConcurrencyLimit, diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx index e60e6276607..5ea114e5da5 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx @@ -709,7 +709,7 @@ function QueuesWithMetricsView() { Limit @@ -855,10 +855,10 @@ function QueuesWithMetricsView() { className={cn( "w-[1%]", queue.paused ? "opacity-50" : undefined, - queue.concurrency?.total?.current != null && - (queue.concurrency.total.running ?? 0) >= + queue.concurrency?.combined?.current != null && + (queue.concurrency.combined.running ?? 0) >= Math.min( - queue.concurrency.total.current, + queue.concurrency.combined.current, environment.concurrencyLimit ) ? "text-warning" @@ -887,14 +887,14 @@ function QueuesWithMetricsView() { ) : ( limit )} - {queue.concurrency?.total?.current != null ? ( + {queue.concurrency?.combined?.current != null ? ( /key ·{" "} {Math.min( - queue.concurrency.total.current, + queue.concurrency.combined.current, environment.concurrencyLimit )}{" "} - total + combined ) : null} @@ -1809,7 +1809,7 @@ function ClassicQueuesView() { Running Limit @@ -1919,10 +1919,10 @@ function ClassicQueuesView() { className={cn( "w-[1%] pl-16 tabular-nums", queue.paused ? "opacity-50" : undefined, - queue.concurrency?.total?.current != null && - (queue.concurrency.total.running ?? 0) >= + queue.concurrency?.combined?.current != null && + (queue.concurrency.combined.running ?? 0) >= Math.min( - queue.concurrency.total.current, + queue.concurrency.combined.current, environment.concurrencyLimit ) ? "text-warning" @@ -1941,14 +1941,14 @@ function ClassicQueuesView() { )} > {limit} - {queue.concurrency?.total?.current != null ? ( + {queue.concurrency?.combined?.current != null ? ( /key ·{" "} {Math.min( - queue.concurrency.total.current, + queue.concurrency.combined.current, environment.concurrencyLimit )}{" "} - total + combined ) : null} diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues_.$queueParam/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues_.$queueParam/route.tsx index 22279ffcaba..7b3bddec64a 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues_.$queueParam/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues_.$queueParam/route.tsx @@ -396,7 +396,7 @@ export default function Page() { ids={ids} timeRange={timeRange} queueName={fullName} - hasTotalLimit={queue.concurrency?.total?.current != null} + hasTotalLimit={queue.concurrency?.combined?.current != null} /> )} @@ -494,25 +494,25 @@ function OverviewCharts({ /> {hasTotalLimit ? ( Runs in flight across ALL concurrency keys ( - ) versus the queue's total limit ( + ) versus the queue's combined limit ( ). } showLegend className="aspect-[2/1]" - query={`SELECT timeBucket() AS t, max(max_total_running) AS running, least(max(max_total_limit), max(max_env_limit)) AS cap\nFROM queue_metrics\nGROUP BY t\nORDER BY t`} + query={`SELECT timeBucket() AS t, max(max_combined_running) AS running, least(max(max_combined_limit), max(max_env_limit)) AS cap\nFROM queue_metrics\nGROUP BY t\nORDER BY t`} fillGaps minBucketSeconds={SYNCED_CHART_MIN_BUCKET_SECONDS} ids={ids} timeRange={timeRange} queueName={queueName} series={[ - { key: "cap", label: "Total limit", color: COLORS.limit }, + { key: "cap", label: "Combined limit", color: COLORS.limit }, { key: "running", label: "Running", color: COLORS.running }, ]} thresholdStroke={{ diff --git a/apps/webapp/app/v3/querySchemas.ts b/apps/webapp/app/v3/querySchemas.ts index 8267a0f8020..05cd7f0b394 100644 --- a/apps/webapp/app/v3/querySchemas.ts +++ b/apps/webapp/app/v3/querySchemas.ts @@ -770,19 +770,19 @@ const queueMetricsSchema: TableSchema = { fillMode: "carry", }), }, - max_total_running: { - name: "max_total_running", + max_combined_running: { + name: "max_combined_running", ...column("UInt32", { description: "Peak in-flight runs across ALL concurrency keys of the queue in the bucket (only emitted for keyed queues). Aggregate with max().", fillMode: "carry", }), }, - max_total_limit: { - name: "max_total_limit", + max_combined_limit: { + name: "max_combined_limit", ...column("UInt32", { description: - "The queue's total concurrency limit across all keys, as stored (0 = no cap; clamp against max_env_limit). Aggregate with max().", + "The queue's combined concurrency limit across all keys, as stored (0 = no cap; clamp against max_env_limit). Aggregate with max().", fillMode: "carry", }), }, diff --git a/apps/webapp/app/v3/queueMetricsMapping.ts b/apps/webapp/app/v3/queueMetricsMapping.ts index d341f4a63cd..f093dc3f027 100644 --- a/apps/webapp/app/v3/queueMetricsMapping.ts +++ b/apps/webapp/app/v3/queueMetricsMapping.ts @@ -131,8 +131,8 @@ export function mapEntryToRows( throttled: num(f.thr), ck_backlogged: num(f.ckq), ck_max_wait_ms: num(f.ckw), - total_running: num(f.tcc), - total_limit: num(f.tlim), + combined_running: num(f.tcc), + combined_limit: num(f.tlim), }, ]; } diff --git a/internal-packages/clickhouse/schema/042_add_queue_metrics_total_concurrency.sql b/internal-packages/clickhouse/schema/042_add_queue_metrics_combined_concurrency.sql similarity index 91% rename from internal-packages/clickhouse/schema/042_add_queue_metrics_total_concurrency.sql rename to internal-packages/clickhouse/schema/042_add_queue_metrics_combined_concurrency.sql index f45e6421ae3..03cb133799a 100644 --- a/internal-packages/clickhouse/schema/042_add_queue_metrics_total_concurrency.sql +++ b/internal-packages/clickhouse/schema/042_add_queue_metrics_combined_concurrency.sql @@ -1,22 +1,22 @@ -- +goose Up --- Total-concurrency gauges: total_running is the in-flight count across ALL --- concurrency-key variants of a queue (the groupConcurrency set), total_limit the +-- Total-concurrency gauges: combined_running is the in-flight count across ALL +-- concurrency-key variants of a queue (the groupConcurrency set), combined_limit the -- RAW stored total cap (0 = none, readers clamp against max_env_limit). Emitted on -- base-queue gauge rows only. Per-key gauge rows now carry the EFFECTIVE per-key -- limit in queue_limit (override-aware), surfaced in the ck tier as max_limit. ALTER TABLE trigger_dev.queue_metrics_raw_v1 - ADD COLUMN IF NOT EXISTS total_running UInt32 DEFAULT 0, - ADD COLUMN IF NOT EXISTS total_limit UInt32 DEFAULT 0; + ADD COLUMN IF NOT EXISTS combined_running UInt32 DEFAULT 0, + ADD COLUMN IF NOT EXISTS combined_limit UInt32 DEFAULT 0; ALTER TABLE trigger_dev.queue_metrics_v1 - ADD COLUMN IF NOT EXISTS max_total_running SimpleAggregateFunction(max, UInt32), - ADD COLUMN IF NOT EXISTS max_total_limit SimpleAggregateFunction(max, UInt32); + ADD COLUMN IF NOT EXISTS max_combined_running SimpleAggregateFunction(max, UInt32), + ADD COLUMN IF NOT EXISTS max_combined_limit SimpleAggregateFunction(max, UInt32); ALTER TABLE trigger_dev.queue_metrics_5m_v1 - ADD COLUMN IF NOT EXISTS max_total_running SimpleAggregateFunction(max, UInt32), - ADD COLUMN IF NOT EXISTS max_total_limit SimpleAggregateFunction(max, UInt32); + ADD COLUMN IF NOT EXISTS max_combined_running SimpleAggregateFunction(max, UInt32), + ADD COLUMN IF NOT EXISTS max_combined_limit SimpleAggregateFunction(max, UInt32); ALTER TABLE trigger_dev.queue_metrics_ck_v1 ADD COLUMN IF NOT EXISTS max_limit SimpleAggregateFunction(max, UInt32); @@ -45,8 +45,8 @@ SELECT max(env_limit) AS max_env_limit, max(ck_backlogged) AS max_ck_backlogged, max(ck_max_wait_ms) AS max_ck_wait_ms, - max(total_running) AS max_total_running, - max(total_limit) AS max_total_limit, + max(combined_running) AS max_combined_running, + max(combined_limit) AS max_combined_limit, sumIf(wait_ms, op = 'started' AND concurrency_key = '') AS wait_ms_sum, countIf(op = 'started' AND wait_ms > 0 AND concurrency_key = '') AS wait_ms_count, quantilesStateIf(0.5, 0.9, 0.95, 0.99)(wait_ms, op = 'started' AND wait_ms > 0 AND concurrency_key = '') AS wait_quantiles @@ -73,8 +73,8 @@ SELECT max(env_limit) AS max_env_limit, max(ck_backlogged) AS max_ck_backlogged, max(ck_max_wait_ms) AS max_ck_wait_ms, - max(total_running) AS max_total_running, - max(total_limit) AS max_total_limit, + max(combined_running) AS max_combined_running, + max(combined_limit) AS max_combined_limit, sumIf(wait_ms, op = 'started' AND concurrency_key = '') AS wait_ms_sum, countIf(op = 'started' AND wait_ms > 0 AND concurrency_key = '') AS wait_ms_count, quantilesStateIf(0.5, 0.9, 0.95, 0.99)(wait_ms, op = 'started' AND wait_ms > 0 AND concurrency_key = '') AS wait_quantiles @@ -104,9 +104,9 @@ DROP VIEW IF EXISTS trigger_dev.queue_metrics_ck_mv_v1; DROP VIEW IF EXISTS trigger_dev.queue_metrics_5m_mv_v1; DROP VIEW IF EXISTS trigger_dev.queue_metrics_mv_v1; ALTER TABLE trigger_dev.queue_metrics_ck_v1 DROP COLUMN IF EXISTS max_limit; -ALTER TABLE trigger_dev.queue_metrics_5m_v1 DROP COLUMN IF EXISTS max_total_running, DROP COLUMN IF EXISTS max_total_limit; -ALTER TABLE trigger_dev.queue_metrics_v1 DROP COLUMN IF EXISTS max_total_running, DROP COLUMN IF EXISTS max_total_limit; -ALTER TABLE trigger_dev.queue_metrics_raw_v1 DROP COLUMN IF EXISTS total_running, DROP COLUMN IF EXISTS total_limit; +ALTER TABLE trigger_dev.queue_metrics_5m_v1 DROP COLUMN IF EXISTS max_combined_running, DROP COLUMN IF EXISTS max_combined_limit; +ALTER TABLE trigger_dev.queue_metrics_v1 DROP COLUMN IF EXISTS max_combined_running, DROP COLUMN IF EXISTS max_combined_limit; +ALTER TABLE trigger_dev.queue_metrics_raw_v1 DROP COLUMN IF EXISTS combined_running, DROP COLUMN IF EXISTS combined_limit; -- Recreate the pre-042 materialized views (the definitions from 036) so ingestion keeps -- feeding every aggregate table after a rollback. diff --git a/internal-packages/clickhouse/src/queueMetrics.ts b/internal-packages/clickhouse/src/queueMetrics.ts index aa3cf5296d2..f3a6be695e4 100644 --- a/internal-packages/clickhouse/src/queueMetrics.ts +++ b/internal-packages/clickhouse/src/queueMetrics.ts @@ -21,8 +21,8 @@ export const QueueMetricsRawV1Input = z.object({ throttled: z.number().optional(), ck_backlogged: z.number().optional(), ck_max_wait_ms: z.number().optional(), - total_running: z.number().optional(), - total_limit: z.number().optional(), + combined_running: z.number().optional(), + combined_limit: z.number().optional(), wait_ms: z.number().optional(), cumulative: z.number().optional(), }); diff --git a/packages/core/src/v3/schemas/queues.ts b/packages/core/src/v3/schemas/queues.ts index f3039f8075a..9ca282fb33d 100644 --- a/packages/core/src/v3/schemas/queues.ts +++ b/packages/core/src/v3/schemas/queues.ts @@ -45,16 +45,16 @@ export const QueueItem = z.object({ overriddenAt: z.coerce.date().nullable(), /** Who overrode the concurrency limit (will be null if overridden via the API) */ overriddenBy: z.string().nullable(), - /** The total concurrency cap across all concurrencyKey values of the queue */ - total: z + /** The combined concurrency cap across all concurrencyKey values of the queue */ + combined: z .object({ - /** The effective/current total concurrency limit (null = no cap) */ + /** The effective/current combined concurrency limit (null = no cap) */ current: z.number().nullable(), - /** The declared total limit an override reverts to on reset */ + /** The declared combined limit an override reverts to on reset */ base: z.number().nullable(), - /** The overridden total limit, when an override is active */ + /** The overridden combined limit, when an override is active */ override: z.number().nullable(), - /** When the total override was applied */ + /** When the combined override was applied */ overriddenAt: z.coerce.date().nullable(), /** Runs currently in flight across all concurrencyKey values */ running: z.number().nullable(), From 8cbe65706162b0a03dec68d6d918260d294247fc Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Sat, 29 Aug 2026 19:35:52 +0100 Subject: [PATCH 09/28] feat(webapp): bracketed combined limit in the Limit column Queues that set a combinedConcurrencyLimit show it bracketed next to the per-key limit with a fine dashed underline and an explanatory tooltip; the Limit header tooltip is width-capped. Queues without one are unchanged. --- .../route.tsx | 74 ++++++++++++++----- 1 file changed, 56 insertions(+), 18 deletions(-) diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx index 5ea114e5da5..dfed5f35590 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx @@ -709,7 +709,8 @@ function QueuesWithMetricsView() { Limit @@ -888,14 +889,32 @@ function QueuesWithMetricsView() { limit )} {queue.concurrency?.combined?.current != null ? ( - - /key ·{" "} - {Math.min( - queue.concurrency.combined.current, - environment.concurrencyLimit - )}{" "} - combined - + + ( + {Math.min( + queue.concurrency.combined.current, + environment.concurrencyLimit + )} + ) + + } + content={ + <> + Combined limit: at most{" "} + {Math.min( + queue.concurrency.combined.current, + environment.concurrencyLimit + )}{" "} + runs across all concurrency keys of this queue. The main limit + applies to each key separately. + + } + className="max-w-[260px]" + /> ) : null} Running Limit @@ -1942,14 +1962,32 @@ function ClassicQueuesView() { > {limit} {queue.concurrency?.combined?.current != null ? ( - - /key ·{" "} - {Math.min( - queue.concurrency.combined.current, - environment.concurrencyLimit - )}{" "} - combined - + + ( + {Math.min( + queue.concurrency.combined.current, + environment.concurrencyLimit + )} + ) + + } + content={ + <> + Combined limit: at most{" "} + {Math.min( + queue.concurrency.combined.current, + environment.concurrencyLimit + )}{" "} + runs across all concurrency keys of this queue. The main limit + applies to each key separately. + + } + className="max-w-[260px]" + /> ) : null} Date: Sat, 29 Aug 2026 19:40:12 +0100 Subject: [PATCH 10/28] fix(webapp): combined-limit tooltip renders beside the cell link The tooltip trigger is a button, so nesting it in the Limit cell's link made clicking it navigate; it now renders as the cell's trailing adornment. --- .../route.tsx | 61 ++++++++++--------- 1 file changed, 33 insertions(+), 28 deletions(-) diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx index dfed5f35590..cc6d7a8674f 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx @@ -877,6 +877,39 @@ function QueuesWithMetricsView() { queue.paused ? "opacity-50" : undefined, queue.concurrency?.overriddenAt && "font-medium text-text-bright" )} + // The combined-limit hint is a tooltip button, so it renders beside the + // link (trailing) rather than nested inside the ; the number stays the + // link. + trailingContent={ + queue.concurrency?.combined?.current != null ? ( + + ( + {Math.min( + queue.concurrency.combined.current, + environment.concurrencyLimit + )} + ) + + } + content={ + <> + Combined limit: at most{" "} + {Math.min( + queue.concurrency.combined.current, + environment.concurrencyLimit + )}{" "} + runs across all concurrency keys of this queue. The main limit + applies to each key separately. + + } + className="max-w-[260px]" + /> + ) : undefined + } > {queue.concurrencyLimitOverridePercent !== null ? ( <> @@ -888,34 +921,6 @@ function QueuesWithMetricsView() { ) : ( limit )} - {queue.concurrency?.combined?.current != null ? ( - - ( - {Math.min( - queue.concurrency.combined.current, - environment.concurrencyLimit - )} - ) - - } - content={ - <> - Combined limit: at most{" "} - {Math.min( - queue.concurrency.combined.current, - environment.concurrencyLimit - )}{" "} - runs across all concurrency keys of this queue. The main limit - applies to each key separately. - - } - className="max-w-[260px]" - /> - ) : null} Date: Mon, 31 Aug 2026 10:30:33 +0100 Subject: [PATCH 11/28] Better tooltip message --- .../route.tsx | 550 +++++++++++++----- 1 file changed, 402 insertions(+), 148 deletions(-) diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx index cc6d7a8674f..4c109e80194 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx @@ -8,7 +8,10 @@ import { } from "@heroicons/react/20/solid"; import { DialogClose } from "@radix-ui/react-dialog"; import { Form, useNavigation } from "@remix-run/react"; -import { type ActionFunctionArgs, type LoaderFunctionArgs } from "@remix-run/server-runtime"; +import { + type ActionFunctionArgs, + type LoaderFunctionArgs, +} from "@remix-run/server-runtime"; import type { RuntimeEnvironmentType } from "@trigger.dev/database"; import { useEffect, useMemo, useState, type ReactNode } from "react"; import { QueuesIcon } from "~/assets/icons/QueuesIcon"; @@ -22,10 +25,19 @@ import { PageBody, PageContainer } from "~/components/layout/AppLayout"; import { MetricsLayout } from "~/components/layout/MetricsLayout"; import { Badge } from "~/components/primitives/Badge"; import { Button, LinkButton } from "~/components/primitives/Buttons"; -import { Dialog, DialogContent, DialogHeader, DialogTrigger } from "~/components/primitives/Dialog"; +import { + Dialog, + DialogContent, + DialogHeader, + DialogTrigger, +} from "~/components/primitives/Dialog"; import { FormButtons } from "~/components/primitives/FormButtons"; import { Header3 } from "~/components/primitives/Headers"; -import { NavBar, PageAccessories, PageTitle } from "~/components/primitives/PageHeader"; +import { + NavBar, + PageAccessories, + PageTitle, +} from "~/components/primitives/PageHeader"; import { PaginationControls } from "~/components/primitives/Pagination"; import { Paragraph } from "~/components/primitives/Paragraph"; import { PopoverMenuItem } from "~/components/primitives/Popover"; @@ -55,7 +67,10 @@ import { useAutoRevalidate } from "~/hooks/useAutoRevalidate"; import { useEnvironment } from "~/hooks/useEnvironment"; import { useOrganization } from "~/hooks/useOrganizations"; import { useProject } from "~/hooks/useProject"; -import { redirectWithErrorMessage, redirectWithSuccessMessage } from "~/models/message.server"; +import { + redirectWithErrorMessage, + redirectWithSuccessMessage, +} from "~/models/message.server"; import { findProjectBySlug } from "~/models/project.server"; import { findEnvironmentBySlug } from "~/models/runtimeEnvironment.server"; import { EnvironmentQueuePresenter } from "~/presenters/v3/EnvironmentQueuePresenter.server"; @@ -64,12 +79,18 @@ import { QueueMetricsPresenter, type QueueListMetric, } from "~/presenters/v3/QueueMetricsPresenter.server"; -import { TimeFilter, timeFilterFromTo } from "~/components/runs/v3/SharedFilters"; +import { + TimeFilter, + timeFilterFromTo, +} from "~/components/runs/v3/SharedFilters"; import { useSearchParams } from "~/hooks/useSearchParam"; import { parseFiniteInt } from "~/utils/searchParams"; import { MiniLineChart } from "~/components/metrics/MiniLineChart"; import { buildActivityTimeAxis } from "~/components/primitives/charts/activityTimeAxis"; -import { Chart, type ChartConfig } from "~/components/primitives/charts/ChartCompound"; +import { + Chart, + type ChartConfig, +} from "~/components/primitives/charts/ChartCompound"; import { ChartCard } from "~/components/primitives/charts/ChartCard"; import { ChartSyncProvider } from "~/components/primitives/charts/ChartSyncContext"; import { useZoomToTimeFilter } from "~/hooks/useZoomToTimeFilter"; @@ -115,6 +136,7 @@ import { import { queueMetricsMaxPeriodDays } from "~/components/queues/queueMetricsPeriod.server"; import { isQueueAtCapacity } from "~/components/queues/queue-thresholds"; import { pageMeta } from "~/utils/pageTitle"; +import { InlineCode } from "~/components/code/InlineCode"; const SearchParamsSchema = z.object({ query: z.string().optional(), @@ -145,14 +167,19 @@ export const meta = pageMeta("Queues"); export const loader = async ({ request, params }: LoaderFunctionArgs) => { const userId = await requireUserId(request); - const { organizationSlug, projectParam, envParam } = EnvironmentParamSchema.parse(params); + const { organizationSlug, projectParam, envParam } = + EnvironmentParamSchema.parse(params); const url = new URL(request.url); const { page, query, period, from, to, sort } = SearchParamsSchema.parse( - Object.fromEntries(url.searchParams) + Object.fromEntries(url.searchParams), ); - const project = await findProjectBySlug(organizationSlug, projectParam, userId); + const project = await findProjectBySlug( + organizationSlug, + projectParam, + userId, + ); if (!project) { throw new Response(undefined, { status: 404, @@ -181,7 +208,7 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { : QUEUE_METRICS_RETENTION_DAYS; const defaultPeriod = clampQueueMetricsPeriod( queueMetricsPeriodFromRequest(request), - maxPeriodDays + maxPeriodDays, ); try { @@ -213,18 +240,23 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { try { const presenter = new QueueMetricsPresenter(); const queueNames = queues.queues.map((q) => - q.type === "task" ? `task/${q.name}` : q.name + q.type === "task" ? `task/${q.name}` : q.name, ); const timeRange = clipQueueMetricsWindow( timeFilterFromTo({ period: - resolveQueueMetricsPeriod({ period, from, to, defaultPeriod, maxPeriodDays }) ?? - undefined, + resolveQueueMetricsPeriod({ + period, + from, + to, + defaultPeriod, + maxPeriodDays, + }) ?? undefined, from: parseFiniteInt(from), to: parseFiniteInt(to), defaultPeriod, }), - maxPeriodDays + maxPeriodDays, ); const queueMetrics = queueNames.length > 0 @@ -243,18 +275,25 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { }; } } catch (error) { - logger.warn("Queue list metrics unavailable, rendering without them", { error }); + logger.warn("Queue list metrics unavailable, rendering without them", { + error, + }); } } // Allocation summary (Environment limit + Allocated tiles) is additive; a presenter // failure must not 400 the page, so fail open to null like the metrics block above. - let allocation: Awaited> | null = null; + let allocation: Awaited< + ReturnType + > | null = null; if (queueMetricsUiEnabled) { try { allocation = await new QueueAllocationPresenter().call({ environment }); } catch (error) { - logger.warn("Queue allocation summary unavailable, rendering without it", { error }); + logger.warn( + "Queue allocation summary unavailable, rendering without it", + { error }, + ); } } @@ -272,7 +311,8 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { console.error(error); throw new Response(undefined, { status: 400, - statusText: "Something went wrong, if this problem persists please contact support.", + statusText: + "Something went wrong, if this problem persists please contact support.", }); } }; @@ -283,13 +323,18 @@ export const action = async ({ request, params }: ActionFunctionArgs) => { return redirectWithErrorMessage( `/orgs/${params.organizationSlug}/projects/${params.projectParam}/env/${params.envParam}/queues`, request, - "Wrong method" + "Wrong method", ); } - const { organizationSlug, projectParam, envParam } = EnvironmentParamSchema.parse(params); + const { organizationSlug, projectParam, envParam } = + EnvironmentParamSchema.parse(params); - const project = await findProjectBySlug(organizationSlug, projectParam, userId); + const project = await findProjectBySlug( + organizationSlug, + projectParam, + userId, + ); if (!project) { throw new Response(undefined, { status: 404, @@ -312,7 +357,11 @@ export const action = async ({ request, params }: ActionFunctionArgs) => { const redirectPath = `/orgs/${organizationSlug}/projects/${projectParam}/env/${envParam}/queues${url.search}`; if (environment.archivedAt) { - return redirectWithErrorMessage(redirectPath, request, "This branch is archived"); + return redirectWithErrorMessage( + redirectPath, + request, + "This branch is archived", + ); } // Per-queue actions (pause/resume/override/remove-override) are shared with the queue detail @@ -335,7 +384,11 @@ export const action = async ({ request, params }: ActionFunctionArgs) => { if (!result.success) { return redirectWithErrorMessage(redirectPath, request, result.error); } - return redirectWithSuccessMessage(redirectPath, request, "Environment paused"); + return redirectWithSuccessMessage( + redirectPath, + request, + "Environment paused", + ); } case "environment-resume": { const resumeService = new PauseEnvironmentService(); @@ -343,10 +396,18 @@ export const action = async ({ request, params }: ActionFunctionArgs) => { if (!result.success) { return redirectWithErrorMessage(redirectPath, request, result.error); } - return redirectWithSuccessMessage(redirectPath, request, "Environment resumed"); + return redirectWithSuccessMessage( + redirectPath, + request, + "Environment resumed", + ); } default: - return redirectWithErrorMessage(redirectPath, request, "Something went wrong"); + return redirectWithErrorMessage( + redirectPath, + request, + "Something went wrong", + ); } }; @@ -359,14 +420,19 @@ function getEnvConcurrencyLimitStatus(environment: { burstFactor: number; }) { const limitStatus = - environment.running === environment.concurrencyLimit * environment.burstFactor + environment.running === + environment.concurrencyLimit * environment.burstFactor ? "limit" : environment.running > environment.concurrencyLimit ? "burst" : "within"; const limitClassName = - limitStatus === "burst" ? "text-warning" : limitStatus === "limit" ? "text-error" : undefined; + limitStatus === "burst" + ? "text-warning" + : limitStatus === "limit" + ? "text-error" + : undefined; return { limitStatus, limitClassName }; } @@ -375,7 +441,11 @@ export default function Page() { // Per-org flag decides which whole page renders. Off => the classic Queues page, // byte-for-byte the pre-metrics UI. Each branch is its own component (own hooks). const { queueMetricsUiEnabled } = useTypedLoaderData(); - return queueMetricsUiEnabled ? : ; + return queueMetricsUiEnabled ? ( + + ) : ( + + ); } function QueuesWithMetricsView() { @@ -435,20 +505,23 @@ function QueuesWithMetricsView() { defaultPeriod: QUEUE_LIVE_BLOCKS_PERIOD, fillGaps: false, refreshIntervalMs: 15_000, - } + }, ); const lastLiveBlockRow = liveBlockRows.length > 0 ? liveBlockRows[liveBlockRows.length - 1] : null; // Only trust the gauge while its newest bucket is fresh. A row painted from the hook's cache on // client-side nav-back (responseCache), or a quiet env whose latest bucket is minutes old, must // not override the loader's Redis-exact live values with a stale count. - const lastLiveBucketMs = lastLiveBlockRow ? tileTimeToMs(lastLiveBlockRow.t) : NaN; + const lastLiveBucketMs = lastLiveBlockRow + ? tileTimeToMs(lastLiveBlockRow.t) + : NaN; const liveBlockIsFresh = useIsMetricResponseFresh( responseReceivedAt, lastLiveBucketMs, - LIVE_GAUGE_FRESH_MS + LIVE_GAUGE_FRESH_MS, ); - const freshLiveBlockRow = lastLiveBlockRow && liveBlockIsFresh ? lastLiveBlockRow : null; + const freshLiveBlockRow = + lastLiveBlockRow && liveBlockIsFresh ? lastLiveBlockRow : null; const envQueuedLive = freshLiveBlockRow ? tileNumber(freshLiveBlockRow.env_queued) : environment.queued; @@ -461,7 +534,8 @@ function QueuesWithMetricsView() { const envLimit = environment.concurrencyLimit; const burstLimit = Math.round(envLimit * environment.burstFactor); const allocated = allocation?.allocated ?? 0; - const allocationPct = envLimit > 0 ? Math.round((allocated / envLimit) * 100) : 0; + const allocationPct = + envLimit > 0 ? Math.round((allocated / envLimit) * 100) : 0; const overAllocated = allocated > envLimit; // Running-block tinting (burst/limit) tracks the live running value, not the loader snapshot. @@ -523,7 +597,11 @@ function QueuesWithMetricsView() { paused : undefined} + suffix={ + env.paused ? ( + paused + ) : undefined + } animate accessory={ @@ -541,7 +619,9 @@ function QueuesWithMetricsView() { /> } - valueClassName={env.paused ? "text-warning tabular-nums" : "tabular-nums"} + valueClassName={ + env.paused ? "text-warning tabular-nums" : "tabular-nums" + } compactThreshold={1000000} /> - Including {envRunningLive - environment.concurrencyLimit} burst runs{" "} - + Including {envRunningLive - environment.concurrencyLimit}{" "} + burst runs ) : limitStatus === "limit" ? ( "At concurrency limit" @@ -594,13 +677,21 @@ function QueuesWithMetricsView() { value={allocation ? allocated : undefined} formattedValue={allocation ? undefined : "–"} valueClassName={cn(allocation && overAllocated && "text-warning")} - suffix={allocation ? `${allocationPct}% of the environment limit` : undefined} + suffix={ + allocation + ? `${allocationPct}% of the environment limit` + : undefined + } suffixClassName="text-text-dimmed" /> 1 ? `bursts up to ${burstLimit}` : undefined} + suffix={ + environment.burstFactor > 1 + ? `bursts up to ${burstLimit}` + : undefined + } suffixClassName="text-text-dimmed" accessory={ plan ? ( @@ -615,7 +706,10 @@ function QueuesWithMetricsView() { ) : ( Limit @@ -721,16 +816,17 @@ function QueuesWithMetricsView() { tooltip={

- Environment: uses the environment - limit of {environment.concurrencyLimit}. + Environment: + uses the environment limit of{" "} + {environment.concurrencyLimit}.

- User: a limit you set in your - code. + User: a limit + you set in your code.

- Override: a limit you set here or - via the API. + Override: a + limit you set here or via the API.

} @@ -750,8 +846,8 @@ function QueuesWithMetricsView() { disableTooltipHoverableContent tooltip={ <> - How many runs were waiting, over the selected time. marks - where the queue was throttled. + How many runs were waiting, over the selected time.{" "} + marks where the queue was throttled. } > @@ -765,16 +861,22 @@ function QueuesWithMetricsView() { {queueRows.length > 0 ? ( queueRows.map((queue) => { - const limit = queue.concurrencyLimit ?? environment.concurrencyLimit; + const limit = + queue.concurrencyLimit ?? environment.concurrencyLimit; const isAtConcurrencyLimit = queue.running >= limit; const isAtQueueLimit = environment.queueSizeLimit !== null && queue.queued >= environment.queueSizeLimit; const queueFilterableName = queueMetricsKey(queue); const queueMetric = metricsByQueue[queueFilterableName]; - const queueDetailPath = v3QueuePath(organization, project, env, { - friendlyId: queue.id, - }); + const queueDetailPath = v3QueuePath( + organization, + project, + env, + { + friendlyId: queue.id, + }, + ); return ( ) : ( ) @@ -813,7 +915,9 @@ function QueuesWithMetricsView() { trailingContent={ isAtConcurrencyLimit ? ( } + button={ + + } content="At concurrency limit: this queue is running as many runs as its limit allows; new runs wait in the backlog." className="max-w-[230px]" disableHoverableContent @@ -822,11 +926,16 @@ function QueuesWithMetricsView() { } > - + {queue.name} {queue.paused ? ( - + Paused ) : null} @@ -844,7 +953,7 @@ function QueuesWithMetricsView() { className={cn( "w-[1%]", queue.paused ? "opacity-50" : undefined, - isAtQueueLimit && "text-error" + isAtQueueLimit && "text-error", )} > {queue.queued} @@ -860,10 +969,10 @@ function QueuesWithMetricsView() { (queue.concurrency.combined.running ?? 0) >= Math.min( queue.concurrency.combined.current, - environment.concurrencyLimit + environment.concurrencyLimit, ) ? "text-warning" - : queue.running > 0 && "text-text-bright" + : queue.running > 0 && "text-text-bright", )} > {queue.running} @@ -875,7 +984,8 @@ function QueuesWithMetricsView() { className={cn( "w-[1%]", queue.paused ? "opacity-50" : undefined, - queue.concurrency?.overriddenAt && "font-medium text-text-bright" + queue.concurrency?.overriddenAt && + "font-medium text-text-bright", )} // The combined-limit hint is a tooltip button, so it renders beside the // link (trailing) rather than nested inside the ; the number stays the @@ -890,7 +1000,7 @@ function QueuesWithMetricsView() { ( {Math.min( queue.concurrency.combined.current, - environment.concurrencyLimit + environment.concurrencyLimit, )} ) @@ -900,10 +1010,11 @@ function QueuesWithMetricsView() { Combined limit: at most{" "} {Math.min( queue.concurrency.combined.current, - environment.concurrencyLimit + environment.concurrencyLimit, )}{" "} - runs across all concurrency keys of this queue. The main limit - applies to each key separately. + runs across all concurrency keys of this + queue. The main limit applies to each key + separately. } className="max-w-[260px]" @@ -915,7 +1026,11 @@ function QueuesWithMetricsView() { <> {limit} - ({formatOverridePercent(queue.concurrencyLimitOverridePercent)}%) + ( + {formatOverridePercent( + queue.concurrencyLimitOverridePercent, + )} + %) ) : ( @@ -926,7 +1041,10 @@ function QueuesWithMetricsView() { to={queueDetailPath} alignment="right" actionClassName="pl-16" - className={cn("w-[1%]", queue.paused ? "opacity-50" : undefined)} + className={cn( + "w-[1%]", + queue.paused ? "opacity-50" : undefined, + )} // Keep the whole row navigable: the override explainer is a tooltip // button, so it renders beside the link (trailing) rather than nested // inside the , and the label itself stays the link. @@ -936,7 +1054,7 @@ function QueuesWithMetricsView() { content={ queue.concurrencyLimitOverridePercent !== null ? `Overridden at ${formatOverridePercent( - queue.concurrencyLimitOverridePercent + queue.concurrencyLimitOverridePercent, )}% of the environment limit.` : `This queue's concurrency limit has been manually overridden to ${limit}.` } @@ -996,7 +1114,9 @@ function QueuesWithMetricsView() { peakTooltip={ queueMetric && queueMetric.throttledTotal > 0 ? `Peak queued; this queue was throttled ${queueMetric.throttledTotal.toLocaleString()} ${ - queueMetric.throttledTotal === 1 ? "time" : "times" + queueMetric.throttledTotal === 1 + ? "time" + : "times" } in this period` : "Peak queued in this period" } @@ -1004,8 +1124,16 @@ function QueuesWithMetricsView() { } - hiddenButtons={!queue.paused && } + visibleButtons={ + queue.paused && ( + + ) + } + hiddenButtons={ + !queue.paused && ( + + ) + } popoverContent={ <> {queue.paused ? ( @@ -1058,7 +1186,9 @@ function QueuesWithMetricsView() { /> } @@ -1071,7 +1201,9 @@ function QueuesWithMetricsView() {
- {hasFilters ? "No queues found matching your filters" : "No queues found"} + {hasFilters + ? "No queues found matching your filters" + : "No queues found"}
@@ -1101,7 +1233,8 @@ function EnvironmentPauseResumeButton({ }, [navigation.state]); const isLoading = Boolean( - navigation.formData?.get("action") === (env.paused ? "environment-resume" : "environment-pause") + navigation.formData?.get("action") === + (env.paused ? "environment-resume" : "environment-pause"), ); return ( @@ -1116,7 +1249,9 @@ function EnvironmentPauseResumeButton({ type="button" variant="secondary/small" LeadingIcon={env.paused ? PlayIcon : PauseIcon} - leadingIconClassName={env.paused ? "text-success" : "text-warning"} + leadingIconClassName={ + env.paused ? "text-success" : "text-warning" + } className={ env.paused ? "border-success/60 text-success [&_span]:text-success hover:border-success" @@ -1144,13 +1279,15 @@ function EnvironmentPauseResumeButton({
- {env.paused ? "Resume environment?" : "Pause environment?"} + + {env.paused ? "Resume environment?" : "Pause environment?"} +
{env.paused ? `This will allow runs to be dequeued in ${environmentFullTitle(env)} again.` : `This will pause all runs from being dequeued in ${environmentFullTitle( - env + env, )}. Any executing runs will continue to run.`}
setIsOpen(false)}> @@ -1166,7 +1303,13 @@ function EnvironmentPauseResumeButton({ disabled={isLoading} variant={env.paused ? "primary/medium" : "danger/medium"} LeadingIcon={ - isLoading ? : env.paused ? PlayIcon : PauseIcon + isLoading ? ( + + ) : env.paused ? ( + PlayIcon + ) : ( + PauseIcon + ) } shortcut={{ modifiers: ["mod"], key: "enter" }} > @@ -1190,7 +1333,7 @@ function EnvironmentPauseResumeButton({ export function isEnvironmentPauseResumeFormSubmission( formMethod: string | undefined, - formData: FormData | undefined + formData: FormData | undefined, ) { if (!formMethod || !formData) { return false; @@ -1204,7 +1347,13 @@ export function isEnvironmentPauseResumeFormSubmission( } export function QueueFilters() { - return ; + return ( + + ); } type MetricTileRow = Record; @@ -1279,7 +1428,10 @@ function tileTimeToMs(value: number | string | null): number { /** Peak of a series, ignoring the buckets it has nothing to say about. */ function peakOf(points: TilePoint[]): number { - return points.reduce((max, p) => (p.value === null ? max : Math.max(max, p.value)), 0); + return points.reduce( + (max, p) => (p.value === null ? max : Math.max(max, p.value)), + 0, + ); } const SCHEDULING_DELAY_QUERY = `SELECT timeBucket() AS t,\n round(quantilesTDigestMerge(0.5, 0.9, 0.95, 0.99)(wait_quantiles)[3]) AS p95,\n sum(wait_ms_count) AS samples\nFROM env_metrics\nGROUP BY t\nORDER BY t`; @@ -1292,8 +1444,8 @@ const QUEUE_HEADER_TILES: QueueHeaderTile[] = [ label: "Env saturation", description: ( <> - How much of the environment's concurrency is in use. Turns above 100%, - when it's into burst capacity. + How much of the environment's concurrency is in use. Turns{" "} + above 100%, when it's into burst capacity. ), color: "var(--color-queues-chart)", @@ -1302,17 +1454,23 @@ const QUEUE_HEADER_TILES: QueueHeaderTile[] = [ { color: "var(--color-warning)", label: "Over limit" }, ], query: `SELECT timeBucket() AS t,\n max(max_env_running) AS running,\n max(max_env_limit) AS env_limit\nFROM env_metrics\nGROUP BY t\nORDER BY t`, - formatValue: (v) => (v > 100 ? `${v}% — over the environment limit` : `${v}%`), + formatValue: (v) => + v > 100 ? `${v}% — over the environment limit` : `${v}%`, formatAxis: (v) => `${v}%`, derive: (rows) => { const points = rows.map((r) => { const limit = tileNumber(r.env_limit); return { bucket: tileTimeToMs(r.t), - value: limit > 0 ? Math.round((tileNumber(r.running) / limit) * 100) : 0, + value: + limit > 0 ? Math.round((tileNumber(r.running) / limit) * 100) : 0, }; }); - return { points, total: peakOf(points), formatTotal: (v) => `${v}% peak` }; + return { + points, + total: peakOf(points), + formatTotal: (v) => `${v}% peak`, + }; }, }, { @@ -1326,7 +1484,11 @@ const QUEUE_HEADER_TILES: QueueHeaderTile[] = [ bucket: tileTimeToMs(r.t), value: tileNumber(r.queued), })); - return { points, total: peakOf(points), formatTotal: (v) => `${v.toLocaleString()} peak` }; + return { + points, + total: peakOf(points), + formatTotal: (v) => `${v.toLocaleString()} peak`, + }; }, }, { @@ -1334,8 +1496,8 @@ const QUEUE_HEADER_TILES: QueueHeaderTile[] = [ label: "Scheduling delay p95", description: ( <> - How long runs wait before they start (95% start faster than this). Turns {" "} - above 1 minute. + How long runs wait before they start (95% start faster than this). Turns{" "} + above 1 minute. ), totalTooltip: "The worst p95 in the selected window.", @@ -1364,8 +1526,9 @@ const QUEUE_HEADER_TILES: QueueHeaderTile[] = [ */ derive: (rows) => { const worst = rows.reduce( - (max, r) => (tileNumber(r.samples) > 0 ? Math.max(max, tileNumber(r.p95)) : max), - 0 + (max, r) => + tileNumber(r.samples) > 0 ? Math.max(max, tileNumber(r.p95)) : max, + 0, ); return { total: worst, @@ -1379,7 +1542,8 @@ const QUEUE_HEADER_TILES: QueueHeaderTile[] = [ id: "throttled", label: "Throttled", description: "How often runs were held back by a limit.", - totalTooltip: "The share of the selected window with at least one blocked dequeue.", + totalTooltip: + "The share of the selected window with at least one blocked dequeue.", color: "var(--color-queues-chart)", legend: [{ color: "var(--color-warning)", label: "Throttled" }], query: THROTTLED_QUERY, @@ -1400,7 +1564,8 @@ const QUEUE_HEADER_TILES: QueueHeaderTile[] = [ */ derive: (rows) => { const nonzero = rows.filter((r) => tileNumber(r.throttled) > 0).length; - const pct = rows.length > 0 ? Math.round((nonzero / rows.length) * 100) : 0; + const pct = + rows.length > 0 ? Math.round((nonzero / rows.length) * 100) : 0; return { total: pct, formatTotal: (v) => `${v}% of current period`, @@ -1466,10 +1631,13 @@ function QueueEnvMetricChart({ const derived = tile.derive(rows); const points = derived.points; - const plottedBucketMs = points.length > 1 ? points[1]!.bucket - points[0]!.bucket : 0; + const plottedBucketMs = + points.length > 1 ? points[1]!.bucket - points[0]!.bucket : 0; const floorWidenedBuckets = - plottedBucketMs > 0 && plottedBucketMs <= HERO_CHART_MIN_BUCKET_SECONDS * 1000; - const readoutQuery = tile.readout && floorWidenedBuckets ? tile.readout.query : ""; + plottedBucketMs > 0 && + plottedBucketMs <= HERO_CHART_MIN_BUCKET_SECONDS * 1000; + const readoutQuery = + tile.readout && floorWidenedBuckets ? tile.readout.query : ""; const readoutResult = useMetricResourceQuery(readoutQuery, sharedOptions); const { total, formatTotal, totalClassName } = tile.readout @@ -1488,11 +1656,12 @@ function QueueEnvMetricChart({ const chartConfig = useMemo( () => ({ [tile.id]: { label: tile.label, color: lineColor } }), - [tile.id, tile.label, lineColor] + [tile.id, tile.label, lineColor], ); const { tickFormatter, tooltipLabelFormatter } = buildActivityTimeAxis(data); - const hasData = data.length > 0 && data.some((p) => Number(p[tile.id] ?? 0) > 0); + const hasData = + data.length > 0 && data.some((p) => Number(p[tile.id] ?? 0) > 0); // Peak readout lives in the card title (ChartCard has no dedicated value slot). A zero/empty // total renders no readout at all (skipping "0% peak", "0 peak", "0" and the p95 "–" placeholder) @@ -1527,7 +1696,7 @@ function QueueEnvMetricChart({ {peak} @@ -1541,7 +1710,7 @@ function QueueEnvMetricChart({ {peak} @@ -1588,7 +1757,9 @@ function QueueEnvMetricChart({ thresholdStroke={thresholdStroke} warningOverlay={warningOverlay} xAxisProps={{ tickFormatter }} - yAxisProps={tile.formatAxis ? { tickFormatter: tile.formatAxis } : undefined} + yAxisProps={ + tile.formatAxis ? { tickFormatter: tile.formatAxis } : undefined + } tooltipLabelFormatter={tooltipLabelFormatter} tooltipValueFormatter={tile.formatValue} /> @@ -1620,11 +1791,21 @@ type QueueHealth = { limit: number; }; -type QueueHealthLabel = "Paused" | "At capacity" | "Backlogged" | "Active" | "Idle"; +type QueueHealthLabel = + | "Paused" + | "At capacity" + | "Backlogged" + | "Active" + | "Idle"; // Single source of truth for the queue health decision, shared by the badge and the table's // health-column sort so the sorted order always matches the labels shown. -function queueHealthLabel({ paused, running, queued, limit }: QueueHealth): QueueHealthLabel { +function queueHealthLabel({ + paused, + running, + queued, + limit, +}: QueueHealth): QueueHealthLabel { if (paused) return "Paused"; if (isQueueAtCapacity({ running, queued, limit })) return "At capacity"; if (queued > 0) return "Backlogged"; @@ -1635,8 +1816,10 @@ function queueHealthLabel({ paused, running, queued, limit }: QueueHealth): Queu // Tint + colored text, sized like the error status chips (see ErrorStatusBadge). const QUEUE_HEALTH_STYLES: Record = { Paused: "bg-warning/10 text-warning system:bg-warning system:text-white", - "At capacity": "bg-warning/10 text-warning system:bg-warning system:text-white", - Backlogged: "bg-blue-500/10 text-blue-500 system:bg-blue-500 system:text-white", + "At capacity": + "bg-warning/10 text-warning system:bg-warning system:text-white", + Backlogged: + "bg-blue-500/10 text-blue-500 system:bg-blue-500 system:text-white", Active: "bg-success/10 text-success system:bg-success system:text-white", Idle: "bg-charcoal-500/10 text-text-dimmed system:bg-charcoal-500 system:text-white", }; @@ -1647,7 +1830,7 @@ function QueueHealthBadge(health: QueueHealth) { {label} @@ -1669,14 +1852,21 @@ function formatWaitMs(ms: number): string { // Drop a trailing ".00" from whole percentages so "50.00" reads as "50" but "12.50" is preserved. function formatOverridePercent(percent: number): string { - return Number.isInteger(percent) ? percent.toString() : percent.toFixed(2).replace(/\.?0+$/, ""); + return Number.isInteger(percent) + ? percent.toString() + : percent.toFixed(2).replace(/\.?0+$/, ""); } // Classic Queues page, restored verbatim from before the Queue Metrics feature. Rendered // when queueMetricsUiEnabled is off so a gated org sees exactly the pre-metrics UI. function ClassicQueuesView() { - const { environment, queues, pagination, hasFilters, autoReloadPollIntervalMs } = - useTypedLoaderData(); + const { + environment, + queues, + pagination, + hasFilters, + autoReloadPollIntervalMs, + } = useTypedLoaderData(); const organization = useOrganization(); const project = useProject(); @@ -1685,7 +1875,8 @@ function ClassicQueuesView() { useAutoRevalidate({ interval: autoReloadPollIntervalMs, onFocus: true }); - const { limitStatus, limitClassName } = getEnvConcurrencyLimitStatus(environment); + const { limitStatus, limitClassName } = + getEnvConcurrencyLimitStatus(environment); return ( @@ -1710,7 +1901,11 @@ function ClassicQueuesView() { paused : undefined} + suffix={ + env.paused ? ( + paused + ) : undefined + } animate accessory={
@@ -1732,7 +1927,9 @@ function ClassicQueuesView() { />
} - valueClassName={env.paused ? "text-warning tabular-nums" : "tabular-nums"} + valueClassName={ + env.paused ? "text-warning tabular-nums" : "tabular-nums" + } compactThreshold={1000000} /> - Including {environment.running - environment.concurrencyLimit} burst runs{" "} - + Including{" "} + {environment.running - environment.concurrencyLimit} burst + runs ) : limitStatus === "limit" ? ( "At concurrency limit" @@ -1781,17 +1979,19 @@ function ClassicQueuesView() { - Burst limit {environment.burstFactor * environment.concurrencyLimit}{" "} + Burst limit{" "} + {environment.burstFactor * environment.concurrencyLimit}{" "} ) : undefined } accessory={ plan ? ( - plan?.v3Subscription?.plan?.limits.concurrentRuns.canExceed ? ( + plan?.v3Subscription?.plan?.limits.concurrentRuns + .canExceed ? ( ) : (
@@ -1833,7 +2042,7 @@ function ClassicQueuesView() { Running Limit @@ -1849,8 +2058,8 @@ function ClassicQueuesView() { className="text-wrap! text-text-dimmed" spacing > - This queue is limited by your environment's concurrency limit of{" "} - {environment.concurrencyLimit}. + This queue is limited by your environment's + concurrency limit of {environment.concurrencyLimit}.
@@ -1860,7 +2069,8 @@ function ClassicQueuesView() { className="text-wrap! text-text-dimmed" spacing > - This queue is limited by a concurrency limit set in your code. + This queue is limited by a concurrency limit set in + your code.
@@ -1870,8 +2080,8 @@ function ClassicQueuesView() { className="text-wrap! text-text-dimmed" spacing > - This queue's concurrency limit has been manually overridden from the - dashboard or API. + This queue's concurrency limit has been manually + overridden from the dashboard or API.
@@ -1887,7 +2097,8 @@ function ClassicQueuesView() { {queues.length > 0 ? ( queues.map((queue) => { - const limit = queue.concurrencyLimit ?? environment.concurrencyLimit; + const limit = + queue.concurrencyLimit ?? environment.concurrencyLimit; const isAtConcurrencyLimit = queue.running >= limit; const isAtQueueLimit = environment.queueSizeLimit !== null && @@ -1903,7 +2114,10 @@ function ClassicQueuesView() { {queue.concurrency?.overriddenAt ? ( + Concurrency limit overridden } @@ -1913,17 +2127,26 @@ function ClassicQueuesView() { /> ) : null} {queue.paused ? ( - + Paused ) : null} {isAtQueueLimit ? ( - + At queue limit ) : null} {isAtConcurrencyLimit ? ( - + At concurrency limit ) : null} @@ -1934,7 +2157,7 @@ function ClassicQueuesView() { className={cn( "w-[1%] pl-16 tabular-nums", queue.paused ? "opacity-50" : undefined, - isAtQueueLimit && "text-error" + isAtQueueLimit && "text-error", )} > {queue.queued} @@ -1948,11 +2171,11 @@ function ClassicQueuesView() { (queue.concurrency.combined.running ?? 0) >= Math.min( queue.concurrency.combined.current, - environment.concurrencyLimit + environment.concurrencyLimit, ) ? "text-warning" : queue.running > 0 && "text-text-bright", - isAtConcurrencyLimit && "text-warning" + isAtConcurrencyLimit && "text-warning", )} > {queue.running} @@ -1962,7 +2185,8 @@ function ClassicQueuesView() { className={cn( "w-[1%] pl-16 tabular-nums", queue.paused ? "opacity-50" : undefined, - queue.concurrency?.overriddenAt && "font-medium text-text-bright" + queue.concurrency?.overriddenAt && + "font-medium text-text-bright", )} > {limit} @@ -1975,7 +2199,7 @@ function ClassicQueuesView() { ( {Math.min( queue.concurrency.combined.current, - environment.concurrencyLimit + environment.concurrencyLimit, )} ) @@ -1985,10 +2209,11 @@ function ClassicQueuesView() { Combined limit: at most{" "} {Math.min( queue.concurrency.combined.current, - environment.concurrencyLimit + environment.concurrencyLimit, )}{" "} - runs across all concurrency keys of this queue. The main limit - applies to each key separately. + runs across all concurrency keys of this + queue. The main limit applies to each key + separately. } className="max-w-[260px]" @@ -2001,7 +2226,8 @@ function ClassicQueuesView() { "w-[1%] pl-16", queue.paused ? "opacity-50" : undefined, isAtConcurrencyLimit && "text-warning", - queue.concurrency?.overriddenAt && "font-medium text-text-bright" + queue.concurrency?.overriddenAt && + "font-medium text-text-bright", )} > {queue.concurrency?.overriddenAt ? ( @@ -2014,8 +2240,16 @@ function ClassicQueuesView() { } - hiddenButtons={!queue.paused && } + visibleButtons={ + queue.paused && ( + + ) + } + hiddenButtons={ + !queue.paused && ( + + ) + } popoverContent={ <> {queue.paused ? ( @@ -2068,7 +2302,9 @@ function ClassicQueuesView() { /> } @@ -2081,7 +2317,9 @@ function ClassicQueuesView() {
- {hasFilters ? "No queues found matching your filters" : "No queues found"} + {hasFilters + ? "No queues found matching your filters" + : "No queues found"}
@@ -2112,3 +2350,19 @@ function BurstFactorTooltip({ /> ); } + +const limitTooltip = ( + <> + + How many runs can execute at once.{" "} + + + 1 (20) means 1 run + per concurrency key, but at most 20 runs across all keys. Set using{" "} + + combinedConcurrencyLimit + {" "} + in your code. + + +); From 1edebceaa715c28e23f871b05c5a7dbedb8eae52 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Mon, 31 Aug 2026 10:37:07 +0100 Subject: [PATCH 12/28] docs(core): combined.current is the declared cap, clamped at admit time Also reflows an import to the formatter's current output. --- .../route.tsx | 512 +++++------------- 1 file changed, 144 insertions(+), 368 deletions(-) diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx index 4c109e80194..66a188e810e 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx @@ -8,10 +8,7 @@ import { } from "@heroicons/react/20/solid"; import { DialogClose } from "@radix-ui/react-dialog"; import { Form, useNavigation } from "@remix-run/react"; -import { - type ActionFunctionArgs, - type LoaderFunctionArgs, -} from "@remix-run/server-runtime"; +import { type ActionFunctionArgs, type LoaderFunctionArgs } from "@remix-run/server-runtime"; import type { RuntimeEnvironmentType } from "@trigger.dev/database"; import { useEffect, useMemo, useState, type ReactNode } from "react"; import { QueuesIcon } from "~/assets/icons/QueuesIcon"; @@ -25,19 +22,10 @@ import { PageBody, PageContainer } from "~/components/layout/AppLayout"; import { MetricsLayout } from "~/components/layout/MetricsLayout"; import { Badge } from "~/components/primitives/Badge"; import { Button, LinkButton } from "~/components/primitives/Buttons"; -import { - Dialog, - DialogContent, - DialogHeader, - DialogTrigger, -} from "~/components/primitives/Dialog"; +import { Dialog, DialogContent, DialogHeader, DialogTrigger } from "~/components/primitives/Dialog"; import { FormButtons } from "~/components/primitives/FormButtons"; import { Header3 } from "~/components/primitives/Headers"; -import { - NavBar, - PageAccessories, - PageTitle, -} from "~/components/primitives/PageHeader"; +import { NavBar, PageAccessories, PageTitle } from "~/components/primitives/PageHeader"; import { PaginationControls } from "~/components/primitives/Pagination"; import { Paragraph } from "~/components/primitives/Paragraph"; import { PopoverMenuItem } from "~/components/primitives/Popover"; @@ -67,10 +55,7 @@ import { useAutoRevalidate } from "~/hooks/useAutoRevalidate"; import { useEnvironment } from "~/hooks/useEnvironment"; import { useOrganization } from "~/hooks/useOrganizations"; import { useProject } from "~/hooks/useProject"; -import { - redirectWithErrorMessage, - redirectWithSuccessMessage, -} from "~/models/message.server"; +import { redirectWithErrorMessage, redirectWithSuccessMessage } from "~/models/message.server"; import { findProjectBySlug } from "~/models/project.server"; import { findEnvironmentBySlug } from "~/models/runtimeEnvironment.server"; import { EnvironmentQueuePresenter } from "~/presenters/v3/EnvironmentQueuePresenter.server"; @@ -79,18 +64,12 @@ import { QueueMetricsPresenter, type QueueListMetric, } from "~/presenters/v3/QueueMetricsPresenter.server"; -import { - TimeFilter, - timeFilterFromTo, -} from "~/components/runs/v3/SharedFilters"; +import { TimeFilter, timeFilterFromTo } from "~/components/runs/v3/SharedFilters"; import { useSearchParams } from "~/hooks/useSearchParam"; import { parseFiniteInt } from "~/utils/searchParams"; import { MiniLineChart } from "~/components/metrics/MiniLineChart"; import { buildActivityTimeAxis } from "~/components/primitives/charts/activityTimeAxis"; -import { - Chart, - type ChartConfig, -} from "~/components/primitives/charts/ChartCompound"; +import { Chart, type ChartConfig } from "~/components/primitives/charts/ChartCompound"; import { ChartCard } from "~/components/primitives/charts/ChartCard"; import { ChartSyncProvider } from "~/components/primitives/charts/ChartSyncContext"; import { useZoomToTimeFilter } from "~/hooks/useZoomToTimeFilter"; @@ -167,19 +146,14 @@ export const meta = pageMeta("Queues"); export const loader = async ({ request, params }: LoaderFunctionArgs) => { const userId = await requireUserId(request); - const { organizationSlug, projectParam, envParam } = - EnvironmentParamSchema.parse(params); + const { organizationSlug, projectParam, envParam } = EnvironmentParamSchema.parse(params); const url = new URL(request.url); const { page, query, period, from, to, sort } = SearchParamsSchema.parse( - Object.fromEntries(url.searchParams), + Object.fromEntries(url.searchParams) ); - const project = await findProjectBySlug( - organizationSlug, - projectParam, - userId, - ); + const project = await findProjectBySlug(organizationSlug, projectParam, userId); if (!project) { throw new Response(undefined, { status: 404, @@ -208,7 +182,7 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { : QUEUE_METRICS_RETENTION_DAYS; const defaultPeriod = clampQueueMetricsPeriod( queueMetricsPeriodFromRequest(request), - maxPeriodDays, + maxPeriodDays ); try { @@ -240,7 +214,7 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { try { const presenter = new QueueMetricsPresenter(); const queueNames = queues.queues.map((q) => - q.type === "task" ? `task/${q.name}` : q.name, + q.type === "task" ? `task/${q.name}` : q.name ); const timeRange = clipQueueMetricsWindow( timeFilterFromTo({ @@ -256,7 +230,7 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { to: parseFiniteInt(to), defaultPeriod, }), - maxPeriodDays, + maxPeriodDays ); const queueMetrics = queueNames.length > 0 @@ -283,17 +257,12 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { // Allocation summary (Environment limit + Allocated tiles) is additive; a presenter // failure must not 400 the page, so fail open to null like the metrics block above. - let allocation: Awaited< - ReturnType - > | null = null; + let allocation: Awaited> | null = null; if (queueMetricsUiEnabled) { try { allocation = await new QueueAllocationPresenter().call({ environment }); } catch (error) { - logger.warn( - "Queue allocation summary unavailable, rendering without it", - { error }, - ); + logger.warn("Queue allocation summary unavailable, rendering without it", { error }); } } @@ -311,8 +280,7 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { console.error(error); throw new Response(undefined, { status: 400, - statusText: - "Something went wrong, if this problem persists please contact support.", + statusText: "Something went wrong, if this problem persists please contact support.", }); } }; @@ -323,18 +291,13 @@ export const action = async ({ request, params }: ActionFunctionArgs) => { return redirectWithErrorMessage( `/orgs/${params.organizationSlug}/projects/${params.projectParam}/env/${params.envParam}/queues`, request, - "Wrong method", + "Wrong method" ); } - const { organizationSlug, projectParam, envParam } = - EnvironmentParamSchema.parse(params); + const { organizationSlug, projectParam, envParam } = EnvironmentParamSchema.parse(params); - const project = await findProjectBySlug( - organizationSlug, - projectParam, - userId, - ); + const project = await findProjectBySlug(organizationSlug, projectParam, userId); if (!project) { throw new Response(undefined, { status: 404, @@ -357,11 +320,7 @@ export const action = async ({ request, params }: ActionFunctionArgs) => { const redirectPath = `/orgs/${organizationSlug}/projects/${projectParam}/env/${envParam}/queues${url.search}`; if (environment.archivedAt) { - return redirectWithErrorMessage( - redirectPath, - request, - "This branch is archived", - ); + return redirectWithErrorMessage(redirectPath, request, "This branch is archived"); } // Per-queue actions (pause/resume/override/remove-override) are shared with the queue detail @@ -384,11 +343,7 @@ export const action = async ({ request, params }: ActionFunctionArgs) => { if (!result.success) { return redirectWithErrorMessage(redirectPath, request, result.error); } - return redirectWithSuccessMessage( - redirectPath, - request, - "Environment paused", - ); + return redirectWithSuccessMessage(redirectPath, request, "Environment paused"); } case "environment-resume": { const resumeService = new PauseEnvironmentService(); @@ -396,18 +351,10 @@ export const action = async ({ request, params }: ActionFunctionArgs) => { if (!result.success) { return redirectWithErrorMessage(redirectPath, request, result.error); } - return redirectWithSuccessMessage( - redirectPath, - request, - "Environment resumed", - ); + return redirectWithSuccessMessage(redirectPath, request, "Environment resumed"); } default: - return redirectWithErrorMessage( - redirectPath, - request, - "Something went wrong", - ); + return redirectWithErrorMessage(redirectPath, request, "Something went wrong"); } }; @@ -420,19 +367,14 @@ function getEnvConcurrencyLimitStatus(environment: { burstFactor: number; }) { const limitStatus = - environment.running === - environment.concurrencyLimit * environment.burstFactor + environment.running === environment.concurrencyLimit * environment.burstFactor ? "limit" : environment.running > environment.concurrencyLimit ? "burst" : "within"; const limitClassName = - limitStatus === "burst" - ? "text-warning" - : limitStatus === "limit" - ? "text-error" - : undefined; + limitStatus === "burst" ? "text-warning" : limitStatus === "limit" ? "text-error" : undefined; return { limitStatus, limitClassName }; } @@ -441,11 +383,7 @@ export default function Page() { // Per-org flag decides which whole page renders. Off => the classic Queues page, // byte-for-byte the pre-metrics UI. Each branch is its own component (own hooks). const { queueMetricsUiEnabled } = useTypedLoaderData(); - return queueMetricsUiEnabled ? ( - - ) : ( - - ); + return queueMetricsUiEnabled ? : ; } function QueuesWithMetricsView() { @@ -505,23 +443,20 @@ function QueuesWithMetricsView() { defaultPeriod: QUEUE_LIVE_BLOCKS_PERIOD, fillGaps: false, refreshIntervalMs: 15_000, - }, + } ); const lastLiveBlockRow = liveBlockRows.length > 0 ? liveBlockRows[liveBlockRows.length - 1] : null; // Only trust the gauge while its newest bucket is fresh. A row painted from the hook's cache on // client-side nav-back (responseCache), or a quiet env whose latest bucket is minutes old, must // not override the loader's Redis-exact live values with a stale count. - const lastLiveBucketMs = lastLiveBlockRow - ? tileTimeToMs(lastLiveBlockRow.t) - : NaN; + const lastLiveBucketMs = lastLiveBlockRow ? tileTimeToMs(lastLiveBlockRow.t) : NaN; const liveBlockIsFresh = useIsMetricResponseFresh( responseReceivedAt, lastLiveBucketMs, - LIVE_GAUGE_FRESH_MS, + LIVE_GAUGE_FRESH_MS ); - const freshLiveBlockRow = - lastLiveBlockRow && liveBlockIsFresh ? lastLiveBlockRow : null; + const freshLiveBlockRow = lastLiveBlockRow && liveBlockIsFresh ? lastLiveBlockRow : null; const envQueuedLive = freshLiveBlockRow ? tileNumber(freshLiveBlockRow.env_queued) : environment.queued; @@ -534,8 +469,7 @@ function QueuesWithMetricsView() { const envLimit = environment.concurrencyLimit; const burstLimit = Math.round(envLimit * environment.burstFactor); const allocated = allocation?.allocated ?? 0; - const allocationPct = - envLimit > 0 ? Math.round((allocated / envLimit) * 100) : 0; + const allocationPct = envLimit > 0 ? Math.round((allocated / envLimit) * 100) : 0; const overAllocated = allocated > envLimit; // Running-block tinting (burst/limit) tracks the live running value, not the loader snapshot. @@ -597,11 +531,7 @@ function QueuesWithMetricsView() { paused - ) : undefined - } + suffix={env.paused ? paused : undefined} animate accessory={ @@ -619,9 +549,7 @@ function QueuesWithMetricsView() { /> } - valueClassName={ - env.paused ? "text-warning tabular-nums" : "tabular-nums" - } + valueClassName={env.paused ? "text-warning tabular-nums" : "tabular-nums"} compactThreshold={1000000} /> - Including {envRunningLive - environment.concurrencyLimit}{" "} - burst runs + Including {envRunningLive - environment.concurrencyLimit} burst runs{" "} + ) : limitStatus === "limit" ? ( "At concurrency limit" @@ -677,21 +602,13 @@ function QueuesWithMetricsView() { value={allocation ? allocated : undefined} formattedValue={allocation ? undefined : "–"} valueClassName={cn(allocation && overAllocated && "text-warning")} - suffix={ - allocation - ? `${allocationPct}% of the environment limit` - : undefined - } + suffix={allocation ? `${allocationPct}% of the environment limit` : undefined} suffixClassName="text-text-dimmed" /> 1 - ? `bursts up to ${burstLimit}` - : undefined - } + suffix={environment.burstFactor > 1 ? `bursts up to ${burstLimit}` : undefined} suffixClassName="text-text-dimmed" accessory={ plan ? ( @@ -706,10 +623,7 @@ function QueuesWithMetricsView() { ) : (

- Environment: - uses the environment limit of{" "} - {environment.concurrencyLimit}. + Environment: uses the environment + limit of {environment.concurrencyLimit}.

- User: a limit - you set in your code. + User: a limit you set in your + code.

- Override: a - limit you set here or via the API. + Override: a limit you set here or + via the API.

} @@ -846,8 +758,8 @@ function QueuesWithMetricsView() { disableTooltipHoverableContent tooltip={ <> - How many runs were waiting, over the selected time.{" "} - marks where the queue was throttled. + How many runs were waiting, over the selected time. marks + where the queue was throttled. } > @@ -861,22 +773,16 @@ function QueuesWithMetricsView() { {queueRows.length > 0 ? ( queueRows.map((queue) => { - const limit = - queue.concurrencyLimit ?? environment.concurrencyLimit; + const limit = queue.concurrencyLimit ?? environment.concurrencyLimit; const isAtConcurrencyLimit = queue.running >= limit; const isAtQueueLimit = environment.queueSizeLimit !== null && queue.queued >= environment.queueSizeLimit; const queueFilterableName = queueMetricsKey(queue); const queueMetric = metricsByQueue[queueFilterableName]; - const queueDetailPath = v3QueuePath( - organization, - project, - env, - { - friendlyId: queue.id, - }, - ); + const queueDetailPath = v3QueuePath(organization, project, env, { + friendlyId: queue.id, + }); return ( ) : ( ) @@ -915,9 +821,7 @@ function QueuesWithMetricsView() { trailingContent={ isAtConcurrencyLimit ? ( - } + button={} content="At concurrency limit: this queue is running as many runs as its limit allows; new runs wait in the backlog." className="max-w-[230px]" disableHoverableContent @@ -926,16 +830,11 @@ function QueuesWithMetricsView() { } > - + {queue.name} {queue.paused ? ( - + Paused ) : null} @@ -953,7 +852,7 @@ function QueuesWithMetricsView() { className={cn( "w-[1%]", queue.paused ? "opacity-50" : undefined, - isAtQueueLimit && "text-error", + isAtQueueLimit && "text-error" )} > {queue.queued} @@ -969,10 +868,10 @@ function QueuesWithMetricsView() { (queue.concurrency.combined.running ?? 0) >= Math.min( queue.concurrency.combined.current, - environment.concurrencyLimit, + environment.concurrencyLimit ) ? "text-warning" - : queue.running > 0 && "text-text-bright", + : queue.running > 0 && "text-text-bright" )} > {queue.running} @@ -984,8 +883,7 @@ function QueuesWithMetricsView() { className={cn( "w-[1%]", queue.paused ? "opacity-50" : undefined, - queue.concurrency?.overriddenAt && - "font-medium text-text-bright", + queue.concurrency?.overriddenAt && "font-medium text-text-bright" )} // The combined-limit hint is a tooltip button, so it renders beside the // link (trailing) rather than nested inside the ; the number stays the @@ -1000,7 +898,7 @@ function QueuesWithMetricsView() { ( {Math.min( queue.concurrency.combined.current, - environment.concurrencyLimit, + environment.concurrencyLimit )} ) @@ -1010,11 +908,10 @@ function QueuesWithMetricsView() { Combined limit: at most{" "} {Math.min( queue.concurrency.combined.current, - environment.concurrencyLimit, + environment.concurrencyLimit )}{" "} - runs across all concurrency keys of this - queue. The main limit applies to each key - separately. + runs across all concurrency keys of this queue. The main limit + applies to each key separately. } className="max-w-[260px]" @@ -1026,10 +923,7 @@ function QueuesWithMetricsView() { <> {limit} - ( - {formatOverridePercent( - queue.concurrencyLimitOverridePercent, - )} + ({formatOverridePercent(queue.concurrencyLimitOverridePercent)} %) @@ -1041,10 +935,7 @@ function QueuesWithMetricsView() { to={queueDetailPath} alignment="right" actionClassName="pl-16" - className={cn( - "w-[1%]", - queue.paused ? "opacity-50" : undefined, - )} + className={cn("w-[1%]", queue.paused ? "opacity-50" : undefined)} // Keep the whole row navigable: the override explainer is a tooltip // button, so it renders beside the link (trailing) rather than nested // inside the , and the label itself stays the link. @@ -1054,7 +945,7 @@ function QueuesWithMetricsView() { content={ queue.concurrencyLimitOverridePercent !== null ? `Overridden at ${formatOverridePercent( - queue.concurrencyLimitOverridePercent, + queue.concurrencyLimitOverridePercent )}% of the environment limit.` : `This queue's concurrency limit has been manually overridden to ${limit}.` } @@ -1114,9 +1005,7 @@ function QueuesWithMetricsView() { peakTooltip={ queueMetric && queueMetric.throttledTotal > 0 ? `Peak queued; this queue was throttled ${queueMetric.throttledTotal.toLocaleString()} ${ - queueMetric.throttledTotal === 1 - ? "time" - : "times" + queueMetric.throttledTotal === 1 ? "time" : "times" } in this period` : "Peak queued in this period" } @@ -1124,16 +1013,8 @@ function QueuesWithMetricsView() { - ) - } - hiddenButtons={ - !queue.paused && ( - - ) - } + visibleButtons={queue.paused && } + hiddenButtons={!queue.paused && } popoverContent={ <> {queue.paused ? ( @@ -1186,9 +1067,7 @@ function QueuesWithMetricsView() { /> } @@ -1201,9 +1080,7 @@ function QueuesWithMetricsView() {
- {hasFilters - ? "No queues found matching your filters" - : "No queues found"} + {hasFilters ? "No queues found matching your filters" : "No queues found"}
@@ -1233,8 +1110,7 @@ function EnvironmentPauseResumeButton({ }, [navigation.state]); const isLoading = Boolean( - navigation.formData?.get("action") === - (env.paused ? "environment-resume" : "environment-pause"), + navigation.formData?.get("action") === (env.paused ? "environment-resume" : "environment-pause") ); return ( @@ -1249,9 +1125,7 @@ function EnvironmentPauseResumeButton({ type="button" variant="secondary/small" LeadingIcon={env.paused ? PlayIcon : PauseIcon} - leadingIconClassName={ - env.paused ? "text-success" : "text-warning" - } + leadingIconClassName={env.paused ? "text-success" : "text-warning"} className={ env.paused ? "border-success/60 text-success [&_span]:text-success hover:border-success" @@ -1279,15 +1153,13 @@ function EnvironmentPauseResumeButton({ - - {env.paused ? "Resume environment?" : "Pause environment?"} - + {env.paused ? "Resume environment?" : "Pause environment?"}
{env.paused ? `This will allow runs to be dequeued in ${environmentFullTitle(env)} again.` : `This will pause all runs from being dequeued in ${environmentFullTitle( - env, + env )}. Any executing runs will continue to run.`} setIsOpen(false)}> @@ -1303,13 +1175,7 @@ function EnvironmentPauseResumeButton({ disabled={isLoading} variant={env.paused ? "primary/medium" : "danger/medium"} LeadingIcon={ - isLoading ? ( - - ) : env.paused ? ( - PlayIcon - ) : ( - PauseIcon - ) + isLoading ? : env.paused ? PlayIcon : PauseIcon } shortcut={{ modifiers: ["mod"], key: "enter" }} > @@ -1333,7 +1199,7 @@ function EnvironmentPauseResumeButton({ export function isEnvironmentPauseResumeFormSubmission( formMethod: string | undefined, - formData: FormData | undefined, + formData: FormData | undefined ) { if (!formMethod || !formData) { return false; @@ -1347,13 +1213,7 @@ export function isEnvironmentPauseResumeFormSubmission( } export function QueueFilters() { - return ( - - ); + return ; } type MetricTileRow = Record; @@ -1428,10 +1288,7 @@ function tileTimeToMs(value: number | string | null): number { /** Peak of a series, ignoring the buckets it has nothing to say about. */ function peakOf(points: TilePoint[]): number { - return points.reduce( - (max, p) => (p.value === null ? max : Math.max(max, p.value)), - 0, - ); + return points.reduce((max, p) => (p.value === null ? max : Math.max(max, p.value)), 0); } const SCHEDULING_DELAY_QUERY = `SELECT timeBucket() AS t,\n round(quantilesTDigestMerge(0.5, 0.9, 0.95, 0.99)(wait_quantiles)[3]) AS p95,\n sum(wait_ms_count) AS samples\nFROM env_metrics\nGROUP BY t\nORDER BY t`; @@ -1444,8 +1301,8 @@ const QUEUE_HEADER_TILES: QueueHeaderTile[] = [ label: "Env saturation", description: ( <> - How much of the environment's concurrency is in use. Turns{" "} - above 100%, when it's into burst capacity. + How much of the environment's concurrency is in use. Turns above 100%, + when it's into burst capacity. ), color: "var(--color-queues-chart)", @@ -1454,16 +1311,14 @@ const QUEUE_HEADER_TILES: QueueHeaderTile[] = [ { color: "var(--color-warning)", label: "Over limit" }, ], query: `SELECT timeBucket() AS t,\n max(max_env_running) AS running,\n max(max_env_limit) AS env_limit\nFROM env_metrics\nGROUP BY t\nORDER BY t`, - formatValue: (v) => - v > 100 ? `${v}% — over the environment limit` : `${v}%`, + formatValue: (v) => (v > 100 ? `${v}% — over the environment limit` : `${v}%`), formatAxis: (v) => `${v}%`, derive: (rows) => { const points = rows.map((r) => { const limit = tileNumber(r.env_limit); return { bucket: tileTimeToMs(r.t), - value: - limit > 0 ? Math.round((tileNumber(r.running) / limit) * 100) : 0, + value: limit > 0 ? Math.round((tileNumber(r.running) / limit) * 100) : 0, }; }); return { @@ -1496,8 +1351,8 @@ const QUEUE_HEADER_TILES: QueueHeaderTile[] = [ label: "Scheduling delay p95", description: ( <> - How long runs wait before they start (95% start faster than this). Turns{" "} - above 1 minute. + How long runs wait before they start (95% start faster than this). Turns {" "} + above 1 minute. ), totalTooltip: "The worst p95 in the selected window.", @@ -1526,9 +1381,8 @@ const QUEUE_HEADER_TILES: QueueHeaderTile[] = [ */ derive: (rows) => { const worst = rows.reduce( - (max, r) => - tileNumber(r.samples) > 0 ? Math.max(max, tileNumber(r.p95)) : max, - 0, + (max, r) => (tileNumber(r.samples) > 0 ? Math.max(max, tileNumber(r.p95)) : max), + 0 ); return { total: worst, @@ -1542,8 +1396,7 @@ const QUEUE_HEADER_TILES: QueueHeaderTile[] = [ id: "throttled", label: "Throttled", description: "How often runs were held back by a limit.", - totalTooltip: - "The share of the selected window with at least one blocked dequeue.", + totalTooltip: "The share of the selected window with at least one blocked dequeue.", color: "var(--color-queues-chart)", legend: [{ color: "var(--color-warning)", label: "Throttled" }], query: THROTTLED_QUERY, @@ -1564,8 +1417,7 @@ const QUEUE_HEADER_TILES: QueueHeaderTile[] = [ */ derive: (rows) => { const nonzero = rows.filter((r) => tileNumber(r.throttled) > 0).length; - const pct = - rows.length > 0 ? Math.round((nonzero / rows.length) * 100) : 0; + const pct = rows.length > 0 ? Math.round((nonzero / rows.length) * 100) : 0; return { total: pct, formatTotal: (v) => `${v}% of current period`, @@ -1631,13 +1483,10 @@ function QueueEnvMetricChart({ const derived = tile.derive(rows); const points = derived.points; - const plottedBucketMs = - points.length > 1 ? points[1]!.bucket - points[0]!.bucket : 0; + const plottedBucketMs = points.length > 1 ? points[1]!.bucket - points[0]!.bucket : 0; const floorWidenedBuckets = - plottedBucketMs > 0 && - plottedBucketMs <= HERO_CHART_MIN_BUCKET_SECONDS * 1000; - const readoutQuery = - tile.readout && floorWidenedBuckets ? tile.readout.query : ""; + plottedBucketMs > 0 && plottedBucketMs <= HERO_CHART_MIN_BUCKET_SECONDS * 1000; + const readoutQuery = tile.readout && floorWidenedBuckets ? tile.readout.query : ""; const readoutResult = useMetricResourceQuery(readoutQuery, sharedOptions); const { total, formatTotal, totalClassName } = tile.readout @@ -1656,12 +1505,11 @@ function QueueEnvMetricChart({ const chartConfig = useMemo( () => ({ [tile.id]: { label: tile.label, color: lineColor } }), - [tile.id, tile.label, lineColor], + [tile.id, tile.label, lineColor] ); const { tickFormatter, tooltipLabelFormatter } = buildActivityTimeAxis(data); - const hasData = - data.length > 0 && data.some((p) => Number(p[tile.id] ?? 0) > 0); + const hasData = data.length > 0 && data.some((p) => Number(p[tile.id] ?? 0) > 0); // Peak readout lives in the card title (ChartCard has no dedicated value slot). A zero/empty // total renders no readout at all (skipping "0% peak", "0 peak", "0" and the p95 "–" placeholder) @@ -1696,7 +1544,7 @@ function QueueEnvMetricChart({ {peak} @@ -1710,7 +1558,7 @@ function QueueEnvMetricChart({ {peak} @@ -1757,9 +1605,7 @@ function QueueEnvMetricChart({ thresholdStroke={thresholdStroke} warningOverlay={warningOverlay} xAxisProps={{ tickFormatter }} - yAxisProps={ - tile.formatAxis ? { tickFormatter: tile.formatAxis } : undefined - } + yAxisProps={tile.formatAxis ? { tickFormatter: tile.formatAxis } : undefined} tooltipLabelFormatter={tooltipLabelFormatter} tooltipValueFormatter={tile.formatValue} /> @@ -1791,21 +1637,11 @@ type QueueHealth = { limit: number; }; -type QueueHealthLabel = - | "Paused" - | "At capacity" - | "Backlogged" - | "Active" - | "Idle"; +type QueueHealthLabel = "Paused" | "At capacity" | "Backlogged" | "Active" | "Idle"; // Single source of truth for the queue health decision, shared by the badge and the table's // health-column sort so the sorted order always matches the labels shown. -function queueHealthLabel({ - paused, - running, - queued, - limit, -}: QueueHealth): QueueHealthLabel { +function queueHealthLabel({ paused, running, queued, limit }: QueueHealth): QueueHealthLabel { if (paused) return "Paused"; if (isQueueAtCapacity({ running, queued, limit })) return "At capacity"; if (queued > 0) return "Backlogged"; @@ -1816,10 +1652,8 @@ function queueHealthLabel({ // Tint + colored text, sized like the error status chips (see ErrorStatusBadge). const QUEUE_HEALTH_STYLES: Record = { Paused: "bg-warning/10 text-warning system:bg-warning system:text-white", - "At capacity": - "bg-warning/10 text-warning system:bg-warning system:text-white", - Backlogged: - "bg-blue-500/10 text-blue-500 system:bg-blue-500 system:text-white", + "At capacity": "bg-warning/10 text-warning system:bg-warning system:text-white", + Backlogged: "bg-blue-500/10 text-blue-500 system:bg-blue-500 system:text-white", Active: "bg-success/10 text-success system:bg-success system:text-white", Idle: "bg-charcoal-500/10 text-text-dimmed system:bg-charcoal-500 system:text-white", }; @@ -1830,7 +1664,7 @@ function QueueHealthBadge(health: QueueHealth) { {label} @@ -1852,21 +1686,14 @@ function formatWaitMs(ms: number): string { // Drop a trailing ".00" from whole percentages so "50.00" reads as "50" but "12.50" is preserved. function formatOverridePercent(percent: number): string { - return Number.isInteger(percent) - ? percent.toString() - : percent.toFixed(2).replace(/\.?0+$/, ""); + return Number.isInteger(percent) ? percent.toString() : percent.toFixed(2).replace(/\.?0+$/, ""); } // Classic Queues page, restored verbatim from before the Queue Metrics feature. Rendered // when queueMetricsUiEnabled is off so a gated org sees exactly the pre-metrics UI. function ClassicQueuesView() { - const { - environment, - queues, - pagination, - hasFilters, - autoReloadPollIntervalMs, - } = useTypedLoaderData(); + const { environment, queues, pagination, hasFilters, autoReloadPollIntervalMs } = + useTypedLoaderData(); const organization = useOrganization(); const project = useProject(); @@ -1875,8 +1702,7 @@ function ClassicQueuesView() { useAutoRevalidate({ interval: autoReloadPollIntervalMs, onFocus: true }); - const { limitStatus, limitClassName } = - getEnvConcurrencyLimitStatus(environment); + const { limitStatus, limitClassName } = getEnvConcurrencyLimitStatus(environment); return ( @@ -1901,11 +1727,7 @@ function ClassicQueuesView() { paused - ) : undefined - } + suffix={env.paused ? paused : undefined} animate accessory={
@@ -1927,9 +1749,7 @@ function ClassicQueuesView() { />
} - valueClassName={ - env.paused ? "text-warning tabular-nums" : "tabular-nums" - } + valueClassName={env.paused ? "text-warning tabular-nums" : "tabular-nums"} compactThreshold={1000000} /> - Including{" "} - {environment.running - environment.concurrencyLimit} burst - runs + Including {environment.running - environment.concurrencyLimit} burst runs{" "} + ) : limitStatus === "limit" ? ( "At concurrency limit" @@ -1979,19 +1798,17 @@ function ClassicQueuesView() { - Burst limit{" "} - {environment.burstFactor * environment.concurrencyLimit}{" "} + Burst limit {environment.burstFactor * environment.concurrencyLimit}{" "} ) : undefined } accessory={ plan ? ( - plan?.v3Subscription?.plan?.limits.concurrentRuns - .canExceed ? ( + plan?.v3Subscription?.plan?.limits.concurrentRuns.canExceed ? ( ) : (
@@ -2058,8 +1866,8 @@ function ClassicQueuesView() { className="text-wrap! text-text-dimmed" spacing > - This queue is limited by your environment's - concurrency limit of {environment.concurrencyLimit}. + This queue is limited by your environment's concurrency limit of{" "} + {environment.concurrencyLimit}.
@@ -2069,8 +1877,7 @@ function ClassicQueuesView() { className="text-wrap! text-text-dimmed" spacing > - This queue is limited by a concurrency limit set in - your code. + This queue is limited by a concurrency limit set in your code.
@@ -2080,8 +1887,8 @@ function ClassicQueuesView() { className="text-wrap! text-text-dimmed" spacing > - This queue's concurrency limit has been manually - overridden from the dashboard or API. + This queue's concurrency limit has been manually overridden from the + dashboard or API.
@@ -2097,8 +1904,7 @@ function ClassicQueuesView() { {queues.length > 0 ? ( queues.map((queue) => { - const limit = - queue.concurrencyLimit ?? environment.concurrencyLimit; + const limit = queue.concurrencyLimit ?? environment.concurrencyLimit; const isAtConcurrencyLimit = queue.running >= limit; const isAtQueueLimit = environment.queueSizeLimit !== null && @@ -2114,10 +1920,7 @@ function ClassicQueuesView() { {queue.concurrency?.overriddenAt ? ( + Concurrency limit overridden } @@ -2127,26 +1930,17 @@ function ClassicQueuesView() { /> ) : null} {queue.paused ? ( - + Paused ) : null} {isAtQueueLimit ? ( - + At queue limit ) : null} {isAtConcurrencyLimit ? ( - + At concurrency limit ) : null} @@ -2157,7 +1951,7 @@ function ClassicQueuesView() { className={cn( "w-[1%] pl-16 tabular-nums", queue.paused ? "opacity-50" : undefined, - isAtQueueLimit && "text-error", + isAtQueueLimit && "text-error" )} > {queue.queued} @@ -2171,11 +1965,11 @@ function ClassicQueuesView() { (queue.concurrency.combined.running ?? 0) >= Math.min( queue.concurrency.combined.current, - environment.concurrencyLimit, + environment.concurrencyLimit ) ? "text-warning" : queue.running > 0 && "text-text-bright", - isAtConcurrencyLimit && "text-warning", + isAtConcurrencyLimit && "text-warning" )} > {queue.running} @@ -2185,8 +1979,7 @@ function ClassicQueuesView() { className={cn( "w-[1%] pl-16 tabular-nums", queue.paused ? "opacity-50" : undefined, - queue.concurrency?.overriddenAt && - "font-medium text-text-bright", + queue.concurrency?.overriddenAt && "font-medium text-text-bright" )} > {limit} @@ -2199,7 +1992,7 @@ function ClassicQueuesView() { ( {Math.min( queue.concurrency.combined.current, - environment.concurrencyLimit, + environment.concurrencyLimit )} ) @@ -2209,11 +2002,10 @@ function ClassicQueuesView() { Combined limit: at most{" "} {Math.min( queue.concurrency.combined.current, - environment.concurrencyLimit, + environment.concurrencyLimit )}{" "} - runs across all concurrency keys of this - queue. The main limit applies to each key - separately. + runs across all concurrency keys of this queue. The main limit + applies to each key separately. } className="max-w-[260px]" @@ -2226,8 +2018,7 @@ function ClassicQueuesView() { "w-[1%] pl-16", queue.paused ? "opacity-50" : undefined, isAtConcurrencyLimit && "text-warning", - queue.concurrency?.overriddenAt && - "font-medium text-text-bright", + queue.concurrency?.overriddenAt && "font-medium text-text-bright" )} > {queue.concurrency?.overriddenAt ? ( @@ -2240,16 +2031,8 @@ function ClassicQueuesView() { - ) - } - hiddenButtons={ - !queue.paused && ( - - ) - } + visibleButtons={queue.paused && } + hiddenButtons={!queue.paused && } popoverContent={ <> {queue.paused ? ( @@ -2302,9 +2085,7 @@ function ClassicQueuesView() { /> } @@ -2317,9 +2098,7 @@ function ClassicQueuesView() {
- {hasFilters - ? "No queues found matching your filters" - : "No queues found"} + {hasFilters ? "No queues found matching your filters" : "No queues found"}
@@ -2357,12 +2136,9 @@ const limitTooltip = ( How many runs can execute at once.{" "} - 1 (20) means 1 run - per concurrency key, but at most 20 runs across all keys. Set using{" "} - - combinedConcurrencyLimit - {" "} - in your code. + 1 (20) means 1 run per concurrency key, + but at most 20 runs across all keys. Set using{" "} + combinedConcurrencyLimit in your code. ); From f38b2965a15d2105b64755e419c1ba827b7bab6f Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Mon, 31 Aug 2026 10:47:42 +0100 Subject: [PATCH 13/28] fix(run-engine): sample the combined gauge after batch admission The dequeue gauge ran before the loop, so a queue's first batch from idle and its final drain were never sampled with their runs in flight and the combined chart under-reported. The successful path now re-samples after admissions; early returns keep the entry sample. Also documents that combined.current is the declared cap, clamped at admit time. --- internal-packages/run-engine/src/run-queue/index.ts | 4 ++++ packages/core/src/v3/schemas/queues.ts | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/internal-packages/run-engine/src/run-queue/index.ts b/internal-packages/run-engine/src/run-queue/index.ts index e4dc84e1ddc..84e39c6d23e 100644 --- a/internal-packages/run-engine/src/run-queue/index.ts +++ b/internal-packages/run-engine/src/run-queue/index.ts @@ -5261,6 +5261,10 @@ else redis.call('ZADD', masterQueueKey, earliestIdx[2], ckWildcardName) end +-- Re-sample the gauge so the emitted snapshot includes this batch's admissions; +-- the top-of-script sample only covers the early returns where nothing was admitted. +${QUEUE_METRICS_CK_DEQUEUE_GAUGE_LUA} + return __qmret(results) `, }); diff --git a/packages/core/src/v3/schemas/queues.ts b/packages/core/src/v3/schemas/queues.ts index 9ca282fb33d..cf604c3c298 100644 --- a/packages/core/src/v3/schemas/queues.ts +++ b/packages/core/src/v3/schemas/queues.ts @@ -48,7 +48,7 @@ export const QueueItem = z.object({ /** The combined concurrency cap across all concurrencyKey values of the queue */ combined: z .object({ - /** The effective/current combined concurrency limit (null = no cap) */ + /** The current combined concurrency limit as declared or overridden (null = no cap). Enforcement clamps it to the environment concurrency limit at admit time. */ current: z.number().nullable(), /** The declared combined limit an override reverts to on reset */ base: z.number().nullable(), From b2bc5934eb88530485b03eb68ca36c1d49679624 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Mon, 31 Aug 2026 12:59:17 +0100 Subject: [PATCH 14/28] refactor(run-engine,webapp): drop per-key override admit reads and limit column Removes the override-aware admit and gauge reads from the CK Lua scripts and the per-key limit column, following the removal of runtime per-key overrides from this stack. --- .../route.tsx | 29 +-- apps/webapp/app/v3/querySchemas.ts | 2 +- ...add_queue_metrics_combined_concurrency.sql | 2 +- .../run-engine/src/run-queue/index.ts | 170 ++---------------- 4 files changed, 16 insertions(+), 187 deletions(-) diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues_.$queueParam/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues_.$queueParam/route.tsx index 7b3bddec64a..aa7400452a7 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues_.$queueParam/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues_.$queueParam/route.tsx @@ -407,13 +407,7 @@ export default function Page() { {view === "keys" && hasKeys ? ( <> - + {selectedKey ? ( @@ -966,15 +960,10 @@ function KeyStatsTable({ ids, timeRange, queueName, - defaultKeyLimit, - envLimit, }: { ids: Ids; timeRange: TimeRangeParams; queueName: string; - /** The limit a key inherits when it has no override (the queue's limit, else the env limit). */ - defaultKeyLimit: number; - envLimit: number; }) { const { value, replace, del } = useSearchParams(); const selectedKey = value("key"); @@ -1017,12 +1006,6 @@ function KeyStatsTable({ Key Queued now Running now - - Limit - Oldest wait Started Peak backlog @@ -1031,11 +1014,11 @@ function KeyStatsTable({ {showLoading ? ( - + Loading… ) : rows.length === 0 ? ( - + {search ? `No keys match “${search}”` : "No concurrency keys"} ) : ( @@ -1049,12 +1032,6 @@ function KeyStatsTable({ {row.key} {row.queued.toLocaleString()} {row.running.toLocaleString()} - - {Math.min(row.limitOverride ?? defaultKeyLimit, envLimit).toLocaleString()} - {row.oldestWaitMs === null ? "–" : formatWaitMs(row.oldestWaitMs)} diff --git a/apps/webapp/app/v3/querySchemas.ts b/apps/webapp/app/v3/querySchemas.ts index 05cd7f0b394..3ec1523c83f 100644 --- a/apps/webapp/app/v3/querySchemas.ts +++ b/apps/webapp/app/v3/querySchemas.ts @@ -1426,7 +1426,7 @@ const queueMetricsByKeySchema: TableSchema = { name: "max_limit", ...column("UInt32", { description: - "The effective concurrency limit for this key (the queue limit, or its per-key override). Aggregate with max().", + "The queue concurrency limit that applied to this key in the bucket. Aggregate with max().", fillMode: "carry", }), }, diff --git a/internal-packages/clickhouse/schema/042_add_queue_metrics_combined_concurrency.sql b/internal-packages/clickhouse/schema/042_add_queue_metrics_combined_concurrency.sql index 03cb133799a..57c6b290efc 100644 --- a/internal-packages/clickhouse/schema/042_add_queue_metrics_combined_concurrency.sql +++ b/internal-packages/clickhouse/schema/042_add_queue_metrics_combined_concurrency.sql @@ -4,7 +4,7 @@ -- concurrency-key variants of a queue (the groupConcurrency set), combined_limit the -- RAW stored total cap (0 = none, readers clamp against max_env_limit). Emitted on -- base-queue gauge rows only. Per-key gauge rows now carry the EFFECTIVE per-key --- limit in queue_limit (override-aware), surfaced in the ck tier as max_limit. +-- limit in queue_limit, surfaced in the ck tier as max_limit. ALTER TABLE trigger_dev.queue_metrics_raw_v1 ADD COLUMN IF NOT EXISTS combined_running UInt32 DEFAULT 0, diff --git a/internal-packages/run-engine/src/run-queue/index.ts b/internal-packages/run-engine/src/run-queue/index.ts index 84e39c6d23e..a20473a6080 100644 --- a/internal-packages/run-engine/src/run-queue/index.ts +++ b/internal-packages/run-engine/src/run-queue/index.ts @@ -114,18 +114,12 @@ local function __gateReconcile(setKey, msgKeyPrefix, reconcileKeyPrefix) end end -local function __gatesHaveCapacity(gatesKeyPrefix, msg, messageId, envLimit, msgKeyPrefix, ckOverridesEnabled) +local function __gatesHaveCapacity(gatesKeyPrefix, msg, messageId, envLimit, msgKeyPrefix) if not msg.gates then return true end for _, gate in ipairs(msg.gates) do local base, variant, gateKey = __gateKeys(gatesKeyPrefix, msg, gate) local occupancy = tonumber(redis.call('SCARD', variant .. ':currentConcurrency') or '0') local perKeyLimit = math.min(tonumber(redis.call('GET', base .. ':concurrency') or '1000000'), envLimit) - if ckOverridesEnabled and gateKey and gateKey ~= '' then - local gateOverride = redis.call('HGET', base .. ':ckLimits', string.sub(variant, #gatesKeyPrefix + 1)) - if gateOverride then - perKeyLimit = math.min(tonumber(gateOverride), envLimit) - end - end if occupancy >= perKeyLimit and redis.call('SISMEMBER', variant .. ':currentConcurrency', messageId) == 0 then __gateReconcile(variant .. ':currentConcurrency', msgKeyPrefix, gatesKeyPrefix) return false @@ -227,8 +221,7 @@ const QUEUE_METRICS_CK_ENQUEUE_GAUGE_LUA = createMetricsGaugeComputeLua({ enabledArg: "ARGV[#ARGV] == '1'", queued: "redis.call('ZCARD', queueKey)", running: "redis.call('SCARD', queueCurrentConcurrencyKey)", - queueLimit: - "redis.call('HGET', ckLimitsKey, queueName) or redis.call('GET', queueConcurrencyLimitKey) or '1000000'", + queueLimit: "redis.call('GET', queueConcurrencyLimitKey) or '1000000'", envQueued: "redis.call('ZCARD', envQueueKey)", envRunning: "redis.call('SCARD', envCurrentConcurrencyKey)", envLimit: "redis.call('GET', envConcurrencyLimitKey) or defaultEnvConcurrencyLimit", @@ -275,13 +268,6 @@ export interface RunQueueMetricsEmitter { emitGauge(shardKey: string, fields: Record): void; } -export class RunQueueConcurrencyKeyLimitExceededError extends Error { - constructor(message: string) { - super(message); - this.name = "RunQueueConcurrencyKeyLimitExceededError"; - } -} - export type RunQueueOptions = { name: string; tracer: Tracer; @@ -346,7 +332,6 @@ export type RunQueueOptions = { */ gatesEnabled?: boolean; /** Cap on per-concurrency-key limit overrides stored per queue. Default 1000. */ - maxConcurrencyKeyOverridesPerQueue?: number; workerOptions?: { pollIntervalMs?: number; immediatePollIntervalMs?: number; @@ -460,7 +445,6 @@ export class RunQueue { private queueSelectionStrategy: RunQueueSelectionStrategy; private shardCount: number; private counterTtlSeconds: number; - private maxConcurrencyKeyOverridesPerQueue: number; private abortController: AbortController; private worker: Worker; private workerQueueResolver: WorkerQueueResolver; @@ -471,7 +455,6 @@ export class RunQueue { constructor(public readonly options: RunQueueOptions) { this.shardCount = options.shardCount ?? 2; this.counterTtlSeconds = options.counterTtlSeconds ?? 86400; - this.maxConcurrencyKeyOverridesPerQueue = options.maxConcurrencyKeyOverridesPerQueue ?? 1000; this.retryOptions = options.retryOptions ?? defaultRetrySettings; this.redis = createRedisClient(options.redis, { onError: (error) => { @@ -666,85 +649,6 @@ export class RunQueue { return this.redis.scard(this.keys.queueGroupConcurrencyKey(env, queue)); } - /** - * Sets a per-concurrency-key limit override for a queue. The stored value is the - * raw requested limit; admit paths clamp to the environment limit at read time. - * Throws RunQueueConcurrencyKeyLimitExceededError when a NEW key would push the - * queue past maxConcurrencyKeyOverridesPerQueue (updates to existing keys always - * succeed). - */ - public async updateQueueConcurrencyKeyLimit( - env: MinimalAuthenticatedEnvironment, - queue: string, - concurrencyKey: string, - limit: number - ) { - const result = await this.redis.setQueueConcurrencyKeyLimit( - this.keys.queueCkLimitsKey(env, queue), - this.keys.queueKey(env, queue, concurrencyKey), - String(limit), - String(this.maxConcurrencyKeyOverridesPerQueue) - ); - - if (result === 0) { - throw new RunQueueConcurrencyKeyLimitExceededError( - `Cannot add a concurrency key override to queue ${queue}: the queue already has ${this.maxConcurrencyKeyOverridesPerQueue} overrides` - ); - } - } - - public async removeQueueConcurrencyKeyLimit( - env: MinimalAuthenticatedEnvironment, - queue: string, - concurrencyKey: string - ) { - return this.redis.hdel( - this.keys.queueCkLimitsKey(env, queue), - this.keys.queueKey(env, queue, concurrencyKey) - ); - } - - /** Returns the raw per-concurrency-key limit overrides for a queue, keyed by concurrency key value. */ - public async getQueueConcurrencyKeyLimits( - env: MinimalAuthenticatedEnvironment, - queue: string - ): Promise> { - const raw = await this.redis.hgetall(this.keys.queueCkLimitsKey(env, queue)); - - const limits: Record = {}; - for (const [variantName, value] of Object.entries(raw)) { - const ckIndex = variantName.indexOf(":ck:"); - if (ckIndex === -1) { - continue; - } - limits[variantName.slice(ckIndex + 4)] = Number(value); - } - return limits; - } - - /** Per-key limit overrides for just the given keys: one HMGET, O(keys) not O(overrides). */ - public async getQueueConcurrencyKeyLimitsForKeys( - env: MinimalAuthenticatedEnvironment, - queue: string, - concurrencyKeys: string[] - ): Promise> { - if (concurrencyKeys.length === 0) { - return {}; - } - - const fields = concurrencyKeys.map((key) => this.keys.queueKey(env, queue, key)); - const values = await this.redis.hmget(this.keys.queueCkLimitsKey(env, queue), ...fields); - - const limits: Record = {}; - concurrencyKeys.forEach((key, index) => { - const value = values[index]; - if (value != null) { - limits[key] = Number(value); - } - }); - return limits; - } - /** Batch variant of totalConcurrencyOfQueue: one pipeline of group SCARDs. */ public async totalConcurrencyOfQueues( env: MinimalAuthenticatedEnvironment, @@ -2522,7 +2426,6 @@ export class RunQueue { const totalConcurrencyLimitKey = this.keys.queueTotalConcurrencyLimitKeyFromQueue( message.queue ); - const ckLimitsKey = this.keys.queueCkLimitsKeyFromQueue(message.queue); const totalConcurrencyEnabledArg = this.options.totalConcurrencyEnabled ? "1" : "0"; if (ttlInfo) { @@ -2546,7 +2449,6 @@ export class RunQueue { baseQueueKey, groupConcurrencyKey, totalConcurrencyLimitKey, - ckLimitsKey, // args queueName, messageId, @@ -2586,7 +2488,6 @@ export class RunQueue { baseQueueKey, groupConcurrencyKey, totalConcurrencyLimitKey, - ckLimitsKey, // args queueName, messageId, @@ -2877,7 +2778,6 @@ export class RunQueue { runningCounterKey, this.keys.queueGroupConcurrencyKeyFromQueue(ckWildcardQueue), this.keys.queueTotalConcurrencyLimitKeyFromQueue(ckWildcardQueue), - this.keys.queueCkLimitsKeyFromQueue(ckWildcardQueue), //args ckWildcardQueue, String(Date.now()), @@ -3774,7 +3674,7 @@ if enableFastPath == '1' then local okDecode, decoded = pcall(cjson.decode, messageData) if okDecode and type(decoded) == 'table' and decoded.gates then gateMsg = decoded - gatesAllowFastPath = __gatesHaveCapacity(keyPrefix, decoded, messageId, envLimit, nil, totalConcurrencyEnabled) + gatesAllowFastPath = __gatesHaveCapacity(keyPrefix, decoded, messageId, envLimit, nil) end end @@ -3889,7 +3789,7 @@ if enableFastPath == '1' then local okDecode, decoded = pcall(cjson.decode, messageData) if okDecode and type(decoded) == 'table' and decoded.gates then gateMsg = decoded - gatesAllowFastPath = __gatesHaveCapacity(keyPrefix, decoded, messageId, envLimit, nil, totalConcurrencyEnabled) + gatesAllowFastPath = __gatesHaveCapacity(keyPrefix, decoded, messageId, envLimit, nil) end end @@ -4168,7 +4068,7 @@ return __qmret(0) // *Tracked variants of dequeueMessageFromKey and the ack/nack/dlq/release/clear // scripts. this.redis.defineCommand("enqueueMessageCkTracked", { - numberOfKeys: 18, + numberOfKeys: 17, lua: ` local masterQueueKey = KEYS[1] local queueKey = KEYS[2] @@ -4190,7 +4090,6 @@ local baseQueueKey = KEYS[15] -- Total-cap keys (KEYS 16-17) local groupConcurrencyKey = KEYS[16] local totalConcurrencyLimitKey = KEYS[17] -local ckLimitsKey = KEYS[18] local queueName = ARGV[1] local messageId = ARGV[2] @@ -4229,10 +4128,6 @@ if enableFastPath == '1' then envLimit ) if totalConcurrencyEnabled then - local perKeyOverride = redis.call('HGET', ckLimitsKey, queueName) - if perKeyOverride then - queueLimit = math.min(tonumber(perKeyOverride), envLimit) - end end if queueCurrent < queueLimit then @@ -4256,7 +4151,7 @@ if enableFastPath == '1' then local okDecode, decoded = pcall(cjson.decode, messageData) if okDecode and type(decoded) == 'table' and decoded.gates then gateMsg = decoded - gatesAllowFastPath = __gatesHaveCapacity(keyPrefix, decoded, messageId, envLimit, nil, totalConcurrencyEnabled) + gatesAllowFastPath = __gatesHaveCapacity(keyPrefix, decoded, messageId, envLimit, nil) end end @@ -4345,7 +4240,7 @@ return __qmret(0) }); this.redis.defineCommand("enqueueMessageWithTtlCkTracked", { - numberOfKeys: 19, + numberOfKeys: 18, lua: ` local masterQueueKey = KEYS[1] local queueKey = KEYS[2] @@ -4368,7 +4263,6 @@ local baseQueueKey = KEYS[16] -- Total-cap keys (KEYS 17-18) local groupConcurrencyKey = KEYS[17] local totalConcurrencyLimitKey = KEYS[18] -local ckLimitsKey = KEYS[19] local queueName = ARGV[1] local messageId = ARGV[2] @@ -4409,10 +4303,6 @@ if enableFastPath == '1' then envLimit ) if totalConcurrencyEnabled then - local perKeyOverride = redis.call('HGET', ckLimitsKey, queueName) - if perKeyOverride then - queueLimit = math.min(tonumber(perKeyOverride), envLimit) - end end if queueCurrent < queueLimit then @@ -4434,7 +4324,7 @@ if enableFastPath == '1' then local okDecode, decoded = pcall(cjson.decode, messageData) if okDecode and type(decoded) == 'table' and decoded.gates then gateMsg = decoded - gatesAllowFastPath = __gatesHaveCapacity(keyPrefix, decoded, messageId, envLimit, nil, totalConcurrencyEnabled) + gatesAllowFastPath = __gatesHaveCapacity(keyPrefix, decoded, messageId, envLimit, nil) end end @@ -4837,7 +4727,7 @@ for i = 1, #messages, 2 do else local gatesAllow = true if gatesEnabled then - gatesAllow = __gatesHaveCapacity(keyPrefix, messageData, messageId, envConcurrencyLimit, messageKeyPrefix, totalConcurrencyEnabled) + gatesAllow = __gatesHaveCapacity(keyPrefix, messageData, messageId, envConcurrencyLimit, messageKeyPrefix) end if gatesAllow then @@ -5039,7 +4929,7 @@ return results // (normal dequeue, TTL-expired, or stale-orphan path — all of which were // counted at enqueue time). this.redis.defineCommand("dequeueMessagesFromCkQueueTracked", { - numberOfKeys: 14, + numberOfKeys: 13, lua: ` local ckIndexKey = KEYS[1] local queueConcurrencyLimitKey = KEYS[2] @@ -5054,7 +4944,6 @@ local lengthCounterKey = KEYS[10] local runningCounterKey = KEYS[11] local groupConcurrencyKey = KEYS[12] local totalConcurrencyLimitKey = KEYS[13] -local ckLimitsKey = KEYS[14] local ckWildcardName = ARGV[1] local currentTime = tonumber(ARGV[2]) @@ -5146,12 +5035,6 @@ for _, ckQueueName in ipairs(ckQueues) do local ckCurrentConcurrency = tonumber(redis.call('SCARD', ckConcurrencyKey) or '0') local perKeyLimit = queueConcurrencyLimit - if totalConcurrencyEnabled then - local perKeyOverride = redis.call('HGET', ckLimitsKey, ckQueueName) - if perKeyOverride then - perKeyLimit = math.min(tonumber(perKeyOverride), envConcurrencyLimit) - end - end if ckCurrentConcurrency >= perKeyLimit then -- Back a blocked variant off so it cannot pin the bounded candidate window @@ -5186,7 +5069,7 @@ for _, ckQueueName in ipairs(ckQueues) do else local gatesAllow = true if gatesEnabled then - gatesAllow = __gatesHaveCapacity(keyPrefix, messageData, messageId, envConcurrencyLimit, messageKeyPrefix, totalConcurrencyEnabled) + gatesAllow = __gatesHaveCapacity(keyPrefix, messageData, messageId, envConcurrencyLimit, messageKeyPrefix) end if not gatesAllow then blockedByGates = true @@ -6088,26 +5971,6 @@ __gatesRelease(keyPrefix, redis.call('GET', messageKey), messageId) `, }); - this.redis.defineCommand("setQueueConcurrencyKeyLimit", { - numberOfKeys: 1, - lua: ` -local ckLimitsKey = KEYS[1] - -local fieldName = ARGV[1] -local limit = ARGV[2] -local maxFields = tonumber(ARGV[3]) - -if redis.call('HEXISTS', ckLimitsKey, fieldName) == 0 then - if redis.call('HLEN', ckLimitsKey) >= maxFields then - return 0 - end -end - -redis.call('HSET', ckLimitsKey, fieldName, limit) -return 1 -`, - }); - this.redis.defineCommand("updateEnvironmentConcurrencyLimits", { numberOfKeys: 2, lua: ` @@ -6472,14 +6335,6 @@ declare module "@internal/redis" { callback?: Callback ): Result; - setQueueConcurrencyKeyLimit( - ckLimitsKey: string, - fieldName: string, - limit: string, - maxFields: string, - callback?: Callback - ): Result; - updateEnvironmentConcurrencyLimits( // keys envConcurrencyLimitKey: string, @@ -6666,7 +6521,6 @@ declare module "@internal/redis" { baseQueueKey: string, groupConcurrencyKey: string, totalConcurrencyLimitKey: string, - ckLimitsKey: string, queueName: string, messageId: string, messageData: string, @@ -6704,7 +6558,6 @@ declare module "@internal/redis" { baseQueueKey: string, groupConcurrencyKey: string, totalConcurrencyLimitKey: string, - ckLimitsKey: string, queueName: string, messageId: string, messageData: string, @@ -6739,7 +6592,6 @@ declare module "@internal/redis" { runningCounterKey: string, groupConcurrencyKey: string, totalConcurrencyLimitKey: string, - ckLimitsKey: string, ckWildcardName: string, currentTime: string, defaultEnvConcurrencyLimit: string, From 925d1398a7c56c4b5ae87b83867ce0ffdebc783c Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Mon, 31 Aug 2026 12:59:38 +0100 Subject: [PATCH 15/28] refactor(webapp): concurrency keys resource stops reading per-key overrides --- .../app/routes/resources.queues.concurrency-keys.ts | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/apps/webapp/app/routes/resources.queues.concurrency-keys.ts b/apps/webapp/app/routes/resources.queues.concurrency-keys.ts index 8c590554e51..67c2b9f500a 100644 --- a/apps/webapp/app/routes/resources.queues.concurrency-keys.ts +++ b/apps/webapp/app/routes/resources.queues.concurrency-keys.ts @@ -43,8 +43,6 @@ export type ConcurrencyKeyRow = { peakBacklog: number; peakRunning: number; meanWaitMs: number; - /** Per-key concurrency limit override, when one is set for this key (null = inherits the queue limit). */ - limitOverride: number | null; }; export type ConcurrencyKeysResponse = @@ -153,11 +151,8 @@ export const action = async ({ request }: ActionFunctionArgs) => { const total = rankingRows?.[0]?.ranked_total ?? 0; const keys = (rankingRows ?? []).map((r) => r.concurrency_key); - // Enrich just this page's keys with live "now" counts and any per-key limit overrides from Redis. - const [live, keyLimitOverrides] = await Promise.all([ - engine.concurrencyKeyLiveStats(environment, queueName, keys), - engine.runQueue.getQueueConcurrencyKeyLimitsForKeys(environment, queueName, keys), - ]); + // Enrich just this page's keys with live "now" counts from Redis. + const live = await engine.concurrencyKeyLiveStats(environment, queueName, keys); const loadedAt = Date.now(); const rows: ConcurrencyKeyRow[] = (rankingRows ?? []).map((r) => { @@ -173,7 +168,6 @@ export const action = async ({ request }: ActionFunctionArgs) => { peakBacklog: r.peak_backlog, peakRunning: r.peak_running, meanWaitMs: r.mean_wait_ms, - limitOverride: keyLimitOverrides[r.concurrency_key] ?? null, }; }); From eeeca71ae0daa0da8a53c13a105074e65cc1cbba Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Mon, 31 Aug 2026 13:11:29 +0100 Subject: [PATCH 16/28] refactor(run-engine): drop the now-unreferenced ck-limits key builders --- internal-packages/run-engine/src/run-queue/keyProducer.ts | 8 -------- internal-packages/run-engine/src/run-queue/types.ts | 3 --- 2 files changed, 11 deletions(-) diff --git a/internal-packages/run-engine/src/run-queue/keyProducer.ts b/internal-packages/run-engine/src/run-queue/keyProducer.ts index 120e04f8c38..98028f5af7b 100644 --- a/internal-packages/run-engine/src/run-queue/keyProducer.ts +++ b/internal-packages/run-engine/src/run-queue/keyProducer.ts @@ -366,14 +366,6 @@ export class RunQueueFullKeyProducer implements RunQueueKeyProducer { return `${this.baseQueueKeyFromQueue(queue)}:${constants.TOTAL_CONCURRENCY_LIMIT_PART}`; } - queueCkLimitsKey(env: RunQueueKeyProducerEnvironment, queue: string): string { - return `${this.queueKey(env, queue)}:ckLimits`; - } - - queueCkLimitsKeyFromQueue(queue: string): string { - return `${this.baseQueueKeyFromQueue(queue)}:ckLimits`; - } - isCkWildcard(queue: string): boolean { return queue.endsWith(":ck:*"); } diff --git a/internal-packages/run-engine/src/run-queue/types.ts b/internal-packages/run-engine/src/run-queue/types.ts index 2961b642314..2cbfe40c775 100644 --- a/internal-packages/run-engine/src/run-queue/types.ts +++ b/internal-packages/run-engine/src/run-queue/types.ts @@ -111,9 +111,6 @@ export interface RunQueueKeyProducer { queueTotalConcurrencyLimitKey(env: RunQueueKeyProducerEnvironment, queue: string): string; queueTotalConcurrencyLimitKeyFromQueue(queue: string): string; - queueCkLimitsKey(env: RunQueueKeyProducerEnvironment, queue: string): string; - queueCkLimitsKeyFromQueue(queue: string): string; - //env oncurrency envCurrentConcurrencyKey(env: EnvDescriptor): string; envCurrentConcurrencyKey(env: RunQueueKeyProducerEnvironment): string; From f39119756b21455a94fd7322cb4cd3b9646205a6 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Mon, 31 Aug 2026 13:44:36 +0100 Subject: [PATCH 17/28] chore: lift the run-queue knip ignore The class the ignore covered is deleted at this level, so the merged result carries no dead-code exemption. --- knip.json | 3 --- 1 file changed, 3 deletions(-) diff --git a/knip.json b/knip.json index c8e7f4027dd..84456756ca1 100644 --- a/knip.json +++ b/knip.json @@ -27,9 +27,6 @@ ], "ignoreDependencies": ["@sentry/cli", "assert", "util"] }, - "internal-packages/run-engine": { - "ignore": ["src/run-queue/index.ts"] - }, "internal-packages/dashboard-agent": { "entry": ["trigger.config.ts", "src/investigation-sweep.ts", "src/maintenance.ts"], "ignoreBinaries": ["rg"] From d9ad08560c149fcdc911d7f4a36f76c7e6fb8aa4 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Mon, 31 Aug 2026 14:03:19 +0100 Subject: [PATCH 18/28] fix(run-engine,webapp,clickhouse): review fixes for the metrics tier The plain dequeue gauge re-samples after admissions like the keyed one, the repair path clears a keyed run's variant and group slots by concurrency key, pause responses include the combined limit, stale wording and a leftover changeset from before the rename are cleaned up, and two empty flag blocks are removed from the fast-path scripts. --- apps/webapp/app/v3/querySchemas.ts | 2 +- .../app/v3/services/pauseQueue.server.ts | 3 ++ ...add_queue_metrics_combined_concurrency.sql | 5 +-- .../run-engine/src/engine/index.ts | 2 ++ .../run-engine/src/run-queue/index.ts | 34 +++++++++++++------ 5 files changed, 32 insertions(+), 14 deletions(-) diff --git a/apps/webapp/app/v3/querySchemas.ts b/apps/webapp/app/v3/querySchemas.ts index 3ec1523c83f..b33b205cc3c 100644 --- a/apps/webapp/app/v3/querySchemas.ts +++ b/apps/webapp/app/v3/querySchemas.ts @@ -1426,7 +1426,7 @@ const queueMetricsByKeySchema: TableSchema = { name: "max_limit", ...column("UInt32", { description: - "The queue concurrency limit that applied to this key in the bucket. Aggregate with max().", + "The queue concurrency limit that applied to this key in the bucket (1000000 = no explicit limit). Aggregate with max().", fillMode: "carry", }), }, diff --git a/apps/webapp/app/v3/services/pauseQueue.server.ts b/apps/webapp/app/v3/services/pauseQueue.server.ts index aa3e21f9727..97cc598daf0 100644 --- a/apps/webapp/app/v3/services/pauseQueue.server.ts +++ b/apps/webapp/app/v3/services/pauseQueue.server.ts @@ -92,6 +92,9 @@ export class PauseQueueService extends BaseService { concurrencyLimitOverriddenAt: updatedQueue.concurrencyLimitOverriddenAt ?? null, concurrencyLimitOverriddenBy: queue.concurrencyLimitOverriddenBy ?? null, paused: updatedQueue.paused, + totalConcurrencyLimit: updatedQueue.totalConcurrencyLimit ?? null, + totalConcurrencyLimitBase: updatedQueue.totalConcurrencyLimitBase ?? null, + totalConcurrencyLimitOverriddenAt: updatedQueue.totalConcurrencyLimitOverriddenAt ?? null, }), }; } catch (error) { diff --git a/internal-packages/clickhouse/schema/042_add_queue_metrics_combined_concurrency.sql b/internal-packages/clickhouse/schema/042_add_queue_metrics_combined_concurrency.sql index 57c6b290efc..dd0c4c53b34 100644 --- a/internal-packages/clickhouse/schema/042_add_queue_metrics_combined_concurrency.sql +++ b/internal-packages/clickhouse/schema/042_add_queue_metrics_combined_concurrency.sql @@ -3,8 +3,9 @@ -- Total-concurrency gauges: combined_running is the in-flight count across ALL -- concurrency-key variants of a queue (the groupConcurrency set), combined_limit the -- RAW stored total cap (0 = none, readers clamp against max_env_limit). Emitted on --- base-queue gauge rows only. Per-key gauge rows now carry the EFFECTIVE per-key --- limit in queue_limit, surfaced in the ck tier as max_limit. +-- base-queue gauge rows only. Per-key gauge rows carry the queue concurrency +-- limit that applied in queue_limit, surfaced in the ck tier as max_limit +-- (1000000 = no explicit limit). ALTER TABLE trigger_dev.queue_metrics_raw_v1 ADD COLUMN IF NOT EXISTS combined_running UInt32 DEFAULT 0, diff --git a/internal-packages/run-engine/src/engine/index.ts b/internal-packages/run-engine/src/engine/index.ts index 504673bf052..bdd4c30adb7 100644 --- a/internal-packages/run-engine/src/engine/index.ts +++ b/internal-packages/run-engine/src/engine/index.ts @@ -2993,6 +2993,7 @@ export class RunEngine { { select: { queue: true, + concurrencyKey: true, }, }, this.prisma @@ -3014,6 +3015,7 @@ export class RunEngine { runId, orgId: latestSnapshot.organizationId, queue: taskRun.queue, + concurrencyKey: taskRun.concurrencyKey ?? undefined, env: { id: latestSnapshot.environmentId, type: latestSnapshot.environmentType, diff --git a/internal-packages/run-engine/src/run-queue/index.ts b/internal-packages/run-engine/src/run-queue/index.ts index a20473a6080..f451d8d9be6 100644 --- a/internal-packages/run-engine/src/run-queue/index.ts +++ b/internal-packages/run-engine/src/run-queue/index.ts @@ -331,7 +331,6 @@ export type RunQueueOptions = { * the total cap covering releases from builds without the mirror. */ gatesEnabled?: boolean; - /** Cap on per-concurrency-key limit overrides stored per queue. Default 1000. */ workerOptions?: { pollIntervalMs?: number; immediatePollIntervalMs?: number; @@ -1537,6 +1536,7 @@ export class RunQueue { runId: string; orgId: string; queue: string; + concurrencyKey?: string; env: RunQueueKeyProducerEnvironment; }) { return this.#callClearMessageFromConcurrencySets(params); @@ -3068,18 +3068,30 @@ export class RunQueue { runId, orgId, queue, + concurrencyKey, env, }: { runId: string; orgId: string; queue: string; + concurrencyKey?: string; env: RunQueueKeyProducerEnvironment; }) { const messageId = runId; const messageKey = this.keys.messageKey(orgId, messageId); - const queueCurrentConcurrencyKey = this.keys.queueCurrentConcurrencyKey(env, queue); + /** + * Callers pass the bare TaskRun queue name plus its concurrencyKey; the run's + * slots live on the ck variant, and the tracked clear additionally mirrors the + * group set and counters that only keyed queues maintain. + */ + const fullQueue = concurrencyKey ? this.keys.queueKey(env, queue, concurrencyKey) : queue; + const queueCurrentConcurrencyKey = this.keys.queueCurrentConcurrencyKey( + env, + queue, + concurrencyKey + ); const envCurrentConcurrencyKey = this.keys.envCurrentConcurrencyKey(env); - const queueCurrentDequeuedKey = this.keys.queueCurrentDequeuedKey(env, queue); + const queueCurrentDequeuedKey = this.keys.queueCurrentDequeuedKey(env, queue, concurrencyKey); const envCurrentDequeuedKey = this.keys.envCurrentDequeuedKey(env); this.logger.debug("Calling clearMessageFromConcurrencySets", { @@ -3094,15 +3106,15 @@ export class RunQueue { service: this.name, }); - if (queue.includes(":ck:")) { + if (fullQueue.includes(":ck:")) { return this.redis.clearMessageFromConcurrencySetsTracked( queueCurrentConcurrencyKey, envCurrentConcurrencyKey, queueCurrentDequeuedKey, envCurrentDequeuedKey, - this.keys.queueRunningCounterKeyFromQueue(queue), - this.keys.ckIndexKeyFromQueue(queue), - this.keys.queueGroupConcurrencyKeyFromQueue(queue), + this.keys.queueRunningCounterKeyFromQueue(fullQueue), + this.keys.ckIndexKeyFromQueue(fullQueue), + this.keys.queueGroupConcurrencyKeyFromQueue(fullQueue), messageKey, messageId, this.options.redis.keyPrefix ?? "", @@ -4127,8 +4139,6 @@ if enableFastPath == '1' then tonumber(redis.call('GET', queueConcurrencyLimitKey) or '1000000'), envLimit ) - if totalConcurrencyEnabled then - end if queueCurrent < queueLimit then -- Total-cap gate: a fast-path admit consumes a group slot, so it must @@ -4302,8 +4312,6 @@ if enableFastPath == '1' then tonumber(redis.call('GET', queueConcurrencyLimitKey) or '1000000'), envLimit ) - if totalConcurrencyEnabled then - end if queueCurrent < queueLimit then -- Total-cap gate: see enqueueMessageCkTracked. @@ -4769,6 +4777,10 @@ else redis.call('ZADD', masterQueueKey, earliestMessage[2], queueName) end +-- Re-sample the gauge so the emitted snapshot includes this batch's admissions; +-- the top-of-script sample only covers the early returns where nothing was admitted. +${QUEUE_METRICS_GAUGE_LUA} + -- Return results as a flat array: [messageId1, messageScore1, messagePayload1, messageId2, messageScore2, messagePayload2, ...] return __qmret(results) `, From 35972f550f4a210272eb723bb7ee6ced71b86e0c Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Mon, 31 Aug 2026 14:03:43 +0100 Subject: [PATCH 19/28] chore: drop the pre-rename changeset superseded by the combined one --- .changeset/queue-total-concurrency-stats.md | 5 ----- 1 file changed, 5 deletions(-) delete mode 100644 .changeset/queue-total-concurrency-stats.md diff --git a/.changeset/queue-total-concurrency-stats.md b/.changeset/queue-total-concurrency-stats.md deleted file mode 100644 index a70da24d1fb..00000000000 --- a/.changeset/queue-total-concurrency-stats.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -"@trigger.dev/core": patch ---- - -Queue retrieve and list API responses now report total concurrency usage. When a queue has a `totalConcurrencyLimit`, `concurrency.total` includes the effective cap, the declared base, any active override, and how many runs are in flight across all concurrency keys. From 99338476060a2e4272730ef8a9d77667c7f61417 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Mon, 31 Aug 2026 17:25:54 +0100 Subject: [PATCH 20/28] perf(run-engine): share the combined-limit read between the admit gate and gauges The CK enqueue and dequeue scripts read the combined concurrency limit key once for admission and again for the metrics gauge tail. A per-call memo makes whichever runs first do the single GET; limits cannot change mid-script, so the value stays exact. Group cardinality remains a fresh read because gauges must reflect post-admission state. --- .../run-engine/src/run-queue/index.ts | 34 +++++++++++++++---- 1 file changed, 28 insertions(+), 6 deletions(-) diff --git a/internal-packages/run-engine/src/run-queue/index.ts b/internal-packages/run-engine/src/run-queue/index.ts index f451d8d9be6..6baca919430 100644 --- a/internal-packages/run-engine/src/run-queue/index.ts +++ b/internal-packages/run-engine/src/run-queue/index.ts @@ -209,11 +209,12 @@ const QUEUE_METRICS_CK_GAUGE_EXTRAS = { }; // Total-concurrency tail (gauge[10]/gauge[11]): live group cardinality + raw stored cap. -// Requires groupConcurrencyKey/totalConcurrencyLimitKey locals; the CK scripts that actually -// run (the Tracked variants and the CK dequeue) all declare them for the total-cap gate. +// Requires the groupConcurrencyKey local and the __totalLimitRaw memo (one GET shared with +// the total-cap gate); the CK scripts that run this (the Tracked variants and the CK +// dequeue) declare both. The group SCARD stays a fresh read: it must be post-admission. const QUEUE_METRICS_TOTAL_GAUGE_EXTRAS = { totalRunning: "redis.call('SCARD', groupConcurrencyKey)", - totalLimit: "redis.call('GET', totalConcurrencyLimitKey) or '0'", + totalLimit: "__totalLimitRaw() or '0'", }; // CK enqueue variants of the two gauges above, extended with the CK-health tail. @@ -4102,6 +4103,13 @@ local baseQueueKey = KEYS[15] -- Total-cap keys (KEYS 16-17) local groupConcurrencyKey = KEYS[16] local totalConcurrencyLimitKey = KEYS[17] +local __rawTotalLimit = nil +local function __totalLimitRaw() + if __rawTotalLimit == nil then + __rawTotalLimit = redis.call('GET', totalConcurrencyLimitKey) or false + end + return __rawTotalLimit +end local queueName = ARGV[1] local messageId = ARGV[2] @@ -4146,7 +4154,7 @@ if enableFastPath == '1' then -- slow path (the message queues; the dequeue gate holds it). local totalAllowsFastPath = true if totalConcurrencyEnabled then - local rawTotalLimit = redis.call('GET', totalConcurrencyLimitKey) + local rawTotalLimit = __totalLimitRaw() if rawTotalLimit then local totalLimit = math.min(tonumber(rawTotalLimit), envLimit) if tonumber(redis.call('SCARD', groupConcurrencyKey) or '0') >= totalLimit then @@ -4273,6 +4281,13 @@ local baseQueueKey = KEYS[16] -- Total-cap keys (KEYS 17-18) local groupConcurrencyKey = KEYS[17] local totalConcurrencyLimitKey = KEYS[18] +local __rawTotalLimit = nil +local function __totalLimitRaw() + if __rawTotalLimit == nil then + __rawTotalLimit = redis.call('GET', totalConcurrencyLimitKey) or false + end + return __rawTotalLimit +end local queueName = ARGV[1] local messageId = ARGV[2] @@ -4317,7 +4332,7 @@ if enableFastPath == '1' then -- Total-cap gate: see enqueueMessageCkTracked. local totalAllowsFastPath = true if totalConcurrencyEnabled then - local rawTotalLimit = redis.call('GET', totalConcurrencyLimitKey) + local rawTotalLimit = __totalLimitRaw() if rawTotalLimit then local totalLimit = math.min(tonumber(rawTotalLimit), envLimit) if tonumber(redis.call('SCARD', groupConcurrencyKey) or '0') >= totalLimit then @@ -4956,6 +4971,13 @@ local lengthCounterKey = KEYS[10] local runningCounterKey = KEYS[11] local groupConcurrencyKey = KEYS[12] local totalConcurrencyLimitKey = KEYS[13] +local __rawTotalLimit = nil +local function __totalLimitRaw() + if __rawTotalLimit == nil then + __rawTotalLimit = redis.call('GET', totalConcurrencyLimitKey) or false + end + return __rawTotalLimit +end local ckWildcardName = ARGV[1] local currentTime = tonumber(ARGV[2]) @@ -4998,7 +5020,7 @@ local actualMaxCount = math.min(maxCount, envAvailableCapacity) -- behind, and blocking on it would deadlock the run against itself). local totalHeadroom = nil if totalConcurrencyEnabled then - local rawTotalLimit = redis.call('GET', totalConcurrencyLimitKey) + local rawTotalLimit = __totalLimitRaw() if rawTotalLimit then local totalConcurrencyLimit = math.min(tonumber(rawTotalLimit), envConcurrencyLimit) local groupCurrentConcurrency = tonumber(redis.call('SCARD', groupConcurrencyKey) or '0') From e2153e2f4d99dc56b95d69128000d7d263713589 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Mon, 31 Aug 2026 17:52:44 +0100 Subject: [PATCH 21/28] perf(run-engine): dequeue gauges sample once, at return The gauge slot is single-valued and the last write wins, so on the success path the post-admission resample made the entry sample pure waste. A return wrapper computes the gauge exactly once per call at exit, keeping every emitted value identical while dropping the discarded reads (about six per plain dequeue, ten per keyed dequeue at full sampling). --- .../run-engine/src/run-queue/index.ts | 31 +++++++++++++------ 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/internal-packages/run-engine/src/run-queue/index.ts b/internal-packages/run-engine/src/run-queue/index.ts index 6baca919430..fe2b3f28efe 100644 --- a/internal-packages/run-engine/src/run-queue/index.ts +++ b/internal-packages/run-engine/src/run-queue/index.ts @@ -173,8 +173,9 @@ local __qm_g = false local function __qmret(r) if r == nil then r = false end return {r, __qm_g} end`; // Fresh-read gauge for splice points with no reusable locals: enqueue slow-path (before -// return 0) and the base dequeue top. Gated on the last ARGV so it is inert unless the -// caller opts in. CK queues emit per-subqueue depth (queue_name aggregates via the MV). +// return 0) and the base dequeue's sample-at-return wrapper. Gated on the last ARGV so it +// is inert unless the caller opts in. CK queues emit per-subqueue depth (queue_name +// aggregates via the MV). const QUEUE_METRICS_GAUGE_LUA = createMetricsGaugeComputeLua({ enabledArg: "ARGV[#ARGV] == '1'", queued: "redis.call('ZCARD', queueKey)", @@ -4680,7 +4681,16 @@ local gatesEnabled = ARGV[7] == '1' local totalConcurrencyEnabled = ARGV[8] == '1' ${QUEUE_METRICS_GAUGE_PRELUDE} ${QUEUE_GATES_LUA_HELPERS} +-- Sample-at-return: the gauge is computed once, by the return wrapper, so every +-- exit emits the state as of that exit (post-admission on the success path) and +-- no path pays for a sample that a later one would overwrite. +local function __qmsample() ${QUEUE_METRICS_GAUGE_LUA} +end +do + local __qmret_inner = __qmret + __qmret = function(r) __qmsample() return __qmret_inner(r) end +end -- Check current env concurrency against the limit local envCurrentConcurrency = tonumber(redis.call('SCARD', envCurrentConcurrencyKey) or '0') @@ -4792,10 +4802,6 @@ else redis.call('ZADD', masterQueueKey, earliestMessage[2], queueName) end --- Re-sample the gauge so the emitted snapshot includes this batch's admissions; --- the top-of-script sample only covers the early returns where nothing was admitted. -${QUEUE_METRICS_GAUGE_LUA} - -- Return results as a flat array: [messageId1, messageScore1, messagePayload1, messageId2, messageScore2, messagePayload2, ...] return __qmret(results) `, @@ -4989,7 +4995,16 @@ local totalConcurrencyEnabled = ARGV[7] == '1' local gatesEnabled = ARGV[8] == '1' ${QUEUE_METRICS_GAUGE_PRELUDE} ${QUEUE_GATES_LUA_HELPERS} +-- Sample-at-return: the gauge is computed once, by the return wrapper, so every +-- exit emits the state as of that exit (post-admission on the success path) and +-- no path pays for a sample that a later one would overwrite. +local function __qmsample() ${QUEUE_METRICS_CK_DEQUEUE_GAUGE_LUA} +end +do + local __qmret_inner = __qmret + __qmret = function(r) __qmsample() return __qmret_inner(r) end +end local function decrLengthCounter() if tonumber(redis.call('GET', lengthCounterKey) or '0') > 0 then @@ -5178,10 +5193,6 @@ else redis.call('ZADD', masterQueueKey, earliestIdx[2], ckWildcardName) end --- Re-sample the gauge so the emitted snapshot includes this batch's admissions; --- the top-of-script sample only covers the early returns where nothing was admitted. -${QUEUE_METRICS_CK_DEQUEUE_GAUGE_LUA} - return __qmret(results) `, }); From 1beeb54975830ae375c9e2bb054f60fd96163d65 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Mon, 31 Aug 2026 18:07:55 +0100 Subject: [PATCH 22/28] test(run-engine): pin dequeue-emitted gauges so a sampling regression fails the suite The gauge assertions were all satisfiable by enqueue-emitted gauges, so breaking the dequeue scripts' sample-at-return wrapper left the suite green. The base test now requires the post-admission reading (running 1, queued 0) and the CK test requires the wildcard aggregate only the CK dequeue emits. Verified by mutation: disabling the wrapper fails both. --- .../run-engine/src/run-queue/metrics.test.ts | 22 ++++++++++++++----- 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/internal-packages/run-engine/src/run-queue/metrics.test.ts b/internal-packages/run-engine/src/run-queue/metrics.test.ts index ebfc295470e..d16b8048d95 100644 --- a/internal-packages/run-engine/src/run-queue/metrics.test.ts +++ b/internal-packages/run-engine/src/run-queue/metrics.test.ts @@ -123,7 +123,10 @@ describe("RunQueue queue-metrics emission", () => { const entries = await waitForEntries(redis, definition, (es) => { const seen = es.map((e) => e.fields.op); - return ["enqueue", "gauge", "started", "ack"].every((o) => seen.includes(o)); + if (!["enqueue", "gauge", "started", "ack"].every((o) => seen.includes(o))) return false; + return es.some( + (e) => e.fields.op === "gauge" && e.fields.cc === "1" && e.fields.ql === "0" + ); }); const ops = entries.map((e) => e.fields.op); expect(ops).toContain("enqueue"); @@ -141,6 +144,14 @@ describe("RunQueue queue-metrics emission", () => { expect(gauge!.fields.ckq).toBeUndefined(); expect(gauge!.fields.ckw).toBeUndefined(); + // Pins the dequeue script's sample-at-return wrapper: only the dequeue emits the + // post-admission reading (running 1, queued 0); the enqueue gauge sees the inverse. + const dequeueGauge = entries.find( + (e) => e.fields.op === "gauge" && e.fields.cc === "1" && e.fields.ql === "0" + ); + assertGauge(dequeueGauge); + expect(dequeueGauge!.fields.q).toContain("task/my-task"); + // The first counter emission also seeds a cum=0 baseline (no wait); the real reading // carries wait. Pick the reading (cum > 0). const started = entries.find((e) => e.fields.op === "started" && Number(e.fields.cum) > 0); @@ -283,14 +294,13 @@ describe("RunQueue queue-metrics emission", () => { expect(dequeued?.messageId).toBe(message.runId); const entries = await waitForEntries(redis, definition, (es) => - es.some( - (e) => e.fields.op === "gauge" && e.fields.q.includes(":ck:") && e.fields.thr === "0" - ) + es.some((e) => e.fields.op === "gauge" && e.fields.q.includes(":ck:*")) ); const gauges = entries.filter((e) => e.fields.op === "gauge"); expect(gauges.length).toBeGreaterThan(0); - // The aggregate CK dequeue gauge targets the CK wildcard and never sets thr. - const aggregate = gauges.find((e) => e.fields.q.includes(":ck:") && e.fields.thr === "0"); + // The aggregate gauge targets the CK wildcard and only the CK dequeue script emits + // it, so this pins that script's sample-at-return wrapper. + const aggregate = gauges.find((e) => e.fields.q.includes(":ck:*")); assertGauge(aggregate); expect(Number(aggregate!.fields.ql)).toBeGreaterThanOrEqual(0); expect(Number(aggregate!.fields.cc)).toBeGreaterThanOrEqual(0); From 10941acd4ff34ca335c8f66c7ef97bd4d185a4fc Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Mon, 31 Aug 2026 18:33:33 +0100 Subject: [PATCH 23/28] test(run-engine): wait for the metrics emitter connection before exercising it The emitter drops emissions until its Redis client is ready, and the tests enqueued immediately after constructing it, so the first counter entry was occasionally lost and the suite flaked roughly two runs in eighty. Awaiting readiness makes every emission land. --- internal-packages/run-engine/src/run-queue/metrics.test.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/internal-packages/run-engine/src/run-queue/metrics.test.ts b/internal-packages/run-engine/src/run-queue/metrics.test.ts index d16b8048d95..efd00cd6aa1 100644 --- a/internal-packages/run-engine/src/run-queue/metrics.test.ts +++ b/internal-packages/run-engine/src/run-queue/metrics.test.ts @@ -81,6 +81,7 @@ describe("RunQueue queue-metrics emission", () => { definition, flag: { enabled: () => true }, }); + await emitter.waitUntilReady(); const queue = new RunQueue({ name: "rq", @@ -183,6 +184,7 @@ describe("RunQueue queue-metrics emission", () => { definition, flag: { enabled: () => true }, }); + await emitter.waitUntilReady(); const queue = new RunQueue({ name: "rq", tracer: trace.getTracer("rq"), @@ -255,6 +257,7 @@ describe("RunQueue queue-metrics emission", () => { maxLen: 1000, }; const emitter = new MetricsStreamEmitter({ redis, definition, flag: { enabled: () => true } }); + await emitter.waitUntilReady(); const queue = new RunQueue({ name: "rq", tracer: trace.getTracer("rq"), @@ -354,6 +357,7 @@ describe("RunQueue queue-metrics emission", () => { flag: { enabled: () => true }, gaugeSampleRate: 0, }); + await emitter.waitUntilReady(); const queue = new RunQueue({ name: "rq", tracer: trace.getTracer("rq"), From 15193400e0e34d3475cbdbb4fa7be3b5d76855cf Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Mon, 31 Aug 2026 18:49:08 +0100 Subject: [PATCH 24/28] test(run-engine,metrics-pipeline): bound emitter-readiness waits and cover the consumer round-trip An unreachable Redis leaves waitUntilReady pending forever, so the bounded wait fails fast with a descriptive error instead of burning the test timeout. The consumer round-trip test gains the same readiness wait its gauge sibling already had, closing the remaining first-emission drop flake. --- .../metrics-pipeline/src/consumer.test.ts | 1 + .../run-engine/src/run-queue/metrics.test.ts | 19 +++++++++++++++---- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/internal-packages/metrics-pipeline/src/consumer.test.ts b/internal-packages/metrics-pipeline/src/consumer.test.ts index 672fa426999..cb111ee1534 100644 --- a/internal-packages/metrics-pipeline/src/consumer.test.ts +++ b/internal-packages/metrics-pipeline/src/consumer.test.ts @@ -43,6 +43,7 @@ redisTest( }); await consumer.start(); + await emitter.waitUntilReady(); emitter.emit("queueA", { op: "enqueue", q: "queueA" }); emitter.emit("queueB", { op: "started", q: "queueB", wait: 42 }); diff --git a/internal-packages/run-engine/src/run-queue/metrics.test.ts b/internal-packages/run-engine/src/run-queue/metrics.test.ts index efd00cd6aa1..8b3872ddc2f 100644 --- a/internal-packages/run-engine/src/run-queue/metrics.test.ts +++ b/internal-packages/run-engine/src/run-queue/metrics.test.ts @@ -24,6 +24,17 @@ const authenticatedEnvDev = { organization: { id: "o1234" }, }; +// A dead Redis leaves waitUntilReady() pending forever (the client retries +// indefinitely), which would burn the whole test timeout with no diagnostic. +async function emitterReady(emitter: MetricsStreamEmitter) { + await Promise.race([ + emitter.waitUntilReady(), + setTimeout(15_000).then(() => { + throw new Error("metrics emitter Redis connection never became ready"); + }), + ]); +} + async function readAllEntries( redisOptions: { host: string; @@ -81,7 +92,7 @@ describe("RunQueue queue-metrics emission", () => { definition, flag: { enabled: () => true }, }); - await emitter.waitUntilReady(); + await emitterReady(emitter); const queue = new RunQueue({ name: "rq", @@ -184,7 +195,7 @@ describe("RunQueue queue-metrics emission", () => { definition, flag: { enabled: () => true }, }); - await emitter.waitUntilReady(); + await emitterReady(emitter); const queue = new RunQueue({ name: "rq", tracer: trace.getTracer("rq"), @@ -257,7 +268,7 @@ describe("RunQueue queue-metrics emission", () => { maxLen: 1000, }; const emitter = new MetricsStreamEmitter({ redis, definition, flag: { enabled: () => true } }); - await emitter.waitUntilReady(); + await emitterReady(emitter); const queue = new RunQueue({ name: "rq", tracer: trace.getTracer("rq"), @@ -357,7 +368,7 @@ describe("RunQueue queue-metrics emission", () => { flag: { enabled: () => true }, gaugeSampleRate: 0, }); - await emitter.waitUntilReady(); + await emitterReady(emitter); const queue = new RunQueue({ name: "rq", tracer: trace.getTracer("rq"), From 306ea8aaf48de6f8eb46dce786812eae26ada2d4 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Mon, 31 Aug 2026 18:51:27 +0100 Subject: [PATCH 25/28] test(run-engine): abort the readiness race timer so its losing branch cannot reject unhandled --- .../run-engine/src/run-queue/metrics.test.ts | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/internal-packages/run-engine/src/run-queue/metrics.test.ts b/internal-packages/run-engine/src/run-queue/metrics.test.ts index 8b3872ddc2f..4c3f17125cf 100644 --- a/internal-packages/run-engine/src/run-queue/metrics.test.ts +++ b/internal-packages/run-engine/src/run-queue/metrics.test.ts @@ -26,13 +26,16 @@ const authenticatedEnvDev = { // A dead Redis leaves waitUntilReady() pending forever (the client retries // indefinitely), which would burn the whole test timeout with no diagnostic. +// Races values, not throws: a rejection in the losing branch of a settled race +// is an unhandled rejection, so the timer is aborted and swallowed instead. async function emitterReady(emitter: MetricsStreamEmitter) { - await Promise.race([ - emitter.waitUntilReady(), - setTimeout(15_000).then(() => { - throw new Error("metrics emitter Redis connection never became ready"); - }), - ]); + const abort = new AbortController(); + const timedOut = setTimeout(15_000, "timeout", { signal: abort.signal }).catch(() => "aborted"); + const winner = await Promise.race([emitter.waitUntilReady().then(() => "ready"), timedOut]); + abort.abort(); + if (winner === "timeout") { + throw new Error("metrics emitter Redis connection never became ready"); + } } async function readAllEntries( From 5d26c215dd60baa28c70d7cd36e716cecc8c841d Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Mon, 31 Aug 2026 18:56:07 +0100 Subject: [PATCH 26/28] test(run-engine): close the emitter when the readiness wait times out --- internal-packages/run-engine/src/run-queue/metrics.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/internal-packages/run-engine/src/run-queue/metrics.test.ts b/internal-packages/run-engine/src/run-queue/metrics.test.ts index 4c3f17125cf..d82ddb35305 100644 --- a/internal-packages/run-engine/src/run-queue/metrics.test.ts +++ b/internal-packages/run-engine/src/run-queue/metrics.test.ts @@ -34,6 +34,7 @@ async function emitterReady(emitter: MetricsStreamEmitter) { const winner = await Promise.race([emitter.waitUntilReady().then(() => "ready"), timedOut]); abort.abort(); if (winner === "timeout") { + await emitter.close().catch(() => {}); throw new Error("metrics emitter Redis connection never became ready"); } } From 5d5ac6080a756b8883cf24479de6a0415761c114 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Mon, 31 Aug 2026 19:01:29 +0100 Subject: [PATCH 27/28] test(metrics-pipeline,run-engine): readiness wait for the per-stream test, honest timer comment The per-stream batches test carried the same first-emission drop race as its siblings; it now waits for the emitter connection too. The readiness helper's comment claimed a losing race branch rejects unhandled, which is not how Promise.race behaves (it handles every input); the abort's real benefit is releasing the timer promptly. --- internal-packages/metrics-pipeline/src/consumer.test.ts | 1 + internal-packages/run-engine/src/run-queue/metrics.test.ts | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/internal-packages/metrics-pipeline/src/consumer.test.ts b/internal-packages/metrics-pipeline/src/consumer.test.ts index cb111ee1534..f9f59335249 100644 --- a/internal-packages/metrics-pipeline/src/consumer.test.ts +++ b/internal-packages/metrics-pipeline/src/consumer.test.ts @@ -157,6 +157,7 @@ redisTest( }); await consumer.start(); + await emitter.waitUntilReady(); emitter.emit(a, { op: "enqueue", q: a }); emitter.emit(b, { op: "enqueue", q: b }); await waitFor(() => inserted.flatMap((i) => i.rows).length >= 2); diff --git a/internal-packages/run-engine/src/run-queue/metrics.test.ts b/internal-packages/run-engine/src/run-queue/metrics.test.ts index d82ddb35305..1b47238db89 100644 --- a/internal-packages/run-engine/src/run-queue/metrics.test.ts +++ b/internal-packages/run-engine/src/run-queue/metrics.test.ts @@ -26,8 +26,8 @@ const authenticatedEnvDev = { // A dead Redis leaves waitUntilReady() pending forever (the client retries // indefinitely), which would burn the whole test timeout with no diagnostic. -// Races values, not throws: a rejection in the losing branch of a settled race -// is an unhandled rejection, so the timer is aborted and swallowed instead. +// The abort releases the losing timer promptly so it cannot hold an event +// loop open for the remaining 15s after a fast ready. async function emitterReady(emitter: MetricsStreamEmitter) { const abort = new AbortController(); const timedOut = setTimeout(15_000, "timeout", { signal: abort.signal }).catch(() => "aborted"); From d9458dbec488373833ec7cf4b154179689b88f28 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Mon, 31 Aug 2026 19:10:57 +0100 Subject: [PATCH 28/28] test(run-engine): fire-and-forget the emitter close on readiness timeout A quit written to a socket that accepted but never completes the handshake never settles, which made the diagnostic throw unreachable. Closing without awaiting keeps the fast, descriptive failure. --- internal-packages/run-engine/src/run-queue/metrics.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal-packages/run-engine/src/run-queue/metrics.test.ts b/internal-packages/run-engine/src/run-queue/metrics.test.ts index 1b47238db89..edae6f8cc30 100644 --- a/internal-packages/run-engine/src/run-queue/metrics.test.ts +++ b/internal-packages/run-engine/src/run-queue/metrics.test.ts @@ -34,7 +34,7 @@ async function emitterReady(emitter: MetricsStreamEmitter) { const winner = await Promise.race([emitter.waitUntilReady().then(() => "ready"), timedOut]); abort.abort(); if (winner === "timeout") { - await emitter.close().catch(() => {}); + void emitter.close().catch(() => {}); throw new Error("metrics emitter Redis connection never became ready"); } }